diff --git a/.github/workflows/application-acl-postgres16-proof.yml b/.github/workflows/application-acl-postgres16-proof.yml new file mode 100644 index 0000000000..a925a68566 --- /dev/null +++ b/.github/workflows/application-acl-postgres16-proof.yml @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +name: Application ACL PostgreSQL 16 Proof + +on: + pull_request: + branches: [main, dev, develop] + paths: + - ".github/workflows/application-acl-postgres16-proof.yml" + - "docker/application-acl-proof/**" + - "scripts/generate_application_database_acl.py" + - "scripts/ci/prove_application_database_acl.py" + - "src/omnibase_infra/validation/application_database_acl.py" + - "src/omnibase_infra/validation/enums/**" + - "src/omnibase_infra/validation/models/**" + - "src/omnibase_infra/validation/types/**" + - "tests/fixtures/application_database_acl/**" + - "tests/unit/validation/test_application_database_acl.py" + - "tests/ci/test_application_database_acl_contract.py" + - "tests/integration/test_application_acl_postgres16_proof.py" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + rebuilt-postgres16-proof: + name: Rebuilt PostgreSQL 16 ACL and rollback proof + env: + COMPOSE_PROJECT_NAME: application-acl-proof-${{ github.run_id }}-${{ github.run_attempt }} + permissions: + contents: read + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON(vars.OMNI_DOCKER_CI_RUNS_ON_JSON || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Verify Docker engine + run: docker info + + - name: Rebuild and prove ACL matrix + run: >- + docker compose + -f docker/application-acl-proof/compose.yml + up --build --abort-on-container-exit --exit-code-from proof + + - name: Tear down synthetic proof + if: always() + run: >- + docker compose + -f docker/application-acl-proof/compose.yml + down --volumes --remove-orphans diff --git a/.github/workflows/artifact-reconciliation-webhook.yml b/.github/workflows/artifact-reconciliation-webhook.yml index 44c05ca13e..382da2a3d0 100644 --- a/.github/workflows/artifact-reconciliation-webhook.yml +++ b/.github/workflows/artifact-reconciliation-webhook.yml @@ -21,14 +21,22 @@ on: pull_request: types: [opened, synchronize, closed, reopened] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # OMN-15730 / CodeRabbit: this job publishes a durable Kafka event with no + # reconciliation/replay path downstream. cancel-in-progress:true can kill + # the publish step mid-flight on a rapid-fire event sequence (e.g. + # synchronize immediately followed by closed), permanently dropping that + # lifecycle event. Do not cancel in-flight runs of this workflow. + cancel-in-progress: false + jobs: publish-pr-webhook: name: Publish PR Webhook Event runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index debc4df99f..809a54ba74 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -58,6 +58,22 @@ permissions: contents: write pull-requests: write +concurrency: + # OMN-15730 / CodeRabbit: check_suite runs do not populate + # github.event.pull_request, so the || github.ref fallback resolves to the + # default branch ref for EVERY check_suite-triggered run regardless of + # which PR's checks completed -- unrelated PRs' auto-merge attempts landed + # in one shared group and cancelled each other. Use the check_suite PR + # list explicitly before falling back to the run id. + group: >- + ${{ github.workflow }}-${{ + github.event.pull_request.number || + github.event.inputs.pr_number || + (github.event.check_suite.pull_requests[0].number) || + github.run_id + }} + cancel-in-progress: true + jobs: auto-merge: timeout-minutes: 30 @@ -127,7 +143,7 @@ jobs: - name: Set up Python 3.12 if: steps.resolve.outputs.skip != 'true' && steps.resolve.outputs.actor == 'jonahgabriel' - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: "3.12" diff --git a/.github/workflows/auto-tag-on-merge.yml b/.github/workflows/auto-tag-on-merge.yml index 59f920dbe7..397c65a369 100644 --- a/.github/workflows/auto-tag-on-merge.yml +++ b/.github/workflows/auto-tag-on-merge.yml @@ -38,12 +38,15 @@ jobs: startsWith(github.event.pull_request.title, 'chore: release') || startsWith(github.event.pull_request.title, 'chore(release)')) timeout-minutes: 15 - # OMNI_RUNNER_SELECTOR_V1 — a lightweight git tag + workflow dispatch; run on - # the public/GitHub-hosted pool so it is not blocked by self-hosted fleet - # saturation (this is the pool the prior reusable already used for - # pull_request events). + # A merged PR from a fork remains on GitHub-hosted compute. Releases from + # this repository use the trusted CI selector. runs-on: >- - ${{ fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') }} + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: diff --git a/.github/workflows/baselines-scheduler.yml b/.github/workflows/baselines-scheduler.yml index f161757921..31a5d195ad 100644 --- a/.github/workflows/baselines-scheduler.yml +++ b/.github/workflows/baselines-scheduler.yml @@ -47,7 +47,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/build-and-push-migrate-image.yml b/.github/workflows/build-and-push-migrate-image.yml index 7c38abbca9..d08df97065 100644 --- a/.github/workflows/build-and-push-migrate-image.yml +++ b/.github/workflows/build-and-push-migrate-image.yml @@ -54,7 +54,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python for AWS CLI - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.12" diff --git a/.github/workflows/build-and-push-runtime.yml b/.github/workflows/build-and-push-runtime.yml index 79da80212b..c9b5328668 100644 --- a/.github/workflows/build-and-push-runtime.yml +++ b/.github/workflows/build-and-push-runtime.yml @@ -103,7 +103,7 @@ jobs: fi - name: Set up Python for AWS CLI - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} @@ -192,6 +192,26 @@ jobs: severity: 'CRITICAL,HIGH' version: 'v0.69.3' + # OMN-15676: prove the required config files are actually IN the image + # before it can be pushed. Three incidents (grants fixture, routing_tiers, + # runner_fleet) shipped images whose repo tree was perfectly correct and + # whose runtime boot-crashed on a missing /app/config file. The registry + # this reads is src/omnibase_infra/runtime/required_image_config_paths.py; + # it runs `test -f` INSIDE the built image, not against the checkout, and + # fails closed on a probe that cannot run. + - name: Assert required config paths exist in the built image (OMN-15676) + run: | + set -euo pipefail + # OMN-15860: this step's python3 is the bare interpreter set up by + # "Setup Python for AWS CLI" above -- no uv/dependency-install + # infrastructure exists anywhere else in this job. required_image_config_paths.py + # (imported by the script below) imports pydantic; without this install + # every run since OMN-15676 added that import has crashed here with + # ModuleNotFoundError, before the image is ever pushed to ECR. + python3 -m pip install --quiet "pydantic>=2.11.7,<3.0.0" + python3 scripts/ci/assert_image_config_paths.py \ + --image "${{ steps.meta.outputs.image_repo }}:${{ github.sha }}" + - name: Push image to Amazon ECR if: success() shell: bash diff --git a/.github/workflows/build-workspace-candidate-runtime.yml b/.github/workflows/build-workspace-candidate-runtime.yml index 3d54e1661e..53c5c805a1 100644 --- a/.github/workflows/build-workspace-candidate-runtime.yml +++ b/.github/workflows/build-workspace-candidate-runtime.yml @@ -59,9 +59,10 @@ jobs: permissions: id-token: write contents: read - # OMNI_RUNNER_SELECTOR_V1 — trusted Docker job on self-hosted omnibase-ci. - runs-on: >- - ${{ fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} + # OMN-14974: use clean cloud egress for ECR. The .201 self-hosted Docker + # daemon is a proven-unreliable push lane (the clean-main runtime workflow + # moved off it in OMN-13747 for the same EOF/stalled-upload failure mode). + runs-on: ubuntu-latest timeout-minutes: 75 outputs: @@ -89,7 +90,7 @@ jobs: driver: docker - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -147,7 +148,7 @@ jobs: cat workspace/sibling-vcs-provenance.json - name: Set up Python for AWS CLI - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -287,6 +288,18 @@ jobs: [ "$PC" = "stability-candidate" ] || { echo "::error::promotion_class label must be stability-candidate"; exit 1; } [ "$NL" = "true" ] || { echo "::error::non_main_lineage label must be true"; exit 1; } + # OMN-15676: prove the required config files are actually IN the candidate + # image before it can be pushed and pinned. The onex-dev runtime digest is + # pinned from exactly this workflow's output (e.g. omninode_infra#802), so + # this is the step that stops a missing COPY from reaching the cluster. + # Reads src/omnibase_infra/runtime/required_image_config_paths.py and runs + # `test -f` INSIDE the built image; fails closed if the probe cannot run. + - name: Assert required config paths exist in the built image (OMN-15676) + run: | + set -euo pipefail + python3 scripts/ci/assert_image_config_paths.py \ + --image "${{ steps.meta.outputs.image_repo }}:${{ steps.meta.outputs.candidate_tag }}" + - name: Push candidate image to Amazon ECR if: success() shell: bash diff --git a/.github/workflows/call-occ-companion-effect.yml b/.github/workflows/call-occ-companion-effect.yml index 4b77f3a63f..08f3cb4490 100644 --- a/.github/workflows/call-occ-companion-effect.yml +++ b/.github/workflows/call-occ-companion-effect.yml @@ -55,12 +55,15 @@ # OMN-14941 PR merges (the SHA does not exist yet), so `@dev` is the only # viable resolvable pin today. # -# SEQUENCING: this pin resolves only AFTER the omniclaude OMN-14941 PR (adding -# call-occ-companion-effect-reusable.yml) merges to omniclaude dev. Until -# then, this workflow fails at parse ("workflow was not found") and the -# tests/ci/test_workflow_uses_refs_resolve.py gate is RED on this repo — -# merge the omniclaude PR first. That gate enforces the ordering -# mechanically instead of by convention. +# SEQUENCING (SATISFIED — OMN-15248): this pin resolved only AFTER the +# omniclaude OMN-14941 PR (adding call-occ-companion-effect-reusable.yml) +# merged to omniclaude dev. That merge has happened — verified 2026-07-27: +# `gh api repos/OmniNode-ai/omniclaude/contents/.github/workflows/call-occ-companion-effect-reusable.yml?ref=dev` +# -> 200, blob c540d981 (autobind sibling -> 5f8f64e4). The resolution gate +# (tests/integration/ci/test_workflow_uses_refs_resolve_live.py, live half; +# tests/ci/test_workflow_uses_refs_resolve.py, static half) is therefore +# expected GREEN. A red there now means a pin genuinely broke — re-pin it; +# there is no standing red-authorization for this workflow. # # RE-PIN FOLLOW-UP (OMN-14812-style): once the reusable promotes to omniclaude # main, re-pin this `uses:` to `@main` or a main SHA so the caller no longer diff --git a/.github/workflows/call-reject-skip.yml b/.github/workflows/call-reject-skip.yml index 7202a444fb..aab6f1f635 100644 --- a/.github/workflows/call-reject-skip.yml +++ b/.github/workflows/call-reject-skip.yml @@ -30,5 +30,5 @@ permissions: jobs: call-reject-skip-token: - uses: OmniNode-ai/omniclaude/.github/workflows/reject-deploy-gate-skip.yml@ff230264ac3300d7ced43564dc921f44558110fe + uses: OmniNode-ai/omniclaude/.github/workflows/reject-deploy-gate-skip.yml@80de61fd1fee04abdeb6918e7f91cf820717e6a8 secrets: inherit diff --git a/.github/workflows/canonical-inference-gate.yml b/.github/workflows/canonical-inference-gate.yml index 4ed3145110..39a5220a25 100644 --- a/.github/workflows/canonical-inference-gate.yml +++ b/.github/workflows/canonical-inference-gate.yml @@ -53,12 +53,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 - trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest. runs-on: >- ${{ - (github.event_name == 'pull_request' && github.base_ref == 'dev') - && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') - || ( (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - ) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/check-env-reads-gate.yml b/.github/workflows/check-env-reads-gate.yml index 80acae22b3..26e6008b91 100644 --- a/.github/workflows/check-env-reads-gate.yml +++ b/.github/workflows/check-env-reads-gate.yml @@ -47,9 +47,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 - trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest. runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/check-handshake.yml b/.github/workflows/check-handshake.yml index a399b8caa9..a2209c77b0 100644 --- a/.github/workflows/check-handshake.yml +++ b/.github/workflows/check-handshake.yml @@ -19,9 +19,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6591bb747f..e3338f8e09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,19 @@ on: branches: [main] pull_request: branches: [main, dev] + # OMN-14241: `edited` is required so a PR-body-only edit (the common case + # is stamping `Evidence-Source: OCC#` after the companion OCC PR + # merges) retriggers this workflow's `occ-preflight` job. The reusable + # occ-preflight.yml already live-fetches the current PR body via + # `gh pr view` on every run -- the defect was never a frozen-body read, + # it was that no new run fired at all, so the last commit-triggered + # FAILURE (correct at that instant, before the stamp existed) sat as the + # permanent status for `occ-preflight / eligibility` and CI Summary + # failed closed on it forever (live case: infra#2696, #2694). GitHub's + # implicit default when `types:` is omitted is + # [opened, synchronize, reopened], which is what this workflow ran on + # before this fix. + types: [opened, edited, synchronize, reopened] merge_group: workflow_dispatch: @@ -59,14 +72,14 @@ jobs: # evidence is missing or red, no runner-heavy validation jobs fan out. occ-preflight: # self-gating-ok: target is in omnibase_core, not a required check on omnibase_infra main - uses: OmniNode-ai/omnibase_core/.github/workflows/occ-preflight.yml@789d175d78a7a802f4f0f4aa2af7083bdfd312c2 + uses: OmniNode-ai/omnibase_core/.github/workflows/occ-preflight.yml@de01ec0964a2e666d739835b02957ae0f06d6a25 permissions: contents: read pull-requests: read with: contracts-dir: contracts receipts-dir: drift/dod_receipts - core-ref: 8a47e092002dd1338599fa0733feda469436acbf + core-ref: dev # ============================================================================= # Phase 0 - Zone filter (OMN-10475) @@ -95,9 +108,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -281,6 +293,39 @@ jobs: --receipts-dir scripts/ci/adequacy_receipts \ --full + # Fail-closed flip-bundle SEAM gate fan-out (OMN-14809 / OMN-15340 / OMN-15344). + # The ratchet above proves a node's SHAPE. It is structurally blind to whether the + # PROOF a flip carries is coherent, and — since OMN-15340 — to whether the declared + # hand-flip parity tests are RED-for-the-right-reason at the receipt's own base_ref. + # A parity test that is ALREADY GREEN on the pre-change tree discriminates nothing, + # and the flip is credited with a behavior proof it never earned. + # + # That mechanism shipped in omnibase_core#1519 but ran in omnibase_core's CI ONLY, + # while ALL 22 committed hand-flip receipts live in this repo (15) and + # omnimarket/omnibase_infra — zero coverage over 100% of the population it was + # built for. This step wires it HERE, in the same job, from the sibling + # .ci/omnibase_core checkout already cloned above: no new job, no new workflow, no new + # hook (net-negative-surface). Flags mirror the ratchet's so it reads THIS repo's + # tree/receipts/baseline, not the ephemeral core checkout's — same --receipts-dir + # reason as OMN-14633 above. + # + # NOT path-filtered, deliberately. The gate self-scopes to nodes NEWLY flipped vs + # origin/dev and exits 0 when there are none (measured 2.4s on .200), so an + # everyday PR pays nothing; a path filter on a fail-closed proof gate is a bypass. + # + # It also enforces scripts/ci/parity_red_on_base_baseline.py as a SHRINK-ONLY + # ratchet: growth hard-fails, a stale entry hard-fails, and removing an entry + # re-arms assertion 7 for that node and EXECUTES it. The inline fetch makes + # origin/dev's tip available for flip-discovery (this checkout is shallow); the + # gate NOTEs and exits 0 if it is still unavailable, exactly as in core. + - name: Canonical-shape flip-bundle seam gate (OMN-15344 fan-out) + run: | + git fetch --no-tags --prune origin +refs/heads/dev:refs/remotes/origin/dev || true + uv run python .ci/omnibase_core/scripts/ci/verify_flip_bundle.py \ + --package omnibase_infra --src-root src \ + --baseline scripts/ci/canonical_handler_shape_baseline.py \ + --receipts-dir scripts/ci/adequacy_receipts + - name: Run mypy type checking run: uv run mypy src/omnibase_infra @@ -301,9 +346,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -351,9 +395,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -389,6 +432,21 @@ jobs: echo "================================================================" uv run pytest tests/audit/test_io_violations.py::TestCIGateIOPurity -v --tb=short + # OMN-15547: every enforcement guard must replay a REAL incident. + # Rides this job on purpose — "ONEX Validators" is already in + # ci_summary_gate.STRICT_GATE_JOBS, so the check gates merge through the + # sole required context without adding a workflow, a job, or a required + # context (net-negative-surface rule). Blocking from the first run: the + # debt is baselined in the registry, so the only way to fail on day one is + # to wire a NEW guard with no real regression case — which is the exact + # behaviour this gate exists to stop. + - name: Incident-replay coverage (OMN-15547) + run: | + echo "================================================================" + echo "Incident-Replay Coverage (OMN-15547)" + echo "================================================================" + uv run python scripts/ci/check_incident_replay_coverage.py + # ARCH-004 imperative-orchestrator ratchet (OMN-13472). # Full report over the repo: surfaces every contract-declared-but-unbound # orchestrator FSM driven imperatively by a handler (the delegation-shaped @@ -470,9 +528,14 @@ jobs: infra-node-handler-ownership: needs: occ-preflight name: Infra Node Handler Ownership - # Pure repository ownership lint. Keep it hosted so self-hosted saturation - # cannot cancel this short job and falsely fail CI Summary. - runs-on: ubuntu-latest + # OMNI_RUNNER_SELECTOR_V1 — only public-fork PRs use hosted compute. + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} timeout-minutes: 5 steps: @@ -490,9 +553,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -524,9 +586,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -559,9 +620,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -590,9 +650,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -626,9 +685,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -663,6 +721,163 @@ jobs: --check-placeholders \ --scan-dir src/omnibase_infra + # Pin reachability -- every cross-repo pin (workflow `uses:`/`with:` refs, + # pyproject [tool.uv.sources] revs, uv.lock git revs) must name a commit + # reachable from a PROTECTED branch of the target repo (OMN-15538). + # + # Why this is a required gate and not a sweep: on 2026-07-30 two pins that + # every existing validator accepted took production down twice. ci.yml + # pinned a deleted omnimarket branch head and startup-failed for ~2.5h with + # every open PR unmergeable (OMN-15536); omnimarket@dev pinned an unlanded + # omnibase_core branch head straight past a prose comment forbidding it. The + # pre-existing validators check hex SHAPE (`[0-9a-f]{40}`) and conclude + # "immutable, therefore safe". Immutable is not reachable. + pin-reachability: + needs: occ-preflight + if: always() + name: Pin Reachability (OMN-15538) + # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + # Raised from 10 to 18 (2026-08-07, PR #2679 comment 5211687650): a + # 38-run fleet measurement of THIS job found setup time (job start -> + # script step start) alone had median 317s and max 613s under the + # existing cache-enabled: "false" -- 6/38 runs were killed by the old + # timeout-minutes: 10 (600s) DURING setup, before the script step ever + # ran. ``cache-enabled: "false"`` is intentionally NOT flipped here: this + # repo's self-hosted-fleet CI already replaced the per-job uv cache with + # the shared host-local dependency-environment canary (see + # ``docs/ci/versioned-ci-env-canary.md``), which explicitly says "Do not + # enable setup-uv cache save for direct jobs that intentionally run + # ``uv sync --no-cache``; the post-job cache upload can outlive the + # actual check and cancel merge-group CI while adding no useful reuse" -- + # and ``tests/ci/test_ci_workflow_resilience.py::test_short_gates_can_disable_uv_cache_cleanup`` + # enforces ``cache-enabled: "false"`` on every ci.yml job uniformly for + # exactly that reason. The fix here is timeout-minutes alone; the + # derivation below is built on the measured fleet MAX (not an assumed + # cache improvement) so it is correct with or without any future cache + # change: + # 613s (measured max setup, fleet n=38) + # + 150s (script _RUN_DEADLINE_SECONDS, see check_pin_reachability.py) + # + 150s (worst-case post-deadline tail: one in-flight rate-limited call, + # 3 * 30s timeouts + 2 * 30s capped backoffs) + # = 913s < 1080s, ~167s (~15%) real margin. + # See scripts/ci/check_pin_reachability.py's _RUN_DEADLINE_SECONDS + # comment for the full script-side derivation, and + # tests/ci/test_pin_reachability_omn15538.py::test_pin_reachability_job_timeout_covers_measured_budget + # for the binding test that parses this value from ci.yml and fails + # closed if it stops covering the budget (or is gamed absurdly high). + timeout-minutes: 18 + + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Setup Python and uv + uses: ./.github/actions/setup-python-uv + with: + python-version: ${{ env.PYTHON_VERSION }} + uv-version: ${{ env.UV_VERSION }} + cache-version: ${{ env.CACHE_VERSION }} + cache-enabled: "false" + + - name: Every cross-repo pin must be reachable from dev or main + env: + # Read-only, public repos: the compare endpoint works unauthenticated + # (60 req/hr). The ambient token is attached purely for rate-limit + # headroom so a busy PR queue cannot turn the gate UNDETERMINED. + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "================================================================" + echo "Pin Reachability (OMN-15538)" + echo "================================================================" + # --min-pins is the anti-vacuity floor: this tree carries >20 pins, so + # an extractor that suddenly finds fewer than 5 is broken, not clean. + # No --allow-undetermined: the script REFUSES that flag when CI is set, + # so an unresolvable pin fails the gate instead of skipping silently. + uv run python scripts/ci/check_pin_reachability.py --min-pins 5 + + # Dep-provenance content-lineage gate (OMN-15604) -- a [tool.uv.sources] + # git-pinned rev must build the SAME src/ tree as the released tag its + # PyPI version string declares, even when the line carries a + # `# raw-override-ok:` escape token. + # + # Why this is a separate gate from `Dep Provenance Gate` (OMN-13873, + # .github/workflows/dep-provenance-gate.yml): that gate forbids a git + # source unconditionally but exempts any line with a non-empty escape + # token -- shape-only, never checked against content. Live incident this + # closes: omnibase_infra@dev declared omnibase-core==0.46.8 while pinning + # git rev 3d51b047 (escaped via `# raw-override-ok: OMN-15414`) whose + # src/ tree measurably DIFFERED from released tag v0.46.8 (9 files, + # +381/-27 lines) -- invisible to every existing check because the escape + # token exempted the only gate that looks at this file at all. This job + # lives in ci.yml (not the standalone dep-provenance-gate.yml workflow) + # so it is immediately observable by the `CI Summary` STRICT_GATE_JOBS + # assertion without needing the EXPECTED_EXTERNAL_CONTEXTS admission + # rule's historical report-rate measurement, which a brand-new context + # cannot satisfy on day one. + dep-provenance-lineage-gate: + needs: occ-preflight + if: always() + name: Dep Provenance Lineage Gate (OMN-15604) + # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 10 + + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Setup Python and uv + uses: ./.github/actions/setup-python-uv + with: + python-version: ${{ env.PYTHON_VERSION }} + uv-version: ${{ env.UV_VERSION }} + cache-version: ${{ env.CACHE_VERSION }} + cache-enabled: "false" + + - name: A git-pinned override must build the released tree of its declared version + env: + # Read-only, public repos: the commits/trees endpoints work + # unauthenticated (60 req/hr). The ambient token is attached purely + # for rate-limit headroom so a busy PR queue cannot turn the gate + # UNDETERMINED. + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "================================================================" + echo "Dep Provenance Lineage Gate (OMN-15604)" + echo "================================================================" + # No --allow-undetermined-lineage: the script REFUSES that flag + # when CI is set, so an unresolvable pin fails the gate instead of + # skipping silently. --check-lineage runs the network-dependent + # half in addition to the existing offline forbid-git-source check. + uv run python scripts/check_dep_provenance.py --check-lineage + + - name: A raw-override-ok escape token must not be expired or cite a closed ticket + env: + # Optional: the ticket-status half of this check gracefully + # degrades (skips, does not fail) when LINEAR_API_KEY is unset -- + # same posture as this org's other already-shipped, LINEAR_API_KEY- + # gated Linear check (see stale-todo-gate.yml). The until= date + # half of the check needs no credential and always runs. + LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }} + run: | + echo "================================================================" + echo "Dep Provenance Escape-Token Reconciliation (OMN-15604 AC3)" + echo "================================================================" + uv run python scripts/check_dep_provenance.py --check-token-expiry + # Topic drift check -- CI gate validating SUFFIX_* constants against contracts (OMN-5248/OMN-5258) # Ensures static topic constants stay in sync with contract.yaml topic declarations. # Cross-repo scanning (OMN-5258): checks out sibling repos to resolve topic references. @@ -673,9 +888,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -745,9 +959,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -859,9 +1072,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -893,9 +1105,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -927,9 +1138,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 -- trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -965,9 +1175,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1000,9 +1209,8 @@ jobs: name: Contract Path Pre-Flight runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1049,9 +1257,8 @@ jobs: name: Contract Sync Gate (Wave C) [OMN-8915] runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1090,9 +1297,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1175,9 +1381,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1225,6 +1430,18 @@ jobs: - name: Run pytest (smart selection) if: vars.ENABLE_SMART_TESTS == 'true' && needs.detect-changes.outputs.is_full_suite == 'false' + # OMN-15233: tests/integration/ci/test_workflow_uses_refs_resolve_live.py + # resolves every cross-repo `uses:` pin against the GitHub contents API + # and FAILS CLOSED when a pin is unverifiable. Without a token those + # requests are unauthenticated and share the runner egress IP's 60/hr + # budget, so they return HTTP 403 "API rate limit exceeded" and the + # whole split goes red for a reason that has nothing to do with the + # pins. The test's own failure message prescribes this fix ("thread + # GH_TOKEN into the test step"); it reads GH_TOKEN or GITHUB_TOKEN. No + # test in this suite skips on token presence, so this only authenticates + # existing requests — it does not activate new ones. + env: + GH_TOKEN: ${{ github.token }} run: | paths=$(echo '${{ needs.detect-changes.outputs.selected_paths }}' | jq -r '.[]') if [ -z "$paths" ]; then @@ -1253,8 +1470,25 @@ jobs: - name: Run pytest (full suite) if: vars.ENABLE_SMART_TESTS != 'true' || needs.detect-changes.outputs.is_full_suite == 'true' + # OMN-15233: see the smart-selection step above — the live `uses:` pin + # resolution test needs an authenticated token or it fails closed on + # HTTP 403 rate limiting. This step is the one that surfaced it. + # + # OMN-15410: NO positional path is passed to pytest here, deliberately. + # This step used to say `uv run pytest tests/`, which silently made + # this workflow — not pyproject.toml — the real definition of "the full + # suite": four collocated test roots (scripts/ci/tests, scripts/tests, + # scripts/runtime_build/tests, and the agent_actions root) existed on + # disk and were collectable, but no CI job ever named them, so 366 + # tests never ran and 4 of them had been RED since 2026-04-02 + # undetected. With no positional argument pytest inherits + # `testpaths` from pyproject.toml verbatim, so there is exactly one + # place to add a root. scripts/validation/validate_test_root_collection.py + # fails closed if a positional path is re-introduced here. + env: + GH_TOKEN: ${{ github.token }} run: | - uv run pytest tests/ \ + uv run pytest \ --ignore=tests/integration/docker \ -m "not slow and not chaos and not kafka and not performance" \ --splits ${{ needs.detect-changes.outputs.split_count }} \ @@ -1296,9 +1530,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1367,9 +1600,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1428,9 +1660,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1463,9 +1694,8 @@ jobs: name: Cross-Repo Migration Conflicts runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1485,9 +1715,8 @@ jobs: name: Contract Compliance Check runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1579,9 +1808,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1820,9 +2048,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — runners are resolved from centralized selector variables. runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1871,7 +2098,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 - trusted events default to self-hosted omnibase-ci; pull_request defaults to ubuntu-latest runs-on: >- ${{ - github.event_name == 'pull_request' + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -1882,7 +2110,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} @@ -1926,13 +2154,19 @@ jobs: # ratchet fails the required "CI Summary" check (no default-deny race). Installs # the omnibase_core validator from the pinned core dev SHA and ratchets # src/omnibase_infra's frozen allowlist DOWN toward zero. - runs-on: ubuntu-latest + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} timeout-minutes: 10 steps: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - name: Install uv @@ -1948,6 +2182,370 @@ jobs: --allowlist .onex_ratchets/noncanonical_class_allowlist.yaml \ src/omnibase_infra + occ-companion-merged: + name: OCC Companion Merged Gate (OMN-15214) + # OMN-15214: makes the 2026-07-26 OCC hygiene-sweep trigger state — an OPEN + # onex_change_control evidence companion whose product PR already MERGED — + # unreachable via the merge path on this repo. The sweep closed five open + # companions (OCC#5012–#5016) whose product PRs had merged, destroying three + # evidence chains; the durable fix is ordering: the companion must MERGE + # BEFORE the product PR can. This job polls the PR's live Evidence-Source + # (live body, not event payload — occ-autobind PATCHes it in after open) and + # greens ONLY when the cited OCC companion PR is MERGED or the cited OCC + # commit SHA is an ancestor of onex_change_control dev/main. A companion + # CLOSED without merging fails immediately (that IS the incident state). + # + # STRICT CI Summary gate (registered in scripts/ci/ci_summary_gate.py + # STRICT_GATE_JOBS) — deliberately NOT a new top-level required context: a + # context that does not report on every PR shape wedges merges indefinitely + # under branch protection, and occ-autobind itself is network-flaky + # (omninode_infra#679, 2026-07-26). Unconditional (no needs/if) so CI + # Summary always WAITS for it and a skip fails closed; internal event + # scoping no-ops on push/workflow_dispatch (mirrors occ-preflight). + # The poller is read-only, but must follow the fork-safe selector used by + # the rest of CI rather than pinning all internal PRs to hosted compute. + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 30 + permissions: + contents: read + pull-requests: read + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + MERGE_GROUP_HEAD_REF: ${{ github.event.merge_group.head_ref }} + # Poll window (seconds) — bounded well under ci-summary's 90m deadline so + # this gate reaches a terminal state first; PENDING converts to FAIL. + DEADLINE_SECONDS: "1500" + POLL_INTERVAL_SECONDS: "30" + steps: + - name: Checkout (gate script) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Require cited OCC evidence to be merged/durable before product merge + run: python3 scripts/ci/check_occ_companion_merged.py + + # OMN-15378 AC3: scripts/deploy-agent's own pytest root (201 tests) ran in a + # SEPARATELY-TRIGGERED workflow, so the CI Summary poller — which reads + # `actions/runs/${RUN_ID}/jobs` for its own run only — structurally could not + # observe it: a RED deploy-agent run left the sole required context on this + # branch ("CI Summary") green. Visibility without enforcement. Calling the + # same workflow from HERE puts its job in ci.yml's own run as the check-run + # "Deploy Agent Tests (OMN-15378) / deploy-agent-tests", which is registered + # in scripts/ci/ci_summary_gate.py STRICT_GATE_JOBS (pinned by + # tests/ci/test_ci_summary_gate.py) — so absent/red/skipped now fails the + # required context. Unconditional (no needs/if) exactly as a STRICT gate + # requires, which also closes the old path-filter hole where a breaking + # change outside scripts/deploy-agent/** produced no signal at all. + # DO NOT rename this job — the display name is the first segment of the + # STRICT_GATE_JOBS entry; a rename makes the gate permanently PENDING. + deploy-agent-tests: + name: Deploy Agent Tests (OMN-15378) + permissions: + contents: read + uses: ./.github/workflows/deploy-agent-tests.yml + + # OMN-15484 (fan-out of OMN-15483). The merge sweep lands PRs while + # adversarial verification is still running against them. OMN-15483 built the + # fix — a required check that fails while a PR carries a hold marker in its + # title or labels, so a held PR can never be required-green for ANY consumer + # (the Codex controller, `gh pr merge`, auto-merge, a merge queue, or the ONEX + # merge node) — but shipped it in omnimarket ONLY. omnibase_infra carries + # incident §C (#2560) in that ticket's table and had zero coverage. + # + # NO VOCABULARY IS DECLARED HERE, deliberately. The hold tokens live in + # exactly one place fleet-wide — omnimarket's + # src/omnimarket/merge_control/hold_marker.py — and the reusable workflow + # reads that module at run time. Copying it here would rebuild across repos + # the bug OMN-15483 round 1 found inside omnimarket: two divergent + # `_DO_NOT_MERGE_RE` definitions, neither a superset of the other. The shared + # workflow scans THIS repo on every run and fails if a second vocabulary + # appears (AC1), proves the gate actually fires here by driving a held vector + # through it (AC3), and checks the context name is not itself a hold token + # (AC5). + # + # UNCONDITIONAL BY DESIGN (AC4) — no `needs:`, no `if:`, no paths filter, same + # posture as `deploy-agent-tests` above and for the same reason. A hold gate + # an unrelated upstream failure can cascade-skip is not a gate. + # + # DO NOT rename this job, and do not add a `name:` — the check-run is + # " / ", i.e. + # "merge-hold-gate / evaluate", which is the exact string registered in + # scripts/ci/ci_summary_gate.py STRICT_GATE_JOBS. A rename on either side + # makes the required CI Summary context permanently PENDING (the same trap + # documented on deploy-agent-tests). tests/ci/test_merge_hold_gate_omn15484.py + # pins both halves and the STRICT registration itself. + # + # `vocabulary_ref` MUST equal the ref in `uses:`: the first selects the + # workflow file, the second selects the source it reads, and drifting them + # would run one vintage of the gate against another vintage of the tokens. + # + # PIN AN ANCESTOR OF omnimarket `dev`, NEVER A PR-HEAD SHA (OMN-15484 step 6). + # The original fan-out pinned 879d6fc6 — the PR HEAD of omnimarket#1976. That + # SHA was 40-hex and therefore satisfied every local check, but it lived only + # on the PR branch. When #1976 squash-merged at 2026-07-30T18:29:59Z the branch + # was deleted and the ref stopped resolving for Actions, so THIS ENTIRE + # WORKFLOW FILE failed to load: runs after 18:59Z reported `conclusion=failure` + # with 0 jobs and `name` degraded to the raw path. That takes down the required + # `CI Summary` context by ABSENCE rather than by failure, which branch + # protection treats as never-satisfied — every omnibase_infra dev PR based on + # 27478af4 or later was unmergeable until this repin. + # + # 54356a83 is the current reusable workflow's pinned dev commit, reachable from + # `dev` forever. + merge-hold-gate: + permissions: + contents: read + pull-requests: read + uses: OmniNode-ai/omnimarket/.github/workflows/merge-hold-gate-reusable.yml@54356a831e3d8876c69373cac884a3df2a5653f7 + with: + vocabulary_ref: 54356a831e3d8876c69373cac884a3df2a5653f7 + context_name: merge-hold-gate / evaluate + # OMN-15361: P0/P8 fail-closed application database domain enforcement. + # Unconditional after OCC eligibility and STRICT in ci_summary_gate.py: validators, + # seeded counterexamples, and rebuilt PostgreSQL 16 proofs report in this + # workflow's run and therefore gate the sole required CI Summary context. + # Deployment activation remains separately BLOCKED in the typed contract; + # this job performs no live database, secret, grant, deploy, or runtime action. + application-database-domain-enforcement: + name: Application Database Domain Enforcement (OMN-15361) + needs: occ-preflight + permissions: + contents: read + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_DOCKER_CI_RUNS_ON_JSON || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 60 + steps: + - name: Checkout enforcement source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Checkout exact sanitized legacy fixture dependency + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: OmniNode-ai/omnibase_infra + ref: 95351f5d8e806fcf7fa2c276d9065df93ccf92b9 + path: .proof-dependencies/legacy-fixture + persist-credentials: false + + - name: Checkout exact authenticated domain-adapter dependency + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: OmniNode-ai/omnibase_infra + ref: 2a2cfb275810b34197d4d5baf55bdcddc443e6dc + path: .proof-dependencies/domain-adapter + persist-credentials: false + + - name: Resolve omnimarket migration source ref + id: resolve-omnimarket-ref + shell: bash + run: python3 scripts/resolve_node_migration_source_ref.py + + - name: Live-resolve omnimarket dev HEAD + # OMN-15703: no hardcoded fallback SHA. When the trailer is absent + # (resolve-omnimarket-ref emits the literal "dev"), resolve + # omnimarket's live dev HEAD via the GitHub API at run time instead + # of pinning to a snapshot that goes stale the moment omnimarket + # dev moves again (the OMN-15701 incident). Fails loudly (non-zero + # exit) rather than silently falling back to a stale pin. + # + # Token: fork-triggered pull_request runs receive no org secrets, so + # secrets.CROSS_REPO_PAT is empty there and this step must not + # hard-depend on it. omnimarket is a PUBLIC repo, so reading its + # commits API needs no elevated scope -- github.token (always present, + # including on fork PRs) is sufficient. Prefer CROSS_REPO_PAT when + # present (same-repo/trusted runs) to stay on the higher, shared + # cross-repo rate-limit budget those runs already use elsewhere in + # this workflow (see lines ~1508/1665/1696); fall back to + # github.token so the fork lane keeps functioning rather than + # aborting before its fork-lane proof steps below. + id: resolve-omnimarket-live-head + if: steps.resolve-omnimarket-ref.outputs.ref == 'dev' + shell: bash + env: + GH_TOKEN: ${{ secrets.CROSS_REPO_PAT || github.token }} + run: | + set -euo pipefail + sha="$(gh api repos/OmniNode-ai/omnimarket/commits/dev --jq .sha)" + if [[ ! "$sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::live omnimarket dev HEAD resolution returned an invalid sha: '${sha}'" >&2 + exit 1 + fi + echo "sha=${sha}" >> "$GITHUB_OUTPUT" + + - name: Checkout exact typed registry dependency + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: OmniNode-ai/omnimarket + ref: ${{ steps.resolve-omnimarket-ref.outputs.ref == 'dev' && steps.resolve-omnimarket-live-head.outputs.sha || steps.resolve-omnimarket-ref.outputs.ref }} + path: .proof-dependencies/omnimarket + persist-credentials: false + + - name: Checkout exact service ownership dependency + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: OmniNode-ai/omninode_infra + ref: 39033d55147ef22a061b665345b506246d3aa543 + path: .proof-dependencies/omninode-infra + token: ${{ secrets.CROSS_REPO_PAT }} + persist-credentials: false + + - name: Setup Python and uv + uses: ./.github/actions/setup-python-uv + with: + python-version: ${{ env.PYTHON_VERSION }} + uv-version: ${{ env.UV_VERSION }} + cache-version: ${{ env.CACHE_VERSION }} + cache-enabled: "false" + + - name: Execute mandatory source assertions and seeded RED controls + run: >- + uv run pytest -q + tests/unit/validation/test_application_relation_ownership.py + tests/unit/validation/test_application_database_acl.py + tests/unit/validation/test_application_database_domain_enforcement.py + tests/unit/validation/test_application_database_sql_enforcement_regressions.py + tests/unit/topology/test_application_database_topology.py + tests/unit/topology/test_application_database_table_grants.py + tests/unit/runtime/auto_wiring/test_projection_domain_adapters.py + tests/ci/test_application_database_domain_enforcement_contract.py + tests/ci/test_application_database_sql_authority_isolation.py + tests/ci/test_application_database_sql_gate.py + + # OMN-15656: the shipped topology's TABLE grants are a projection of node + # contract db_io.db_tables. --check fails on drift in either direction (a + # declared table with no grant, or a grant no contract declares); --prove + # resolves every declaration through the real wiring validator against + # every shipped profile. A contract added upstream without a matching + # grant now fails here instead of failing a deploy. + - name: Enforce contract-derived application-database TABLE grants + run: >- + uv run python scripts/generate_application_database_table_grants.py + --contracts-root .proof-dependencies/omnimarket/src/omnimarket/nodes + --check --prove + + - name: Enforce schema qualification in changed SQL (trusted) + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + env: + APPLICATION_SQL_BASE_REVISION: >- + ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || '7228ce0c0934ae096dd6effd0f84ff1913fec6c0' }} + run: >- + uv run python scripts/ci/check_application_database_sql.py + --base-revision "$APPLICATION_SQL_BASE_REVISION" + --head-revision HEAD + --ownership-manifest .proof-dependencies/omnimarket/scripts/application-relation-ownership.yaml + --ownership-manifest .proof-dependencies/omninode-infra/db/migrations/application-relation-ownership.yaml + --ownership-manifest .proof-dependencies/omninode-infra/k8s/migrations/application-relation-ownership.yaml + + # Public-fork pull_request jobs cannot receive organization secrets. The + # fork lane deliberately omits the private manifest authority: any changed + # SQL targeting one of those objects is rejected as undeclared. The + # synthetic Docker-proof authority is also isolated from both production + # lanes and is proved independently by mandatory source and Docker gates. + - name: Enforce schema qualification in changed SQL (public fork, fail closed) + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository + env: + APPLICATION_SQL_BASE_REVISION: ${{ github.event.pull_request.base.sha }} + run: >- + uv run python scripts/ci/check_application_database_sql.py + --base-revision "$APPLICATION_SQL_BASE_REVISION" + --head-revision HEAD + --ownership-manifest .proof-dependencies/omnimarket/scripts/application-relation-ownership.yaml + + - name: Verify Docker engine + run: docker info + + - name: Rebuild and prove application-domain catalog gates + env: + COMPOSE_PROJECT_NAME: omn15361-domain-${{ github.run_id }}-${{ github.run_attempt }} + run: >- + docker compose + -f docker/application-domain-enforcement/compose.yml + up --build --abort-on-container-exit --exit-code-from proof + + - name: Clean application-domain proof + if: always() + env: + COMPOSE_PROJECT_NAME: omn15361-domain-${{ github.run_id }}-${{ github.run_attempt }} + run: >- + docker compose + -f docker/application-domain-enforcement/compose.yml + down --volumes --remove-orphans + + - name: Rebuild and prove authenticated domain adapters + working-directory: .proof-dependencies/domain-adapter + env: + COMPOSE_PROJECT_NAME: omn15361-adapter-${{ github.run_id }}-${{ github.run_attempt }} + run: >- + docker compose + -f docker/domain-adapter-proof/compose.yml + up --build --abort-on-container-exit --exit-code-from proof + + - name: Clean domain-adapter proof + if: always() + working-directory: .proof-dependencies/domain-adapter + env: + COMPOSE_PROJECT_NAME: omn15361-adapter-${{ github.run_id }}-${{ github.run_attempt }} + run: >- + docker compose + -f docker/domain-adapter-proof/compose.yml + down --volumes --remove-orphans + + - name: Rebuild and prove generated role and default-privilege gates + env: + COMPOSE_PROJECT_NAME: omn15361-acl-${{ github.run_id }}-${{ github.run_attempt }} + run: >- + docker compose + -f docker/application-acl-proof/compose.yml + up --build --abort-on-container-exit --exit-code-from proof + + - name: Clean generated ACL proof + if: always() + env: + COMPOSE_PROJECT_NAME: omn15361-acl-${{ github.run_id }}-${{ github.run_attempt }} + run: >- + docker compose + -f docker/application-acl-proof/compose.yml + down --volumes --remove-orphans + + - name: Rebuild and prove exact sanitized fresh and legacy fixture + working-directory: .proof-dependencies/legacy-fixture + env: + COMPOSE_PROJECT_NAME: omn15361-legacy-${{ github.run_id }}-${{ github.run_attempt }} + run: >- + docker compose + -f docker/legacy-rds-fixture/compose.yml + up --build --abort-on-container-exit --exit-code-from proof + + - name: Clean sanitized fresh and legacy fixture proof + if: always() + working-directory: .proof-dependencies/legacy-fixture + env: + COMPOSE_PROJECT_NAME: omn15361-legacy-${{ github.run_id }}-${{ github.run_attempt }} + run: >- + docker compose + -f docker/legacy-rds-fixture/compose.yml + down --volumes --remove-orphans + ci-summary: name: CI Summary # OMN-14127: REQUIRED branch-protection context — the single umbrella gate for @@ -1961,19 +2559,36 @@ jobs: # success/failure within a bounded deadline. # # DEFAULT-DENY, FAIL-CLOSED: green ONLY when every strict + skippable - # aggregate gate is present+good AND no non-allowlisted job failed. The - # auditable gating set + soft-allowlist live in scripts/ci/ci_summary_gate.py + # aggregate gate is present+good, no non-allowlisted job failed, AND every + # declared EXTERNAL context on the PR head is present+success. The auditable + # gating set + soft-allowlist live in scripts/ci/ci_summary_gate.py # (unit-tested in tests/ci/test_ci_summary_gate.py); it reproduces the exact # strictness of the old needs-based condition (13 strict `== success` gates, # 4 `success||skipped` gates) and adds a strictly-stronger default-deny sweep. - # Pure hosted GitHub-API polling: ZERO self-hosted/LAN dependency, so it is - # immune to fleet saturation. This also supersedes the (unset, no-op) - # OMNI_REQUIRED_CI_RUNS_ON_JSON merge_group runner-override that used to sit on - # this job's runs-on selector. + # + # OMN-15496: the job-list poll below only ever sees THIS run. Checks emitted + # by other workflow files are invisible to it and are ALSO absent from branch + # protection (dev requires exactly ["CI Summary"], strict=false), so they were + # enforced by neither layer — 59 such contexts on a real merged PR head, incl. + # deploy-gate, Receipt Gate, CodeRabbit Thread Check and pr-title. #2555 merged + # with this context GREEN while `deploy-gate / deploy-gate` was RED on the same + # SHA. The poll step now also asserts EXPECTED_EXTERNAL_CONTEXTS against + # commits//check-runs, fail-closed. That tuple is measured, not guessed: + # a context is admitted only at 100% merge-time report rate over recent merged + # dev PRs, because a context that does not report on every PR shape wedges the + # branch. Do NOT add names to it from a workflow file alone. + # Public fork PRs run on hosted compute; trusted events use the CI selector + # so internal PRs are not silently routed to hosted minutes. # # DO NOT rename — the job name IS the required context; a rename re-wedges # every PR. - runs-on: ubuntu-latest + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} if: always() permissions: actions: read @@ -1997,8 +2612,28 @@ jobs: GH_REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} + # OMN-15496: the PR head SHA whose CROSS-WORKFLOW check-runs are asserted + # against EXPECTED_EXTERNAL_CONTEXTS. The jobs API above only ever sees + # THIS run; every other workflow file's context is invisible to it and is + # also absent from branch protection (dev requires exactly "CI Summary"), + # so those checks previously blocked nothing. + EVENT_NAME: ${{ github.event_name }} + # OMN-15532 — PR author login. Drops ONLY the contexts that this + # author's PRs structurally cannot produce (ACTOR_CONDITIONAL_CONTEXTS: + # cr-thread-gate-caller.yml skips dependabot[bot], so + # "gate / CodeRabbit Thread Check" is never created on its PRs and + # would otherwise burn the full deadline then fail closed). Empty on + # non-PR events, which drops nothing. + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + HEAD_SHA: >- + ${{ github.event.pull_request.head.sha + || github.event.merge_group.head_sha + || github.sha }} # Bounded poll window (minutes) — kept under the run-cancel window so - # the required context always reaches a terminal state first. + # the required context always reaches a terminal state first. Measured + # headroom for the external wait (OMN-15496): over the 16 dev PRs merged + # 2026-07-29→30, the slowest asserted external context terminalized + # 24.9 min after this job started, so 90 min is not a new timeout risk. DEADLINE_MINUTES: "90" POLL_INTERVAL_SECONDS: "45" run: | @@ -2011,7 +2646,7 @@ jobs: attempt=0 while true; do if [ "$(date +%s)" -ge "${deadline}" ]; then - echo "::error::CI Summary poll deadline (${DEADLINE_MINUTES}m) reached with gating jobs still pending or the jobs API unreachable — failing closed" + echo "::error::CI Summary poll deadline (${DEADLINE_MINUTES}m) reached with gating jobs or external contexts still pending, or the API unreachable — failing closed" python3 scripts/ci/ci_summary_gate.py --jobs-file jobs.json --run-attempt "${RUN_ATTEMPT}" --report-only 2>/dev/null || true exit 1 fi @@ -2023,12 +2658,29 @@ jobs: sleep "${POLL_INTERVAL_SECONDS}" continue fi - python3 scripts/ci/ci_summary_gate.py --jobs-file jobs.json --run-attempt "${RUN_ATTEMPT}" + # Cross-workflow check-runs for the exact head. On a failed fetch the + # file is REMOVED, not left stale: the gate reads an absent file as + # "no external context observed" → PENDING → fail-closed at the + # deadline. It must never resolve green off a stale or partial file. + rm -f check_runs.json + if [ "${EVENT_NAME}" = "pull_request" ]; then + if ! gh api --paginate \ + "repos/${GH_REPO}/commits/${HEAD_SHA}/check-runs?per_page=100" \ + --jq '.check_runs[]' | jq -s '.' > check_runs.json; then + echo "check-runs fetch failed (attempt ${attempt}); retrying after ${POLL_INTERVAL_SECONDS}s" + rm -f check_runs.json + sleep "${POLL_INTERVAL_SECONDS}" + continue + fi + fi + python3 scripts/ci/ci_summary_gate.py --jobs-file jobs.json --run-attempt "${RUN_ATTEMPT}" \ + --check-runs-file check_runs.json --event-name "${EVENT_NAME}" \ + --pr-author "${PR_AUTHOR:-}" rc=$? case "${rc}" in - 0) echo "::notice::CI Summary = SUCCESS (all gating jobs provably passed)"; exit 0 ;; - 1) echo "::error::CI Summary = FAILURE (fail-closed: a gating job failed/cancelled)"; exit 1 ;; - 2) : ;; # PENDING — gating jobs still running; keep polling + 0) echo "::notice::CI Summary = SUCCESS (all gating jobs and asserted external contexts provably passed)"; exit 0 ;; + 1) echo "::error::CI Summary = FAILURE (fail-closed: a gating job or an asserted external context failed/cancelled)"; exit 1 ;; + 2) : ;; # PENDING — gating jobs or external contexts still running; keep polling *) echo "::error::ci_summary_gate.py returned unexpected code ${rc}; failing closed"; exit 1 ;; esac sleep "${POLL_INTERVAL_SECONDS}" @@ -2091,9 +2743,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -2197,9 +2848,8 @@ jobs: name: Writer-Migration Coupling Check runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -2220,6 +2870,46 @@ jobs: - name: Check writer-migration coupling run: uv run python scripts/check_migration_required.py --ci + # Node migration declaration check -- every vendored node migration under + # docker/migrations/forward/nodes/ must carry a checked-in stream/domain + # declaration (application-migrations.tsv), explicit block + # (application-migration-blocks.tsv), or audited historical declaration + # (legacy-node-migrations.tsv) BEFORE it can reach a runtime. + # scripts/validation/validate_application_migration_manifest.py already + # implemented this exact check (plus checksum + duplicate + shape + # validation) but was never wired anywhere -- only its own test file + # exercised it directly. OMN-15717: node_pr_review_bot's + # 001_create_review_bot_bypass_log.sql shipped undeclared and this + # pre-existing, unwired validator would have caught it; it only failed + # closed at a live bootstrap.sql deploy, weeks after merge. + node-migration-declaration-check: + needs: occ-preflight + timeout-minutes: 30 + name: Node Migration Declaration Check + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: Setup Python and uv + uses: ./.github/actions/setup-python-uv + with: + python-version: ${{ env.PYTHON_VERSION }} + uv-version: ${{ env.UV_VERSION }} + cache-version: ${{ env.CACHE_VERSION }} + cache-enabled: "false" + install-args: '--reinstall-package omnibase-core --reinstall-package omnibase-spi --frozen' + + - name: Check node migration declarations + run: uv run python scripts/validation/validate_application_migration_manifest.py + # Migration integration test -- spins up real Postgres, applies all migrations, # and asserts every manifest table exists (OMN-3529) migration-integration: @@ -2228,9 +2918,8 @@ jobs: if: needs.occ-preflight.result == 'success' && (needs.zone-filter.outputs.docs_only != 'true') runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -2316,6 +3005,55 @@ jobs: run: | uv run python scripts/run-migrations.py + # OMN-15376: node-migration shape drift. Runs HERE, in the one job that + # already owns a real Postgres, because the class is only observable by + # EXECUTING the SQL against a pre-existing table of the wrong shape -- + # a CREATE TABLE IF NOT EXISTS silently no-ops and the next + # column-dependent statement kills the deploy. The suite creates and drops + # its own throwaway databases on this service; it never touches + # omnibase_infra. The static half of the gate runs in the main test job. + # + # psql is required because the proof must use the SAME invocation the + # deploy runners use (psql -v ON_ERROR_STOP=1 -f). Container runners may + # lack it and may lack sudo (same constraint reusable-runtime-boot.yml + # documents), so the client install is best-effort and the pytest step is + # gated on the result -- an unobtainable psql degrades this to the static + # gate rather than wedging a required job on a missing package. + - name: Ensure psql client (best effort) + id: ensure_psql + shell: bash + run: | + set -uo pipefail + if ! command -v psql >/dev/null 2>&1; then + if command -v sudo >/dev/null 2>&1; then + sudo apt-get update -qq \ + && sudo apt-get install -y -qq --no-install-recommends postgresql-client \ + || true + elif command -v apt-get >/dev/null 2>&1; then + apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends postgresql-client \ + || true + fi + fi + if command -v psql >/dev/null 2>&1 || ls /opt/homebrew/opt/postgresql@*/bin/psql >/dev/null 2>&1; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "::warning::psql client unavailable — the OMN-15376 shape-drift EXECUTION proof cannot run on this runner; the static gate in tests/ci/test_node_migration_shape_reconciliation.py still applies." + echo "available=false" >> "$GITHUB_OUTPUT" + fi + + - name: Node migration shape-drift proof (RED/GREEN + convergence) + if: steps.ensure_psql.outputs.available == 'true' + env: + OMNIBASE_INFRA_DB_URL: postgresql://postgres:test_password@${{ steps.postgres_host.outputs.host }}:${{ job.services.postgres.ports['5432'] }}/omnibase_infra + # Fail-closed inside the suite: with a Postgres in hand, a SKIP is a + # vacuous green and is treated as an error. + OMN15376_REQUIRE_PG: "1" + run: | + uv run pytest \ + tests/integration/migrations/test_node_migration_shape_drift_omn15376.py \ + -v --tb=short -p no:randomly + - name: Assert manifest tables exist env: POSTGRES_HOST: ${{ steps.postgres_host.outputs.host }} @@ -2400,8 +3138,9 @@ jobs: # via a silent skip). Detection-only sweeps get ignored, so this ships as a # mechanical CI gate + pre-commit hook in the same PR (Operating Rule #5). # - # This job PROVISIONS Postgres (mirroring migration-integration), applies all - # migrations, exports the connection env, and runs a CURATED set of + # This job PROVISIONS Postgres (explicit pull + run steps — see OMN-15249 note + # on the job below for why NOT a `services:` block), applies all migrations, + # exports the connection env, and runs a CURATED set of # Postgres-only integration proofs, then asserts via check_integration_skips.py # that (1) none skipped citing a *provisioned* service's absence and (2) at # least one integration test actually executed (zero-collection is itself a @@ -2425,30 +3164,104 @@ jobs: if: needs.occ-preflight.result == 'success' && (needs.zone-filter.outputs.docs_only != 'true') runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} timeout-minutes: 15 - # Same Postgres service block as migration-integration: a real DB so the - # @pytest.mark.integration real-DB proofs EXECUTE instead of self-skipping. - services: - postgres: - image: postgres:16-alpine - env: - POSTGRES_PASSWORD: test_password - POSTGRES_DB: omnibase_infra - ports: - - 5432/tcp - options: >- - --health-cmd pg_isready - --health-interval 5s - --health-timeout 5s - --health-retries 10 + # OMN-15249: Postgres is provisioned by EXPLICIT steps below, NOT a + # GitHub-managed `services:` block. On #2492 (heads 1729c7c3 / dbfc0c98) the + # services image pull timed out against registry-1.docker.io; that pull runs + # inside GitHub's own "Initialize containers" step, which this repo cannot + # name, cannot bound, and — critically — which steps carrying `if: always()` + # still run past. The job therefore reached the silent-skip enforcement step + # with no toolchain installed and terminated on `uv: command not found` / + # `exit 127`, several steps removed from the registry timeout that caused it. + # Owning the pull makes the failure fatal AT the pull, with the registry, + # image, and timeout named in the annotation. + # Proof: tests/ci/test_integration_guard_pull_fatality.py. + env: + GUARD_PG_IMAGE: postgres:16-alpine + GUARD_PG_REGISTRY: registry-1.docker.io + GUARD_PG_PULL_ATTEMPTS: "3" + GUARD_PG_PULL_TIMEOUT_SECONDS: "120" + GUARD_PG_PULL_BACKOFF_SECONDS: "8" + GUARD_PG_READY_TIMEOUT_SECONDS: "120" + GUARD_PG_CONTAINER: omn14172-guard-pg-${{ github.run_id }}-${{ github.run_attempt }} steps: + # FIRST step by design: nothing after a failed image pull can be trusted, + # so the pull runs before checkout/toolchain setup are even attempted. + # Bounded retry, fail-closed on exhaustion — never fall through. + - name: Pull Postgres image (fail-closed, OMN-15249) + id: pull_postgres + run: | + set -euo pipefail + + attempt=0 + while [ "${attempt}" -lt "${GUARD_PG_PULL_ATTEMPTS}" ]; do + attempt=$((attempt + 1)) + echo "Pull attempt ${attempt}/${GUARD_PG_PULL_ATTEMPTS}: ${GUARD_PG_IMAGE} from ${GUARD_PG_REGISTRY} (per-attempt timeout ${GUARD_PG_PULL_TIMEOUT_SECONDS}s)" + if timeout "${GUARD_PG_PULL_TIMEOUT_SECONDS}" docker pull "${GUARD_PG_IMAGE}"; then + echo "Pulled ${GUARD_PG_IMAGE} on attempt ${attempt}/${GUARD_PG_PULL_ATTEMPTS}." + exit 0 + fi + echo "Attempt ${attempt}/${GUARD_PG_PULL_ATTEMPTS} failed." + if [ "${attempt}" -lt "${GUARD_PG_PULL_ATTEMPTS}" ]; then + sleep $((attempt * GUARD_PG_PULL_BACKOFF_SECONDS)) + fi + done + + echo "::error title=Postgres image pull failed (OMN-15249)::Could not pull image ${GUARD_PG_IMAGE} from registry ${GUARD_PG_REGISTRY} after ${GUARD_PG_PULL_ATTEMPTS} bounded attempts (per-attempt timeout ${GUARD_PG_PULL_TIMEOUT_SECONDS}s). Failing closed AT the pull: the Integration Silent-Skip Guard emits no verdict for a run whose Postgres container never materialized. This is a registry/network failure, not a missing binary and not a test failure." + exit 1 + + # Replaces the `services:` health/port wiring: same image, same health + # probe, same ephemeral published port. Fails closed if the container never + # reports healthy, again with a named annotation. + - name: Start Postgres container (fail-closed, OMN-15249) + id: start_postgres + run: | + set -euo pipefail + + docker rm -f "${GUARD_PG_CONTAINER}" >/dev/null 2>&1 || true + docker run --detach \ + --name "${GUARD_PG_CONTAINER}" \ + --env POSTGRES_PASSWORD=test_password \ + --env POSTGRES_DB=omnibase_infra \ + --health-cmd pg_isready \ + --health-interval 5s \ + --health-timeout 5s \ + --health-retries 10 \ + --publish 5432 \ + "${GUARD_PG_IMAGE}" + + deadline=$(( $(date +%s) + GUARD_PG_READY_TIMEOUT_SECONDS )) + while true; do + status="$(docker inspect -f '{{.State.Health.Status}}' "${GUARD_PG_CONTAINER}" 2>/dev/null || echo unknown)" + if [ "${status}" = "healthy" ]; then + break + fi + if [ "$(date +%s)" -ge "${deadline}" ]; then + echo "::error title=Postgres container never became healthy (OMN-15249)::Container ${GUARD_PG_CONTAINER} (${GUARD_PG_IMAGE}) was still '${status}' after ${GUARD_PG_READY_TIMEOUT_SECONDS}s. Failing closed: no guard verdict is emitted without a provisioned Postgres." + docker logs "${GUARD_PG_CONTAINER}" || true + exit 1 + fi + sleep 2 + done + + # Docker publishes 5432 on an ephemeral host port, exactly as the + # `services:` block did; the next step resolves which interface the + # runner can actually reach it on. + published="$(docker port "${GUARD_PG_CONTAINER}" 5432/tcp | head -n 1)" + port="${published##*:}" + if [ -z "${port}" ]; then + echo "::error title=Postgres port unresolved (OMN-15249)::docker port ${GUARD_PG_CONTAINER} 5432/tcp returned '${published}'." + exit 1 + fi + echo "port=${port}" >> "$GITHUB_OUTPUT" + echo "Postgres healthy on published host port ${port}." + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Setup Python and uv @@ -2463,7 +3276,7 @@ jobs: - name: Resolve Postgres service host id: postgres_host env: - POSTGRES_PORT: ${{ job.services.postgres.ports['5432'] }} + POSTGRES_PORT: ${{ steps.start_postgres.outputs.port }} run: | python - <<'PY' >> "$GITHUB_OUTPUT" import os @@ -2501,8 +3314,9 @@ jobs: PY - name: Apply all migrations + id: apply_migrations env: - OMNIBASE_INFRA_DB_URL: postgresql://postgres:test_password@${{ steps.postgres_host.outputs.host }}:${{ job.services.postgres.ports['5432'] }}/omnibase_infra + OMNIBASE_INFRA_DB_URL: postgresql://postgres:test_password@${{ steps.postgres_host.outputs.host }}:${{ steps.start_postgres.outputs.port }}/omnibase_infra run: | uv run python scripts/run-migrations.py @@ -2513,9 +3327,10 @@ jobs: # the guard stays correct if the curated set broadens to tests that read # discrete vars. - name: Export integration Postgres env + id: export_pg_env env: RESOLVED_HOST: ${{ steps.postgres_host.outputs.host }} - RESOLVED_PORT: ${{ job.services.postgres.ports['5432'] }} + RESOLVED_PORT: ${{ steps.start_postgres.outputs.port }} run: | { echo "OMNIBASE_INFRA_DB_URL=postgresql://postgres:test_password@${RESOLVED_HOST}:${RESOLVED_PORT}/omnibase_infra" @@ -2531,9 +3346,10 @@ jobs: # Run ONLY the curated Postgres-only integration proofs (kept in sync with # integration_skip_guard.yaml curated_test_paths). The selected proof is # Postgres-only and self-contained against the freshly-migrated DB. Real integration - # FAILURES fail this step directly; the silent-skip enforcement below runs - # regardless (if: always()). + # FAILURES fail this step directly; the silent-skip enforcement below still + # runs on that path (that is the whole point of the guard). - name: Run curated Postgres integration proofs (Postgres provisioned) + id: run_curated_proofs run: | uv run pytest \ tests/integration/handlers/test_registration_storage_postgres_uuid_cast.py::TestRegistrationStoragePostgresUuidCast::test_query_round_trip_produces_stdlib_uuid \ @@ -2542,20 +3358,38 @@ jobs: # Fail-closed silent-skip enforcement. RED if any integration test skipped # citing a provisioned service's absence, or if zero integration tests ran. + # + # OMN-15249: deliberately NOT a bare `if: always()`. This step must fire when + # the curated proofs RAN (success or failure) and must NOT fire when they were + # never reached — a verdict about silent skips is meaningless for a run whose + # Postgres container never materialized, and on #2492 the bare always() form + # ran with no toolchain installed and surfaced `exit 127` as the job's terminal + # error, hiding the registry pull timeout that actually caused the failure. - name: Enforce no missing-service silent-skips (OMN-14172) - if: always() + id: enforce_no_silent_skips + if: always() && steps.run_curated_proofs.conclusion != 'skipped' run: | uv run python scripts/ci/check_integration_skips.py \ --junit integration-guard.xml - name: Upload integration-guard results - if: always() + id: upload_guard_results + if: always() && steps.run_curated_proofs.conclusion != 'skipped' uses: actions/upload-artifact@v4 with: name: integration-guard-results path: integration-guard.xml retention-days: 7 + # Unconditional: a container started before a later step failed must never + # be leaked onto a self-hosted runner. `|| true` keeps teardown from + # inventing a second, misattributed failure on an already-failing job. + - name: Stop Postgres container + id: stop_postgres + if: always() + run: | + docker rm --force "${GUARD_PG_CONTAINER}" >/dev/null 2>&1 || true + ai-slop-check: needs: occ-preflight timeout-minutes: 30 @@ -2563,9 +3397,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/contract-validation.yml b/.github/workflows/contract-validation.yml index f0e6ec3c9e..f22acba438 100644 --- a/.github/workflows/contract-validation.yml +++ b/.github/workflows/contract-validation.yml @@ -42,9 +42,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/cr-thread-gate-caller.yml b/.github/workflows/cr-thread-gate-caller.yml index 8f60e3e816..af69c10b79 100644 --- a/.github/workflows/cr-thread-gate-caller.yml +++ b/.github/workflows/cr-thread-gate-caller.yml @@ -13,6 +13,38 @@ on: merge_group: types: [checks_requested] +concurrency: + # OMN-15730 / CodeRabbit: issue_comment and merge_group events do not + # populate github.event.pull_request, so the || github.ref fallback + # resolved to one shared identity (default branch ref, or the merge_group + # ref) across unrelated PRs' comment/merge-queue-triggered runs, letting + # them cancel each other. Use the event-specific identity before falling + # back to the run id. + # + # OMN-15815: that fix still coalesces a pull_request run and an + # issue_comment run on the SAME PR into one group (both resolve to the + # same PR number), with no github.event_name discriminator. When + # CodeRabbit edits its summary comment (an issue_comment event) while the + # real pull_request-triggered gate run is still in flight, + # cancel-in-progress cancels that in-flight run; the CodeRabbit-actored + # run then skips its own gate job (see the `if:` actor filter below), so + # no replacement check-run is ever emitted and the required + # "gate / CodeRabbit Thread Check" context is left stuck cancelled with + # no successor. Prefix the group with github.event_name so pull_request, + # issue_comment, pull_request_review, pull_request_review_comment, and + # merge_group runs of this workflow can never share a concurrency group + # with each other, while same-event-type runs on the same PR still + # coalesce (two rapid pushes to one PR still supersede each other; rapid + # consecutive comment edits on one PR still dedup among themselves). + group: >- + ${{ github.workflow }}-${{ github.event_name }}-${{ + github.event.pull_request.number || + github.event.issue.number || + github.event.merge_group.head_sha || + github.run_id + }} + cancel-in-progress: true + jobs: gate: if: >- @@ -26,9 +58,12 @@ jobs: github.actor != 'github-actions[bot]' && !contains(github.actor, 'coderabbit')) concurrency: - group: cr-thread-gate-${{ github.event.pull_request.number || github.event.issue.number || github.event.merge_group.head_sha || github.run_id }} + # OMN-15815: same event-scoping as the workflow-level group above — + # this job-level group predates OMN-15730 and was never touched by + # that fix, so it carried the identical same-PR cross-event collision. + group: cr-thread-gate-${{ github.event_name }}-${{ github.event.pull_request.number || github.event.issue.number || github.event.merge_group.head_sha || github.run_id }} cancel-in-progress: true - uses: OmniNode-ai/omniclaude/.github/workflows/cr-thread-gate.yml@70718ddb45f6df5eb5e07159f2ae812efb1da7db + uses: OmniNode-ai/omniclaude/.github/workflows/cr-thread-gate.yml@80de61fd1fee04abdeb6918e7f91cf820717e6a8 with: pr-number: ${{ format('{0}', github.event.pull_request.number || github.event.issue.number || 0) }} secrets: diff --git a/.github/workflows/cr-thread-gate.yml b/.github/workflows/cr-thread-gate.yml index 3830ecdfe7..0a86c79be7 100644 --- a/.github/workflows/cr-thread-gate.yml +++ b/.github/workflows/cr-thread-gate.yml @@ -35,9 +35,8 @@ jobs: name: CodeRabbit Thread Check runs-on: >- ${{ - (inputs.pr-base-ref == 'dev' || - (github.event.pull_request && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/dep-provenance-gate.yml b/.github/workflows/dep-provenance-gate.yml index 542b6e4791..8142a06106 100644 --- a/.github/workflows/dep-provenance-gate.yml +++ b/.github/workflows/dep-provenance-gate.yml @@ -39,15 +39,18 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: dep-provenance-gate: name: Dep Provenance Gate # OMNI_RUNNER_SELECTOR_V1 - trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest. runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/dependency-cascade.yml b/.github/workflows/dependency-cascade.yml index fc36005d5c..6da673f392 100644 --- a/.github/workflows/dependency-cascade.yml +++ b/.github/workflows/dependency-cascade.yml @@ -127,6 +127,22 @@ jobs: token: ${{ secrets.cross-repo-pat || secrets.CROSS_REPO_PAT }} fetch-depth: 1 + # OMN-15604 AC4: `uv lock --upgrade-package` cannot move a + # [tool.uv.sources] git pin -- checked out into a side directory (the + # downstream-repo checkout above overwrote the job's default + # workspace, so this repo's own scripts/ are not otherwise present) + # so the "Create branch and upgrade lockfile" step below can fail loud + # BEFORE attempting a lock that is guaranteed to be a silent no-op. + - name: Checkout omnibase_infra dep-provenance script + if: steps.token_check.outputs.has_token == 'true' + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: OmniNode-ai/omnibase_infra + path: _dep_provenance_gate + sparse-checkout: | + scripts/check_dep_provenance.py + sparse-checkout-cone-mode: false + - name: Set up uv if: steps.token_check.outputs.has_token == 'true' uses: astral-sh/setup-uv@v7 @@ -155,6 +171,21 @@ jobs: PKG_HYPHEN="${{ steps.vars.outputs.pkg_hyphen }}" VER="${{ steps.vars.outputs.version }}" + # OMN-15604 AC4: `uv lock --upgrade-package` cannot move a + # [tool.uv.sources] git pin -- uv always prefers an explicit + # source override over registry resolution, so re-locking against + # the SAME override typically re-resolves to a byte-identical + # uv.lock. Left unchecked, the SKIP branch below would misreport + # that as "already on latest" when this repo is actually still + # stuck on the git pin. Fail loud and explicit BEFORE attempting + # the (guaranteed no-op) lock, instead of after. + if ! python3 "${GITHUB_WORKSPACE}/_dep_provenance_gate/scripts/check_dep_provenance.py" \ + --pyproject pyproject.toml \ + --check-movable "$PKG_HYPHEN"; then + echo "::error::Dependency cascade cannot move ${PKG_HYPHEN} in ${{ matrix.repo }} -- it is pinned via a [tool.uv.sources] git override. 'uv lock --upgrade-package' will silently re-resolve against the SAME override and report no lockfile change, masking a no-op cascade (OMN-15604). Remove the [tool.uv.sources] override for ${PKG_HYPHEN} in ${{ matrix.repo }}'s pyproject.toml first, then re-run this cascade." + exit 1 + fi + # PyPI's upload -> public-index propagation can lag several minutes. # The old 12x10s (=120s) retry ceiling was too short and silently # failed the cascade (OMN-14468). Poll the PyPI JSON API until the diff --git a/.github/workflows/deploy-agent-tests.yml b/.github/workflows/deploy-agent-tests.yml new file mode 100644 index 0000000000..de410dcbab --- /dev/null +++ b/.github/workflows/deploy-agent-tests.yml @@ -0,0 +1,98 @@ +name: Deploy Agent Tests (OMN-15378) + +# OMN-15378: scripts/deploy-agent is a standalone uv sub-project (its own +# pyproject.toml + uv.lock, deployed onto the .201 host independently of the +# main omnibase_infra dependency tree — see scripts/deploy-agent/pyproject.toml +# and scripts/deploy-agent/deploy/deploy-agent.service.d). Its tests +# (scripts/deploy-agent/tests/) were never wired into ANY CI job: the main +# suite only ever collects `tests/` (pyproject.toml testpaths = ["tests"]), +# and folding these files into that tree is not a drop-in move — both trees +# declare a top-level `tests` package (tests/__init__.py vs +# scripts/deploy-agent/tests/__init__.py), so collecting them together in one +# pytest session raises ImportPathMismatchError (verified locally 2026-07-29). +# The tests sat completely uncollected for ~5 weeks; one of them (RED against +# the OMN-12988->OMN-12990 superseded literal) went unnoticed the entire time +# and was only fixed by hand, incidentally, in PR #2536 (OMN-14968). +# +# This workflow runs them the same way their own `dod_evidence` proof commands +# always have — `uv run --project scripts/deploy-agent`, using deploy-agent's +# OWN pyproject.toml/uv.lock, not the root project's. This is deliberate: the +# deploy-agent runtime on the .201 host has its own minimal dependency set, +# and testing it through the main repo's heavier venv could mask a real +# dependency mismatch between test and deploy environments. +# +# scripts/validation/validate_test_root_collection.py (wired into the main +# suite via tests/ci/test_validate_test_root_collection.py) registers this +# workflow as the CI wiring behind scripts/deploy-agent's +# STANDALONE_PROJECT_ROOTS entry — if this file is renamed/removed without +# updating that registration, the guard fails closed. +# +# OMN-15378 AC3 (enforcement, not just visibility): this workflow is +# `workflow_call`-only and is invoked by the `deploy-agent-tests` job in +# .github/workflows/ci.yml, so its job surfaces in ci.yml's OWN run as the +# check-run "Deploy Agent Tests (OMN-15378) / deploy-agent-tests" and is +# registered in scripts/ci/ci_summary_gate.py STRICT_GATE_JOBS. That is what +# makes it GATE merge: the CI Summary poller reads +# `actions/runs/${RUN_ID}/jobs` for its own run only, so while these tests +# lived in a separately-triggered workflow a RED result left the sole required +# context ("CI Summary") green — the tests ran but blocked nothing. +# +# DO NOT re-add `pull_request:`/`push:`/`merge_group:` triggers here — the +# caller in ci.yml already covers those events, and self-triggering would run +# the suite twice per PR (duplicate producer). DO NOT rename this file or the +# `deploy-agent-tests` job: the file path is the guard's registration key and +# the job name is the second segment of the STRICT_GATE_JOBS entry. +# +# The caller's path filters are also deliberately gone: the previous +# `paths: [scripts/deploy-agent/**]` filter meant a change ANYWHERE ELSE that +# broke these tests produced no signal at all. The suite is 201 tests in ~6s, +# so it now runs unconditionally on every ci.yml event. +# +# No `concurrency:` block: in a called workflow `github.workflow` resolves to +# the CALLER's workflow name, so a group key here would shadow ci.yml's own +# (already-correct) PR-keyed cancel-in-progress group. + +on: + workflow_call: + workflow_dispatch: + +env: + UV_VERSION: "0.6.14" + PYTHON_VERSION: "3.12" + CACHE_VERSION: "0.6.0" + +jobs: + deploy-agent-tests: + name: deploy-agent-tests + permissions: + contents: read + # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 10 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Setup Python and uv (deploy-agent sub-project) + uses: ./.github/actions/setup-python-uv + with: + python-version: ${{ env.PYTHON_VERSION }} + uv-version: ${{ env.UV_VERSION }} + cache-version: ${{ env.CACHE_VERSION }} + cache-enabled: "false" + working-directory: scripts/deploy-agent + install-args: "--extra dev" + + - name: Run deploy-agent unit tests (OMN-15378) + run: | + uv run --project scripts/deploy-agent --extra dev \ + pytest scripts/deploy-agent/tests -v --tb=short diff --git a/.github/workflows/dev-baseline-publisher.yml b/.github/workflows/dev-baseline-publisher.yml index f0653ad21c..58fa18e1df 100644 --- a/.github/workflows/dev-baseline-publisher.yml +++ b/.github/workflows/dev-baseline-publisher.yml @@ -54,7 +54,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/dev-lane-liveness.yml b/.github/workflows/dev-lane-liveness.yml new file mode 100644 index 0000000000..84c5f60fb8 --- /dev/null +++ b/.github/workflows/dev-lane-liveness.yml @@ -0,0 +1,111 @@ +# dev-lane-liveness.yml — scheduled lab/dev lane liveness probe (OMN-15190) +# +# WHY THIS EXISTS +# The lab/dev lane (compose project `omnibase-infra` on the `.201` host) has +# been GC/idle-reclaimed to zero containers at least five times (2026-07-13, +# 07-14, 07-24, 07-26 x2). Every single occurrence was discovered +# REACTIVELY, by whichever PR happened to hit the resulting org-wide +# occ-autobind / occ-companion-effect connection-refused cascade into +# occ-preflight / Receipt Gate. Nothing watched the lane. +# +# Operator ruling 2026-07-29 (WS-4): the lab lane is KEEP-ALIVE — testing +# things live is the point of the lab — so lane-down is a DEFECT, not the +# ephemeral-by-design state OMN-13414 documented. This workflow is the +# surface that says so. +# +# The probe logic is NOT new: `scripts/system_health_check.sh` is the repo's +# canonical health gate and already carried most of it. It had zero call +# sites. This wires it and adds the lane-specific check (`--lane` mode -> +# dev_lane_liveness + redpanda + runtime_containers). +# +# WHY A SEPARATE FILE FROM runner-fleet-canary.yml +# Same 15-minute cadence, adjacent concern, deliberately NOT the same file. +# The fleet canary carries an invariant asserted by +# tests/ci/test_runner_listener_liveness.py: EVERY job in that workflow runs +# on GitHub-hosted compute, because a canary that shares fate with the fleet +# it watches proves nothing. This job must run self-hosted (below), so +# folding it in would have meant narrowing that safety test inside a feature +# PR — the thing OMN-15234 explicitly split into its own ticket when it had +# to narrow a repo-wide gate. Separate file, guard untouched, and the two +# red signals stay distinguishable: "the fleet is dying" and "the lane is +# dying" are different pages. +# +# WHY SELF-HOSTED IS CORRECT HERE (this is not the fleet canary's mistake) +# The fate boundary that matters is COMPOSE PROJECT, not host. The watched +# thing is project `omnibase-infra`; the watcher is the `omnibase-deploy` +# runner in a different project on the same host. The lane has repeatedly +# died with the runner fully alive — that IS the observed failure mode. +# And it must probe from where the CI publishers actually publish: the +# lane's broker host-port is on the tailnet, unreachable from GitHub-hosted +# compute, so an ubuntu-latest job could only ever assert "I cannot see it." +# `omnibase-deploy` is the one runner with both docker.sock and the +# host-gateway alias (docker/docker-compose.runners.yml). +# +# Residual, stated rather than hidden: if the whole `.201` host dies this +# job never gets scheduled and reports nothing. That case is covered by +# runner-fleet-canary.yml, which runs on GitHub-hosted compute and goes red +# on the same cadence. +# +# PR-TIME GUARD +# Deliberately no `pull_request` trigger: the deploy runner is not a PR +# runner, so a PR-triggered job could only skip its own probe. The PR-time +# guard is the hermetic suite `tests/ci/test_dev_lane_liveness.py`, which +# drives the real bash artifact and asserts this file's wiring (the `--lane` +# invocation, `LANE_PROBE_HOST`, `ONEX_LANE_KEEPALIVE`, the schedule, and +# the runs-on on both this job and the fleet canary's). +name: dev-lane-liveness + +on: + schedule: + # Every 15 minutes, offset 7 past the quarter to avoid a thundering herd + # with runner-fleet-canary.yml (*/15) — repo cron convention. + - cron: "7,22,37,52 * * * *" + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + dev-lane-liveness: + name: dev-lane-liveness + # See "WHY SELF-HOSTED IS CORRECT HERE" above before changing this. + runs-on: [self-hosted, omnibase-deploy] + timeout-minutes: 10 + steps: + - name: Checkout (health gate script + rendered runtime policy) + uses: actions/checkout@v4 + with: + # No git operations after checkout — do not leave the Actions token + # in local git config (zizmor artipacked). + persist-credentials: false + + - name: Probe lab/dev lane liveness + env: + # Inside the deploy-runner container `localhost` is the container + # itself, so the script's on-host default would probe the wrong + # network namespace and manufacture a false RED. Set explicitly here + # rather than relying on the live container's compose env, which is + # only refreshed on recreate (OMN-14958 / OMN-14900). + LANE_PROBE_HOST: host.docker.internal + # Operator ruling 2026-07-29 (WS-4): lane-down is a defect. + ONEX_LANE_KEEPALIVE: "1" + run: | + set -euo pipefail + bash scripts/system_health_check.sh --lane --ci | tee lane-health.json + + - name: Publish lane verdict to job summary + if: always() + run: | + set -uo pipefail + { + echo "## Lab/dev lane liveness (OMN-15190)" + echo "" + echo '```json' + cat lane-health.json 2>/dev/null \ + || echo '{"overall":"unknown","detail":"probe produced no output"}' + echo '```' + echo "" + echo "RED here means the lab lane is down or degraded. It is KEEP-ALIVE" + echo "by operator ruling — recovery path:" + echo "\`docs/runbooks/cold-lane-full-bringup.md\`." + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/dispatch-parity-gate.yml b/.github/workflows/dispatch-parity-gate.yml index 6b2cf0e6d9..915802f210 100644 --- a/.github/workflows/dispatch-parity-gate.yml +++ b/.github/workflows/dispatch-parity-gate.yml @@ -62,9 +62,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted; public forks use ubuntu-latest. runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/dispatcher-route-coverage.yml b/.github/workflows/dispatcher-route-coverage.yml index e014e8f4b2..80fbda8fc4 100644 --- a/.github/workflows/dispatcher-route-coverage.yml +++ b/.github/workflows/dispatcher-route-coverage.yml @@ -59,9 +59,8 @@ jobs: name: dispatcher-route-coverage runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 2e38441409..29a7705f33 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -73,6 +73,45 @@ jobs: - name: Validate Dockerfile plugin pin ranges run: python scripts/check_dockerfile_pins.py --dockerfile docker/Dockerfile.runtime + # OMN-15421: prove the tenant/internal/catalog operation split from a rebuilt + # source image against PostgreSQL 16. This is deliberately independent from + # the large runtime image build so a narrow adapter regression gets a narrow, + # actionable verdict. + domain-adapter-proof: + name: Projection Domain Adapter Proof (OMN-15421) + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON(vars.OMNI_DOCKER_CI_RUNS_ON_JSON || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 15 + + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Verify Docker socket access + run: docker info + + - name: Run rebuilt-image PostgreSQL 16 proof + env: + COMPOSE_PROJECT_NAME: omn15421-${{ github.run_id }}-${{ github.run_attempt }} + run: >- + docker compose + -f docker/domain-adapter-proof/compose.yml + up --build --abort-on-container-exit --exit-code-from proof + + - name: Remove proof containers and volumes + if: always() + env: + COMPOSE_PROJECT_NAME: omn15421-${{ github.run_id }}-${{ github.run_attempt }} + run: >- + docker compose + -f docker/domain-adapter-proof/compose.yml + down --volumes --remove-orphans + # Build and validate Docker image docker-build: name: Build Runtime Image diff --git a/.github/workflows/duplication-sweep.yml b/.github/workflows/duplication-sweep.yml index b72bd3636f..66094bd8ce 100644 --- a/.github/workflows/duplication-sweep.yml +++ b/.github/workflows/duplication-sweep.yml @@ -53,12 +53,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 - trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest. runs-on: >- ${{ - (github.event_name == 'pull_request' && github.base_ref == 'dev') - && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') - || ( (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - ) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/fresh-deploy-fitness.yml b/.github/workflows/fresh-deploy-fitness.yml index dfa353a16d..a5108a2748 100644 --- a/.github/workflows/fresh-deploy-fitness.yml +++ b/.github/workflows/fresh-deploy-fitness.yml @@ -48,9 +48,8 @@ jobs: name: release-identity runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -80,9 +79,8 @@ jobs: name: terminal-cost-completeness runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -90,7 +88,7 @@ jobs: steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: ${{ env.PYTHON_VERSION }} - name: Forbid hardcoded cost_usd=0.0 on terminal cost paths @@ -100,9 +98,8 @@ jobs: name: context-field-presence runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -110,7 +107,7 @@ jobs: steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: ${{ env.PYTHON_VERSION }} - name: Install PyYAML @@ -133,9 +130,8 @@ jobs: # is correct. runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -170,9 +166,8 @@ jobs: # pre-existing pins are refreshed. runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -183,7 +178,7 @@ jobs: steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: ${{ env.PYTHON_VERSION }} - name: Build-provenance version-skew guard (advisory) diff --git a/.github/workflows/hostile-reviewer.yml b/.github/workflows/hostile-reviewer.yml index 9099169160..1ce2b8ddf4 100644 --- a/.github/workflows/hostile-reviewer.yml +++ b/.github/workflows/hostile-reviewer.yml @@ -26,26 +26,35 @@ name: Hostile Reviewer on: pull_request: branches: [main, dev] - types: [opened, synchronize, reopened] + # OMN-14241: `edited` added so a PR-body-only edit (e.g. stamping + # `Evidence-Source: OCC#`) retriggers this workflow's own + # `occ-preflight` job -- see ci.yml's identical trigger for the full + # rationale. Without it, `occ-preflight / eligibility` (workflowName + # "Hostile Reviewer") sits at a stale pre-stamp FAILURE forever. + types: [opened, edited, synchronize, reopened] permissions: pull-requests: write contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: occ-preflight: # self-gating-ok: centralized OCC preflight reusable is the canonical cross-repo gate for non-OCC repositories. # omnibase_infra has no local occ-preflight.yml (unlike omnibase_core, which # hosts the reusable) -- reference it remotely, matching the pinned SHA # already used by this repo's own call-occ-preflight.yml / auto-merge.yml. - uses: OmniNode-ai/omnibase_core/.github/workflows/occ-preflight.yml@79c620f904526f773f6b92eb98743b011fe2eb12 + uses: OmniNode-ai/omnibase_core/.github/workflows/occ-preflight.yml@de01ec0964a2e666d739835b02957ae0f06d6a25 permissions: contents: read pull-requests: read with: contracts-dir: contracts receipts-dir: drift/dod_receipts - core-ref: 8a47e092002dd1338599fa0733feda469436acbf + core-ref: dev hostile-review: needs: occ-preflight @@ -54,8 +63,7 @@ jobs: runs-on: >- ${{ (github.event_name == 'pull_request' && - (github.event.pull_request.head.repo.full_name != github.repository || - github.event.pull_request.base.ref == 'dev')) + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -433,8 +441,7 @@ jobs: runs-on: >- ${{ (github.event_name == 'pull_request' && - (github.event.pull_request.head.repo.full_name != github.repository || - github.event.pull_request.base.ref == 'dev')) + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/integration-test-check.yml b/.github/workflows/integration-test-check.yml index cf216f4ebd..7151865052 100644 --- a/.github/workflows/integration-test-check.yml +++ b/.github/workflows/integration-test-check.yml @@ -39,9 +39,8 @@ jobs: name: Integration Test Coverage runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -122,9 +121,8 @@ jobs: name: Integration Test Removal Gate runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/legacy-rds-fixture-proof.yml b/.github/workflows/legacy-rds-fixture-proof.yml new file mode 100644 index 0000000000..2d6d6391b6 --- /dev/null +++ b/.github/workflows/legacy-rds-fixture-proof.yml @@ -0,0 +1,60 @@ +name: Sanitized Legacy RDS Fixture Proof + +on: + pull_request: + branches: [main, dev] + paths: + - ".github/workflows/legacy-rds-fixture-proof.yml" + - "docker/legacy-rds-fixture/**" + - "docker/migrations/**" + - "scripts/run-forward-migrations.sh" + push: + branches: [main] + paths: + - ".github/workflows/legacy-rds-fixture-proof.yml" + - "docker/legacy-rds-fixture/**" + - "docker/migrations/**" + - "scripts/run-forward-migrations.sh" + workflow_dispatch: + +concurrency: + group: legacy-rds-fixture-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + fixture-proof: + name: PostgreSQL 16 Fresh + Legacy Fixture + # OMNI_RUNNER_SELECTOR_V1 - trusted Docker proofs default to self-hosted omnibase-ci; public forks default to ubuntu-latest. + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON(vars.OMNI_DOCKER_CI_RUNS_ON_JSON || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 15 + + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + # This proof is intentionally independent of OCC preflight. It validates + # sanitized fixture behavior on draft PRs; it grants no release or merge + # authorization, and the repository's OCC gates remain fail-closed. + - name: Sanitized legacy-RDS Docker proof + run: | + docker compose \ + --project-name "omn15422-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + -f docker/legacy-rds-fixture/compose.yml \ + up --build --abort-on-container-exit --exit-code-from proof + + - name: Clean sanitized legacy-RDS Docker proof + if: always() + run: | + docker compose \ + --project-name "omn15422-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + -f docker/legacy-rds-fixture/compose.yml \ + down --volumes --remove-orphans diff --git a/.github/workflows/main-target-guard.yml b/.github/workflows/main-target-guard.yml index 08f94419f1..47f23099a7 100644 --- a/.github/workflows/main-target-guard.yml +++ b/.github/workflows/main-target-guard.yml @@ -25,11 +25,8 @@ jobs: name: main-target-guard runs-on: >- ${{ - (github.event_name == 'pull_request' && github.base_ref == 'dev') - && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') - || ( - (github.event.pull_request.head.repo.full_name != github.repository) - ) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/nightly-integration.yml b/.github/workflows/nightly-integration.yml index fa45ee3924..349dd956ae 100644 --- a/.github/workflows/nightly-integration.yml +++ b/.github/workflows/nightly-integration.yml @@ -37,7 +37,6 @@ env: OMNI_INFRA_HOST: "localhost" POSTGRES_HOST: "localhost" REDPANDA_ADVERTISE_HOST: "localhost" - INTEGRATION_POSTGRES_PORT: "5433" INTEGRATION_POSTGRES_USER: "postgres" # NB: "test-password" is the default in docker/docker-compose.e2e.yml # (POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-test-password}) — this is NOT a @@ -47,9 +46,13 @@ env: # checkov:skip=CKV_SECRET_4:Test-only ephemeral docker-compose.e2e.yml password INTEGRATION_POSTGRES_PASSWORD: ${{ secrets.INTEGRATION_POSTGRES_PASSWORD || 'test-password' }} INTEGRATION_POSTGRES_DB: "omnibase_infra" - OMNIBASE_INFRA_DB_URL: ${{ secrets.OMNIBASE_INFRA_DB_URL || format('postgresql://postgres:{0}@localhost:5433/omnibase_infra', secrets.INTEGRATION_POSTGRES_PASSWORD || 'test-password') }} - # E2E kafka exposed on host port - KAFKA_BOOTSTRAP_SERVERS: "localhost:19092" + # OMN-15565: INTEGRATION_POSTGRES_PORT / KAFKA_BOOTSTRAP_SERVERS / + # OMNIBASE_INFRA_DB_URL are deliberately NOT pinned here. The e2e stack's + # former defaults (5433 / 19092) collide with live .201 lane bindings on this + # shared self-hosted runner host, so the ports are allocated free at run time + # by the "Derive isolated e2e namespace" step below and exported via + # $GITHUB_ENV. Re-pinning them here would be shadowed by that step and would + # only mislead a reader into thinking the ports are fixed. ONEX_EVENT_BUS_TYPE: "kafka" ONEX_ENVIRONMENT: "test" LLM_ENDPOINT_CIDR_ALLOWLIST: "10.0.0.0/8" @@ -95,7 +98,32 @@ jobs: && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} - timeout-minutes: 60 + # OMN-15567: was 60. The single `pytest tests/integration/` step now has + # up to three independent full-Dockerfile.runtime build invocations that + # can each legitimately run right up to their own pytest-timeout ceiling + # before this job-level timeout would matter -- + # test_build_succeeds_with_public_deps, test_build_uses_buildkit_cache_mounts + # (both explicit, in TestDockerBuild), and the module-scoped + # built_test_image fixture's build (shared by TestDockerSecurity / + # TestDockerRuntime / TestDockerHealthCheck / TestDockerImageLabels, but + # only actually built once per module, by whichever of those is the + # first consumer). Worst-case budget sum, using the per-test ceiling + # each build test is marked with (BUILD_TIMEOUT=1200s default + + # BUILD_TEST_TIMEOUT_MARGIN_SECONDS=60s = 1260s), not the empirically + # observed cache-warm durations that D2 already showed are not safe to + # assume: + # 3 x 1260s (worst-case cold build each) = 3780s (63.0 min) + # + rest-of-suite baseline, ~793s observed 2026-08-04 + # (993s total "Run integration tests" step minus + # the 200.88s attributed to test_build_succeeds_with_public_deps), + # rounded up for headroom against suite growth = 900s (15.0 min) + # + fixed step overhead observed the same run + # (checkout/python/uv/deps/e2e-stack-up/health-wait + # + teardown/upload, ~60s + ~3s), rounded up = 120s ( 2.0 min) + # ------------------------------------------------------------------ + # worst-case total = 4800s (80.0 min) + # 90 gives ~10 minutes of margin above that worst-case sum. + timeout-minutes: 90 steps: - name: Checkout code @@ -104,7 +132,7 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} @@ -122,27 +150,312 @@ jobs: - name: Install dependencies run: uv sync --reinstall-package omnibase-core --reinstall-package omnibase-spi + # OMN-15565: this workflow runs on a self-hosted runner that shares the + # .201 Docker daemon with the live lab/dev lane. Before this step existed, + # the compose invocations below carried no project name, so they resolved + # to docker-compose.e2e.yml's default project -- which WAS the lab lane's + # own project `omnibase-infra`. The teardown's `down -v --remove-orphans` + # then deleted the lane's containers and its data volumes, nightly. + # + # Everything the Docker daemon namespaces globally (project, container + # names, volume names, network name) is derived from a run-scoped id here, + # mirroring .github/workflows/reusable-runtime-boot.yml. Host ports are + # allocated free because the e2e defaults (19092/18082/...) are bound by + # the live lane. + - name: Derive isolated e2e namespace + run: | + set -euo pipefail + e2e_id="e2e-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}" + e2e_id="$(echo "$e2e_id" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9_-' '-')" + project="omnibase-infra-${e2e_id}" + + pick_free_port() { + python3 - <<'PY' + import socket + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + print(sock.getsockname()[1]) + PY + } + + postgres_port="$(pick_free_port)" + kafka_port="$(pick_free_port)" + + db_url="postgresql://postgres:${INTEGRATION_POSTGRES_PASSWORD}@localhost:${postgres_port}/omnibase_infra" + + # KAFKA_BOOTSTRAP_SERVERS / OMNIBASE_INFRA_DB_URL / INTEGRATION_POSTGRES_HOST + # below are "localhost" defaults, matching the historical (pre-OMN-15567) + # shape so test_nightly_database_url_uses_the_ephemeral_postgres_port + # keeps passing unmodified. OMN-15567: on this self-hosted runner + # "localhost" is not reliably reachable from the runner process, so the + # later "Resolve reachable e2e connectivity host" step re-derives and + # overrides all of these (later $GITHUB_ENV writes win) once the actual + # stack is up and its real reachability has been probed. + { + echo "OMNIBASE_INFRA_COMPOSE_PROJECT=${project}" + echo "OMNIBASE_INFRA_POSTGRES_CONTAINER=${project}-postgres" + echo "OMNIBASE_INFRA_REDPANDA_CONTAINER=${project}-redpanda" + echo "OMNIBASE_INFRA_TOPIC_MANAGER_CONTAINER=${project}-topic-manager" + echo "OMNIBASE_INFRA_VALKEY_CONTAINER=${project}-valkey" + echo "OMNIBASE_INFRA_INFISICAL_CONTAINER=${project}-infisical" + echo "OMNIBASE_INFRA_RUNTIME_CONTAINER=${project}-runtime" + echo "OMNIBASE_INFRA_NETWORK=${project}-network" + echo "OMNIBASE_INFRA_POSTGRES_VOLUME=${project}-postgres-data" + echo "OMNIBASE_INFRA_REDPANDA_VOLUME=${project}-redpanda-data" + echo "OMNIBASE_INFRA_VALKEY_VOLUME=${project}-valkey-data" + echo "OMNIBASE_INFRA_RUNTIME_LOG_VOLUME=${project}-runtime-logs" + echo "POSTGRES_PORT=${postgres_port}" + echo "KAFKA_PORT=${kafka_port}" + echo "REDPANDA_ADMIN_PORT=$(pick_free_port)" + echo "REDPANDA_SCHEMA_REGISTRY_PORT=$(pick_free_port)" + echo "REDPANDA_PANDAPROXY_PORT=$(pick_free_port)" + echo "VALKEY_PORT=$(pick_free_port)" + echo "INTEGRATION_POSTGRES_PORT=${postgres_port}" + echo "KAFKA_BOOTSTRAP_SERVERS=localhost:${kafka_port}" + echo "OMNIBASE_INFRA_DB_URL=${db_url}" + } >> "$GITHUB_ENV" + + echo "e2e compose project: ${project}" + echo "e2e postgres port: ${postgres_port}, kafka port: ${kafka_port}" + + # OMN-15567: reusable-runtime-boot.yml:250-270 documents that this class + # of self-hosted runner executes inside a container sharing the host + # Docker daemon, so host-published ports are not reachable at + # "localhost" from the runner process -- only from the actual Docker + # host. nightly-integration.yml never detected or handled this. Mirror + # reusable-runtime-boot.yml's detection here (same runner_container_id + # heuristic, same /proc/net/route default-gateway read) so redpanda's + # external listener can advertise the Docker host gateway instead of + # "localhost" when that fallback is needed by the probe below. + - name: Detect runner network topology + run: | + set -euo pipefail + runner_container_id="" + # NB: intentionally NOT `mapfile`/`readarray` -- those are bash 4+ + # builtins. This step's shell is invoked directly by pytest in + # tests/unit/docker/test_nightly_e2e_runner_connectivity.py via + # `bash -c`, and the system `/bin/bash` on macOS gate hosts is + # 3.2.57 (Apple ships no newer bash for licensing reasons). A + # `while read` loop reading from the same process substitution is + # bash-3-compatible and behaves identically on the Linux runner. + runner_candidates=() + while IFS= read -r candidate; do + runner_candidates+=("$candidate") + done < <( + { + grep -Eo '[0-9a-f]{64}' /proc/self/cgroup 2>/dev/null || true + hostname + } | awk 'NF && !seen[$0]++' + ) + # bash 3.2 (unlike 4.4+) raises "unbound variable" under `set -u` + # when expanding "${arr[@]}" on a zero-length array, even one that + # was explicitly declared empty -- guard the count first. + if [[ ${#runner_candidates[@]} -gt 0 ]]; then + for candidate in "${runner_candidates[@]}"; do + if docker inspect --type container "$candidate" >/dev/null 2>&1; then + runner_container_id="$(docker inspect --type container --format '{{.Id}}' "$candidate")" + break + fi + done + fi + + e2e_advertise_host="localhost" + if [[ -n "$runner_container_id" ]]; then + docker_host_gateway="$( + python3 - <<'PY' + import socket + import struct + + try: + with open("/proc/net/route", encoding="utf-8") as route_file: + next(route_file) + for line in route_file: + fields = line.split() + if len(fields) >= 3 and fields[1] == "00000000": + gateway = int(fields[2], 16) + print(socket.inet_ntoa(struct.pack("> "$GITHUB_ENV" + + # Fail closed at run time, not just at review time: the protected set is + # read from the lane census manifest, so a lane added later is covered + # without editing this workflow. + - name: Assert e2e project is not a live lane + run: | + set -euo pipefail + uv run python - <<'PY' + import os + import sys + + import yaml + + project = os.environ["OMNIBASE_INFRA_COMPOSE_PROJECT"] + manifest = yaml.safe_load(open("deploy/lane-census/lane-manifest.yaml", encoding="utf-8")) + protected = { + lane["compose_project"] + for lane in manifest["lanes"].values() + if lane.get("compose_project") + } + if project in protected: + sys.exit( + f"::error::refusing to run the e2e stack in live lane project " + f"{project!r} (censused lanes: {sorted(protected)})" + ) + print(f"e2e project {project!r} is not a censused lane (checked {sorted(protected)})") + PY + - name: Spin up e2e stack run: | - docker compose -f docker/docker-compose.e2e.yml up -d \ + docker compose -p "${OMNIBASE_INFRA_COMPOSE_PROJECT}" \ + -f docker/docker-compose.e2e.yml up -d \ postgres redpanda valkey redpanda-topic-manager - name: Wait for health checks run: | echo "Waiting for postgres..." - timeout 120 bash -c 'until docker exec omnibase-infra-postgres pg_isready -U postgres -d omnibase_infra; do sleep 2; done' + timeout 120 bash -c 'until docker exec "$OMNIBASE_INFRA_POSTGRES_CONTAINER" pg_isready -U postgres -d omnibase_infra; do sleep 2; done' echo "Waiting for redpanda..." - timeout 120 bash -c 'until docker exec omnibase-infra-redpanda rpk cluster health 2>/dev/null | grep -q "Healthy:.*true"; do sleep 2; done' + timeout 120 bash -c 'until docker exec "$OMNIBASE_INFRA_REDPANDA_CONTAINER" rpk cluster health 2>/dev/null | grep -q "Healthy:.*true"; do sleep 2; done' echo "Waiting for valkey..." - timeout 60 bash -c 'until docker exec omnibase-infra-valkey valkey-cli ping | grep -q PONG; do sleep 2; done' + timeout 60 bash -c 'until docker exec "$OMNIBASE_INFRA_VALKEY_CONTAINER" valkey-cli ping | grep -q PONG; do sleep 2; done' echo "Waiting for topic manager to complete..." - timeout 60 bash -c 'until [ "$(docker inspect -f "{{.State.Status}}" omnibase-infra-topic-manager 2>/dev/null)" = "exited" ]; do sleep 2; done' + timeout 60 bash -c 'until [ "$(docker inspect -f "{{.State.Status}}" "$OMNIBASE_INFRA_TOPIC_MANAGER_CONTAINER" 2>/dev/null)" = "exited" ]; do sleep 2; done' echo "All services healthy" + # OMN-15567: the stack being Healthy (previous step) does not prove the + # runner process itself can reach it -- "Wait for health checks" only + # probes from inside the containers via `docker exec`. Confirmed live: + # on runs 30681782952 (workflow_dispatch) and 30685774570 (schedule), + # both post-OMN-15565, redpanda/postgres came up Healthy in ~5s but the + # runner's own connection to "localhost:" failed + # outright (aiokafka KafkaConnectionError / asyncpg OSError "Connect + # call failed"), because this runner does not share a network namespace + # with the Docker host. Mirror reusable-runtime-boot.yml's connectivity + # probe: try Docker DNS (compose network) first, then localhost, then + # the Docker host gateway; fail closed with diagnostics if none work + # instead of letting pytest fail on opaque connection-refused errors. + - name: Resolve reachable e2e connectivity host + run: | + set -euo pipefail + + can_connect() { + python3 - "$1" "$2" <<'PY' + import socket + import sys + + host = sys.argv[1] + port = int(sys.argv[2]) + try: + with socket.create_connection((host, port), timeout=2): + pass + except OSError: + raise SystemExit(1) + PY + } + + runner_on_compose_network=false + if [[ -n "${OMNIBASE_INFRA_RUNNER_CONTAINER_ID:-}" ]]; then + docker network connect "${OMNIBASE_INFRA_NETWORK}" "${OMNIBASE_INFRA_RUNNER_CONTAINER_ID}" 2>/dev/null || true + if docker inspect --type container --format '{{json .NetworkSettings.Networks}}' "${OMNIBASE_INFRA_RUNNER_CONTAINER_ID}" \ + | grep -q "\"${OMNIBASE_INFRA_NETWORK}\""; then + runner_on_compose_network=true + fi + fi + + # resolved_host is the POSTGRES host; resolved_kafka_host is the + # REDPANDA host. In the localhost/gateway branches these are always + # the same value (one Docker-published endpoint, disambiguated only + # by port). The Docker DNS branch is the one case where postgres + # and redpanda are DIFFERENT container names on the compose + # network -- a single shared variable there previously cross-wired + # REDPANDA_ADVERTISE_HOST/OMNI_INFRA_HOST to the *postgres* + # container name. + resolved_host="" + resolved_kafka_host="" + resolved_pg_port="" + if $runner_on_compose_network \ + && can_connect "${OMNIBASE_INFRA_POSTGRES_CONTAINER}" 5432 \ + && can_connect "${OMNIBASE_INFRA_REDPANDA_CONTAINER}" 9092; then + echo "e2e stack reachable by generated container names (Docker DNS)" + resolved_host="${OMNIBASE_INFRA_POSTGRES_CONTAINER}" + resolved_kafka_host="${OMNIBASE_INFRA_REDPANDA_CONTAINER}" + resolved_pg_port="5432" + kafka_host_port="${OMNIBASE_INFRA_REDPANDA_CONTAINER}:9092" + pg_host_port="${OMNIBASE_INFRA_POSTGRES_CONTAINER}:5432" + elif can_connect localhost "${POSTGRES_PORT}" && can_connect localhost "${KAFKA_PORT}"; then + echo "e2e stack reachable by host-published ports (localhost)" + resolved_host="localhost" + resolved_kafka_host="localhost" + resolved_pg_port="${POSTGRES_PORT}" + kafka_host_port="localhost:${KAFKA_PORT}" + pg_host_port="localhost:${POSTGRES_PORT}" + elif [[ "${E2E_REDPANDA_ADVERTISE_HOST}" != "localhost" ]] \ + && can_connect "${E2E_REDPANDA_ADVERTISE_HOST}" "${POSTGRES_PORT}" \ + && can_connect "${E2E_REDPANDA_ADVERTISE_HOST}" "${KAFKA_PORT}"; then + echo "e2e stack reachable by Docker host gateway (${E2E_REDPANDA_ADVERTISE_HOST})" + resolved_host="${E2E_REDPANDA_ADVERTISE_HOST}" + resolved_kafka_host="${E2E_REDPANDA_ADVERTISE_HOST}" + resolved_pg_port="${POSTGRES_PORT}" + kafka_host_port="${E2E_REDPANDA_ADVERTISE_HOST}:${KAFKA_PORT}" + pg_host_port="${E2E_REDPANDA_ADVERTISE_HOST}:${POSTGRES_PORT}" + else + echo "::error::e2e stack is Healthy but unreachable from the runner by Docker DNS, localhost, or Docker host gateway" + echo "runner_container_id=${OMNIBASE_INFRA_RUNNER_CONTAINER_ID:-} runner_on_compose_network=${runner_on_compose_network} e2e_redpanda_advertise_host=${E2E_REDPANDA_ADVERTISE_HOST}" + exit 1 + fi + + if [[ "$resolved_host" == "192.168.86.201" || "$resolved_kafka_host" == "192.168.86.201" ]]; then + echo "::error::resolved e2e connectivity host is the live .201 instance; refusing" + exit 1 + fi + + { + echo "KAFKA_BOOTSTRAP_SERVERS=${kafka_host_port}" + echo "OMNIBASE_INFRA_DB_URL=postgresql://postgres:${INTEGRATION_POSTGRES_PASSWORD}@${pg_host_port}/omnibase_infra" + echo "INTEGRATION_POSTGRES_HOST=${resolved_host}" + echo "INTEGRATION_POSTGRES_PORT=${resolved_pg_port}" + echo "OMNI_INFRA_HOST=${resolved_kafka_host}" + echo "POSTGRES_HOST=${resolved_host}" + # OMN-15567 remediation round 2: POSTGRES_PORT was set once, in + # the earlier "Derive isolated e2e namespace" step, to the + # ephemeral host-published port, and never re-emitted here, so it + # stayed paired with the ORIGINAL host even after POSTGRES_HOST + # was overridden to resolved_host above -- an invalid host/port + # pair under the Docker-DNS and gateway branches. Consumers: + # tests/integration/test_dispatch_roundtrip.py, + # tests/integration/injection_effectiveness/conftest.py, + # tests/integration/migrations/test_node_migration_shape_drift_omn15376.py. + echo "POSTGRES_PORT=${resolved_pg_port}" + echo "REDPANDA_ADVERTISE_HOST=${resolved_kafka_host}" + } >> "$GITHUB_ENV" + - name: Run integration tests run: | THRESHOLD="${{ github.event.inputs.success_threshold || env.SUCCESS_THRESHOLD }}" @@ -155,9 +468,74 @@ jobs: --timeout=300 \ --timeout-method=thread + # OMN-15565: the explicit -p is load-bearing. `-v --remove-orphans` against + # the default project is exactly what deleted the lab lane's containers and + # data volumes every night; scoped to the run-scoped project it can only + # delete this run's own resources. Because this step uses `if: always()`, it + # independently re-derives this run's immutable project and rejects empty, + # unexpected, or censused live-lane targets before the destructive down. - name: Tear down e2e stack if: always() - run: docker compose -f docker/docker-compose.e2e.yml down -v --remove-orphans + run: | + set -euo pipefail + project="${OMNIBASE_INFRA_COMPOSE_PROJECT:-}" + e2e_id="e2e-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}" + e2e_id="$(echo "$e2e_id" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9_-' '-')" + expected_project="omnibase-infra-${e2e_id}" + if [[ -z "$project" ]]; then + echo "::error::OMNIBASE_INFRA_COMPOSE_PROJECT is unset - refusing to run a project-less teardown" + exit 1 + fi + if [[ "$project" != "$expected_project" ]]; then + echo "::error::refusing to tear down unexpected compose project '$project' (expected '$expected_project' for this run)" + exit 1 + fi + uv run python - "$project" <<'PY' + import sys + + import yaml + + project = sys.argv[1] + manifest = yaml.safe_load(open("deploy/lane-census/lane-manifest.yaml", encoding="utf-8")) + protected = { + lane["compose_project"] + for lane in manifest["lanes"].values() + if lane.get("compose_project") + } + if project in protected: + sys.exit( + f"::error::refusing to tear down live lane project {project!r} " + f"(censused lanes: {sorted(protected)})" + ) + PY + + # OMN-15567: "Resolve reachable e2e connectivity host" attaches the + # long-lived self-hosted runner container to this run's network + # (`docker network connect`) so it can reach containers by Docker + # DNS name. Nothing disconnected it before this fix -- every run + # leaked one network attachment (and the network delete below + # then failed with "Resource is still in use", silently, since + # `docker compose down` does not propagate that as a nonzero exit + # for an already-removed set of containers). Disconnect first so + # the network can actually be deleted. + if [[ -n "${OMNIBASE_INFRA_RUNNER_CONTAINER_ID:-}" && -n "${OMNIBASE_INFRA_NETWORK:-}" ]]; then + docker network disconnect "${OMNIBASE_INFRA_NETWORK}" "${OMNIBASE_INFRA_RUNNER_CONTAINER_ID}" 2>/dev/null || true + # OMN-15567 remediation round 2: the disconnect above swallows its + # own exit code on purpose -- a disconnect racing an + # already-gone container/network is benign and must not fail + # this always()-run teardown step -- but that made a REAL + # failure to detach just as silent as the original bug. Verify + # the runner is actually off the network's member list; if not, + # surface it loudly (non-fatal ::error:: annotation) instead of + # reproducing the silence this fix exists to close. + if docker network inspect "${OMNIBASE_INFRA_NETWORK}" \ + --format '{{json .Containers}}' 2>/dev/null \ + | grep -q "${OMNIBASE_INFRA_RUNNER_CONTAINER_ID}"; then + echo "::error::runner container ${OMNIBASE_INFRA_RUNNER_CONTAINER_ID} is still attached to ${OMNIBASE_INFRA_NETWORK} after docker network disconnect -- this run will leak the attachment" + fi + fi + + docker compose -p "$project" -f docker/docker-compose.e2e.yml down -v --remove-orphans - name: Upload test results if: always() diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml index a49becaeb3..a739299981 100644 --- a/.github/workflows/nightly-tests.yml +++ b/.github/workflows/nightly-tests.yml @@ -55,7 +55,7 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/no-bare-compose-teardown.yml b/.github/workflows/no-bare-compose-teardown.yml index 86d210d019..dedf9fb195 100644 --- a/.github/workflows/no-bare-compose-teardown.yml +++ b/.github/workflows/no-bare-compose-teardown.yml @@ -27,9 +27,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 - trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest. runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -39,7 +38,7 @@ jobs: with: # git ls-files (used by the scanner) needs the full tracked index. fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: '3.12' # pytest is imported by the module (for the @pytest.mark decorators), but diff --git a/.github/workflows/node-migration-sync.yml b/.github/workflows/node-migration-sync.yml index e9cfe5ee6f..abd343bc6c 100644 --- a/.github/workflows/node-migration-sync.yml +++ b/.github/workflows/node-migration-sync.yml @@ -40,9 +40,8 @@ jobs: name: node-migration-sync runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -112,9 +111,8 @@ jobs: name: deployed-migration-tree-sync-logic runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/non-dev-base-guard.yml b/.github/workflows/non-dev-base-guard.yml index 2bcc807c26..166c0f7166 100644 --- a/.github/workflows/non-dev-base-guard.yml +++ b/.github/workflows/non-dev-base-guard.yml @@ -29,11 +29,8 @@ jobs: name: non-dev-base-guard runs-on: >- ${{ - (github.event_name == 'pull_request' && github.base_ref == 'dev') - && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') - || ( - (github.event.pull_request.head.repo.full_name != github.repository) - ) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/omni-standards-compliance.yml b/.github/workflows/omni-standards-compliance.yml index d5c2703c15..e3ce08828b 100644 --- a/.github/workflows/omni-standards-compliance.yml +++ b/.github/workflows/omni-standards-compliance.yml @@ -42,9 +42,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -78,9 +77,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -108,9 +106,8 @@ jobs: name: Handler Contract Compliance runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -119,7 +116,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} @@ -207,9 +204,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -246,9 +242,8 @@ jobs: name: CI Naming Convention runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/omnigate.yml b/.github/workflows/omnigate.yml index 1d3852b2ff..8ce7ca6c1d 100644 --- a/.github/workflows/omnigate.yml +++ b/.github/workflows/omnigate.yml @@ -31,7 +31,7 @@ jobs: git cat-file -e "${{ github.event.pull_request.base.sha }}^{commit}" git cat-file -e "${{ github.event.pull_request.head.sha }}^{commit}" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" diff --git a/.github/workflows/plugin-pin-cascade.yml b/.github/workflows/plugin-pin-cascade.yml index a1efd8b890..df80c05d7a 100644 --- a/.github/workflows/plugin-pin-cascade.yml +++ b/.github/workflows/plugin-pin-cascade.yml @@ -59,7 +59,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: '3.12' diff --git a/.github/workflows/pr-merged-event.yml b/.github/workflows/pr-merged-event.yml index bc48ec3d03..eb3a6a084d 100644 --- a/.github/workflows/pr-merged-event.yml +++ b/.github/workflows/pr-merged-event.yml @@ -26,6 +26,10 @@ on: - main - dev +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: publish-pr-merged-event: name: Publish PR Merged Event diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml index f49d8c5352..c0f638943f 100644 --- a/.github/workflows/pr-title-check.yml +++ b/.github/workflows/pr-title-check.yml @@ -16,6 +16,6 @@ concurrency: jobs: pr-title: - uses: OmniNode-ai/onex_change_control/.github/workflows/pr-title-check-reusable.yml@df0e348bebfb893fdebf74189bc0c782b624b02f + uses: OmniNode-ai/onex_change_control/.github/workflows/pr-title-check-reusable.yml@ab886f4cb91827d45f9fed0db0b9dd24a5948d87 permissions: pull-requests: read diff --git a/.github/workflows/precommit-parity-gate.yml b/.github/workflows/precommit-parity-gate.yml index 5e9f0468b8..f79b9dba3d 100644 --- a/.github/workflows/precommit-parity-gate.yml +++ b/.github/workflows/precommit-parity-gate.yml @@ -43,20 +43,22 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: precommit-parity-gate: name: Precommit Parity Gate - # OMN-14668: canonical OMNI_RUNNER_SELECTOR_V1 form for the byte-match parity - # gate is HOSTED-PINNED — `runs-on: ubuntu-latest`, unconditionally, for every - # event. The prior form routed ONLY public-fork PRs to hosted and left every - # trusted internal PR / push / merge_group on the self-hosted omnibase-ci - # fleet — a saturated or absent fleet would leave this required check - # queued/cancelled = a false-green window in the very gate meant to kill - # false-greens. This gate is short and dependency-free (`uv run --no-project - # --with pyyaml` over local files), so it must never depend on self-hosted - # availability. Matches the already-hosted omnibase_spi / omnibase_compat - # parity gates. - runs-on: ubuntu-latest + # Public fork PRs stay on GitHub-hosted compute. Same-repository PRs, pushes, + # and merge-queue runs use the trusted CI fleet like the rest of PR CI. + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} timeout-minutes: 10 steps: diff --git a/.github/workflows/prod-promotion-lineage.yml b/.github/workflows/prod-promotion-lineage.yml index 7ce9b1be24..fef255e8ff 100644 --- a/.github/workflows/prod-promotion-lineage.yml +++ b/.github/workflows/prod-promotion-lineage.yml @@ -56,6 +56,10 @@ env: UV_VERSION: "0.6.14" CACHE_VERSION: "0.6.0" +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: guard-tests: name: Prod promotion-lineage guard tests @@ -105,7 +109,7 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/product-readiness-shadow.yml b/.github/workflows/product-readiness-shadow.yml index 15826e9cc8..22ebe46c5f 100644 --- a/.github/workflows/product-readiness-shadow.yml +++ b/.github/workflows/product-readiness-shadow.yml @@ -113,9 +113,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -139,9 +138,8 @@ jobs: name: typecheck (shadow) runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -166,9 +164,8 @@ jobs: name: tests+coverage (shadow) runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -208,9 +205,8 @@ jobs: if: always() runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -221,7 +217,7 @@ jobs: with: fetch-depth: 1 - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: "3.12" - name: Emit typed reason graph (report-only) diff --git a/.github/workflows/receipt-honesty.yml b/.github/workflows/receipt-honesty.yml index 4e070ae867..7c08cde567 100644 --- a/.github/workflows/receipt-honesty.yml +++ b/.github/workflows/receipt-honesty.yml @@ -37,9 +37,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 - trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest. runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b98d0be714..e5c4fd0b78 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,14 @@ concurrency: jobs: release: - timeout-minutes: 30 + # OMN-16047: 60, not 30. The pin-resolvability gate below installs the built + # wheel's whole transitive closure with --no-cache (135 packages / 242 MB for + # this repo) and now budgets up to 1800s for it, because a *cached* uv sync on + # the self-hosted fleet already costs 110-402s. 30 minutes left no room for + # that install plus build + publish, so the job ceiling has to clear the step + # ceiling -- otherwise a slow install dies as an opaque job kill instead of the + # script's own diagnostic failure. + timeout-minutes: 60 runs-on: >- ${{ fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} outputs: diff --git a/.github/workflows/required-check-skip-guard-caller.yml b/.github/workflows/required-check-skip-guard-caller.yml index a8d9da0637..9b1eb00414 100644 --- a/.github/workflows/required-check-skip-guard-caller.yml +++ b/.github/workflows/required-check-skip-guard-caller.yml @@ -40,7 +40,11 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: required-check-skip-guard: if: ${{ always() }} - uses: OmniNode-ai/omniclaude/.github/workflows/required-check-skip-guard-reusable.yml@70718ddb45f6df5eb5e07159f2ae812efb1da7db + uses: OmniNode-ai/omniclaude/.github/workflows/required-check-skip-guard-reusable.yml@80de61fd1fee04abdeb6918e7f91cf820717e6a8 diff --git a/.github/workflows/runner-broker-dispatch-wedge-rerun.yml b/.github/workflows/runner-broker-dispatch-wedge-rerun.yml new file mode 100644 index 0000000000..ce52319f4e --- /dev/null +++ b/.github/workflows/runner-broker-dispatch-wedge-rerun.yml @@ -0,0 +1,53 @@ +# runner-broker-dispatch-wedge-rerun.yml — targeted signature rerun (OMN-15776) +# +# Scans recent self-hosted-runner jobs across the fleet's repos for the +# proven GitHub Actions broker-dispatch/reconnect race signature (job +# assigned to a self-hosted runner, conclusion failure/cancelled, ZERO +# steps recorded, duration in a tight band around the observed fixed +# ~10m0-1s GitHub-side orphan timeout) and reissues ONLY the matched job. +# +# This is not a blanket retry-on-red policy — a job with real steps or a +# duration outside the narrow signature band is never touched, so a genuine +# content failure is never laundered into a rerun. +# +# HARD REQUIREMENT: runs on GitHub-hosted compute (ubuntu-latest), never +# self-hosted — same rationale as runner-fleet-canary.yml: this must not +# share fate with the fleet it remediates (and specifically must not be +# vulnerable to the exact broker-dispatch race it exists to detect). +name: runner-broker-dispatch-wedge-rerun + +on: + schedule: + - cron: "*/10 * * * *" + workflow_dispatch: + inputs: + dry_run: + description: "Scan and report candidates without issuing reruns" + required: false + default: "false" + +permissions: + contents: read + actions: write # required to POST /actions/jobs/{id}/rerun + +jobs: + wedge-rerun: + name: wedge-rerun + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout (script only) + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Scan for broker-dispatch-wedge signature and rerun matches + env: + RUNNER_GITHUB_TOKEN: ${{ secrets.RUNNER_FLEET_STATUS_TOKEN || secrets.CROSS_REPO_PAT }} + REPOS_CSV: "omnibase_infra,omnibase_core,omniclaude,omnimarket,onex_change_control" + run: | + ARGS="" + if [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then + ARGS="--dry-run" + fi + bash scripts/ci/runner_broker_dispatch_wedge_rerun.sh ${ARGS} diff --git a/.github/workflows/runner-disk-preflight.yml b/.github/workflows/runner-disk-preflight.yml index efd0a9cd6c..ca8f312ee1 100644 --- a/.github/workflows/runner-disk-preflight.yml +++ b/.github/workflows/runner-disk-preflight.yml @@ -55,9 +55,8 @@ jobs: # public fork PRs use ubuntu-latest (no secrets, disk check still runs). runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/runner-routing-audit.yml b/.github/workflows/runner-routing-audit.yml index e6cf614701..d74afa5a69 100644 --- a/.github/workflows/runner-routing-audit.yml +++ b/.github/workflows/runner-routing-audit.yml @@ -19,15 +19,18 @@ permissions: contents: read actions: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: audit: name: Runner Routing Audit # OMNI_RUNNER_SELECTOR_V1 - trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest. runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/runtime-rebuild-trigger.yml b/.github/workflows/runtime-rebuild-trigger.yml index 89f1bd8d65..46a819737a 100644 --- a/.github/workflows/runtime-rebuild-trigger.yml +++ b/.github/workflows/runtime-rebuild-trigger.yml @@ -18,15 +18,18 @@ # # Trigger conditions (evaluated by scripts/trigger_rebuild_on_merge.py): # - PR had the "runtime_change" label, OR -# - Any changed file matches src/omnimarket/** or src/omnibase_infra/nodes/** +# - omniclaude's canonical deploy-gate classifier identifies a runtime path # -# Tickets: OMN-8917 (original auto-trigger), OMN-12573 (re-point to node_redeploy) +# Control-bus routing is independent of the requested runtime lane. Both dev +# and stability-test commands enter through the dev control bus where +# node_redeploy_orchestrator consumes them; runtime_lane in the signed payload +# decides the deployment target. The broker comes from omnimarket's checked-in +# config/ci_bus_lanes.yaml overlay, not an opaque Actions secret (OMN-15009). # -# Required GitHub Secrets: -# KAFKA_BOOTSTRAP_SERVERS -- e.g. pkc-xxx.us-east-1.aws.confluent.cloud:9092 -# KAFKA_SASL_USERNAME -- Confluent Cloud API key (or SASL username) -# KAFKA_SASL_PASSWORD -- Confluent Cloud API secret (or SASL password) -# DEPLOY_AGENT_HMAC_SECRET -- HMAC secret for signing redeploy commands +# Required GitHub Secrets: none. The deploy-agent HMAC belongs to the runtime +# deploy EFFECT that emits rebuild-requested, not this upstream start command. +# +# Tickets: OMN-8917, OMN-12573, OMN-15009 name: Runtime Rebuild Trigger @@ -37,6 +40,10 @@ on: - main - dev +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: trigger-rebuild: name: Trigger node_redeploy Start @@ -60,6 +67,29 @@ jobs: with: separator: "," + - name: Fetch authoritative CI bus lane overlay + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: OmniNode-ai/omnimarket + ref: dev + path: .ci-bus-overlay + persist-credentials: false + sparse-checkout: | + config/ci_bus_lanes.yaml + src/omnimarket/nodes/node_redeploy_orchestrator/models/model_redeploy_start_command.py + sparse-checkout-cone-mode: false + + - name: Fetch canonical deploy-path classifier + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: OmniNode-ai/omniclaude + ref: main + path: .deploy-gate-source + persist-credentials: false + sparse-checkout: | + .github/actions/deploy-gate/validate_pr_deploy_required.py + sparse-checkout-cone-mode: false + - name: Set up CI Python environment uses: ./.github/actions/setup-python-uv with: @@ -73,10 +103,6 @@ jobs: # All potentially attacker-controlled inputs (labels, base_ref, sha) are # passed through environment variables to prevent shell injection. env: - KAFKA_BOOTSTRAP_SERVERS: ${{ secrets.KAFKA_BOOTSTRAP_SERVERS }} - KAFKA_SASL_USERNAME: ${{ secrets.KAFKA_SASL_USERNAME }} - KAFKA_SASL_PASSWORD: ${{ secrets.KAFKA_SASL_PASSWORD }} - DEPLOY_AGENT_HMAC_SECRET: ${{ secrets.DEPLOY_AGENT_HMAC_SECRET }} PR_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} PR_NUMBER: ${{ github.event.pull_request.number }} @@ -90,4 +116,8 @@ jobs: --labels "$PR_LABELS" \ --base-branch "$PR_BASE_BRANCH" \ --source-sha "$PR_MERGE_SHA" \ + --bus-lane "dev" \ + --bus-overlay ".ci-bus-overlay/config/ci_bus_lanes.yaml" \ + --consumer-model ".ci-bus-overlay/src/omnimarket/nodes/node_redeploy_orchestrator/models/model_redeploy_start_command.py" \ + --runtime-path-validator ".deploy-gate-source/.github/actions/deploy-gate/validate_pr_deploy_required.py" \ --requested-by "gha/omnibase_infra/pr-$PR_NUMBER" diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 92edd6c880..7156ed56a2 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -28,9 +28,8 @@ jobs: name: CodeQL runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/seed-provenance-check.yml b/.github/workflows/seed-provenance-check.yml index 1dd61f1b15..a0c9a4592c 100644 --- a/.github/workflows/seed-provenance-check.yml +++ b/.github/workflows/seed-provenance-check.yml @@ -36,9 +36,8 @@ jobs: name: Seed Provenance Advisory Check runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} @@ -49,7 +48,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: "3.12" diff --git a/.github/workflows/shellcheck-gate.yml b/.github/workflows/shellcheck-gate.yml index f9ed1b74fb..809a98a823 100644 --- a/.github/workflows/shellcheck-gate.yml +++ b/.github/workflows/shellcheck-gate.yml @@ -30,9 +30,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/skill-node-mapping-sync.yml b/.github/workflows/skill-node-mapping-sync.yml index 7a33eda647..424c9889b4 100644 --- a/.github/workflows/skill-node-mapping-sync.yml +++ b/.github/workflows/skill-node-mapping-sync.yml @@ -45,9 +45,8 @@ jobs: name: skill-node-mapping-sync runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/stale-todo-gate.yml b/.github/workflows/stale-todo-gate.yml index 13cf32d8bf..da403d2b56 100644 --- a/.github/workflows/stale-todo-gate.yml +++ b/.github/workflows/stale-todo-gate.yml @@ -33,9 +33,8 @@ jobs: name: Stale TODO Gate runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/todo-audit-on-merge.yml b/.github/workflows/todo-audit-on-merge.yml index 6baa46888a..fb8666bcfd 100644 --- a/.github/workflows/todo-audit-on-merge.yml +++ b/.github/workflows/todo-audit-on-merge.yml @@ -30,6 +30,10 @@ on: - main - dev +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: todo-audit: timeout-minutes: 30 diff --git a/.github/workflows/url-authority-gate.yml b/.github/workflows/url-authority-gate.yml index 3fd2dbe087..d0ec756694 100644 --- a/.github/workflows/url-authority-gate.yml +++ b/.github/workflows/url-authority-gate.yml @@ -48,12 +48,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 - trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest. runs-on: >- ${{ - (github.event_name == 'pull_request' && github.base_ref == 'dev') - && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') - || ( (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - ) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/validate-validator-requirements.yml b/.github/workflows/validate-validator-requirements.yml index 7fa342d86c..1ecc315343 100644 --- a/.github/workflows/validate-validator-requirements.yml +++ b/.github/workflows/validate-validator-requirements.yml @@ -36,9 +36,8 @@ jobs: # OMNI_RUNNER_SELECTOR_V1 — runners are resolved from centralized selector variables. runs-on: >- ${{ - ((github.event_name == 'pull_request' && github.base_ref == 'dev') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository)) + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} diff --git a/.github/workflows/validator-runtime-profiles.yml b/.github/workflows/validator-runtime-profiles.yml index 5f5826d0e0..44fe602fa1 100644 --- a/.github/workflows/validator-runtime-profiles.yml +++ b/.github/workflows/validator-runtime-profiles.yml @@ -47,13 +47,8 @@ concurrency: jobs: validate: name: validate - # OMNI_RUNNER_SELECTOR_V1 - trusted events default to self-hosted omnibase-ci; pull_request defaults to ubuntu-latest - runs-on: >- - ${{ - github.event_name == 'pull_request' - && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') - || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') - }} + # workflow_dispatch-only: this trusted maintenance lane never executes PR code. + runs-on: ${{ fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') }} timeout-minutes: 10 steps: @@ -61,7 +56,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/.onex_ratchets/noncanonical_class_allowlist.yaml b/.onex_ratchets/noncanonical_class_allowlist.yaml index ac86b21a2f..57c4cd8883 100644 --- a/.onex_ratchets/noncanonical_class_allowlist.yaml +++ b/.onex_ratchets/noncanonical_class_allowlist.yaml @@ -56,7 +56,6 @@ allowlist: - "omnibase_infra.observability.runner_health.collector_runner_health:CollectorRunnerHealth" - "omnibase_infra.onboarding.adapter_cli_input:AdapterCliInput" - "omnibase_infra.onboarding.adapter_fake_input:AdapterFakeInput" - - "omnibase_infra.runtime.auto_wiring.handler_wiring:SyncPsycopg2Adapter" - "omnibase_infra.runtime.contract_registration_event_router:ContractRegistrationEventRouter" - "omnibase_infra.runtime.emit_daemon.event_registry:EventRegistry" - "omnibase_infra.runtime.freshness_monitor:ServiceFreshnessMonitor" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2559657873..4df1c1cf8b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -107,11 +107,16 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: + # `\.captured$`: verbatim incident-replay artifacts (OMN-15547). These + # hooks are `types: [text]`, so they match on content and would append a + # newline to a captured HTTP body or API response -- silently voiding the + # sha256 that makes it evidence. Observed, not hypothetical: it happened + # to all three captures on their first commit and R1 rejected them. - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - exclude: ^\.github/workflows/ + exclude: ^\.github/workflows/|\.captured$ - id: end-of-file-fixer - exclude: ^\.github/workflows/ + exclude: ^\.github/workflows/|\.captured$ - id: check-merge-conflict - id: check-added-large-files args: [--maxkb=1000] @@ -192,6 +197,21 @@ repos: files: ^src/omnibase_infra/configs/pricing_manifest\.yaml$ pass_filenames: true stages: [pre-commit] + # OMN-15538: reject a pin whose commit is not reachable from a protected + # branch of the target repo. Mirrors the required CI job + # "Pin Reachability (OMN-15538)"; catching a dead pin at commit time is + # what stops the OMN-15536 shape (ci.yml startup-fails, "CI Summary" + # never reports, the whole repo wedges) from reaching the remote at all. + # --allow-undetermined keeps an offline commit unblocked; the script + # REFUSES that flag under CI, so the enforcing surface stays fail-closed + # and the hook can never be the reason a bad pin merges. + - id: check-pin-reachability + name: Cross-repo pins must be reachable from dev/main (OMN-15538) + entry: uv run --frozen python scripts/ci/check_pin_reachability.py --allow-undetermined + language: system + files: ^(\.github/workflows/.*\.ya?ml|pyproject\.toml|uv\.lock)$ + pass_filenames: true + stages: [pre-commit] - id: reject-node-implementations name: Reject unallowlisted node handlers (belong in omnimarket) entry: bash scripts/ci/reject-node-implementations.sh --staged @@ -326,6 +346,17 @@ repos: pass_filenames: false always_run: false stages: [pre-commit] + # OMN-15378: fail closed on any new `tests/` directory pytest can never + # collect (scripts/deploy-agent/tests/ sat uncollected + RED for ~5 + # weeks with zero CI signal). always_run — a new stray root can appear + # anywhere in the tree, not just under files already staged. + - id: onex-validate-test-root-collection + name: ONEX Uncollected Test Root Check (OMN-15378) + entry: uv run --frozen python scripts/validation/validate_test_root_collection.py + language: system + pass_filenames: false + always_run: true + stages: [pre-commit] # OMN-12559 / OMN-13124: vendored omnimarket node migrations must stay in # sync with source. MUST always_run — the prior files:-scoped trigger only # fired when a file under docker/migrations/forward/nodes/ (or the sync @@ -773,6 +804,19 @@ repos: entry: uv run --frozen python scripts/check_migration_required.py --pre-commit pass_filenames: false stages: [pre-commit] + # OMN-15717: every vendored node migration under docker/migrations/forward/nodes/ + # must carry a checked-in stream/domain declaration BEFORE it can reach a + # runtime -- catches the class of gap that let node_pr_review_bot's + # 001_create_review_bot_bypass_log.sql ship undeclared and fail closed at + # bootstrap.sql deploy time instead of at PR time. Wires the pre-existing + # (previously unwired) scripts/validation/validate_application_migration_manifest.py. + - id: onex-check-node-migration-declarations + name: Node Migration Declaration Check + language: system + entry: uv run --frozen python scripts/validation/validate_application_migration_manifest.py + pass_filenames: false + files: ^docker/migrations/forward/(nodes/|_ledger/(application-migrations|application-migration-blocks|legacy-node-migrations)\.tsv) + stages: [pre-commit] # OMN-3617: Block planning docs (belong in omni_home/docs/plans/) - id: no-planning-docs name: Block planning docs (belong in omni_home/docs/plans/) @@ -1007,6 +1051,21 @@ repos: always_run: true stages: [pre-commit] files: ^(\.pre-commit-config\.yaml|\.github/workflows/(canonical-inference-gate|url-authority-gate)\.yml)$ + # Incident-replay coverage (OMN-15547): a guard that has never been run + # against the real thing it exists to catch is decorative. Fires on the + # two surfaces that can create the gap -- wiring a new guard (this config + # or a workflow) and editing the registry/fixtures themselves. Local + # because the failure it prevents is committed BEFORE CI sees it: the + # author who adds the hook line is the only person who still has the real + # artifact to hand. + - id: incident-replay-coverage + name: Incident-replay coverage (OMN-15547) + entry: uv run python scripts/ci/check_incident_replay_coverage.py + language: system + pass_filenames: false + always_run: true + stages: [pre-commit] + files: ^(\.pre-commit-config\.yaml|\.github/workflows/.*\.ya?ml|tests/incident_replays/.*|tests/fixtures/.*|scripts/.*\.(py|sh)|deploy/maintenance/.*)$ # Governed impacted-test selector at PRE-PUSH (OMN-13973 / WS7 OMN-14655 fan-out; # canary = omnibase_core#1451). Runs the fast local IMPACTED SUBSET of the unit # suite once per `git push`, via the SAME selector CI uses diff --git a/CLAUDE.md b/CLAUDE.md index 448f993b8f..0d21841f7e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,27 +6,6 @@ --- -## Table of Contents - -1. [Repo Invariants](#repo-invariants) -2. [Non-Goals](#non-goals) -3. [Service Catalog Architecture](#service-catalog-architecture) -4. [Quick Reference](#quick-reference) -5. [Architecture: Four-Node Pattern](#architecture-four-node-pattern) -6. [Declarative Nodes](#declarative-nodes) -7. [Handler System](#handler-system) -8. [Intent Model Architecture](#intent-model-architecture) -9. [Error Handling](#error-handling) -10. [Infrastructure Patterns](#infrastructure-patterns) -11. [Pydantic Model Standards](#pydantic-model-standards) -12. [Testing and CI](#testing-and-ci) -13. [Contract-Driven Config Discovery](#contract-driven-config-discovery) -14. [Agent-Driven Development](#agent-driven-development) -15. [Common Pitfalls](#common-pitfalls) -16. [Release Process](#release-process) - ---- - ## Repo Invariants These are non-negotiable architectural truths: @@ -45,207 +24,50 @@ These are non-negotiable architectural truths: We explicitly do **NOT** optimize for: -- **Backwards compatibility** - This repo has no external consumers. Schemas, APIs, and interfaces may change without deprecation periods. If something needs to change, change it. No `_deprecated` suffixes, no shims, no compatibility layers. +- **Backwards compatibility** - This repo has no external consumers. Schemas, APIs, and interfaces may change without deprecation periods. No `_deprecated` suffixes, no shims, no compatibility layers. - **Convenience over correctness** - Contract violations fail loudly - **Business logic in nodes** - Nodes coordinate; handlers compute - **Dynamic runtime behavior** - All behavior must be contract-declared -- **Implicit state** - All state transitions are explicit and auditable -- **Tight coupling** - Protocol-based DI enforces loose coupling - **Versioned directories** - NEVER create `v1_0_0/`, `v2/` directories; version through `contract.yaml` fields only -**When you see deprecated or unused code: DELETE IT.** Do not: -- Leave it "for reference" -- Comment it out -- Add deprecation warnings -- Create compatibility shims -- Keep old function signatures with forwarding +**When you see deprecated or unused code: DELETE IT.** Do not leave it "for reference", comment it out, add deprecation warnings, create compatibility shims, or keep old function signatures with forwarding. --- -## Install Model - -`omnibase_infra` ships as both a **pip-installable package** and a **cloneable repository**. -The two serve different purposes. +## Install Model & Service Catalog -### Pip Package (library + runtime CLIs) +`omnibase_infra` ships as both a pip package (library + runtime CLIs; entry points are +declared in `pyproject.toml [project.scripts]`) and a cloneable repo (the operational +`scripts/` are NOT bundled in the pip package — they need a clone). All Docker +infrastructure is generated from typed YAML manifests: `docker/catalog/services/*.yaml` +grouped by `docker/catalog/bundles.yaml`, driven by the `onex` CLI +(`src/omnibase_infra/docker/catalog/cli.py` — `generate` / `validate` / `up` / `down`). +Never hand-edit the generated compose file; never bypass the catalog with raw +`docker compose -f `. -Install via pip for: -- Using `omnibase_infra` as a library dependency in other ONEX services -- Running the bundled runtime CLIs - -```bash -pip install omnibase-infra -# or -uv add omnibase-infra -``` +Full walkthrough (install decision table, bundle composition, adding a new service): +`docs/patterns/service_catalog.md`. -**Bundled CLI entry points** (available after `pip install omnibase-infra`): - -| Command | Entry Point | Purpose | -|---------|-------------|---------| -| `omni-infra` | `omnibase_infra.cli.commands:cli` | General CLI | -| `onex-runtime` | `omnibase_infra.runtime.kernel:main` | Start ONEX runtime | -| `onex-infra-test` | `omnibase_infra.cli.infra_test.cli:cli` | Infra test runner | -| `onex-git-hook-relay` | `omnibase_infra.cli.git_hook_relay:main` | Git hook relay | -| `onex-linear-relay` | `omnibase_infra.cli.linear_relay:main` | Linear relay | -| `onex-status` | `omnibase_infra.tui.__main__:run_status_tui` | Status TUI | - -### Local Clone (operational scripts) - -A **local clone is required** to run the operational scripts in `scripts/`. These scripts -are **not bundled** in the pip package — they live only in the repository source tree. - -```bash -git clone https://github.com/OmniNode-ai/omnibase_infra.git -cd omnibase_infra -uv sync -``` - -**Scripts that require a local clone:** - -| Script | Purpose | Requires Clone | -|--------|---------|---------------| -| `scripts/seed-infisical.py` | Populate Infisical from contract YAMLs | Yes | -| `scripts/bootstrap-infisical.sh` | Full first-time bootstrap sequence | Yes | -| `scripts/provision-infisical.py` | Create machine identities, write credentials back to `~/.omnibase/.env` | Yes | -| `scripts/setup-infisical-identity.sh` | Create runtime/admin machine identities | Yes | -| `scripts/create_kafka_topics.py` | Create Kafka/Redpanda topics | Yes | -| `scripts/validate.py` | Run ONEX validators | Yes | -| All other `scripts/*.py` | Operational, CI, or dev tooling | Yes | - -**Why scripts require a clone:** These scripts scan the repository source tree directly -(e.g., `seed-infisical.py` iterates over `src/omnibase_infra/nodes/*/contract.yaml`), -write back to `~/.omnibase/.env`, or depend on shell tooling co-located with the repo. - -### Decision Summary - -| Use Case | Install Method | -|----------|---------------| -| Add `omnibase_infra` as a library dependency | `pip install omnibase-infra` | -| Run ONEX runtime services | `pip install omnibase-infra` → `onex-runtime` | -| Bootstrap Infisical (first-time setup) | Clone + `scripts/bootstrap-infisical.sh` | -| Seed Infisical from contracts | Clone + `uv run python scripts/seed-infisical.py` | -| Provision machine identities | Clone + `uv run python scripts/provision-infisical.py` | -| Run CI validators | Clone + `uv run python scripts/validate.py` | -| Develop nodes and handlers | Clone (full dev environment) | - -> **Note on `sync-omnibase-env.py`**: This script is **not** part of -> `omnibase_infra`. Use the separately installed environment-sync tooling -> available in your workspace. - ---- - -## Service Catalog Architecture - -The service catalog is the authoritative source for all Docker infrastructure. -Every deployable unit is a typed YAML manifest; the compose file is generated, not hand-edited. - -### Concepts - -| Term | Description | -|------|-------------| -| **Manifest** | Typed YAML declaration of a single deployable service (`docker/catalog/services/.yaml`) | -| **Bundle** | Named group of manifests deployed together (`docker/catalog/bundles.yaml`) | -| **Resolver** | Loads manifests + bundles, resolves transitive `includes`, returns `ResolvedStack` | -| **Generator** | Renders `ResolvedStack` → `docker-compose.generated.yml` | -| **Validator** | Checks that all `required_env` vars are present before start | - -### Bundle Definitions - -| Bundle | Contents | Purpose | -|--------|----------|---------| -| `core` | postgres, redpanda, valkey, infisical | Always-on infrastructure | -| `runtime` | (composed) | Full ONEX runtime stack — includes `core`, `tracing`, and the four sub-bundles below | -| `runtime-core` | omninode-runtime, runtime-effects, agent-actions-consumer, skill-lifecycle-consumer, context-audit-consumer, omninode-contract-resolver, intelligence-api | 7 services with no env requirements beyond the 0.34.0 baseline; deploy correctness fixes without needing new secrets | -| `runtime-integrations` | ci-relay, linear-relay, waitlist-signup-notifier | External-integration services; needs `CI_CALLBACK_TOKEN`, `LINEAR_WEBHOOK_SECRET`, `WAITLIST_NOTIFIER_SLACK_*` | -| `runtime-observability-projections` | injection-effectiveness-consumer, savings-estimation-consumer, llm-cost-aggregation-consumer, consumer-health-projection, decision-store-consumer | Postgres projection consumers; needs `OMNIBASE_INFRA_INJECTION_EFFECTIVENESS_POSTGRES_DSN` | -| `runtime-infrastructure` | forward-migration, migration-gate, intelligence-migration, runtime-worker, retry-worker, autoheal | Migrations, workers, container autoheal | -| `memgraph` | omnibase-infra-memgraph | Graph memory — injects `OMNIMEMORY_*` env vars | -| `observability` | phoenix | LLM observability (Phoenix traces/evals) | -| `tracing` | (none) + observability | Injects OTEL env vars; phoenix pulled in transitively | -| `secrets` | infisical | Secrets management — injects `INFISICAL_ADDR` | -| `auth` | keycloak | Local OIDC/auth | - -**Transitive resolution**: `runtime` includes `core`, `tracing`, and the four runtime sub-bundles; `tracing` includes `observability`. The resolver expands all `includes` before collecting services. - -**Incremental rollout**: when new runtime services land with new secret requirements, operators can deploy `onex up runtime-core` to pick up correctness fixes without also needing those secrets. Once the secrets are seeded (Infisical or `~/.omnibase/.env`), bring up the remaining sub-bundles with `onex up runtime-integrations`, `onex up runtime-observability-projections`, or `onex up runtime-infrastructure`. - -**Env injection**: Each bundle may declare `inject_env` (hardcoded values injected into generated compose) and `inject_required_env` (vars that must be present in the operator environment at start time). - -### onex CLI Commands - -The `onex` CLI (`src/omnibase_infra/docker/catalog/cli.py`) is the primary operator interface. - -```bash -# Generate compose file for one or more bundles -uv run python -m omnibase_infra.docker.catalog.cli generate core -uv run python -m omnibase_infra.docker.catalog.cli generate runtime memgraph - -# Validate env completeness before starting -uv run python -m omnibase_infra.docker.catalog.cli validate runtime -uv run python -m omnibase_infra.docker.catalog.cli validate runtime memgraph - -# Start a bundle (generate + validate + docker compose up) -uv run python -m omnibase_infra.docker.catalog.cli up core -uv run python -m omnibase_infra.docker.catalog.cli up runtime memgraph tracing - -# Stop a running bundle -uv run python -m omnibase_infra.docker.catalog.cli down core -``` - -The shell functions `infra-up`, `infra-up-runtime`, `infra-up-memory`, and `infra-down` (defined in `~/.zshrc`) are backwards-compatible wrappers around `onex up/down`. They remain the preferred operator interface — do not bypass them with raw `docker compose -f `. - -### Shell Function → onex Mapping - -| Shell Function | Equivalent onex Command | -|----------------|------------------------| -| `infra-up` | `onex up core` | -| `infra-up-runtime` | `onex up runtime` | -| `infra-up-memory` | `onex up runtime memgraph` | -| `infra-down` | `onex down ` | - -### Adding a New Service - -1. Create `docker/catalog/services/.yaml` using an existing manifest as template. -2. Set `layer` to one of: `infrastructure`, `runtime`, `observability`, `auth`, `secrets`. -3. Declare all `required_env` vars that the container needs from the operator environment. -4. Add hardcoded container-internal addresses under `hardcoded_env` (never pass host-side env vars for internal addressing). -5. Add the service name to the appropriate bundle(s) in `docker/catalog/bundles.yaml`. -6. Run `uv run python -m omnibase_infra.docker.catalog.cli validate ` to confirm env contract. - -### Env Var Contract - -Three categories of env vars in the catalog: - -| Category | Location | Behavior | -|----------|----------|----------| -| `required_env` | Per-manifest YAML | Must be set in operator env; validated before start | -| `hardcoded_env` | Per-manifest YAML | Container-internal addresses; never overrideable | -| `inject_env` | Per-bundle in `bundles.yaml` | Injected only when that bundle is selected | - -**Rule**: Container-to-container addresses (e.g. `redpanda:9092`, `valkey:6379`) must live in `hardcoded_env`, never in `required_env`. Operator-supplied secrets (`POSTGRES_PASSWORD`, API keys) belong in `required_env`. +**Env var rule (trap)**: Container-to-container addresses (e.g. `redpanda:9092`, +`valkey:6379`) must live in `hardcoded_env`, never in `required_env`. Operator-supplied +secrets (`POSTGRES_PASSWORD`, API keys) belong in `required_env`. --- ## Quick Reference ```bash -# Setup -uv sync && pre-commit install - -# Testing -uv run pytest tests/ # All tests -uv run pytest tests/ -n auto # Parallel execution -uv run pytest tests/ -m unit # Unit tests only -uv run pytest tests/ -m integration # Integration tests only -uv run pytest tests/ --cov # With coverage (60% minimum) - -# Code Quality +uv sync && pre-commit install # Setup +uv run pytest tests/ -n auto # Tests (parallel) +uv run pytest tests/ -m unit # Unit only; -m integration for integration uv run mypy src/omnibase_infra/ # Type checking uv run ruff check src/ tests/ # Linting -pre-commit run --all-files # All hooks +pre-commit run --all-files # All hooks ``` +Coverage minimum is enforced via `fail_under` in `pyproject.toml` — read it there, don't +trust a copied number. + ## SPDX Headers All source files in `src/`, `tests/`, `scripts/`, `examples/` require MIT SPDX headers. @@ -270,11 +92,8 @@ Canonical spec: `omnibase_core/docs/conventions/FILE_HEADERS.md` ### Autonomous mode safety rails -When operating autonomously in this repo: -- Never disable pre-commit hooks, CI checks, or type checkers to make code pass. - Fix the code instead. -- Never write state files to `~/.claude/`; use the workspace `.onex_state/` - directory. +- Never disable pre-commit hooks, CI checks, or type checkers to make code pass. Fix the code instead. +- Never write state files to `~/.claude/`; use the workspace `.onex_state/` directory. - Friction logs go under `.onex_state/friction/` for external observability. ### Contract-first topic definitions @@ -286,8 +105,8 @@ When adding a new Kafka topic: 1. Declare it in the node's contract YAML under `event_bus.publish_topics` or `subscribe_topics` 2. Add the topic to the relevant `topics.yaml` skill file if it is a skill-emitted topic 3. Reference the contract-declared topic name in code via the contract loader -4. Never hardcode topic strings like `"onex.evt.foo.bar.v1"` in Python modules -5. The CI check `check-arch-invariants` enforces this -- hardcoded topic strings will fail CI +4. Never hardcode topic strings like `"onex.evt.foo.bar.v1"` in Python modules — the + `arch-invariants` job in `.github/workflows/ci.yml` fails CI on hardcoded topic strings --- @@ -302,8 +121,6 @@ When adding a new Kafka topic: **Data Flow**: Unidirectional left-to-right. No backwards dependencies. -### Node Types - | Node | Contract Type | Purpose | Primary Output | |------|--------------|---------|----------------| | **EFFECT** | `EFFECT_GENERIC` | External I/O (APIs, DB, files) | `events[]` | @@ -311,24 +128,8 @@ When adding a new Kafka topic: | **REDUCER** | `REDUCER_GENERIC` | FSM state management | `projections[]` | | **ORCHESTRATOR** | `ORCHESTRATOR_GENERIC` | Workflow coordination | `events[]`, `intents[]` | -### Import Path - -```python -from omnibase_core.nodes import ( - NodeEffect, # External I/O operations - NodeCompute, # Pure transformations - NodeReducer, # FSM-driven state - NodeOrchestrator, # Workflow coordination -) -``` - -### Layer Responsibilities - -| Layer | Responsibility | -|-------|---------------| -| `omnibase_core` | Node archetypes, I/O models, enums | -| `omnibase_spi` | Protocol definitions | -| `omnibase_infra` | Infrastructure implementations | +Base classes: `from omnibase_core.nodes import NodeEffect, NodeCompute, NodeReducer, NodeOrchestrator`. +Layering: `omnibase_core` = archetypes/models/enums, `omnibase_spi` = protocols, `omnibase_infra` = implementations. --- @@ -336,111 +137,29 @@ from omnibase_core.nodes import ( **ALL nodes MUST be declarative - no custom Python logic in node.py** -```python -# CORRECT - Declarative node (extends base, no custom logic) -from omnibase_core.nodes import NodeOrchestrator -from omnibase_core.models.container.model_onex_container import ModelONEXContainer - -class NodeRegistrationOrchestrator(NodeOrchestrator): - """Declarative orchestrator - all behavior defined in contract.yaml.""" - - def __init__(self, container: ModelONEXContainer) -> None: - super().__init__(container) - # No custom code - driven entirely by contract -``` - -### Declarative Pattern Requirements - -1. Extend base class from `omnibase_core.nodes` -2. Use `container: ModelONEXContainer` for dependency injection +1. Extend the base class from `omnibase_core.nodes` +2. Use `container: ModelONEXContainer` for dependency injection (call `super().__init__(container)`) 3. Define all behavior in `contract.yaml` (handlers, routing, workflows) 4. `node.py` contains ONLY the class definition extending base - no custom logic -### Canonical Node Directory Structure - -```text -nodes// -├── __init__.py # Public exports -├── contract.yaml # ONEX contract (REQUIRED) -├── node.py # Declarative node class (REQUIRED) -├── models/ # Node-specific Pydantic models -│ ├── __init__.py -│ └── model_.py -├── registry/ # Dependency injection registry -│ ├── __init__.py -│ └── registry_infra_.py -├── handlers/ # Handler implementations (optional) -│ ├── __init__.py -│ └── handler_.py -└── dispatchers/ # Dispatcher adapters (optional) - ├── __init__.py - └── dispatcher_.py -``` - -### Contract Requirements - -| Field | Type | Required | Description | -|-------|------|----------|-------------| -| `name` | string | Yes | Node identifier | -| `node_type` | string | Yes | `EFFECT_GENERIC`, `COMPUTE_GENERIC`, `REDUCER_GENERIC`, `ORCHESTRATOR_GENERIC` | -| `contract_version` | object | Yes | `{major, minor, patch}` | -| `node_version` | string/object | Yes | Semantic version | -| `description` | string | Yes | Node purpose | -| `input_model` | object | Yes | `{name, module, description}` | -| `output_model` | object | Yes | `{name, module, description}` | +For the canonical node directory layout and required contract fields, copy an existing +node under `src/omnibase_infra/nodes/` rather than working from a prose description. --- ## Handler System -### Handler Protocols - -| Protocol | Purpose | Input/Output | -|----------|---------|--------------| -| `ProtocolHandler` | Request/response I/O (HTTP, DB, Kafka) | `ModelProtocolRequest` → `ModelProtocolResponse` | -| `ProtocolMessageHandler` | Category-based (dispatch) | `ModelEventEnvelope` → `ModelHandlerOutput` | - -### Handler Routing Strategies - -**`payload_type_match`** - Routes based on event payload model type (orchestrator handlers): -```yaml -handler_routing: - routing_strategy: "payload_type_match" - handlers: - - event_model: - name: "ModelNodeIntrospectionEvent" - module: "omnibase_infra.models.registration.model_node_introspection_event" - handler: - name: "HandlerNodeIntrospected" - module: "omnibase_infra.nodes.node_registration_orchestrator.handlers.handler_node_introspected" -``` - -**`operation_match`** - Routes based on envelope operation (infrastructure handlers): -```yaml -handler_routing: - routing_strategy: "operation_match" - handlers: - - operation: "register_node" - handler: - name: "HandlerConsulRegister" - module: "omnibase_infra.nodes.node_registry_effect.handlers.handler_consul_register" -``` - -### Handler Classification +Two protocols: `ProtocolHandler` (request/response I/O: `ModelProtocolRequest` → +`ModelProtocolResponse`) and `ProtocolMessageHandler` (dispatch: `ModelEventEnvelope` → +`ModelHandlerOutput`). -Handlers expose two classification properties: +Two routing strategies in `handler_routing` contract blocks: +- `payload_type_match` — routes on event payload model type (orchestrator handlers) +- `operation_match` — routes on envelope operation (infrastructure handlers) -```python -@property -def handler_type(self) -> EnumHandlerType: - """Architectural role: INFRA_HANDLER, NODE_HANDLER, PROJECTION_HANDLER""" - return EnumHandlerType.INFRA_HANDLER - -@property -def handler_category(self) -> EnumHandlerTypeCategory: - """Behavioral classification: EFFECT, COMPUTE, NONDETERMINISTIC_COMPUTE""" - return EnumHandlerTypeCategory.EFFECT -``` +See `docs/patterns/operation_routing.md` and existing node contracts for the YAML shape. +Handlers expose `handler_type` (`EnumHandlerType`) and `handler_category` +(`EnumHandlerTypeCategory`) classification properties. ### Handler No-Publish Constraint @@ -456,121 +175,28 @@ def handler_category(self) -> EnumHandlerTypeCategory: ## Intent Model Architecture -**Overview**: Reducers emit intents that orchestrators route to Effect layer nodes. Payload models extend `BaseModel` directly (since omnibase_core 0.6.2). - -### Two-Layer Intent Structure - -| Layer | Model | Purpose | -|-------|-------|---------| -| 1. Typed Payload | `ModelPayloadConsulRegister` | Domain-specific Pydantic model with `intent_type` field | -| 2. Outer Container | `ModelIntent` | Standard intent envelope with `intent_type="extension"` | - -### Defining Typed Payload Models - -```python -# In nodes/reducers/models/model_payload_consul_register.py -from pydantic import BaseModel, ConfigDict, Field -from typing import Literal -from uuid import UUID - -class ModelPayloadConsulRegister(BaseModel): - """Typed payload for Consul service registration. - - Note: Extends BaseModel directly (ModelIntentPayloadBase was removed in - omnibase_core 0.6.2). - """ - model_config = ConfigDict(frozen=True, extra="forbid") - - intent_type: Literal["consul.register"] = Field(default="consul.register") - correlation_id: UUID - service_id: str - service_name: str - tags: list[str] - health_check: dict[str, str] | None = None -``` - -### Building Intents in Reducers - -```python -from omnibase_core.models.reducer.model_intent import ModelIntent - -# Build typed payload with domain data -consul_payload = ModelPayloadConsulRegister( - correlation_id=correlation_id, - service_id=f"onex-{node_type}-{node_id}", - service_name=f"onex-{node_type}", - tags=["node_type:effect"], -) - -# Return as ModelIntent from reducer -return ModelIntent( - intent_type="extension", - target=f"consul://service/{service_name}", - payload=consul_payload, -) -``` - -### Intent Type Routing - -- `ModelIntent.intent_type` is always `"extension"` for infrastructure intents -- `payload.intent_type` contains the specific routing key (e.g., `"consul.register"`) -- Effect layer routes based on `payload.intent_type` - -### Target URI Convention - -Format: `{protocol}://{resource}/{identifier}` - -Examples: -- `postgres://node_registrations/{node_id}` -- `consul://service/{service_name}` +Reducers emit intents that orchestrators route to Effect layer nodes. + +- **Two layers**: a typed payload model (e.g. `ModelPayloadConsulRegister`, with its own + `intent_type` literal field like `"consul.register"`) wrapped in the standard + `ModelIntent` envelope with `intent_type="extension"`. +- **Routing**: the Effect layer routes on `payload.intent_type`, not the outer envelope. +- **Target URI convention**: `{protocol}://{resource}/{identifier}` (e.g. + `postgres://node_registrations/{node_id}`, `consul://service/{service_name}`). +- **Trap**: infra payload models extend `BaseModel` directly (repo convention — see + `docs/standards/ONEX_TERMINOLOGY.md`). Do NOT justify this as "`ModelIntentPayloadBase` + was removed in omnibase_core 0.6.2" — that claim is false: the class exists in live + core (`omnibase_core.models.reducer.payloads`, present at v0.6.2 and every version + since) and bases core's own closed-set intent payloads. The real 0.6.2 change was + `ModelIntent.payload: dict[str, Any]` → `ProtocolIntentPayload` (OMN-1256). --- ## Error Handling -### Error Hierarchy - -```text -ModelOnexError (omnibase_core) -└── RuntimeHostError (base infrastructure error) - ├── ProtocolConfigurationError - ├── SecretResolutionError - ├── InfraConnectionError (transport-aware codes) - │ ├── InfraConsulError - │ └── InfraVaultError - ├── InfraTimeoutError - ├── InfraAuthenticationError - ├── InfraRateLimitedError - ├── InfraUnavailableError - ├── EnvelopeValidationError - ├── UnknownHandlerTypeError - ├── ContainerWiringError - │ ├── ServiceRegistrationError - │ ├── ServiceResolutionError - │ └── ContainerValidationError - ├── ChainPropagationError - ├── ArchitectureViolationError - ├── BindingResolutionError - ├── RepositoryError - │ ├── RepositoryContractError - │ ├── RepositoryValidationError - │ ├── RepositoryExecutionError - │ └── RepositoryTimeoutError - └── ContractPublisherError -``` - -### Error Class Selection - -| Scenario | Error Class | -|----------|-------------| -| Config invalid | `ProtocolConfigurationError` | -| Connection failed | `InfraConnectionError` | -| Timeout | `InfraTimeoutError` | -| Auth failed | `InfraAuthenticationError` | -| Rate limited | `InfraRateLimitedError` | -| Unavailable | `InfraUnavailableError` | -| Repository operation | `RepositoryError` (or subclass) | -| Container wiring | `ContainerWiringError` (or subclass) | +The infra error hierarchy roots at `RuntimeHostError` (itself under `ModelOnexError`); +see `omnibase_infra.errors` for the concrete tree — pick the narrowest matching class +(`InfraConnectionError`, `InfraTimeoutError`, `RepositoryError`, `ContainerWiringError`, ...). ### Error Context Factory (MANDATORY) @@ -578,117 +204,54 @@ ModelOnexError (omnibase_core) from omnibase_infra.errors import InfraConnectionError, ModelInfraErrorContext from omnibase_infra.enums import EnumInfraTransportType -# Auto-generate correlation_id (new error, no existing ID) +# Auto-generates correlation_id; pass correlation_id=... to propagate an existing one context = ModelInfraErrorContext.with_correlation( transport_type=EnumInfraTransportType.DATABASE, operation="execute_query", ) - -# Propagate existing correlation_id (preserve trace chain) -context = ModelInfraErrorContext.with_correlation( - correlation_id=request.correlation_id, - transport_type=EnumInfraTransportType.DATABASE, - operation="execute_query", -) - raise InfraConnectionError("Failed to connect", context=context) from e ``` ### Error Sanitization -**NEVER include**: passwords, API keys, PII, connection strings with credentials - -**SAFE to include**: service names, operation names, correlation IDs, ports - -Use utility functions from `omnibase_infra.utils.util_error_sanitization`: -- `sanitize_error_message()` - For DLQ/logs -- `sanitize_secret_path()` - For Vault paths -- `sanitize_consul_key()` - For Consul keys +**NEVER include**: passwords, API keys, PII, connection strings with credentials. +Use `sanitize_error_message()` / `sanitize_secret_path()` / `sanitize_consul_key()` from +`omnibase_infra.utils.util_error_sanitization`. --- ## Infrastructure Patterns -### Transport Types - -| Type | Value | Handler/Service | -|------|-------|-----------------| -| `HTTP` | `"http"` | `HandlerHTTP`, `ServiceHealth` | -| `DATABASE` | `"db"` | `HandlerDb`, `PostgresRepositoryRuntime` | -| `KAFKA` | `"kafka"` | `EventBusKafka`, `AdapterProtocolEventPublisherKafka` | -| `CONSUL` | `"consul"` | `HandlerConsul` | -| `VAULT` | `"vault"` | `HandlerVault` | -| `VALKEY` | `"valkey"` | (Planned) | -| `GRPC` | `"grpc"` | (Planned) | -| `RUNTIME` | `"runtime"` | `RuntimeHostProcess` | -| `MCP` | `"mcp"` | `HandlerMCP` | -| `FILESYSTEM` | `"filesystem"` | `HandlerFileSystem` | -| `INMEMORY` | `"inmemory"` | `EventBusInmemory` | -| `QDRANT` | `"qdrant"` | `HandlerQdrant` | -| `GRAPH` | `"graph"` | (Planned - Memgraph/Neo4j) | - -### Circuit Breaker +Transport types are enumerated in `EnumInfraTransportType` — read the enum, not a copied +table. -Use `MixinAsyncCircuitBreaker` for external service integrations: - -```python -class MyAdapter(MixinAsyncCircuitBreaker): - def __init__(self, config): - self._init_circuit_breaker( - threshold=5, - reset_timeout=60.0, - service_name="my-service", - transport_type=EnumInfraTransportType.HTTP, - half_open_successes=1, - ) - - async def connect(self): - async with self._circuit_breaker_lock: - await self._check_circuit_breaker("connect", correlation_id) - # ... operation ... -``` +### Circuit Breaker & Dispatcher Resilience -**States**: CLOSED → OPEN (after threshold failures) → HALF_OPEN (after timeout) → CLOSED (on success) +Use `MixinAsyncCircuitBreaker` for external service integrations (see +`docs/patterns/circuit_breaker_implementation.md`). -### Dispatcher Resilience - -**Dispatchers own their own resilience** - the `MessageDispatchEngine` does NOT wrap dispatchers with circuit breakers. - -Each dispatcher should: -- Implement `MixinAsyncCircuitBreaker` for external service calls -- Configure thresholds appropriate to their transport type -- Raise `InfraUnavailableError` when circuit opens +**Dispatchers own their own resilience** - the `MessageDispatchEngine` does NOT wrap +dispatchers with circuit breakers. Each dispatcher implements `MixinAsyncCircuitBreaker` +for external service calls, configures thresholds appropriate to its transport type, and +raises `InfraUnavailableError` when the circuit opens. See +`docs/patterns/dispatcher_resilience.md`. ### Correlation ID Rules -1. Always propagate from incoming requests -2. Auto-generate with `uuid4()` if missing -3. Include in all error context +Always propagate from incoming requests; auto-generate with `uuid4()` if missing; include +in all error context. --- ## Pydantic Model Standards -### File & Class Naming - -| Type | File Pattern | Class Pattern | -|------|-------------|---------------| -| Model | `model_.py` | `Model` | -| Adapter | `adapter_.py` | `Adapter` | -| Dispatcher | `dispatcher_.py` | `Dispatcher` | -| Enum | `enum_.py` | `Enum` | -| Mixin | `mixin_.py` | `Mixin` | -| Protocol | `protocol_.py` | `Protocol` | -| Service | `service_.py` | `Service` | -| Store | `store_.py` | `Store` | -| Validator | `validator_.py` | `Validator` | -| Registry (node) | `registry_infra_.py` | `RegistryInfra` | -| Registry (standalone) | `registry_.py` | `Registry` | - -### ConfigDict Requirements +File/class naming is mechanical: `model_.py` → `Model`, `enum_.py` → +`Enum`, and likewise for `adapter_`, `dispatcher_`, `mixin_`, `protocol_`, +`service_`, `store_`, `validator_` prefixes. Node registries: `registry_infra_.py` +→ `RegistryInfra`. ```python -# Standard pattern (most common) +# Standard ConfigDict (most common) model_config = ConfigDict( frozen=True, # Immutability for thread safety extra="forbid", # Strict validation @@ -696,106 +259,23 @@ model_config = ConfigDict( ) ``` -### Field Patterns - -```python -# Required field -field_name: FieldType = Field(..., description="Clear description") - -# Optional field (prefer empty string over None for strings) -error_message: str = Field(default="", description="Empty if no error") - -# Collections - use default_factory for mutable defaults -items: list[str] = Field(default_factory=list) - -# Immutable collections - use tuple for frozen models -errors: tuple[ModelError, ...] = Field(default_factory=tuple) -``` - -### Custom `__bool__` for Result Models - -Result models may override `__bool__` for idiomatic conditional checks: - -```python -def __bool__(self) -> bool: - """Allow using result in boolean context. - - Warning: - **Non-standard __bool__ behavior**: Returns ``True`` only when - ``is_valid`` is True. Differs from typical Pydantic behavior. - """ - return self.is_valid -``` - -**Documentation requirement**: Always include a `Warning` section explaining non-standard behavior. +Prefer empty string over `None` for optional strings; use `default_factory` for +collections; use `tuple` for collections on frozen models. A model overriding `__bool__` +must document the non-standard behavior in a `Warning` docstring section. --- ## Testing and CI -### Test Directory Structure - -```text -tests/ -├── conftest.py # Root conftest with shared fixtures -├── helpers/ # Test helper utilities -├── unit/ # Auto-marked with `unit` marker -├── integration/ # Auto-marked with `integration` marker -├── chaos/ # Auto-marked with `chaos` marker -├── replay/ # Auto-marked with `replay` marker -├── performance/ # Auto-marked with `performance` marker -└── ci/ # CI/CD specific tests -``` - -### Pytest Markers - -| Marker | Description | Auto-applied | -|--------|-------------|--------------| -| `unit` | Unit tests in isolation | Yes | -| `integration` | Multi-component tests | Yes | -| `slow` | Tests >1s execution | No | -| `chaos` | Chaos engineering tests | Yes | -| `performance` | Performance/benchmark tests | Yes | -| `consul` | Tests requiring real Consul | No | -| `postgres` | Tests requiring PostgreSQL | No | -| `kafka` | Tests requiring Kafka | No | -| `serial` | Non-parallel tests | No | - -### Running Tests +Test tree: `tests/{unit,integration,chaos,replay,performance}` — directory placement +auto-applies the matching pytest marker; the full marker list lives in `pyproject.toml`. +Service-dependent markers (`consul`, `postgres`, `kafka`) and `slow`/`serial` are manual. ```bash -# All tests -uv run pytest tests/ - -# With coverage (60% minimum required) -uv run pytest tests/ --cov=omnibase_infra --cov-report=html - -# By category -uv run pytest -m unit # Unit tests only -uv run pytest -m integration # Integration tests only -uv run pytest -m "not slow" # Exclude slow tests - -# Parallel execution -uv run pytest tests/ -n auto - -# Debug mode (no parallelism) -uv run pytest tests/ -n 0 -xvs +uv run pytest tests/ -n auto # Parallel +uv run pytest tests/ -n 0 -xvs # Debug mode (no parallelism) ``` -### Coverage Requirement - -**Minimum 60% coverage required** (`fail_under = 60` in pyproject.toml) - -### Common Fixtures - -| Fixture | Purpose | -|---------|---------| -| `mock_container` | MagicMock ONEX container | -| `container_with_registries` | Real ModelONEXContainer with wired services | -| `event_bus` | In-memory event bus with cleanup | -| `cleanup_consul_test_services` | Cleans Consul test registrations | -| `cleanup_postgres_test_projections` | Cleans PostgreSQL test rows | - ### Runtime Startup is a First-Class CI Gate Any PR that touches `auto_wiring/`, `service_kernel.py`, handler `__init__` signatures, or kernel-level registration MUST include a test that: @@ -806,8 +286,8 @@ Any PR that touches `auto_wiring/`, `service_kernel.py`, handler `__init__` sign CI must additionally boot `omninode-runtime` in a compose sandbox and assert: -- the container reaches Docker healthy state within the configured compose `start_period` (currently 600s for `omninode-runtime`, see `docker/docker-compose.infra.yml`), and -- `RestartCount == 0` at the health-ready checkpoint — not a fixed 45s wall-clock window. +- the container reaches Docker healthy state within the compose `start_period` configured for `omninode-runtime` in `docker/docker-compose.infra.yml`, and +- `RestartCount == 0` at the health-ready checkpoint — not a fixed wall-clock window. Ad-hoc short timeouts are forbidden: any PR that shortens the gate below `start_period` must also update the compose healthcheck in the same PR, with justification. @@ -819,83 +299,20 @@ Ad-hoc short timeouts are forbidden: any PR that shortens the gate below `start_ ## Contract-Driven Config Discovery -Infisical-backed configuration management. - -### Overview - -The config discovery system extracts configuration requirements from ONEX -contract YAML files and resolves them from Infisical at runtime. It scans -three Pydantic-backed contract fields: - -1. `metadata.transport_type` -- the transport type declared in metadata -2. `handler_routing.handlers[].handler_type` -- handler-level transport types -3. `dependencies[].type == "environment"` -- explicit env var dependencies - -### Components - -| Component | Location | Purpose | -|-----------|----------|---------| -| `TransportConfigMap` | `runtime/config_discovery/transport_config_map.py` | Maps transport types to Infisical paths | -| `ContractConfigExtractor` | `runtime/config_discovery/contract_config_extractor.py` | Scans contracts for config requirements | -| `ConfigPrefetcher` | `runtime/config_discovery/config_prefetcher.py` | Prefetches values through HandlerInfisical | -| `ModelTransportConfigSpec` | `runtime/config_discovery/models/model_transport_config_spec.py` | Spec for transport config in Infisical | -| `ModelConfigRequirements` | `runtime/config_discovery/models/model_config_requirements.py` | Aggregated requirements from contracts | - -### Infisical Path Convention - -```text -Shared: /shared//KEY -Per-service: /services///KEY -``` - -### Bootstrap Sequence - -```text -Step 1: PostgreSQL starts (POSTGRES_PASSWORD from .env) -Step 2: Valkey starts -Step 3: Infisical starts (depends_on: postgres + valkey healthy) -Step 4: Identity provisioning (first-time only) -Step 5: Seed runs (populates Infisical from contracts + .env values) -Step 6: Runtime services start (prefetch from Infisical) -``` - -### Scripts - -| Script | Purpose | -|--------|---------| -| `scripts/bootstrap-infisical.sh` | Orchestrates the full bootstrap sequence | -| `scripts/seed-infisical.py` | Populates Infisical from contracts (safe by default, `--dry-run`) | -| `scripts/setup-infisical-identity.sh` | Creates machine identities (runtime=read-only, admin=read-write) | - -### Opt-In Behavior - -Config prefetch is **opt-in**: it only runs when `INFISICAL_ADDR` is set in the -environment. Without it, the runtime falls back to standard environment variable -resolution. This means local development works without Infisical. - -### .env Reduction - -The `.env.example` has been reduced from ~660 lines to ~30 lines (bootstrap-only). -The full pre-Infisical config is preserved in `docs/env-example-full.txt`. - ---- - -## Agent-Driven Development - -**ALL CODING TASKS MUST USE SUB-AGENTS - NO EXCEPTIONS** - -| Task Type | Agent | -|-----------|-------| -| Simple tasks | Direct specialist (`agent-commit`, `agent-testing`, `agent-contract-validator`) | -| Complex workflows | `agent-onex-coordinator` → `agent-workflow-coordinator` | -| Multi-domain | `agent-ticket-manager` for planning, orchestrators for execution | - -**Prefer `subagent_type: "general-purpose"`** for ONEX development workflows. - -### Critical Policies - -- **NEVER** use `run_in_background: true` for Task tool -- Parallel execution: call multiple Task tools in a **single message** +Infisical-backed configuration management: config requirements are extracted from ONEX +contract YAMLs (`metadata.transport_type`, handler-level transport types, and +`dependencies[].type == "environment"`) and resolved from Infisical at runtime. +Implementation lives under `src/omnibase_infra/runtime/config_discovery/`. + +- **Path convention**: shared config at `/shared//KEY`, per-service at + `/services///KEY`. +- **Opt-in (trap)**: config prefetch only runs when `INFISICAL_ADDR` is set in the + environment. Without it, the runtime falls back to standard environment variable + resolution — local development works without Infisical. +- **Bootstrap**: `scripts/bootstrap-infisical.sh` orchestrates the full first-time + sequence (identity provisioning + seeding); `scripts/seed-infisical.py` is safe by + default and supports `--dry-run`. The full pre-Infisical env reference is preserved in + `docs/env-example-full.txt`. --- @@ -903,126 +320,58 @@ The full pre-Infisical config is preserved in `docs/env-example-full.txt`. ### Do NOT -1. **Skip base class initialization** - ```python - def __init__(self, container): - pass # WRONG - missing super().__init__(container) - ``` - -2. **Add custom logic to declarative nodes** - ```python - class MyNode(NodeOrchestrator): - def process(self, data): # WRONG - nodes are declarative only - return self._custom_logic(data) - ``` - +1. **Skip base class initialization** — node `__init__` without `super().__init__(container)` is wrong +2. **Add custom logic to declarative nodes** — no `process()`/business methods on node classes 3. **Return result from ORCHESTRATOR** ```python return ModelHandlerOutput.for_orchestrator(result={"status": "done"}) # ValueError! ``` - -4. **Use ModelIntentPayloadBase** (removed in omnibase_core 0.6.2) +4. **Base infra payload DTOs on ModelIntentPayloadBase** — extend `BaseModel` directly + (repo convention). The class itself was never removed: it lives in + `omnibase_core.models.reducer.payloads` and bases core's closed-set intent payloads. ```python - from omnibase_core.models.reducer.payloads import ModelIntentPayloadBase # WRONG - # Use: from pydantic import BaseModel + class ModelPayloadExample(BaseModel): # infra convention; do not subclass core's base ``` ### DO 1. Always call `super().__init__(container)` in node constructors -2. Use `ModelONEXContainer` for dependency injection -3. Use protocol names for DI: `container.get_service("ProtocolEventBus")` -4. Keep nodes declarative - all logic in handlers -5. Use `ModelInfraErrorContext.with_correlation()` for error context +2. Use protocol names for DI: `container.get_service("ProtocolEventBus")` +3. Keep nodes declarative - all logic in handlers +4. Use `ModelInfraErrorContext.with_correlation()` for error context --- ## Handler Plugin Loader -The runtime uses **plugin-based handler loading** from YAML contracts. - -### Contract-Based Handler Declaration - -```yaml -handler_routing: - routing_strategy: "payload_type_match" - handlers: - - event_model: "ModelNodeIntrospectionEvent" - handler_class: "HandlerNodeIntrospected" - handler_module: "omnibase_infra.handlers.handler_node_introspected" -``` - -### Contract File Precedence - -| Filename | Purpose | -|----------|---------| -| `handler_contract.yaml` | Dedicated handler contract (preferred) | -| `contract.yaml` | General ONEX contract with handler fields | - -**FAIL-FAST**: When both files exist in the same directory, loader raises `AMBIGUOUS_CONTRACT_CONFIGURATION` error. - -### Error Codes - -| Code | Description | -|------|-------------| -| `HANDLER_LOADER_006` | `PROTOCOL_NOT_IMPLEMENTED` | -| `HANDLER_LOADER_010` | `MODULE_NOT_FOUND` | -| `HANDLER_LOADER_011` | `CLASS_NOT_FOUND` | -| `HANDLER_LOADER_012` | `IMPORT_ERROR` | -| `HANDLER_LOADER_013` | `NAMESPACE_NOT_ALLOWED` | -| `HANDLER_LOADER_040` | `AMBIGUOUS_CONTRACT_CONFIGURATION` | +The runtime uses plugin-based handler loading from YAML contracts +(see `docs/patterns/handler_plugin_loader.md`). -### Security: Namespace Allowlisting - -```python -# Restrict to trusted namespaces (recommended for production) -loader = HandlerPluginLoader( - allowed_namespaces=["omnibase_infra.", "omnibase_core.", "myapp.handlers."] -) -``` +- **Contract file precedence**: `handler_contract.yaml` (dedicated, preferred) vs + `contract.yaml` (general contract with handler fields). +- **FAIL-FAST (trap)**: when both files exist in the same directory, the loader raises + `AMBIGUOUS_CONTRACT_CONFIGURATION` — it does not silently pick one. +- **Security**: restrict loading with `HandlerPluginLoader(allowed_namespaces=[...])` + in production. --- ## Release Process -### Version Compatibility Matrix - -`src/omnibase_infra/runtime/version_compatibility.py` maintains a runtime -check that verified installed `omnibase_core` and `omnibase_spi` versions match -the constraints declared in `pyproject.toml`. - -**How it works:** +`src/omnibase_infra/runtime/version_compatibility.py` checks at runtime that installed +`omnibase_core` / `omnibase_spi` versions match the constraints in `pyproject.toml`. +`VERSION_MATRIX` is derived automatically from `pyproject.toml` at import time; +`_FALLBACK_MATRIX` (used when no source tree is present) is kept in sync by +`scripts/update_version_matrix.py`. -`VERSION_MATRIX` is derived **automatically at import time** from `pyproject.toml`. -No manual update is required when bumping dependency versions — just update -`pyproject.toml` and the matrix follows. +**Dependency bump checklist:** -A `_FALLBACK_MATRIX` with hardcoded values is used when `pyproject.toml` is -not present (e.g. installed package without source tree). The fallback is kept -in sync with the `scripts/update_version_matrix.py` script. +1. Update `pyproject.toml` bounds, then `uv sync`. +2. `uv run pytest tests/unit/runtime/test_version_compatibility.py` — `test_matrix_matches_pyproject` catches drift. +3. The release workflow runs `scripts/update_version_matrix.py --check` as a pre-build gate (run without `--check` to update the fallback in-place). -**Release checklist for dependency bumps:** - -1. Update `pyproject.toml` with new `>=X.Y.Z,/protection` mutation, dry-run the audit: ```bash bash scripts/audit-branch-protection.sh --repo --dry-run ``` -The script checks two invariants: (A) `required_approving_review_count` must be 0 (solo-dev workflow), and (B) every required status check context must match a check-run name seen on the last 5 commits. A periodic CI job (`.github/workflows/branch-protection-audit.yml`, schedule `23 */4 * * *`) runs this automatically and fails the workflow on any violation. - -### Enforcement + merge-policy parity ratchet (OMN-14288, REPORT-ONLY) - -The two checks above only cover the **ORPHANED** direction (a required context that no longer reports). They do **not** catch the **MISSING** direction — a load-bearing gate that `CLAUDE.md`/doctrine *claims* is enforced but is absent from live `required_status_checks` (the exact hole that left `deploy-gate` + `reject-skip` unenforced on omnimarket/omniclaude `dev`) — nor **merge-policy drift** (a live merge-queue/strict setting that diverges from the decided policy). The parity ratchet closes both: +For enforcement + merge-policy parity (MISSING gates, needs-closure, queue/strict drift): ```bash uv run python scripts/audit_required_context_parity_cli.py report --owner OmniNode-ai ``` -It reads the single, machine-asserted `scripts/enforcement_parity_manifest.yaml` — one `{repo → branch → {load_bearing_gates[], merge_policy}}` file encoding the **full** deterministic branch-protection policy as config-as-data — fetches live branch-protection + aggregator-workflow + merge-queue state, and reports: - -- **Enforcement dimension** (`load_bearing_gates[]`, each `coverage: direct | needs_child`): **MISSING** (declared direct gate absent from `required_status_checks`), **NEEDS_CLOSURE** (a `needs_child` gate not in its aggregator's transitive `needs:` closure), **UNPROTECTED** (declared branch with no protection object). -- **Merge-policy dimension** (`merge_policy: {queue: enabled|disabled, strict: bool}`): **QUEUE_DRIFT** (live merge-queue state ≠ declared) and **STRICT_DRIFT** (live require-branches-up-to-date ≠ declared). The decided policy is **queue disabled on all dev branches** (a merge queue's only unique value is the `merge_group` re-test-against-latest-base, which wedges the saturated self-hosted fleet; required contexts fire on `pull_request`, so disabling loses no enforcement) with **strict on the two dashboards** as the lighter combine-breakage guard. - -The assertion logic lives in `audit_branch_protection_lib.py` (pure, unit-tested); the CLI is the thin `gh`/YAML I/O shell. It runs as a **non-blocking, report-only** step in `branch-protection-audit.yml` — it never mutates branch protection and always exits 0 (report-then-enforce rollout; the enforcing per-PR gate lands separately). This manifest is the machine-checked replacement for the per-repo, honor-system `.github/required-checks.yaml` prose manifests **and** the recurring manual "re-verify branch protection after merges" ritual. +The declared policy lives in `scripts/enforcement_parity_manifest.yaml` (config-as-data: +`{repo → branch → {load_bearing_gates[], merge_policy}}`); assertion logic is in +`scripts/audit_branch_protection_lib.py`. Both audits run on a schedule via +`.github/workflows/branch-protection-audit.yml` (the parity ratchet is report-only and +never mutates protection). --- -**Python**: 3.12+ | **Ready?** → Check `docs/patterns/` for implementation guides - -**Bottom Line**: Declarative nodes, container injection, agent-driven development. No backwards compatibility, no custom node logic. +**Bottom Line**: Declarative nodes, container injection, contracts as source of truth. No backwards compatibility, no custom node logic. diff --git a/config/application_database_domain_enforcement.yaml b/config/application_database_domain_enforcement.yaml new file mode 100644 index 0000000000..1c5bf8124b --- /dev/null +++ b/config/application_database_domain_enforcement.yaml @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: MIT +schema_version: "1.0" +ticket: OMN-15361 +predecessor_pins: + "omnibase_core#1529": 1f4549d71d4d39560ac5a162ac1d39e54d86e688 + "omnibase_infra#2547": 95351f5d8e806fcf7fa2c276d9065df93ccf92b9 + "omnibase_infra#2548": 7228ce0c0934ae096dd6effd0f84ff1913fec6c0 + "omnibase_infra#2558": 2a2cfb275810b34197d4d5baf55bdcddc443e6dc + "omnimarket#1956": 4637e625c99ef17c190aa471a5e51b7f646c6dfd + "omninode_infra#771": 39033d55147ef22a061b665345b506246d3aa543 +gates: + classification: + source_enforcement: mandatory + deployment_enforcement: blocked + source_proofs: + - typed exactly-one-owner, topology-domain, and exact catalog-census validators green + - seeded missing, duplicate, conflicting, incomplete, empty, and public-leak census controls red + source_proof_paths: + - tests/unit/validation/test_application_relation_ownership.py + - tests/unit/validation/test_application_database_domain_enforcement.py + - docker/application-domain-enforcement/compose.yml + - config/application_database_domain_proof_ownership.yaml + seeded_red_controls: + - missing-owner + - duplicate-owner + - conflicting-location + - incomplete-retained-census + - empty-authoritative-relation-set + - public-catalog-leak + deployment_blockers: + - OMN-15423 retained live census and full-day activity evidence remain blocked + - OMN-15423 ownership manifests are not landed on their deployment branches + schema_qualification: + source_enforcement: mandatory + deployment_enforcement: blocked + source_proofs: + - changed-file migration SQL gate rejects public and unqualified reads, writes, DDL, grants, indexes, and references + - deployable production manifests and the ephemeral PostgreSQL proof manifest are independently validated authority universes and are never composed + - kind-aware ownership checks bind non-CREATE targets to exact object classes and routine overloads; procedural dynamic SQL and implicit multirange identities fail closed + - rebuilt PostgreSQL catalog proof contains only typed application schemas + source_proof_paths: + - tests/unit/validation/test_application_database_domain_enforcement.py + - tests/unit/validation/test_application_database_sql_enforcement_regressions.py + - tests/ci/test_application_database_sql_authority_isolation.py + - tests/ci/test_application_database_sql_gate.py + - docker/application-domain-enforcement/compose.yml + seeded_red_controls: + - public-application-table + - unqualified-application-table + - unqualified-application-mutation-target + - unknown-topology-schema + - wrong-object-kind + - wrong-routine-overload + - dynamic-sql-target + - implicit-multirange-identity + deployment_blockers: + - OMN-15356 and OMN-15359 target schema migrations are not landed + - migrated public sources remain until the separately approved P8 retirement + tenant_rls: + source_enforcement: mandatory + deployment_enforcement: blocked + source_proofs: + - typed catalog validator covers UUID identity, no default, ENABLE and FORCE RLS + - exact USING and WITH CHECK plus tenant A/B, unset, malformed view and function behavior execute red + - canonical policy pg_policy.polroles evidence is exact PUBLIC scope; unrelated or missing role scope fails closed + - identity-root creation and enumeration are behaviorally bound to an audited NOLOGIN BYPASSRLS control identity while runtime cross-tenant access remains denied + - SECURITY DEFINER authority fingerprints language, body forms, leakproof/volatility/parallel flags, config, kind, strictness, result shape, and RLS-relevant metadata + source_proof_paths: + - tests/unit/validation/test_application_database_domain_enforcement.py + - tests/ci/test_application_database_sql_gate.py + - docker/application-domain-enforcement/compose.yml + seeded_red_controls: + - tenant-text-key + - tenant-nullable + - tenant-default + - missing-enable-rls + - missing-force-rls + - using-drift + - with-check-drift + - canonical-policy-unrelated-role + - uncontracted-identity-root + - identity-root-runtime-login + - identity-root-unproven-enumeration + - identity-root-runtime-membership + - identity-root-runtime-set-role + - widening-permissive-policy + - owner-security-view + - unsafe-security-definer + - unproven-security-view + - unproven-security-definer + - security-definer-volatility-drift + deployment_blockers: + - OMN-15416 non-owner FORCE-RLS deployed behavior proof is not complete + - tenant target migrations and live grant/RLS application require a separately approved deploy + internal_catalog: + source_enforcement: mandatory + deployment_enforcement: blocked + source_proofs: + - internal and catalog catalog-state validators reject tenant columns, RLS, and authoritative source_tenant_id + - generated-column dependency expansion prevents source_tenant_id aliases from acquiring uniqueness, partition, deduplication, authorization, or write authority + - internal no-GUC behavior is exercised through the rebuilt adapter proof + source_proof_paths: + - tests/unit/validation/test_application_database_domain_enforcement.py + - docker/domain-adapter-proof/compose.yml + seeded_red_controls: + - internal-tenant-id + - catalog-tenant-id + - internal-tenant-policy + - catalog-rls + - uncontracted-source-tenant + - source-tenant-generated-unique-alias + deployment_blockers: + - OMN-15356 internal and catalog transformations are not landed + - internal and catalog live pool secrets and deploy evidence do not exist + role_acl: + source_enforcement: mandatory + deployment_enforcement: blocked + source_proofs: + - generated ACL matrix rejects runtime ownership, DDL, BYPASSRLS, broad and future grants + - foreign tables, aggregate/window routines, and base/range/multirange types use exact PostgreSQL grant classes and catalog kinds; extensions fail closed + - rebuilt PostgreSQL 16 ACL and rollback proof executes seeded role defects + source_proof_paths: + - tests/unit/validation/test_application_database_acl.py + - docker/application-acl-proof/compose.yml + seeded_red_controls: + - public-connect + - public-execute + - runtime-owner + - runtime-ddl + - runtime-bypassrls + - cross-domain-grant + - unsafe-default-privilege + deployment_blockers: + - authorized catalog parity and full-day activity evidence are blocked + - workload role secret, live grant, and deploy preconditions are not authorized + one_database: + source_enforcement: mandatory + deployment_enforcement: blocked + source_proofs: + - checked-in topology resolves all application pools to one physical database + - typed pool evidence requires exact distinct current users + source_proof_paths: + - tests/unit/topology/test_application_database_topology.py + - tests/unit/validation/test_application_database_domain_enforcement.py + seeded_red_controls: + - old-application-database + - duplicate-pool-user + - wrong-pool-user + - missing-pool-binding + deployment_blockers: + - OMN-15358 app_dashboard pool cutover is not landed or deployed + - OMN-15424 onex_api pool cutover is not landed or deployed + - OMN-15425 tenant projection pool cutover is not landed or deployed + - OMN-15426 internal runtime pool cutover is not landed or deployed + adapter: + source_enforcement: mandatory + deployment_enforcement: blocked + source_proofs: + - verified envelope authority binds the tenant UUID transaction-locally + - internal operation proves no resolver and no tenant GUC + source_proof_paths: + - tests/unit/runtime/auto_wiring/test_projection_domain_adapters.py + - docker/domain-adapter-proof/compose.yml + seeded_red_controls: + - untrusted-tenant-selection + - mismatched-signer-binding + - nonlocal-tenant-guc + - leaked-tenant-guc + - internal-resolver-call + - domain-blind-upsert + deployment_blockers: + - dedicated pool secrets are absent from authorized source evidence + - adapter deployment and real-pool readback require a separately approved deploy + topology_parity: + source_enforcement: mandatory + deployment_enforcement: blocked + source_proofs: + - typed topology renders exact Docker projections + - Docker topology and catalog projections are parity-validated; Kubernetes parity remains blocked + source_proof_paths: + - tests/unit/topology/test_application_database_topology.py + - src/omnibase_infra/topology/application_database.py + seeded_red_controls: + - profile-instance-drift + - database-user-drift + - docker-profile-injection-drift + - docker-dsn-consumer-drift + deployment_blockers: + - Kubernetes topology projection source is not landed on deployment branches + - no authorized deployed topology readback has been captured diff --git a/config/application_database_domain_proof_ownership.yaml b/config/application_database_domain_proof_ownership.yaml new file mode 100644 index 0000000000..75a30f0aca --- /dev/null +++ b/config/application_database_domain_proof_ownership.yaml @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +schema_version: "1.0" +service: application_domain_proof +owner_declaration: service:application_domain_proof +target_database_ref: application +db_io: + db_tables: + - name: tenants + database_ref: application + schema: tenant + migration: docker/application-domain-enforcement/seed.sql + access: read_write + role: tenant_identity_root + - name: events + database_ref: application + schema: tenant + migration: docker/application-domain-enforcement/seed.sql + access: read_write + role: tenant_event_proof + - name: runtime_state + database_ref: application + schema: omninode_internal + migration: docker/application-domain-enforcement/seed.sql + access: read_write + role: internal_runtime_proof + - name: feature_flags + database_ref: application + schema: platform_catalog + migration: docker/application-domain-enforcement/seed.sql + access: read_write + role: platform_catalog_proof +relation_evidence: + - name: tenants + kind: table + database_ref: application + schema: tenant + domain: TENANT + owner_declaration: service:application_domain_proof + tenant_identity_column: id + identity_root_contract: canonical_tenant_identity_root + identity_root_control_role: tenant_control_admin + identity_root_control_operations: + - tenant_creation + - cross_tenant_enumeration + canonical_policy_name: tenant_identity_isolation + deduplication_key_columns: [id] + authorization_dependency_columns: [id] + write_eligibility_dependency_columns: [id] + - name: events + kind: table + database_ref: application + schema: tenant + domain: TENANT + owner_declaration: service:application_domain_proof + tenant_identity_column: tenant_id + canonical_policy_name: tenant_isolation + deduplication_key_columns: [event_id] + authorization_dependency_columns: [tenant_id] + write_eligibility_dependency_columns: [tenant_id] + - name: runtime_state + kind: table + database_ref: application + schema: omninode_internal + domain: OMNINODE_INTERNAL + owner_declaration: service:application_domain_proof + source_tenant_provenance_contract: non_authoritative_provenance + deduplication_key_columns: [state_id] + authorization_dependency_columns: [] + write_eligibility_dependency_columns: [] + - name: feature_flags + kind: table + database_ref: application + schema: platform_catalog + domain: PLATFORM_CATALOG + owner_declaration: service:application_domain_proof + deduplication_key_columns: [flag_id] + authorization_dependency_columns: [] + write_eligibility_dependency_columns: [] + - name: events_view + kind: view + database_ref: application + schema: tenant + domain: TENANT + owner_declaration: service:application_domain_proof + readers: [app_dashboard] +database_objects: + - name: safe_report + kind: function + database_ref: application + schema: tenant + domain: TENANT + owner_declaration: service:application_domain_proof + readers: [app_dashboard] + function_signature: "()" + audit_id: OMN-15361:tenant.safe_report:58b47971e3234c0117f153a4d3d7c7d0efdfb611804ba729153dfac19e503cfe + definition_sha256: 58b47971e3234c0117f153a4d3d7c7d0efdfb611804ba729153dfac19e503cfe diff --git a/config/runner_fleet.yaml b/config/runner_fleet.yaml index 69d4e9a869..753bc61664 100644 --- a/config/runner_fleet.yaml +++ b/config/runner_fleet.yaml @@ -9,11 +9,11 @@ github_org: OmniNode-ai runner_host: omninode-pc.tail75df5e.ts.net runner_group: omnibase-ci runner_name_prefix: omninode-runner -# Phase-A scale-up (OMN-14029): 64 always-on steady-state runners, extending the -# reconciled .201 fleet from 48. All 64 are steady (no burst +# Phase-A scale-up (OMN-14029), extended for saturation recovery (OMN-15978): +# 72 always-on steady-state runners. All 72 are steady (no burst # profiles), so burst_count == expected_count (no separate burst tier). -expected_count: 64 -burst_count: 64 +expected_count: 72 +burst_count: 72 # Docker subnet-pool alerting (OMN-12566). The runner host's default address # pool can subnet a bounded number of networks before `docker network create` # fails with "all predefined address pools have been fully subnetted". Alert at diff --git a/config/runner_routing_policy.yaml b/config/runner_routing_policy.yaml index ade8a0b662..680e91e300 100644 --- a/config/runner_routing_policy.yaml +++ b/config/runner_routing_policy.yaml @@ -17,9 +17,11 @@ hosted_runner_allowlist: reason: "OMN-14127: CI Summary is a lightweight hosted fail-closed aggregator so required status always posts during self-hosted saturation." - path: .github/workflows/build-and-push-runtime.yml reason: "OMN-13747: ECR push remains hosted until the self-hosted egress failure is retired." + - path: .github/workflows/build-workspace-candidate-runtime.yml + reason: "OMN-14974: candidate runtime ECR push uses the same hosted clean-egress lane after the self-hosted daemon repeatedly stalled during upload." - path: .github/workflows/omnigate.yml reason: "Fork-only untrusted PR verification must not execute on self-hosted runners." - path: .github/workflows/runner-fleet-canary.yml reason: "OMN-13915: scheduled fleet canary must not share fate with the self-hosted runner fleet it monitors." - - path: .github/workflows/precommit-parity-gate.yml - reason: "OMN-14668: byte-match precommit parity proof is dependency-free and hosted-pinned so self-hosted fleet saturation cannot create a required-check false-green window." + - path: .github/workflows/runner-broker-dispatch-wedge-rerun.yml + reason: "OMN-15776: scheduled broker-dispatch-wedge rerun must not share fate with the self-hosted runner fleet it remediates -- same isolation rationale as runner-fleet-canary.yml above." diff --git a/contracts/OMN-12912.yaml b/contracts/OMN-12912.yaml new file mode 100644 index 0000000000..7d59497646 --- /dev/null +++ b/contracts/OMN-12912.yaml @@ -0,0 +1,50 @@ +schema_version: "1.0.0" +ticket_id: "OMN-12912" +title: "Run the Hybrid Gateway as a real two-broker edge process" +summary: >- + Replaces the dormant/private-envelope gateway prototype with a dedicated tenant-scoped process over canonical ModelEventEnvelope records, separate local and cloud Kafka configurations, fixed topic mirroring, tenant-bound admission, liveness heartbeats, and source-retaining destination retries. The production path uses explicit pull/commit transport semantics: destination acknowledgement is followed by a durable edge-local envelope_id marker before the source offset commits, and the marker survives container restart on a named volume to suppress acknowledged redelivery. +is_seam_ticket: true +interface_change: true +interfaces_touched: + - "events" + - "protocols" + - "topics" +evidence_requirements: + - kind: "tests" + description: "Gateway config, transform, service, runtime, and topic-gate tests pass." + command: >- + uv run pytest tests/unit/nodes/node_bus_forwarder_effect tests/unit/runtime/test_gateway_forwarder_runtime.py tests/unit/idempotency/test_store_sqlite.py tests/unit/event_bus/test_event_bus_kafka_topic_gate.py tests/integration/gateway/test_bus_forwarder_contract_integration.py -q + - kind: "tests" + description: "The forwarder node contract and the complete infra contract set validate." + command: >- + uv run --frozen python scripts/batch_validate_node_contracts.py --directory src/omnibase_infra/nodes/node_bus_forwarder_effect --verbose && uv run --frozen python scripts/validate.py contracts + - kind: "tests" + description: "The permanent gateway deployment requires Roles Anywhere, exposes no inbound port, and waits for container health." + command: >- + uv run pytest tests/unit/infra/test_gateway_compose_credentials.py -q +emergency_bypass: + enabled: false + justification: "" + follow_up_ticket_id: "" +dod_evidence: + - id: "dod-gateway-focused-suite" + description: "The complete changed gateway and durable delivery test surface passes." + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/nodes/node_bus_forwarder_effect tests/unit/runtime/test_gateway_forwarder_runtime.py tests/unit/idempotency/test_store_sqlite.py tests/unit/event_bus/test_event_bus_kafka_topic_gate.py tests/integration/gateway/test_bus_forwarder_contract_integration.py -q + - id: "dod-gateway-contract-validation" + description: "The forwarder and full infrastructure contract sets validate." + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run --frozen python scripts/batch_validate_node_contracts.py --directory src/omnibase_infra/nodes/node_bus_forwarder_effect --verbose && uv run --frozen python scripts/validate.py contracts + - id: "dod-gateway-short-lived-credentials" + description: "The deployment surface rejects static AWS access keys, keeps the edge outbound-only, and is supervised by systemd." + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/infra/test_gateway_compose_credentials.py -q diff --git a/contracts/OMN-14974.yaml b/contracts/OMN-14974.yaml new file mode 100644 index 0000000000..1043c4956d --- /dev/null +++ b/contracts/OMN-14974.yaml @@ -0,0 +1,46 @@ +schema_version: "1.0.0" +ticket_id: "OMN-14974" +title: "Vendor the live-event lifecycle taxonomy backfill" +summary: > + Vendors the node_projection_live_events 0001 lifecycle reclassification migration byte-identically from paired omnimarket PR #1983. The migration repairs durable rows whose inference, evaluation, or delegation lifecycle was previously collapsed into ROUTING or ACTION, and the paired source PR is blocked until this deployed forward-tree copy lands first. + +is_seam_ticket: false +interface_change: false +interfaces_touched: [] +emergency_bypass: + enabled: false + justification: "" + follow_up_ticket_id: "" +dod_evidence: + - id: "dod-001" + description: > + The deployed forward-tree migration is byte-identical to the paired omnimarket PR #1983 source. + + source: "manual" + checks: + - check_type: "command" + check_value: "cmp -s $OMNI_HOME/omni_worktrees/OMN-14974/omnimarket-event-type-taxonomy/src/omnimarket/nodes/node_projection_live_events/migrations/0001_reclassify_event_lifecycle_types.sql docker/migrations/forward/nodes/node_projection_live_events/0001_reclassify_event_lifecycle_types.sql" + - id: "dod-002" + description: > + The full vendored migration tree is in sync with the explicitly paired omnimarket source ref. + + source: "manual" + checks: + - check_type: "command" + check_value: "OMNIMARKET_SRC=$OMNI_HOME/omni_worktrees/OMN-14974/omnimarket-event-type-taxonomy bash scripts/sync-node-migrations.sh --check" + - id: "dod-003" + description: > + Migration discovery, fence parity, and sequence validation tests pass with the new node migration. + + source: "manual" + checks: + - check_type: "command" + check_value: "LC_ALL=C LANG=C uv run pytest tests/scripts/test_node_migration_fence_parity.py tests/ci/test_validate_migration_sequence.py -q" + - id: "dod-deploy" + description: > + Deploy-gate evidence: this PR adds one idempotent node-owned SQL migration to the deployed forward tree. It performs no deployment itself; the governed onex-dev redeploy will apply it through the existing namespaced migration runner before live readback. + + source: "manual" + checks: + - check_type: "command" + check_value: "echo 'deploy-gate: OMN-14974 vendors one byte-identical idempotent node migration; apply only through governed onex-dev redeploy'" diff --git a/contracts/OMN-15009.yaml b/contracts/OMN-15009.yaml new file mode 100644 index 0000000000..e4a09aad7f --- /dev/null +++ b/contracts/OMN-15009.yaml @@ -0,0 +1,64 @@ +# OMN-15009 repo-local deploy contract. Canonical receipts live in +# onex_change_control; this copy carries the product and deploy-gate evidence. +schema_version: '1.0.0' +ticket_id: OMN-15009 +title: 'Repair the post-merge runtime rebuild trigger with overlay-driven bus routing' +summary: >- + The post-merge workflow could never publish because it required absent GitHub Kafka/SASL/HMAC secrets, and its historical payload contained fields rejected by the live extra=forbid ModelRedeployStartCommand consumer. Resolve the reviewable omnimarket dev-bus overlay on the trusted .200 runner, emit exactly the canonical redeploy-start command shape with the merge commit as git_ref, validate producer/consumer field compatibility before publishing, and fail on any transport or delivery error. Keep DEPLOY_AGENT_HMAC_SECRET at the correct downstream trust boundary by injecting it fail-closed into runtime-effects; the deploy effect signs rebuild-requested and the deploy agent verifies it. +is_seam_ticket: true +interface_change: true +interfaces_touched: + - events + - topics + - protocols +evidence_requirements: + - kind: tests + description: >- + Trigger tests prove strict payload compatibility, overlay resolution, transport selection, fail-closed delivery, and zero GitHub secret dependencies for the upstream command. + command: >- + uv run pytest tests/unit/scripts/test_pr_merged_rebuild_trigger.py tests/scripts/test_trigger_redeploy_start.py tests/scripts/test_trigger_producer_effect_assertion.py -q + - kind: tests + description: >- + Catalog, compose-anchor, and render-fixture tests prove runtime-effects receives DEPLOY_AGENT_HMAC_SECRET and fails before boot when it is absent. + command: >- + uv run pytest tests/unit/infra/test_catalog_generator.py tests/unit/infra/test_runtime_bundle_decomposition.py tests/ci/test_runtime_env_anchor.py tests/ci/test_compose_required_env_coverage.py -q && uv run pytest tests/scripts/test_check_required_env_vars.py -q + - kind: ci + description: CI pipeline green on the product PR. + command: gh pr checks --repo OmniNode-ai/omnibase_infra +emergency_bypass: + enabled: false + justification: '' + follow_up_ticket_id: '' +dod_evidence: + - id: dod-strict-redeploy-start-publisher + description: >- + The producer resolves the checked-in dev control-bus broker, supports the local plaintext Redpanda transport, validates against the canonical strict consumer model, emits the exact six-field command, and treats delivery timeout/error as failure. + source: generated + checks: + - check_type: command + check_value: >- + uv run pytest tests/unit/scripts/test_pr_merged_rebuild_trigger.py tests/scripts/test_trigger_redeploy_start.py tests/scripts/test_trigger_producer_effect_assertion.py -q + - id: dod-runtime-effects-hmac-boundary + description: >- + Both the catalog-generated compose and checked-in base compose require DEPLOY_AGENT_HMAC_SECRET for runtime-effects; all registered lane render fixtures include the contract so a future omission fails CI. + source: generated + checks: + - check_type: command + check_value: >- + uv run pytest tests/unit/infra/test_catalog_generator.py tests/unit/infra/test_runtime_bundle_decomposition.py tests/ci/test_runtime_env_anchor.py tests/ci/test_compose_required_env_coverage.py -q && uv run pytest tests/scripts/test_check_required_env_vars.py -q + - id: dod-pre-push-on-200 + description: >- + The repository pre-push gate runs on stickybeatz-studio (.200), the same machine class and process environment as the trusted self-hosted runners. + source: manual + checks: + - check_type: command + check_value: >- + cd /Users/jonah/Code/omni_home/omni_worktrees/OMN-15009/omnibase_infra && pre-commit run --hook-stage pre-push --all-files + - id: dod-post-merge-live-activation + description: >- + After merge and explicit operator approval, recreate runtime-effects with the HMAC env, start the dev-lane deploy-agent service, publish one cumulative redeploy-start command, and verify authoritative terminal completion plus exact deployed git_ref. This evidence is intentionally pending; the PR does not authorize a .201 restart or deployment. + source: manual + checks: + - check_type: command + check_value: >- + operator-approved .201 activation and one-command post-merge readback diff --git a/contracts/services/runtime_policy.contract.yaml b/contracts/services/runtime_policy.contract.yaml index 48e656e929..c34b64ed31 100644 --- a/contracts/services/runtime_policy.contract.yaml +++ b/contracts/services/runtime_policy.contract.yaml @@ -28,10 +28,15 @@ profiles: effects_port: 8086 topic_provisioner_max_partitions: 0 # OMN-14551: boundary DLQ routing (auto-wired consume boundary, - # ONEX_BOUNDARY_DLQ_ENABLED) stays OFF here. This lane is currently cold; - # flip only after a dedicated dev-lane live proof, not bundled with the - # stability-test G6 proof this contract change lands. - boundary_dlq_enabled: false + # ONEX_BOUNDARY_DLQ_ENABLED) flipped ON 2026-08-05 after a dedicated + # dev-lane live proof (the lane is warm, not cold, as of this flip -- + # supersedes the prior "currently cold" framing). Ruling basis: + # never-silent-drop doctrine + the measured live swallow escalation on + # this lane (OMN-15447, 9 boundary_swallow_observed occurrences + # 2026-08-05, dlq_enabled=False on every one). Best-effort DLQ delivery + # only (no nack/redelivery) per the forbid-verify ruling on + # OMN-14507/OMN-14548 -- see stability-test's identical stance above. + boundary_dlq_enabled: true secret_resolver_config_path: /app/data/delegation/secret_resolver.yaml secret_resolver_mappings: - logical_name: llm.openrouter.api_key diff --git a/deploy/lane-census/census-snapshot.json b/deploy/lane-census/census-snapshot.json index 31426bcf95..e7203a4660 100644 --- a/deploy/lane-census/census-snapshot.json +++ b/deploy/lane-census/census-snapshot.json @@ -3,7 +3,7 @@ "event_type": "lane-census-drift", "topic": "onex.evt.infra.lane-census-drift.v1", "host": "omninode-pc.tail75df5e.ts.net", - "emitted_at": "2026-07-22T21:57:05.783732+00:00", + "emitted_at": "2026-08-08T19:39:02.027888+00:00", "severity": "critical", "lanes_checked": [ "stability-test", @@ -11,8 +11,15 @@ "judge", "dev" ], - "drift_count": 9, + "drift_count": 11, "findings": [ + { + "lane": "stability-test", + "kind": "oneshot_failed", + "container": "omnibase-infra-stability-test-forward-migration", + "detail": "migration/init container 'omnibase-infra-stability-test-forward-migration' Exited non-zero (code 3): Exited (3) 46 hours ago", + "severity": "critical" + }, { "lane": "stability-test", "kind": "container_absent", @@ -75,9 +82,16 @@ "container": "omninode-prod-contract-resolver", "detail": "required service container 'omninode-prod-contract-resolver' is not running (desired replicas=1); lane 'prod' is degraded", "severity": "critical" + }, + { + "lane": "dev", + "kind": "unexpected_container", + "container": "omnibase-infra-forward-migration", + "detail": "container 'omnibase-infra-forward-migration' carries com.omninode.lane='dev' but is not declared in the lane manifest", + "severity": "warning" } ], - "alert_key": "lane-census-drift:omninode-pc.tail75df5e.ts.net:d608e21fee5031ab", - "ticket_title": "fix(infra): lane drift [prod, stability-test] \u2014 9x container_absent", - "ticket_body": "## Lane census drift detected\n\nHost: `omninode-pc.tail75df5e.ts.net`\nLanes checked: stability-test, prod, judge, dev\n\nThe desired-state lane census (deploy/lane-census/lane-manifest.yaml)\ndoes not match the live runtime. Drift items below name exactly what is\nmissing or extra. This is the regression class the lane-census ratchet\n(OMN-13011) exists to catch \u2014 it would have fired on 2026-06-11 when prod\nruntime containers and the broker network were silently absent.\n\n## Findings\n\n- **[critical] container_absent** `omninode-stability-test-agent-actions-consumer` (stability-test): required service container 'omninode-stability-test-agent-actions-consumer' is not running (desired replicas=1); lane 'stability-test' is degraded\n- **[critical] container_absent** `omninode-stability-test-skill-lifecycle-consumer` (stability-test): required service container 'omninode-stability-test-skill-lifecycle-consumer' is not running (desired replicas=1); lane 'stability-test' is degraded\n- **[critical] container_absent** `omnibase-stability-test-intelligence-api` (stability-test): required service container 'omnibase-stability-test-intelligence-api' is not running (desired replicas=1); lane 'stability-test' is degraded\n- **[critical] container_absent** `omninode-stability-test-contract-resolver` (stability-test): required service container 'omninode-stability-test-contract-resolver' is not running (desired replicas=1); lane 'stability-test' is degraded\n- **[critical] container_absent** `omninode-prod-runtime-worker` (prod): required service container 'omninode-prod-runtime-worker' is not running (desired replicas=1); lane 'prod' is degraded\n- **[critical] container_absent** `omninode-prod-agent-actions-consumer` (prod): required service container 'omninode-prod-agent-actions-consumer' is not running (desired replicas=1); lane 'prod' is degraded\n- **[critical] container_absent** `omninode-prod-skill-lifecycle-consumer` (prod): required service container 'omninode-prod-skill-lifecycle-consumer' is not running (desired replicas=1); lane 'prod' is degraded\n- **[critical] container_absent** `omnibase-prod-intelligence-api` (prod): required service container 'omnibase-prod-intelligence-api' is not running (desired replicas=1); lane 'prod' is degraded\n- **[critical] container_absent** `omninode-prod-contract-resolver` (prod): required service container 'omninode-prod-contract-resolver' is not running (desired replicas=1); lane 'prod' is degraded\n\n## Action\nBring the lane back to declared state (compose up the absent containers / reattach the network), or update the lane manifest in the same PR if the desired state legitimately changed. Do not silence the check." + "alert_key": "lane-census-drift:omninode-pc.tail75df5e.ts.net:5098b00154cea5af", + "ticket_title": "fix(infra): lane drift [dev, prod, stability-test] \u2014 9x container_absent, 1x oneshot_failed, 1x unexpected_container", + "ticket_body": "## Lane census drift detected\n\nHost: `omninode-pc.tail75df5e.ts.net`\nLanes checked: stability-test, prod, judge, dev\n\nThe desired-state lane census (deploy/lane-census/lane-manifest.yaml)\ndoes not match the live runtime. Drift items below name exactly what is\nmissing or extra. This is the regression class the lane-census ratchet\n(OMN-13011) exists to catch \u2014 it would have fired on 2026-06-11 when prod\nruntime containers and the broker network were silently absent.\n\n## Findings\n\n- **[critical] oneshot_failed** `omnibase-infra-stability-test-forward-migration` (stability-test): migration/init container 'omnibase-infra-stability-test-forward-migration' Exited non-zero (code 3): Exited (3) 46 hours ago\n- **[critical] container_absent** `omninode-stability-test-agent-actions-consumer` (stability-test): required service container 'omninode-stability-test-agent-actions-consumer' is not running (desired replicas=1); lane 'stability-test' is degraded\n- **[critical] container_absent** `omninode-stability-test-skill-lifecycle-consumer` (stability-test): required service container 'omninode-stability-test-skill-lifecycle-consumer' is not running (desired replicas=1); lane 'stability-test' is degraded\n- **[critical] container_absent** `omnibase-stability-test-intelligence-api` (stability-test): required service container 'omnibase-stability-test-intelligence-api' is not running (desired replicas=1); lane 'stability-test' is degraded\n- **[critical] container_absent** `omninode-stability-test-contract-resolver` (stability-test): required service container 'omninode-stability-test-contract-resolver' is not running (desired replicas=1); lane 'stability-test' is degraded\n- **[critical] container_absent** `omninode-prod-runtime-worker` (prod): required service container 'omninode-prod-runtime-worker' is not running (desired replicas=1); lane 'prod' is degraded\n- **[critical] container_absent** `omninode-prod-agent-actions-consumer` (prod): required service container 'omninode-prod-agent-actions-consumer' is not running (desired replicas=1); lane 'prod' is degraded\n- **[critical] container_absent** `omninode-prod-skill-lifecycle-consumer` (prod): required service container 'omninode-prod-skill-lifecycle-consumer' is not running (desired replicas=1); lane 'prod' is degraded\n- **[critical] container_absent** `omnibase-prod-intelligence-api` (prod): required service container 'omnibase-prod-intelligence-api' is not running (desired replicas=1); lane 'prod' is degraded\n- **[critical] container_absent** `omninode-prod-contract-resolver` (prod): required service container 'omninode-prod-contract-resolver' is not running (desired replicas=1); lane 'prod' is degraded\n- **[warning] unexpected_container** `omnibase-infra-forward-migration` (dev): container 'omnibase-infra-forward-migration' carries com.omninode.lane='dev' but is not declared in the lane manifest\n\n## Action\nBring the lane back to declared state (compose up the absent containers / reattach the network), or update the lane manifest in the same PR if the desired state legitimately changed. Do not silence the check." } diff --git a/deploy/maintenance/cron.d/omninode-host-maintenance-sync b/deploy/maintenance/cron.d/omninode-host-maintenance-sync new file mode 100644 index 0000000000..28ed227507 --- /dev/null +++ b/deploy/maintenance/cron.d/omninode-host-maintenance-sync @@ -0,0 +1,6 @@ +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# OMN-15525: fail loudly when a host maintenance artifact drifts from origin/dev. +# Hourly at :37 to stay off the */15 system-report boundary. +37 * * * * root /data/maintenance/bin/omninode-host-maintenance-sync.sh --check --slack diff --git a/deploy/maintenance/cron.d/omninode-system-slack-report b/deploy/maintenance/cron.d/omninode-system-slack-report new file mode 100644 index 0000000000..8195bb1524 --- /dev/null +++ b/deploy/maintenance/cron.d/omninode-system-slack-report @@ -0,0 +1,8 @@ +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# Morning OmniNode system digest to Slack. +5 8 * * * root /data/maintenance/bin/omninode-system-slack-report.sh --mode digest + +# State-change alerts for disk pressure, Docker container health, and runtime endpoint health. +*/15 * * * * root /data/maintenance/bin/omninode-system-slack-report.sh --mode alert diff --git a/deploy/maintenance/omninode-host-maintenance-sync.sh b/deploy/maintenance/omninode-host-maintenance-sync.sh new file mode 100644 index 0000000000..1455301db5 --- /dev/null +++ b/deploy/maintenance/omninode-host-maintenance-sync.sh @@ -0,0 +1,217 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# omninode-host-maintenance-sync.sh — install/verify the `.201` host maintenance +# artifacts that live in this repo but execute from outside any container. +# +# WHY THIS EXISTS (OMN-15525) +# `deploy/maintenance/omninode-system-slack-report.sh` runs as root from +# /data/maintenance/bin/ under /etc/cron.d/. No sanctioned deploy path covered +# that directory: `deploy-runtime.sh` deploys containers, not host files. So +# the script was hand-copied once, drifted for weeks, and the divergence was +# invisible — OMN-15509 fixed the repo copy and changed nothing about what the +# platform actually alarmed on, because nothing installs or checks the host +# copy. `omnibase_infra#2572` merged and the monitor stayed blind. +# +# The same shape already bit a second artifact (OMN-15521: the gateway +# forwarder hand-deployed to root-owned /opt/omninode/gateway). Two host +# artifacts sharing one structural gap is the argument for one install path +# rather than another one-off copy, which is what this is. +# +# Per CLAUDE.md rule 5 and `feedback_a_rule_is_not_a_mechanism`: a runbook step +# saying "remember to copy the file" is not enforcement. `--check` runs on a +# schedule and FAILS — non-zero exit, and a Slack alert with --slack — when an +# installed artifact does not match `origin/dev`. +# +# WHAT IS COMPARED +# The installed file's sha256 against the sha256 of the blob at +# `origin/dev`, read with `git cat-file` after a fetch. Deliberately NOT the +# clone's working tree: /data/omninode/omnibase_infra on .201 sat 40+ commits +# behind `dev` while this was written, so a working-tree comparison would have +# reported "in sync" against a stale checkout — a false green in the checker +# built to catch false greens. +# +# FAIL-CLOSED +# Missing host file, missing repo blob, failed fetch, unresolvable ref, or an +# unreadable path is CRITICAL. "Could not determine" is never "fine". + +set -euo pipefail + +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +INFRA_REPO_ROOT=${OMNINODE_INFRA_REPO_ROOT:-/data/omninode/omnibase_infra} +SYNC_REF=${OMNINODE_MAINTENANCE_SYNC_REF:-origin/dev} +SYNC_REMOTE=${OMNINODE_MAINTENANCE_SYNC_REMOTE:-origin} +SYNC_BRANCH=${OMNINODE_MAINTENANCE_SYNC_BRANCH:-dev} +ENV_FILE=${OMNINODE_ALERT_ENV_FILE:-/data/omninode/omnibase_infra/.env} +# Skip the network round-trip (tests, and any caller that already fetched). +SKIP_FETCH=${OMNINODE_MAINTENANCE_SYNC_SKIP_FETCH:-0} + +# repo-relative path | installed path | mode +# +# Adding a host artifact here is what makes it governed. An artifact absent from +# this manifest is exactly the OMN-15525 condition and will not be checked. +MANIFEST=( + "deploy/maintenance/omninode-system-slack-report.sh|/data/maintenance/bin/omninode-system-slack-report.sh|0755" + # OMN-15550. The reporter shells out to this probe from `collect()`, so an + # un-synced copy is a silently blind detector -- exactly the OMN-15525 + # condition (merged, never deployed, nothing alarms) that this manifest + # exists to make impossible. + "scripts/omninode-ci-required-context-probe.py|/data/maintenance/bin/omninode-ci-required-context-probe.py|0755" + "deploy/maintenance/cron.d/omninode-system-slack-report|/etc/cron.d/omninode-system-slack-report|0644" + "deploy/maintenance/omninode-host-maintenance-sync.sh|/data/maintenance/bin/omninode-host-maintenance-sync.sh|0755" + "deploy/maintenance/cron.d/omninode-host-maintenance-sync|/etc/cron.d/omninode-host-maintenance-sync|0644" +) + +# Optional manifest override: a file of `relpath|hostpath|mode` lines, blank and +# `#` lines ignored. This exists so the detector can be exercised against +# scratch paths — both by the hermetic tests and by the OMN-15525 AC5 proof that +# it actually reddens — WITHOUT pointing a `--install` run at, or otherwise +# touching, the live root-owned artifacts. Never set in the cron unit. +MANIFEST_FILE=${OMNINODE_MAINTENANCE_SYNC_MANIFEST:-} +if [[ -n "$MANIFEST_FILE" ]]; then + if [[ ! -r "$MANIFEST_FILE" ]]; then + echo "FATAL: manifest $MANIFEST_FILE is unreadable" >&2 + exit 2 + fi + MANIFEST=() + while IFS= read -r line; do + [[ -n "$line" && "$line" != \#* ]] || continue + MANIFEST+=("$line") + done <"$MANIFEST_FILE" + (( ${#MANIFEST[@]} > 0 )) || { + echo "FATAL: manifest $MANIFEST_FILE declares no artifacts" >&2 + exit 2 + } +fi + +MODE=check +SLACK=0 +for arg in "$@"; do + case "$arg" in + --check) MODE=check ;; + --install) MODE=install ;; + --slack) SLACK=1 ;; + -h|--help) + sed -n '2,40p' "$0" + exit 0 + ;; + *) + echo "unknown argument: $arg" >&2 + exit 2 + ;; + esac +done + +die() { + echo "FATAL: $*" >&2 + exit 2 +} + +[[ -d "$INFRA_REPO_ROOT/.git" ]] || die "no git clone at $INFRA_REPO_ROOT (set OMNINODE_INFRA_REPO_ROOT)" + +if [[ "$SKIP_FETCH" != "1" ]]; then + git -C "$INFRA_REPO_ROOT" fetch --quiet "$SYNC_REMOTE" \ + "+refs/heads/${SYNC_BRANCH}:refs/remotes/${SYNC_REMOTE}/${SYNC_BRANCH}" \ + || die "fetch of ${SYNC_REMOTE}/${SYNC_BRANCH} failed; cannot compare against $SYNC_REF" +fi + +REF_SHA=$(git -C "$INFRA_REPO_ROOT" rev-parse --verify "$SYNC_REF" 2>/dev/null) \ + || die "cannot resolve $SYNC_REF in $INFRA_REPO_ROOT" + +sha_of_stdin() { sha256sum | awk '{print $1}'; } + +# sha256 of a path as it exists at $SYNC_REF, or empty when the blob is absent. +ref_blob_sha() { + local relpath="$1" + git -C "$INFRA_REPO_ROOT" cat-file blob "${SYNC_REF}:${relpath}" 2>/dev/null | sha_of_stdin +} + +ref_blob_exists() { + git -C "$INFRA_REPO_ROOT" cat-file -e "${SYNC_REF}:$1" 2>/dev/null +} + +installed_sha() { + local path="$1" + [[ -r "$path" ]] || return 1 + sha256sum "$path" | awk '{print $1}' +} + +drift_count=0 +missing_count=0 +report_lines=() + +for entry in "${MANIFEST[@]}"; do + IFS='|' read -r relpath hostpath mode <<<"$entry" + + if ! ref_blob_exists "$relpath"; then + report_lines+=("CRITICAL|$hostpath|blob ${relpath} absent at ${SYNC_REF}") + drift_count=$((drift_count + 1)) + continue + fi + want=$(ref_blob_sha "$relpath") + if [[ -z "$want" ]]; then + report_lines+=("CRITICAL|$hostpath|could not read ${relpath} at ${SYNC_REF}") + drift_count=$((drift_count + 1)) + continue + fi + + if [[ "$MODE" == "install" ]]; then + tmp=$(mktemp) + git -C "$INFRA_REPO_ROOT" cat-file blob "${SYNC_REF}:${relpath}" >"$tmp" \ + || die "failed to extract ${relpath} at ${SYNC_REF}" + install -m "$mode" "$tmp" "${hostpath}.omn-sync.tmp" \ + || die "cannot write ${hostpath}.omn-sync.tmp (root required?)" + # Rename is atomic: a cron run reading the old inode is never handed a + # half-written script. + mv -f "${hostpath}.omn-sync.tmp" "$hostpath" || die "cannot replace $hostpath" + rm -f "$tmp" + fi + + if ! have=$(installed_sha "$hostpath"); then + report_lines+=("CRITICAL|$hostpath|NOT INSTALLED or unreadable (want ${want:0:12})") + missing_count=$((missing_count + 1)) + drift_count=$((drift_count + 1)) + continue + fi + + if [[ "$have" == "$want" ]]; then + report_lines+=("OK|$hostpath|${have:0:12} matches ${SYNC_REF}") + else + report_lines+=("CRITICAL|$hostpath|DRIFT installed=${have:0:12} ${SYNC_REF}=${want:0:12}") + drift_count=$((drift_count + 1)) + fi +done + +echo "omninode host maintenance sync — mode=$MODE ref=$SYNC_REF (${REF_SHA:0:12}) repo=$INFRA_REPO_ROOT" +printf '%s\n' "${report_lines[@]}" +echo "drifted=$drift_count missing=$missing_count checked=${#MANIFEST[@]}" + +if (( drift_count > 0 )) && (( SLACK == 1 )); then + if [[ -f "$ENV_FILE" ]]; then + set -a + set +u + # shellcheck disable=SC1090 + . "$ENV_FILE" + set -u + set +a + fi + channel="${SLACK_CHANNEL_ID:-${SLACK_DEFAULT_CHANNEL:-}}" + if [[ -n "${SLACK_BOT_TOKEN:-}" && -n "$channel" ]]; then + text=$(printf '*OmniNode host maintenance drift*\nHost: %s\n%s host artifact(s) do not match `%s`.\n```\n%s\n```' \ + "$(hostname)" "$drift_count" "$SYNC_REF" "$(printf '%s\n' "${report_lines[@]}")") + payload=$(jq -n --arg channel "$channel" --arg text "$text" \ + '{channel:$channel,text:$text,attachments:[{color:"danger",text:$text,mrkdwn_in:["text"]}]}') + curl -fsS --retry 2 --max-time 10 \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + -H 'Content-Type: application/json; charset=utf-8' \ + -d "$payload" https://slack.com/api/chat.postMessage \ + | jq -e '.ok == true' >/dev/null || echo "WARNING: Slack post failed" >&2 + else + echo "WARNING: --slack requested but no SLACK_BOT_TOKEN/channel in $ENV_FILE" >&2 + fi +fi + +# Non-zero on drift is the enforcement: cron reddens, and any caller that gates +# on this script fails rather than logging a line nobody reads. +(( drift_count == 0 )) || exit 1 diff --git a/deploy/maintenance/omninode-system-slack-report.sh b/deploy/maintenance/omninode-system-slack-report.sh new file mode 100755 index 0000000000..840b91e0d8 --- /dev/null +++ b/deploy/maintenance/omninode-system-slack-report.sh @@ -0,0 +1,636 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# omninode-system-slack-report.sh — the .201 host system-health Slack reporter. +# +# WHAT THIS IS +# This is the producer behind the Slack messages "*OmniNode system alert*", +# "*OmniNode morning system digest*" and "*[OmniNode alert resolved]*". +# It is executed as root on the `.201` host (omninode-pc) by +# /etc/cron.d/omninode-system-slack-report: +# 5 8 * * * root --mode digest +# */15 * * * * root --mode alert +# The deployed copy lives at /data/maintenance/bin/omninode-system-slack-report.sh. +# `deploy/maintenance/cron.d/omninode-system-slack-report` in this repo is the +# as-deployed cron unit. +# +# WHY IT IS IN THIS REPO NOW (OMN-15509) +# Until 2026-07-30 this script existed ONLY on the host, untracked by any +# repository. That is why the defect below survived: there was no diff to +# review, no test to fail, and no grep in any repo that could find it. The +# file is version-controlled here so a change to what the platform alarms on +# is a reviewable diff with a test attached. +# +# THE DEFECT THIS FIXES (OMN-15509) +# The as-deployed `collect()` probed exactly five endpoints: +# runtime-18085, runtime-28085, projection-api-13002, +# deploy-agent-8099, web-3003 +# The dev/lab lane's main runtime port (:8085) was ABSENT. On 2026-07-30 the +# dev runtime sat at Docker `health: starting` with :8085 returning 503, zero +# registered handlers, and dependent containers stuck in `Created`, for at +# least 26 minutes — and the 16:30:02Z Slack message reported every listed +# runtime endpoint as `HTTP 200 (OK)`. The monitor was structurally incapable +# of observing the thing that broke. +# +# Four separate false-green mechanisms are closed here: +# 1. Lane coverage. Every lane's MAIN runtime health endpoint is probed, +# and the lane->port map is read from `docker/runtime-policy.env` (the +# rendered runtime-policy contract that `deploy-runtime.sh` and +# `scripts/system_health_check.sh` already read) rather than being +# hardcoded per call site. A lane cannot be silently dropped. +# 2. Body honesty. A runtime `/health` can return 200 with a body that says +# it is NOT healthy. The old check used `grep -Ei 'healthy|ok'` against +# the body, which MATCHES `{"healthy": false}` — the substring is there. +# Runtime endpoints now resolve a real status from the JSON body and go +# CRITICAL when it is not healthy, or when no status can be resolved. +# 3. `health: starting`. A container whose healthcheck has never once passed +# reports neither `unhealthy` nor `Exited`, so it was invisible. A +# container still `starting` past its own `start_period` is now CRITICAL. +# 4. Exit accounting. `Exited` was reported but excluded from the CRITICAL +# condition, so a crashed container was silent. Non-zero exits are now +# CRITICAL; Exit(0) one-shots (migration/init containers) are not — they +# are expected to finish. +# +# Fail-closed throughout: an endpoint that cannot be probed (connection +# refused, DNS failure, timeout -> code 000) is CRITICAL, never skipped; a +# docker query that fails is CRITICAL, never treated as "nothing wrong". +# +# WHAT OMN-15525 FIXES ON TOP (found by actually deploying the above) +# The OMN-15509 revision was installed on .201 on 2026-07-30T18:12Z and +# immediately reported CRITICAL for all three lanes against a demonstrably +# HEALTHY fleet (all three /health returned 200 with "healthy":true). It was +# rolled back at 18:13:29Z, before the 18:15 cron fired. Two defects: +# +# A. False-RED from a truncated parse. `check_runtime_lane` cut the body to +# 180 bytes for display and then handed THAT to jq. A real runtime +# /health body on .201 is 2644 bytes, so jq always failed with +# "Unfinished string at EOF", the verdict was always "unresolvable", and +# fail-closed correctly turned an unparseable input into CRITICAL. The +# rule was right; the input was mutilated before it got there. The body +# is now parsed whole and truncated only for the reported excerpt. +# This is why the unit fixtures did not catch it: HEALTHY_BODY was 63 +# bytes, comfortably under the cut. Fixtures now carry a realistic +# >180-byte body so the truncation boundary is inside test coverage. +# +# C. The alert could not fire at all. `$issues`, `$issue_keys`, +# `$critical_count` and `$warning_count` were all selected with +# `$2=="CRITICAL"`, but endpoint rows carry their status in `$1` (only +# disk/docker rows use `$2`). So no endpoint failure — no runtime lane, +# projection-api, deploy-agent or web — ever reached `$issues`, and +# `--mode alert` took the "clean" branch and posted nothing. The digest +# text rendered the CRITICAL lines because `format_digest` happened to +# handle both shapes, which is why the .201 run printed three CRITICAL +# lanes directly under `Issues: *0 critical*`. OMN-15509 taught this +# script to SEE a dead lane; defect C is what kept it from SAYING so. +# See `row_status` / `row_key` near the bottom of this file. +# +# B. Silent hardcoded port fallback (rule 8). `policy_env_value` returned +# SUCCESS when the policy file did not exist, and `lane_main_port` +# substituted a literal 8085/18085/28085 for an empty value. A renamed +# key or an unrendered runtime-policy.env degraded to probing guessed +# ports with no signal. An unresolvable lane port is now CRITICAL. +# +# A monitor has two failure directions and both are fatal to it: blind (the +# OMN-15509 defect) and crying wolf (defect A). A permanent CRITICAL on a +# healthy fleet gets the channel muted, after which the blind spot is back +# with extra steps. +# +# PROD IS READ-ONLY +# The prod lane is probed with a plain GET against /health and nothing else. +# This script never mutates any lane. + +set -euo pipefail + +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV_FILE=${OMNINODE_ALERT_ENV_FILE:-/data/omninode/omnibase_infra/.env} +# Repo root that carries the rendered runtime policy. Defaults to the parent of +# ENV_FILE's directory-of-record so the deployed layout needs no extra config. +INFRA_REPO_ROOT=${OMNINODE_INFRA_REPO_ROOT:-/data/omninode/omnibase_infra} +STATE_DIR=${OMNINODE_ALERT_STATE_DIR:-/data/maintenance/state} +LOG_DIR=${OMNINODE_ALERT_LOG_DIR:-/data/maintenance/logs} +LOCK_FILE=${OMNINODE_ALERT_LOCK_FILE:-/run/omninode-system-slack-report.lock} +PROBE_HOST=${OMNINODE_ALERT_PROBE_HOST:-127.0.0.1} # fallback-ok: this reporter runs ON the .201 host as root via cron and probes that host's OWN published lane ports, so the loopback IS the target, not a stand-in for a remote address; same posture as LANE_PROBE_HOST in scripts/system_health_check.sh +# Grace applied to a `health: starting` container whose image declares no +# start_period. Fail-closed: a finite grace, not "never alarm". +STARTING_GRACE_SECONDS=${OMNINODE_ALERT_STARTING_GRACE_SECONDS:-180} +# How much of a response body is quoted into the Slack message / log line. This +# bounds DISPLAY ONLY. It must never be applied before a body is parsed or +# pattern-matched (OMN-15525) — see check_runtime_lane. +BODY_EXCERPT_BYTES=${OMNINODE_ALERT_BODY_EXCERPT_BYTES:-180} +MODE=digest + +if [[ "${1:-}" == "--mode" ]]; then + MODE="${2:-digest}" +elif [[ -n "${1:-}" ]]; then + MODE="$1" +fi + +mkdir -p "$STATE_DIR" "$LOG_DIR" +LOG_FILE="$LOG_DIR/omninode-system-slack-report-$(date -u +%Y%m%dT%H%M%SZ).log" +# dry-run is the human/CI inspection mode: keep its report on stdout instead of +# burying it in a log file, so it can be read (and asserted on) directly. +if [[ "$MODE" != "dry-run" ]]; then + exec >>"$LOG_FILE" 2>&1 +fi +exec 9>"$LOCK_FILE" +if ! flock -n 9; then + echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) another system report is already running" + exit 0 +fi + +if [[ -f "$ENV_FILE" ]]; then + set -a + set +u + # shellcheck disable=SC1090 + . "$ENV_FILE" + set -u + set +a +fi + +if [[ "$MODE" != "dry-run" ]]; then + : "${SLACK_BOT_TOKEN:?SLACK_BOT_TOKEN must be set in $ENV_FILE}" + SLACK_CHANNEL_ID="${SLACK_CHANNEL_ID:-${SLACK_DEFAULT_CHANNEL:-}}" + : "${SLACK_CHANNEL_ID:?SLACK_CHANNEL_ID or SLACK_DEFAULT_CHANNEL must be set in $ENV_FILE}" +else + SLACK_CHANNEL_ID="${SLACK_CHANNEL_ID:-${SLACK_DEFAULT_CHANNEL:-dry-run}}" +fi + +DATA_WARN_PCT=${DATA_WARN_PCT:-95} +DATA_CRIT_PCT=${DATA_CRIT_PCT:-98} +DATA_WARN_FREE_GB=${DATA_WARN_FREE_GB:-200} +DATA_CRIT_FREE_GB=${DATA_CRIT_FREE_GB:-50} +ROOT_WARN_PCT=${ROOT_WARN_PCT:-85} +ROOT_CRIT_PCT=${ROOT_CRIT_PCT:-92} +ROOT_WARN_FREE_GB=${ROOT_WARN_FREE_GB:-50} +ROOT_CRIT_FREE_GB=${ROOT_CRIT_FREE_GB:-20} + +# --------------------------------------------------------------------------- +# Lane -> main runtime port map (OMN-15509 AC2) +# +# Read by targeted key extraction from the rendered runtime policy, NOT by +# `source`: that file is a generated artifact whose key set changes when the +# contract is re-rendered, and sourcing it would let a future render silently +# redefine this script's other variables. Same idiom, same file, and the same +# reasoning as scripts/system_health_check.sh. +# +# There are NO literal fallback ports (OMN-15525). The original revision of this +# block fell back to hardcoded 8085/18085/28085 whenever the policy file was +# missing or a key was renamed, and `policy_env_value` even returned SUCCESS on +# a missing file. That is a silent default masquerading as a resolved contract — +# CLAUDE.md rule 8 forbids exactly this, and it is the same false-green family +# OMN-15509 closed: the probe would keep reporting on a guessed port and the +# operator would never learn the lane map had stopped resolving. An unresolvable +# lane port is now a CRITICAL fact, reported the same as an unreachable endpoint. +# --------------------------------------------------------------------------- +LANE_PORT_UNRESOLVED='__unresolved__' + +runtime_policy_file() { + printf '%s' "${OMNINODE_RUNTIME_POLICY_ENV:-${INFRA_REPO_ROOT}/docker/runtime-policy.env}" +} + +# Echo the value for `key`, or return non-zero when the policy file is absent or +# the key is not present in it. Fail-fast: never returns a substitute value. +policy_env_value() { + local key="$1" file value + file="$(runtime_policy_file)" + [[ -f "$file" ]] || return 1 + value=$(sed -n "s/^${key}=//p" "$file" | tail -n 1 | tr -d "\"'") + [[ -n "$value" ]] || return 1 + printf '%s' "$value" +} + +# label|policy-key -- a pure data table, no comments inside the array (the +# fallback-port guard parses every line in it as lane|key). +# +# Must cover EVERY lane declaring a *_RUNTIME_MAIN_PORT in runtime-policy.env. +# Held in two-way parity by +# tests/unit/scripts/test_omninode_system_slack_report.py, which DERIVES the +# lane set from the policy instead of restating it: a row dropped here, or a +# lane added to the policy and not here, fails that module. +# +# judge (:48085) is read-only and NOT authorized for mutation, but it runs +# seven containers on .201 and was omitted from this table through OMN-15509 +# and OMN-15525, so a dead judge runtime paged nobody (OMN-15556). Probing it +# is a plain GET /health, which is a read. +RUNTIME_LANE_SPECS=( + "dev|DEV_RUNTIME_MAIN_PORT" + "stability-test|STABILITY_TEST_RUNTIME_MAIN_PORT" + "prod|PROD_RUNTIME_MAIN_PORT" + "judge|JUDGE_RUNTIME_MAIN_PORT" +) + +# Resolve a lane's main runtime port, or emit $LANE_PORT_UNRESOLVED. A value that +# is not a bare integer is also unresolved — probing "http://host:garbage/health" +# would just produce a confusing connection error instead of naming the real +# problem (the lane map). +lane_main_port() { + local key="$1" value + if ! value="$(policy_env_value "$key")" || [[ ! "$value" =~ ^[0-9]+$ ]]; then + printf '%s' "$LANE_PORT_UNRESOLVED" + return 0 + fi + printf '%s' "$value" +} + +post_slack() { + local text="$1" + local color="${2:-#439FE0}" + local payload + payload=$(jq -n \ + --arg channel "$SLACK_CHANNEL_ID" \ + --arg text "$text" \ + --arg color "$color" \ + '{channel:$channel,text:$text,attachments:[{color:$color,text:$text,mrkdwn_in:["text"]}]}') + curl -fsS --retry 2 --max-time 10 \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + -H 'Content-Type: application/json; charset=utf-8' \ + -d "$payload" \ + https://slack.com/api/chat.postMessage | jq -e '.ok == true' >/dev/null +} + +df_line() { + local mount="$1" + df -BG --output=target,size,used,avail,pcent "$mount" | awk 'NR==2 {gsub("G","",$4); gsub("%","",$5); print $1"|"$2"|"$3"|"$4"|"$5}' +} + +classify_disk() { + local mount="$1" pct="$2" avail_gb="$3" warn_pct="$4" crit_pct="$5" warn_free="$6" crit_free="$7" + if (( pct >= crit_pct || avail_gb <= crit_free )); then + echo "CRITICAL" + elif (( pct >= warn_pct || avail_gb <= warn_free )); then + echo "WARNING" + else + echo "OK" + fi +} + +check_http() { + local label="$1" url="$2" expect_regex="${3:-}" + local tmp code status body body_excerpt + tmp=$(mktemp) + code=$(curl -sS --max-time 4 -o "$tmp" -w '%{http_code}' "$url" 2>/dev/null || true) + # Match against the WHOLE body; truncate only the excerpt that gets reported. + # See BODY_EXCERPT_BYTES — matching an excerpt makes the verdict depend on + # where the payload happens to be cut. + body=$(tr '\n' ' ' <"$tmp") + body_excerpt=$(printf '%s' "$body" | head -c "$BODY_EXCERPT_BYTES") + rm -f "$tmp" + status="OK" + if [[ ! "$code" =~ ^2 ]]; then + status="CRITICAL" + elif [[ -n "$expect_regex" ]] && ! grep -Eiq "$expect_regex" <<<"$body"; then + status="WARNING" + fi + printf '%s|%s|%s|%s\n' "$status" "$label" "${code:-000}" "$body_excerpt" +} + +# Resolve a runtime /health body to healthy / unhealthy / unresolvable. +# +# Substring matching is NOT usable here: `{"healthy": false}` contains both +# "healthy" and "false", and the old `grep -Ei 'healthy|ok'` scored it OK. The +# body is parsed as JSON and an explicit boolean/status field is required. +# Anything that cannot be resolved to an affirmative healthy signal is +# unresolvable, and unresolvable is CRITICAL (fail-closed) — a monitor that +# cannot tell is not allowed to say green. +runtime_body_verdict() { + local body="$1" verdict + verdict=$(jq -r ' + def norm: if type == "string" then ascii_downcase else . end; + if type != "object" then "unresolvable" + elif (.healthy? != null and (.healthy | type) == "boolean") + then (if .healthy then "healthy" else "unhealthy" end) + elif (.details?.healthy? != null and (.details.healthy | type) == "boolean") + then (if .details.healthy then "healthy" else "unhealthy" end) + elif (.status? != null) + then (if (.status | norm) == "healthy" or (.status | norm) == "ok" or (.status | norm) == "pass" + then "healthy" else "unhealthy" end) + else "unresolvable" + end + ' <<<"$body" 2>/dev/null) || verdict="unresolvable" + [[ -n "$verdict" ]] || verdict="unresolvable" + printf '%s' "$verdict" +} + +# Probe one lane's MAIN runtime health endpoint. Read-only GET on every lane, +# prod included. +check_runtime_lane() { + local lane="$1" port="$2" + local label="runtime-${lane}-${port}" + local tmp code body body_excerpt status detail verdict + tmp=$(mktemp) + code=$(curl -sS -X GET --max-time 4 -o "$tmp" -w '%{http_code}' "http://${PROBE_HOST}:${port}/health" 2>/dev/null || true) + # The verdict is computed from the FULL body; only the reported excerpt is + # truncated. Truncating BEFORE the verdict was OMN-15525: a real .201 runtime + # /health body is ~2.6 KB, so the 180-byte excerpt handed to jq was always + # invalid JSON ("Unfinished string at EOF"), every lane resolved to + # "unresolvable", and fail-closed turned that into CRITICAL on a fully healthy + # fleet. Fail-closed is right; feeding it a mutilated input is not. + body=$(tr '\n' ' ' <"$tmp") + body_excerpt=$(printf '%s' "$body" | head -c "$BODY_EXCERPT_BYTES") + rm -f "$tmp" + code="${code:-000}" + + if [[ ! "$code" =~ ^2 ]]; then + # Covers 5xx AND connection-refused/timeout (code 000). Never skipped. + status="CRITICAL" + detail="$body_excerpt" + else + verdict=$(runtime_body_verdict "$body") + case "$verdict" in + healthy) status="OK"; detail="$body_excerpt" ;; + unhealthy) status="CRITICAL"; detail="HTTP 200 but health body is NOT healthy: $body_excerpt" ;; + *) status="CRITICAL"; detail="HTTP 200 but health status could not be resolved from body (fail-closed): $body_excerpt" ;; + esac + fi + printf '%s|%s|%s|%s\n' "$status" "$label" "$code" "$detail" +} + +# ISO-8601 -> epoch seconds. GNU `date -d` on the deploy host; BSD `date -j -f` +# fallback so the hermetic test can drive this same artifact on macOS (rule 11a +# runs gates on .200). Returns 0 when the timestamp cannot be parsed, which the +# caller treats as "cannot age this container" -> fail-closed. +epoch_from_iso() { + local ts="$1" out + [[ -n "$ts" ]] || { printf '0'; return 0; } + out=$(date -u -d "$ts" +%s 2>/dev/null || true) + if [[ ! "$out" =~ ^[0-9]+$ ]]; then + # Docker emits e.g. 2026-07-30T16:19:02.123456789Z — trim to whole seconds. + local trimmed="${ts%%.*}" + trimmed="${trimmed%Z}" + out=$(date -u -j -f '%Y-%m-%dT%H:%M:%S' "$trimmed" +%s 2>/dev/null || true) + fi + [[ "$out" =~ ^[0-9]+$ ]] || out=0 + printf '%s' "$out" +} + +# Containers stuck in `health: starting` past their own start_period. +# +# This is the state that produced the silent 26-minute outage: a healthcheck +# that has never once passed reports neither `unhealthy` nor `Exited`, so every +# count the old script kept was zero. +starting_past_start_period() { + local names name started start_period_ns grace_s started_epoch now_epoch age_s + now_epoch=$(date -u +%s) + names=$(docker ps --filter 'health=starting' --format '{{.Names}}' 2>/dev/null || echo "__DOCKER_QUERY_FAILED__") + if [[ "$names" == "__DOCKER_QUERY_FAILED__" ]]; then + printf '%s' "__DOCKER_QUERY_FAILED__" + return 0 + fi + local out=() + while IFS= read -r name; do + [[ -n "$name" ]] || continue + started=$(docker inspect -f '{{.State.StartedAt}}' "$name" 2>/dev/null || true) + start_period_ns=$(docker inspect -f '{{if .Config.Healthcheck}}{{.Config.Healthcheck.StartPeriod}}{{else}}0{{end}}' "$name" 2>/dev/null || true) + [[ "$start_period_ns" =~ ^[0-9]+$ ]] || start_period_ns=0 + grace_s=$(( start_period_ns / 1000000000 )) + (( grace_s > 0 )) || grace_s="$STARTING_GRACE_SECONDS" + started_epoch=$(epoch_from_iso "$started") + if (( started_epoch == 0 )); then + # Cannot age the container -> cannot prove it is still inside its grace. + out+=("${name}(age-unknown)") + continue + fi + age_s=$(( now_epoch - started_epoch )) + if (( age_s > grace_s )); then + out+=("${name}(${age_s}s>${grace_s}s)") + fi + done <<<"$names" + printf '%s' "${out[*]:-}" +} + +# Containers that exited non-zero. Exit(0) one-shots (migration/init) are +# expected to finish and must not alarm. +exited_nonzero() { + local raw + raw=$(docker ps -a --format '{{.Names}}\t{{.Status}}' 2>/dev/null || echo "__DOCKER_QUERY_FAILED__") + if [[ "$raw" == "__DOCKER_QUERY_FAILED__" ]]; then + printf '%s' "__DOCKER_QUERY_FAILED__" + return 0 + fi + awk -F'\t' '$2 ~ /Exited \([1-9][0-9]*\)/ {printf "%s ", $1}' <<<"$raw" | sed 's/ $//' +} + +# Required GitHub status contexts that never reported (OMN-15550). +# +# WHY THIS LIVES HERE AND NOT IN GITHUB ACTIONS +# A required check that never reports is ABSENT, not RED. Branch protection +# blocks the PR identically, but an absent context has no row in any list, so +# `gh pr checks` reads all-green while every PR in the repo is unmergeable. +# On 2026-07-30 (OMN-15536) `omnibase_infra`'s ci.yml failed to assemble; +# `CI Summary` is that repo's SOLE required context, so all 7 open PRs wedged +# silently for ~2.5h until a human noticed. A detector living inside the CI +# system it watches would have failed to assemble with it -- so it runs here, +# on a host that does not depend on GitHub Actions. +# +# Folding it into this reporter rather than building a second alerter is the +# net-negative-surface rule: it inherits this script's Slack poster, its +# state-change de-duplication, its resolved-notification and its */15 cron. +# No new cron unit, no second Slack integration. +# +# The probe emits `ci|STATUS|key|detail` rows, which `row_status()` reads at +# column 2 and `row_key()` de-duplicates as `ci|`. A probe failure is a +# WARNING row, never silence: "could not look" must not render as "nothing +# wrong". It is deliberately not CRITICAL -- an unreachable API is not evidence +# that PRs are stranded, and paging on every network blip mutes the channel. +check_ci_required_contexts() { + # Both artifacts are installed side by side in /data/maintenance/bin by the + # host maintenance sync, so `dirname $0` resolves the probe on the host. In + # the repo the probe lives under scripts/ (the env-read gate's approved + # location for operational Python), not next to this file. + local probe="${OMNINODE_CI_PROBE_SCRIPT:-$(dirname "$0")/omninode-ci-required-context-probe.py}" + local python_bin="${OMNINODE_CI_PROBE_PYTHON:-python3}" + + if [[ "${OMNINODE_CI_PROBE_ENABLED:-1}" != "1" ]]; then + return 0 + fi + if [[ ! -r "$probe" ]]; then + printf 'ci|WARNING|required-contexts|probe script missing or unreadable at %s\n' "$probe" + return 0 + fi + if ! command -v "$python_bin" >/dev/null 2>&1; then + printf 'ci|WARNING|required-contexts|%s not found; required-context probe did not run\n' "$python_bin" + return 0 + fi + + local out + # A hung probe must not wedge the whole 15-minute health tick, so it is + # bounded and a timeout is reported as a WARNING row like any other + # "we could not look" outcome. + if ! out=$(timeout "${OMNINODE_CI_PROBE_TIMEOUT:-120}" "$python_bin" "$probe" 2>/dev/null); then + printf 'ci|WARNING|required-contexts|probe exited non-zero or timed out; required-context state unknown\n' + return 0 + fi + if [[ -z "$out" ]]; then + printf 'ci|WARNING|required-contexts|probe produced no rows; required-context state unknown\n' + return 0 + fi + printf '%s\n' "$out" +} + +collect() { + local now host root data root_status data_status running unhealthy restarting dead created + local dangling named_dangling anonymous_dangling docker_status docker_detail + local starting_stuck exited_bad lane spec key port + now=$(date -u +%Y-%m-%dT%H:%M:%SZ) + host=$(hostname) + root=$(df_line /) + data=$(df_line /data) + IFS='|' read -r _ root_size root_used root_avail root_pct <<<"$root" + IFS='|' read -r _ data_size data_used data_avail data_pct <<<"$data" + root_status=$(classify_disk / "$root_pct" "$root_avail" "$ROOT_WARN_PCT" "$ROOT_CRIT_PCT" "$ROOT_WARN_FREE_GB" "$ROOT_CRIT_FREE_GB") + data_status=$(classify_disk /data "$data_pct" "$data_avail" "$DATA_WARN_PCT" "$DATA_CRIT_PCT" "$DATA_WARN_FREE_GB" "$DATA_CRIT_FREE_GB") + + running=$(docker ps --format '{{.Names}}' | wc -l | tr -d ' ') + unhealthy=$(docker ps -a --format '{{.Names}}\t{{.Status}}' | grep -Eci 'unhealthy' || true) + restarting=$(docker ps -a --format '{{.Names}}\t{{.Status}}' | grep -Eci 'Restarting' || true) + dead=$(docker ps -a --format '{{.Names}}\t{{.Status}}' | grep -Eci 'Dead' || true) + created=$(docker ps -a --format '{{.Names}}\t{{.Status}}' | grep -Eci 'Created' || true) + dangling=$(docker volume ls -qf dangling=true | wc -l | tr -d ' ') + anonymous_dangling=$(docker volume ls -qf dangling=true | grep -Ec '^[0-9a-f]{64}$' || true) + named_dangling=$(docker volume ls -qf dangling=true | grep -Evc '^[0-9a-f]{64}$' || true) + + starting_stuck=$(starting_past_start_period) + exited_bad=$(exited_nonzero) + + docker_status=OK + docker_detail="unhealthy=$unhealthy restarting=$restarting dead=$dead created=$created" + if [[ "$starting_stuck" == "__DOCKER_QUERY_FAILED__" || "$exited_bad" == "__DOCKER_QUERY_FAILED__" ]]; then + # Fail-closed: a docker query that did not run is not evidence of health. + docker_status=CRITICAL + docker_detail="$docker_detail docker_query=FAILED" + else + docker_detail="$docker_detail starting_past_start_period=${starting_stuck:-none} exited_nonzero=${exited_bad:-none}" + if [[ "$unhealthy" != 0 || "$restarting" != 0 || "$dead" != 0 || "$created" != 0 \ + || -n "$starting_stuck" || -n "$exited_bad" ]]; then + docker_status=CRITICAL + fi + fi + + { + echo "timestamp|$now" + echo "host|$host" + echo "disk|$root_status|/|${root_used}/${root_size}|${root_avail}G free|${root_pct}%" + echo "disk|$data_status|/data|${data_used}/${data_size}|${data_avail}G free|${data_pct}%" + echo "docker|OK|running_containers|$running" + echo "docker|$docker_status|container_issues|$docker_detail" + echo "docker|OK|dangling_volumes|total=$dangling anonymous=$anonymous_dangling named=$named_dangling" + # Every lane's MAIN runtime health endpoint, from the lane->port map. + for spec in "${RUNTIME_LANE_SPECS[@]}"; do + IFS='|' read -r lane key <<<"$spec" + port=$(lane_main_port "$key") + if [[ "$port" == "$LANE_PORT_UNRESOLVED" ]]; then + # Rule 8 / OMN-15525: refuse to probe a guessed port. An unresolvable + # lane map is itself the outage-shaped fact worth alarming on. + printf 'CRITICAL|runtime-%s-unresolved|000|lane main port unresolvable: key %s missing from %s\n' \ + "$lane" "$key" "$(runtime_policy_file)" + continue + fi + check_runtime_lane "$lane" "$port" + done + check_http projection-api-13002 "http://${PROBE_HOST}:13002/health" 'ok|healthy' + check_http deploy-agent-8099 "http://${PROBE_HOST}:8099/health" 'idle|running|state|ok' + check_http web-3003 "http://${PROBE_HOST}:3003/" '' + check_ci_required_contexts + } +} + +snapshot=$(collect) +if [[ "$MODE" != "dry-run" ]]; then + echo "$snapshot" +fi + +host=$(awk -F'|' '$1=="host"{print $2}' <<<"$snapshot") + +# The snapshot carries TWO row shapes and the status lives in a different +# column in each: +# +# disk|STATUS|name|... <- resource rows, status in $2 +# docker|STATUS|name|... +# STATUS|label|code|detail <- endpoint rows, status in $1 +# +# OMN-15525: every selector below used to test `$2` only, so NO endpoint row +# could ever land in `$issues` / the counters. `format_digest` handled both +# shapes, which is why the rendered text listed `CRITICAL runtime-dev-8085` +# under *Active issues* while the header said `0 critical` — and, far worse, +# why `--mode alert` computed an EMPTY `$issues`, took the "clean" branch, and +# paged nobody. A dead runtime lane could not raise an alert even after the +# probe was fixed: OMN-15509 taught the reporter to SEE the lane, and this is +# what stopped it from SAYING anything. Verified live on .201 — the merged +# revision printed three CRITICAL lanes above `Issues: *0 critical*`. +# +# `row_status` is the single definition of "this row's status" and everything +# downstream keys off it. +row_status='function row_status() { return ($1=="OK" || $1=="WARNING" || $1=="CRITICAL") ? $1 : $2 }' +# Stable identity for alert de-duplication: label + status only. Volatile +# fields (HTTP code, body excerpt, free-GB) are deliberately excluded so a +# flapping 000/503 on one dead lane is one alert, not one per tick. +row_key='function row_key() { return ($1=="OK" || $1=="WARNING" || $1=="CRITICAL") ? $2 : $1 "|" $3 }' + +issues=$(awk -F'|' "$row_status"'{ s=row_status() } s=="WARNING" || s=="CRITICAL" {print}' <<<"$snapshot" || true) +issue_keys=$(awk -F'|' "$row_status$row_key"'{ s=row_status() } s=="WARNING" || s=="CRITICAL" {print row_key() "|" s}' <<<"$snapshot" || true) +critical_count=$(awk -F'|' "$row_status"'{ s=row_status() } s=="CRITICAL" {c++} END {print c+0}' <<<"$snapshot") +warning_count=$(awk -F'|' "$row_status"'{ s=row_status() } s=="WARNING" {c++} END {print c+0}' <<<"$snapshot") +issue_hash=$(printf '%s\n' "$issue_keys" | sha256sum | awk '{print $1}') +state_file="$STATE_DIR/omninode-system-alert.hash" +prev_hash=$(cat "$state_file" 2>/dev/null || true) + +format_digest() { + local title="$1" + local lines endpoint_lines ci_lines issue_lines + lines=$(awk -F'|' '$1=="disk" {printf "- `%s`: %s, %s, %s (%s)\n", $3, $4, $5, $6, $2} $1=="docker" {printf "- Docker `%s`: %s (%s)\n", $3, $4, $2}' <<<"$snapshot") + endpoint_lines=$(awk -F'|' '$1=="OK" || $1=="WARNING" || $1=="CRITICAL" {printf "- `%s`: HTTP %s (%s)\n", $2, $3, $1}' <<<"$snapshot") + # OMN-15550. The heartbeat row renders here even when clean, so a reader can + # tell "scanned N repos, found nothing" apart from "did not scan" -- the + # detection-shelf blindness where a silent section reads as healthy. + ci_lines=$(awk -F'|' '$1=="ci" {printf "- `%s`: %s (%s)\n", $3, $4, $2}' <<<"$snapshot") + [[ -n "$ci_lines" ]] || ci_lines="- No required-context probe rows this tick" + # `next` keeps the three row shapes mutually exclusive so a `ci` row cannot + # also be rendered by the generic column-2 branch below it. + issue_lines=$(awk -F'|' '$1=="ci" && ($2=="WARNING" || $2=="CRITICAL") {printf "- %s `%s`: %s\n", $2, $3, $4; next} $2=="WARNING" || $2=="CRITICAL" {printf "- %s `%s`: %s %s %s\n", $2, $3, $4, $5, $6; next} $1=="WARNING" || $1=="CRITICAL" {printf "- %s `%s`: HTTP %s %s\n", $1, $2, $3, $4}' <<<"$snapshot") + if [[ -z "$issue_lines" ]]; then + issue_lines="- No active warning/critical checks" + fi + cat <"$state_file" + elif [[ "$issue_hash" != "$prev_hash" ]]; then + color='warning' + [[ "$critical_count" != 0 ]] && color='danger' + post_slack "$(format_digest '*OmniNode system alert*')" "$color" + echo "$issue_hash" >"$state_file" + else + echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) issues unchanged; Slack suppressed" + fi + ;; + dry-run) + format_digest '*OmniNode system dry run*' + ;; + *) + echo "unknown mode: $MODE" >&2 + exit 2 + ;; +esac diff --git a/docker/Dockerfile.runtime b/docker/Dockerfile.runtime index cccf73d214..38e47402a5 100644 --- a/docker/Dockerfile.runtime +++ b/docker/Dockerfile.runtime @@ -458,11 +458,25 @@ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ uv-with-retry pip install "setuptools>=78.1.1" -# Security: upgrade cryptography to clear GHSA-537c-gmf6-5ccf (HIGH, fixed in >=48.0.1). -# cryptography 46.0.7 (from transitive deps) is flagged by Trivy scan (ignore-unfixed: true). +# Security: upgrade cryptography to clear GHSA-537c-gmf6-5ccf (HIGH, fixed in +# >=48.0.1) and, as of OMN-15688 (2026-08-04), CVE-2026-69247 (PKCS#7 +# EnvelopedData decryption exposes a Bleichenbacher oracle through +# distinguishable errors, fixed in >=50.0.0) and CVE-2026-69249 (duplicate +# self-signed intermediates cause exponential path-building, fixed in >=49.0.0). +# cryptography from transitive deps is flagged by Trivy scan (ignore-unfixed: +# true) -- both new CVEs report Status: fixed, so ignore-unfixed does not +# suppress them. The 48.0.1 floor that cleared the 2026-06 finding was measured +# RED again by build-workspace-candidate-runtime.yml run 30878868464. # Force upgrade here after all plugin installs to ensure the patched version is present. RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ - uv-with-retry pip install "cryptography>=48.0.1" + uv-with-retry pip install "cryptography>=50.0.0" + +# Security: upgrade aiohttp to clear CVE-2026-69244 (HIGH, out-of-bounds heap +# read in the C HTTP response parser error path on malformed responses, fixed in +# >=3.14.3). The lock resolved 3.14.1; flagged by the same run 30878868464 scan. +# Force upgrade here after all plugin installs -- a plugin may pull an older pin. +RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ + uv-with-retry pip install "aiohttp>=3.14.3" # OMN-15147: upgrade pyasn1 to clear CVE-2026-59885/CVE-2026-59886 (DoS via # crafted ASN.1 OBJECT IDENTIFIER / REAL values, fixed in >=0.6.4) and @@ -742,6 +756,16 @@ RUN set -eu; \ # /usr/lib/node_modules/npm/node_modules/ are flagged CRITICAL/HIGH by Trivy # (severity CRITICAL,HIGH, ignore-unfixed): # cross-spawn 7.0.3 -> CVE-2024-21538 (fixed 7.0.5) +# REMOVED from the pack loop in OMN-15688: npm 11.18.0 +# does not bundle cross-spawn AT ALL. Measured against +# `node:20-slim` + `npm install -g npm@11.18.0`: +# `find $(npm root -g)/npm/node_modules -type d -name +# cross-spawn` returns NOTHING, at any depth. The loop's +# `rm -rf` was a no-op and the `tar -xzf` was creating a +# brand-new, unreferenced top-level copy that nothing +# resolves — dead weight since the npm 10 -> 11 upgrade. +# Trivy does not flag cross-spawn on this image, which is +# consistent: there is no copy of it to flag. # glob 10.4.2 -> CVE-2025-64756 (fixed 10.5.0/11.1.0) # minimatch 9.0.5 -> CVE-2026-26996/27903/27904 (fixed >=9.0.7 / 10.2.x) # tar 6.2.1 -> CVE-2026-23745/23950/24842/ @@ -749,16 +773,31 @@ RUN set -eu; \ # brace-expansion 5.0.7 -> CVE-2026-14257 (HIGH, DoS via crafted pattern, # fixed 5.0.8) — pulled in transitively by the # minimatch/glob bundles above; added OMN-15147. +# brace-expansion 5.0.8 -> CVE-2026-69152 (HIGH, DoS via unbounded intermediate +# arrays that BYPASSES the CVE-2026-14257 mitigation, +# fixed 5.0.9) — OMN-15688. +# ip-address 10.2.0 -> CVE-2026-69192 (HIGH, Address4 decodes leading-zero +# octets as decimal while resolvers decode them as +# octal, fixed 10.3.1) — bundled transitively via +# npm's socks-proxy-agent -> socks chain; added +# OMN-15688. # # Remediation is two-step because these are *bundled inside npm itself*: # 1. Upgrade npm to 11.18.0 (exact pin — reproducible + satisfies # scripts/check_dockerfile_pins.py; npm@ is rejected). npm 11 already # bundles the patched cross-spawn 7.0.6. -# 2. Overwrite npm's OWN bundled copies of the four flagged packages with the +# 2. Overwrite npm's OWN bundled copies of the flagged packages with the # patched releases. `npm install -g @x` installs a SEPARATE global # package and leaves npm's bundled vulnerable copy in place (Trivy would # still flag it), so instead `npm pack` each patched release and extract it # directly over the bundled copy under $(npm root -g)/npm/node_modules. +# The loop is guarded by a presence assert (OMN-15688): every package it +# overwrites must ALREADY exist at that exact path. Without the assert, a +# package npm does not bundle there gets a fresh unreferenced copy while any +# real vulnerable copy survives elsewhere — a green scan over a still- +# vulnerable image. The assert is what caught the dead cross-spawn entry +# above. Verified on npm 11.18.0 that the remaining five are flat top-level +# entries with no nested duplicates. # Exact versions keep the image reproducible (npm pack is not an # `npm install -g` global-CLI install, so it is outside the pin-check). # The Node CLI itself is retained — it backs runtime coding-agent delegation @@ -767,8 +806,14 @@ RUN set -eu; \ npm install -g npm@11.18.0; \ npm cache clean --force; \ NPM_MODULES="$(npm root -g)/npm/node_modules"; \ + for pkg in glob minimatch tar brace-expansion ip-address; do \ + test -d "${NPM_MODULES}/${pkg}" || { \ + echo "ERROR: no bundled ${pkg} at ${NPM_MODULES}/${pkg} — npm's bundle layout changed; extracting there would create an unreferenced copy and leave the vulnerable one in place" >&2; \ + exit 1; \ + }; \ + done; \ tmp="$(mktemp -d)"; \ - for spec in cross-spawn@7.0.6 glob@13.0.6 minimatch@10.2.5 tar@7.5.19 brace-expansion@5.0.8; do \ + for spec in glob@13.0.6 minimatch@10.2.5 tar@7.5.19 brace-expansion@5.0.9 ip-address@10.3.1; do \ pkg="${spec%@*}"; \ npm pack "$spec" --pack-destination "$tmp" >/dev/null; \ tb="$(ls "$tmp/${pkg}"-*.tgz)"; \ @@ -777,7 +822,7 @@ RUN set -eu; \ tar -xzf "$tb" -C "${NPM_MODULES}/${pkg}" --strip-components=1; \ done; \ rm -rf "$tmp"; \ - for pkg in cross-spawn glob minimatch tar brace-expansion; do \ + for pkg in glob minimatch tar brace-expansion ip-address; do \ v="$(node -p "require('${NPM_MODULES}/${pkg}/package.json').version")"; \ echo "npm bundled ${pkg} patched to ${v}"; \ done @@ -848,6 +893,43 @@ COPY --from=builder --chown=omniinfra:omniinfra \ /app/.venv/lib/python*/site-packages/omnibase_core/contracts/runtime_data/ \ /app/contracts/runtime/ +# OMN-15645: Copy the packaged omnimarket routing_tiers.yaml from the installed +# venv into a stable, non-version-embedded in-image path so +# DELEGATION_ROUTING_TIERS_PATH can be pinned in compose (docker-compose.infra.yml +# x-runtime-env) to a fixed literal -- mirroring BIFROST_CONTRACT_PATH (OMN-12864) +# -- instead of a literal python3.X site-packages path that silently breaks on a +# base-image Python version bump (the exact trap OMN-15628's runtime entrypoint +# self-heal, below, exists to correct for a *stale* pin). The `python*` glob +# above is the same technique already used for the omnibase_core runtime +# contracts copy, so this never hardcodes the interpreter minor version either. +# Destination is /app/config/, deliberately NOT /app/contracts/ -- the runtime +# services bind-mount ../contracts:/app/contracts:ro (host content), which +# would shadow anything baked into that path at image-build time. /app/config/ +# carries no volume/bind-mount entry anywhere in the compose lane files, so the +# image-baked content is never hidden or overridden at container start. +COPY --from=builder --chown=omniinfra:omniinfra \ + /app/.venv/lib/python*/site-packages/omnimarket/configs/routing_tiers.yaml \ + /app/config/delegation/routing_tiers.yaml + +# OMN-15676: Bake the runner-fleet config into the image. This file is tracked +# in the repo and every source-level check passed, but no COPY shipped it, so +# the deployed runtime raised +# FileNotFoundError: Runner fleet config not found: /app/config/runner_fleet.yaml +# HandlerRunnerFleetSnapshot.__init__ calls load_runner_fleet_config(), which +# raises rather than falling back to embedded lab values (deliberately -- a +# missing config is a deployment error), and the handler is instantiated during +# auto-wiring, so under ONEX_WIRING_STRICT_MODE=1 this is a boot failure. +# default_runner_fleet_config_path() resolves repo-root-relative from the module +# file; under this image's PYTHONPATH=/app/src that lands on +# /app/config/runner_fleet.yaml, which is exactly the destination below. +# Same /app/config/ rationale as the routing_tiers.yaml COPY above: no lane file +# and no onex-dev manifest mounts anything over /app/config/, so the baked +# content cannot be shadowed at container start. A bind-mount would NOT fix this +# -- the deployed pods have no such mount. +# scripts/ci/assert_image_config_paths.py asserts this path (and every other +# entry in the typed registry) inside the built image before the push step. +COPY --chown=omniinfra:omniinfra config/runner_fleet.yaml /app/config/runner_fleet.yaml + # Copy workspace/release provenance manifest generated by the builder stage. COPY --from=builder --chown=omniinfra:omniinfra /app/build-provenance.json /app/build-provenance.json @@ -855,6 +937,41 @@ COPY --from=builder --chown=omniinfra:omniinfra /app/build-provenance.json /app/ COPY --chown=omniinfra:omniinfra docker/entrypoint-runtime.sh /app/entrypoint-runtime.sh RUN chmod +x /app/entrypoint-runtime.sh +# OMN-16026: Keycloak realm reconciler, invoked as a batch Job -- NOT by the +# runtime kernel. omninode_infra's k8s/onex-dev/jobs/seed-keycloak-clients-job.yaml +# runs `python scripts/seed-keycloak-clients.py` with workingDir /app against +# this same digest-pinned image, which is why the file has to live here rather +# than only in the build context. +# +# Until this COPY existed the Job could not run at all: the `runtime` stage +# carried no `scripts/` directory, so every execution died on +# "python: can't open file '/app/scripts/seed-keycloak-clients.py'" (observed +# live on onex-dev 2026-08-13). The earlier attempt to close this, +# omnibase_infra#2661, was closed without merging, and the stale ordering note +# in that Job manifest kept pointing at it. +# +# Copied as a plain file rather than exposed via `python -m`, for the same +# reason as onex-container-healthcheck above: the module imports nothing but +# the standard library (argparse/json/os/subprocess/sys/urllib/pathlib), so +# running it directly avoids the multi-second omnibase_infra package import. +# tests/unit/scripts/test_seed_keycloak_clients_stdlib_only.py pins the +# stdlib-only property so this stays true. +COPY --chown=omniinfra:omniinfra \ + scripts/seed-keycloak-clients.py \ + /app/scripts/seed-keycloak-clients.py + +# OMN-15217: semantic container healthcheck, installed at a stable path. +# Same file as omnibase_infra/runtime/health/container_healthcheck.py — copied +# rather than invoked via `python -m` on purpose: importing the omnibase_infra +# package costs ~6.8s inside this image (measured in-container 2026-07-27), +# which against a 10s healthcheck timeout is a flap waiting to happen. The +# module imports nothing but the standard library, so running it as a plain +# file starts in ~0.12s. tests/unit/runtime/health/test_container_healthcheck.py +# pins the stdlib-only property so this stays true. +COPY --chown=omniinfra:omniinfra \ + src/omnibase_infra/runtime/health/container_healthcheck.py \ + /usr/local/bin/onex-container-healthcheck + # Create runtime directories with proper permissions. # Avoid a recursive chown over /app here: the venv and copied artifacts already # preserve ownership at COPY time, and the recursive walk can dominate Docker @@ -870,15 +987,28 @@ USER root # This should match the port your runtime listens on for health checks EXPOSE 8085 -# Health check using the runtime's health endpoint. The endpoint returns a -# failing status while runtime_attached=false or is_running=false, so plain -# curl --fail cannot mark a degraded runtime as Docker healthy. +# OMN-15217: semantic health check. +# +# The previous check was `curl --fail --silent .../health`, and the comment here +# claimed it "cannot mark a degraded runtime as Docker healthy". That was wrong, +# and the stability lane proved it: on 2026-07-27T12:58Z +# omninode-stability-test-runtime showed `Up 3 hours (healthy)` while logging +# `Runtime health check: status=DEGRADED contracts=296 errors=4` every five +# minutes. /health returns HTTP 200 for a *running* degraded runtime by design +# (see ServiceHealth._handle_health), so `curl --fail` — which asserts only +# "status code < 400" — is a liveness probe, not a health check. +# +# onex-container-healthcheck reads the runtime's own verdict out of the response +# body (details.runtime_health, published by ServiceRuntimeHealthMonitor) and +# exits non-zero when the runtime reports DEGRADED/CRITICAL. # - interval: Check every 30 seconds -# - timeout: Fail if check takes more than 10 seconds -# - start-period: Allow 120 seconds for startup (Kafka consumer init is slow) +# - timeout: 10 seconds (the check itself starts in ~0.12s; stdlib-only) +# - start-period: 120 seconds — the monitor's first verdict lands one check +# interval (default 300s) after boot, and an absent verdict passes, so a +# booting runtime never flaps on a verdict that has not been computed yet # - retries: Mark unhealthy after 3 consecutive failures HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \ - CMD curl --fail --silent http://localhost:8085/health || exit 1 + CMD python /usr/local/bin/onex-container-healthcheck || exit 1 # Use tini as init to properly handle signals (PID 1 problem) # Without tini, signals like SIGTERM aren't properly forwarded to the Python process, diff --git a/docker/README.md b/docker/README.md index 202b4b0d37..e246b3fc2d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -75,6 +75,8 @@ cp .env.example .env # - POSTGRES_PASSWORD: openssl rand -hex 32 # - INFISICAL_ENCRYPTION_KEY (secrets profile): openssl rand -hex 32 # - INFISICAL_AUTH_SECRET (secrets profile): openssl rand -hex 32 +# - DEV_REDPANDA_ADVERTISE_HOST: 'localhost' for single-host dev, or the +# reachable host/IP for off-host clients (OMN-15173: required, no default) # 3. Deploy via the canonical script (DO NOT use docker compose up directly) cd .. && ./scripts/deploy-runtime.sh --execute --restart @@ -191,8 +193,13 @@ docker compose -f docker-compose.infra.yml --profile runtime up -d --build # Scale workers manually docker compose -f docker-compose.infra.yml --profile runtime up -d --scale runtime-worker=4 -# Or set via environment variable -WORKER_REPLICAS=8 docker compose -f docker-compose.infra.yml --profile runtime up -d +# Or set via environment variable. NOTE (OMN-14968): the knob is the +# lane-prefixed DEV_WORKER_REPLICAS, and it is fail-closed -- the render aborts +# if it is unset, so source the ledgered policy env (which pins it to 1) +# rather than relying on a default. The bare WORKER_REPLICAS name no longer +# exists; it silently resolved to 0 and dropped the worker. +source runtime-policy.env # supplies DEV_WORKER_REPLICAS=1 +DEV_WORKER_REPLICAS=8 docker compose -f docker-compose.infra.yml --profile runtime up -d ``` #### Consul Profile (Service Discovery) @@ -514,6 +521,7 @@ These variables must be set explicitly. The runtime will fail to start if they a | `OMNIBASE_INFRA_DB_URL` | Full PostgreSQL DSN for omnibase_infra. Required for CLI/scripts. For Docker-only usage, `POSTGRES_PASSWORD` alone suffices (the fallback constructs the DSN automatically). Set this explicitly if your password contains special characters (`@`, `:`, `/`, `%`, `#`) since the Docker fallback cannot URL-encode. | Secret | (default) | | `INFISICAL_ENCRYPTION_KEY`| Hex-encoded AES key (32 or 64 hex chars) | Secret | secrets | | `INFISICAL_AUTH_SECRET` | JWT signing secret for authentication | Secret | secrets | +| `DEV_REDPANDA_ADVERTISE_HOST` | Dev lane external Redpanda advertise host (OMN-15173: no silent default — `docker compose config` fails fast when unset). Use `localhost` for pure single-host dev; use the reachable host/IP for off-host clients. | Non-secret | (default) | ### Optional Variables (With Defaults) @@ -521,7 +529,6 @@ These variables must be set explicitly. The runtime will fail to start if they a |------------------------------|------------------------------------|----------------------------------------| | **Kafka/Redpanda** | | | | `KAFKA_BOOTSTRAP_SERVERS` | `localhost:19092` | Kafka/Redpanda broker addresses (host); containers use `redpanda:9092` | -| `DEV_REDPANDA_ADVERTISE_HOST` | `localhost` | Dev lane external Redpanda advertise host | | Stability-test Redpanda advertise host | Contract overlay | Stability-test uses `x-omninode-contract-overlay` in `docker-compose.stability-test.yml`; do not configure it through environment variables | | `PROD_REDPANDA_ADVERTISE_HOST` | Required for prod overlay | Prod external Redpanda advertise host | | **PostgreSQL** | | | diff --git a/docker/application-acl-proof/Dockerfile b/docker/application-acl-proof/Dockerfile new file mode 100644 index 0000000000..9ea7a23252 --- /dev/null +++ b/docker/application-acl-proof/Dockerfile @@ -0,0 +1,34 @@ +# syntax=docker/dockerfile:1.7 +# SPDX-License-Identifier: MIT + +ARG UV_VERSION=0.11.8 + +FROM postgres:16-alpine AS postgres +COPY docker/application-acl-proof/seed.sql /docker-entrypoint-initdb.d/00-acl-seed.sql + +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv-bin + +FROM python:3.12-slim AS proof + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src \ + UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_HTTP_TIMEOUT=600 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential git libpq-dev postgresql-client \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=uv-bin /uv /uvx /usr/local/bin/ +WORKDIR /app +COPY pyproject.toml uv.lock README.md LICENSE ./ +RUN uv sync --frozen --no-dev --no-install-project + +COPY src/ ./src/ +COPY tests/fixtures/application_database_acl/ ./proof/fixtures/ +COPY scripts/ci/prove_application_database_acl.py ./proof/prove.py +COPY docker/application-acl-proof/generated/prechange-fixture-acl.json ./proof/prechange-fixture-acl.json + +CMD ["/app/.venv/bin/python", "/app/proof/prove.py"] diff --git a/docker/application-acl-proof/README.md b/docker/application-acl-proof/README.md new file mode 100644 index 0000000000..2855c23e97 --- /dev/null +++ b/docker/application-acl-proof/README.md @@ -0,0 +1,70 @@ +# Generated application ACL proof + +This rebuilt PostgreSQL 16 harness proves the generated one-database ACL and +default-privilege matrix without touching a deployed database. The seed is wholly +synthetic: it contains no dump-derived value, customer identifier, credential, secret, +or live catalog observation. + +Run it with rebuilt images: + +```bash +docker compose -f docker/application-acl-proof/compose.yml \ + up --build --abort-on-container-exit --exit-code-from proof +docker compose -f docker/application-acl-proof/compose.yml \ + down --volumes --remove-orphans +``` + +The same catalog comparator must detect seeded database/schema/object ownership, +`PUBLIC`, broad-object, per-column, grant-option, unsafe-default, PG16 membership, +undeclared-principal, and cross-domain defects before the matrix is +applied. A transient RED control also proves that an undeclared runtime-owned object +cannot hide outside the typed inventory. The harness covers tables, views, +materialized views, sequences, functions, procedures, +enum/domain/composite/range/multirange types, and the `public` schema. It then applies +the generated SQL twice, proves exact catalog +and behavioral access, restores the generated pre-change fixture snapshot including +grantors and membership options, proves a byte-for-byte semantic snapshot match, +reapplies twice, and proves the final state. Routine targets include their PostgreSQL +identity arguments so overloads cannot collapse or receive a name-wide grant. +The generated script has typed `scaffold` and `full` phases. A scaffold can become +`READY` while the full phase remains honestly `BLOCKED` on objects that have not yet +been materialized. Conversely, FULL can be `READY` while SCAFFOLD is `BLOCKED` when +the exact catalog census proves existing roles need attribute hardening; FULL checks +that observed pre-state and repairs it in its transaction, while the additive phase +refuses to conceal it. The scaffold creates absent-but-catalog-proven workload/owner +roles and the three empty target schemas, adds only grants for roles already proven +safe (or newly created safe), and establishes deny-by-default future-object ACLs +only for owner roles it creates. +It preserves existing role attributes, memberships, database ownership, legacy +`CONNECT`, and legacy `public` access; exact revocation and ownership hardening are +FULL-only after the activity/catalog gates pass. A disposable fresh-database lane +proves its wrong-database guard, atomic failure, idempotent apply, legacy +CONNECT/read/write preservation, zero object mutation, hostile collation/type +collision rejection, the later FULL denial, and exact non-cascading removal before +copy. Every script asserts +`current_database()` before its first mutation. + +Both phases run through the real `psql` path inside one explicit transaction; an +injected mid-apply error must leave the entire pre-change snapshot unchanged. The +rollback path is independently failure-injected and transactional. It restores +grantor chains topologically for database/schema/object/column ACLs and PG16 role +memberships, then removes only still-empty schemas and dependency-free roles—never +with `CASCADE`. Raw `pg_default_acl` row presence is part of the snapshot: rollback +normalizes PostgreSQL's implicit function/type `PUBLIC` defaults before replaying +captured deviations, and behavioral checks create future objects to prove those +built-ins return. An unrelated-schema default-ACL sentinel remains byte-for-byte and +behaviorally unchanged through scaffold, FULL, and rollback. The same run revokes +all database privileges before re-granting exact +`CONNECT` access and proves positive/negative connection isolation across the eight +approved non-system databases: `omnidash_analytics`, `omninode_cloud`, `keycloak`, +`omnibase_infra`, `omniintelligence`, `omniclaude`, `omnimemory`, and `umami`. + +`generated/prechange-fixture-acl.json` is explicitly a sanitized fixture rollback +artifact, not a live RDS ACL capture or a full-day `(datname, usename)` sample. The +locked real-source candidate remains `BLOCKED` and emits no production SQL until an +authorized exact principal census (including explicit cluster-global presence and +absence evidence), immutable source-locked catalog/activity query and result blobs, +a complete 24-hour-or-longer activity window, an +independent typed principal/domain policy, exact function signatures and object-kind +counts, materialized target-location evidence (source and target may coexist), and +the named catalog/activity/ownership blockers are resolved. diff --git a/docker/application-acl-proof/compose.yml b/docker/application-acl-proof/compose.yml new file mode 100644 index 0000000000..1eea0868bd --- /dev/null +++ b/docker/application-acl-proof/compose.yml @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: MIT +services: + postgres: + build: + context: ../.. + dockerfile: docker/application-acl-proof/Dockerfile + target: postgres + environment: + POSTGRES_DB: omnidash_analytics + POSTGRES_PASSWORD: acl-proof-admin-only # pragma: allowlist secret + tmpfs: + - /var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d omnidash_analytics"] + interval: 1s + timeout: 3s + retries: 30 + proof: + build: + context: ../.. + dockerfile: docker/application-acl-proof/Dockerfile + target: proof + depends_on: + postgres: + condition: service_healthy + environment: + ADMIN_DSN: postgresql://postgres:acl-proof-admin-only@postgres:5432/omnidash_analytics # pragma: allowlist secret + UPDATE_PRECHANGE: ${UPDATE_PRECHANGE:-false} + volumes: + - ./generated:/output diff --git a/docker/application-acl-proof/generated/candidate-matrix.yaml b/docker/application-acl-proof/generated/candidate-matrix.yaml new file mode 100644 index 0000000000..8b2d3bfebe --- /dev/null +++ b/docker/application-acl-proof/generated/candidate-matrix.yaml @@ -0,0 +1,6989 @@ +absent_connect_principals: {} +absent_owner_roles: [] +absent_principals: + application: [] +absent_schemas: + application: [] +allowed_connect_principals: {} +allowed_memberships: [] +authorization_scope: deployment +blockers: + - 'ACL policy violation: principal ''app_dashboard'' has cross-domain privileges on platform_catalog.None (PLATFORM_CATALOG)' + - 'ACL policy violation: principal ''app_dashboard'' has cross-domain privileges on tenant.None (TENANT)' + - 'ACL policy violation: principal ''omninode_runtime'' has cross-domain privileges on omninode_internal.None (OMNINODE_INTERNAL)' + - 'ACL policy violation: principal ''onex_api'' has cross-domain privileges on platform_catalog.None (PLATFORM_CATALOG)' + - 'ACL policy violation: principal ''onex_api'' has cross-domain privileges on tenant.None (TENANT)' + - 'ACL policy violation: principal ''tenant_projection_writer'' has cross-domain privileges on tenant.None (TENANT)' + - 'ACL policy violation: routine omninode_internal.refresh_baselines_quality_snapshots_projected_at lacks an exact signature' + - 'ACL policy violation: routine omninode_internal.refresh_baselines_roi_snapshots_projected_at lacks an exact signature' + - 'ACL policy violation: routine omninode_internal.refresh_capsule_store_updated_at lacks an exact signature' + - 'ACL policy violation: routine omninode_internal.refresh_context_roi_scores_updated_at lacks an exact signature' + - 'ACL policy violation: routine omninode_internal.refresh_intent_classification_events_updated_at lacks an exact signature' + - 'ACL policy violation: routine omninode_internal.refresh_session_outcomes_updated_at lacks an exact signature' + - 'ACL policy violation: routine omninode_internal.refresh_voice_sessions_updated_at lacks an exact signature' + - 'ACL policy violation: routine omninode_internal.update_cost_by_repo_snapshots_updated_at lacks an exact signature' + - 'ACL policy violation: routine omninode_internal.update_instruction_eval_aggregate_snapshots_updated_at lacks an exact signature' + - 'ACL policy violation: routine omninode_internal.update_llm_cost_aggregates_updated_at lacks an exact signature' + - 'ACL policy violation: routine omninode_internal.update_skill_execution_snapshots_updated_at lacks an exact signature' + - 'ACL policy violation: routine omninode_internal.update_traces_updated_at lacks an exact signature' + - 'ACL policy violation: routine tenant.refresh_delegation_budget_state_updated_at lacks an exact signature' + - 'ACL policy violation: routine tenant.refresh_savings_estimates_updated_at lacks an exact signature' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.agent_routing_decisions is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.baselines_breakdown is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.baselines_breakdown_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.baselines_comparisons is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.baselines_comparisons_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.baselines_quality_snapshots is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.baselines_roi_snapshots is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.baselines_snapshots is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.baselines_trend is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.baselines_trend_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.capability_scores is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.capability_scores_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.capsule_store is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.context_roi_scores is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.contract_registry is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.contract_registry_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.cost_by_repo_snapshots is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.dep_health_findings is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.dep_health_findings_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.deployment_evidence_projection is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.deployment_readiness_projection is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.event_chain is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.evidence_correlation_trace_projection is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.evidence_dashboard_projection is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.evidence_readiness_aggregate_projection is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.gate_activity is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.gate_activity_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.gate_metrics is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.generation_events is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.instruction_eval_aggregate_snapshots is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.intent_classification_events is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.intent_classification_events_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.live_events is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.llm_call_metrics is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.llm_cost_aggregates is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.llm_delegation_daily_projection is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.llm_routing_decisions is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.mcp_tools is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.merge_state_transitions is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.merge_state_transitions_projection_cursor_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.migrations_log is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.migrations_log_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.nightly_loop_decisions is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.nightly_loop_iterations is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.node_schema_migrations is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.node_service_registry is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.omnimarket_schema_migrations is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.omnimarket_schema_migrations_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.overnight_session_phases is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.overnight_session_phases_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.overnight_sessions is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.pattern_learning_artifacts is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.pr_lifecycle_ledger_entries is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.pr_lifecycle_ledger_entries_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.pr_merged_events is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.pr_merged_events_projection_cursor_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.projection_capsule_effectiveness is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.projection_overnight_readiness is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.projection_routing_decision is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.receipt_gate_rows is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.receipt_gate_rows_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.refresh_baselines_quality_snapshots_projected_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.refresh_baselines_roi_snapshots_projected_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.refresh_capsule_store_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.refresh_context_roi_scores_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.refresh_intent_classification_events_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.refresh_session_outcomes_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.refresh_voice_sessions_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.renderer_capability_projection is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.sandbox_decisions is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.schema_migrations is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.session_outcomes is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.session_replay_snapshots is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.skill_execution_snapshots is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.stripe_webhook_events is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.stripe_webhook_events_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.swarm_runs is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.traces is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.update_cost_by_repo_snapshots_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.update_instruction_eval_aggregate_snapshots_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.update_llm_cost_aggregates_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.update_skill_execution_snapshots_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.update_traces_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.omninode_internal.voice_sessions is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.platform_catalog.plan_entitlements is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.platform_catalog.plan_tiers is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.platform_catalog.usage_privileges is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.bootstrap_tokens is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.bootstrap_tokens_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.delegation_budget_state is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.delegation_events is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.gateway_rate_limit_events is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.gateway_workflows is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.projection_cost_savings_overview is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.projection_delegation_inference_response_text is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.projection_delegation_model_routing is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.projection_delegation_quality_gate is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.projection_delegation_savings is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.projection_delegation_savings_series is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.projection_delegation_summary is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.projection_delegation_token_usage is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.refresh_delegation_budget_state_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.refresh_savings_estimates_updated_at is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.savings_estimates is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.tenant_api_keys is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.tenant_billing is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.tenant_credentials is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.tenant_plans is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.tenants is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.usage_events is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.usage_events_id_seq is not materialized at the rendered location' + - 'ACL policy violation: target object omnidash_analytics.tenant.workflow_results is not materialized at the rendered location' + - 'ACL scaffold policy violation: allowed CONNECT database keys do not cover required CONNECT set' + - 'ACL scaffold policy violation: application: scaffold lacks exact target schema evidence' + - 'ACL scaffold policy violation: application: scaffold lacks presence/absence evidence for declared principals' + - 'ACL scaffold policy violation: keycloak: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: observed CONNECT database keys do not cover required CONNECT set' + - 'ACL scaffold policy violation: observed CONNECT owner database keys do not cover required CONNECT set' + - 'ACL scaffold policy violation: omnibase_infra: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: omniclaude: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: omnidash_analytics: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: omniintelligence: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: omnimemory: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: omninode_cloud: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: principal ''app_dashboard'' has cross-domain scaffold access' + - 'ACL scaffold policy violation: principal ''omninode_runtime'' has cross-domain scaffold access' + - 'ACL scaffold policy violation: principal ''onex_api'' has cross-domain scaffold access' + - 'ACL scaffold policy violation: principal ''tenant_projection_writer'' has cross-domain scaffold access' + - 'ACL scaffold policy violation: scaffold database owner map is incomplete' + - 'ACL scaffold policy violation: scaffold managed owners lack governed owner-role state' + - 'ACL scaffold policy violation: scaffold managed owners lack presence/absence evidence' + - 'ACL scaffold policy violation: scaffold workloads lack governed LOGIN role state' + - 'ACL scaffold policy violation: umami: CONNECT allowlist is empty' + - 'application: requires exactly one independent ACL policy, got []' + - 'application: requires exactly one principal inventory, got []' + - 'keycloak: requires exactly one CONNECT policy, got []' + - 'live catalog object identities differ from the exact typed ownership projection: missing=[(''function'', ''omninode_internal'', ''refresh_baselines_quality_snapshots_projected_at'', ''''), (''function'', ''omninode_internal'', ''refresh_baselines_roi_snapshots_projected_at'', ''''), (''function'', ''omninode_internal'', ''refresh_capsule_store_updated_at'', ''''), (''function'', ''omninode_internal'', ''refresh_context_roi_scores_updated_at'', ''''), (''function'', ''omninode_internal'', ''refresh_intent_classification_events_updated_at'', ''''), (''function'', ''omninode_internal'', ''refresh_session_outcomes_updated_at'', ''''), (''function'', ''omninode_internal'', ''refresh_voice_sessions_updated_at'', ''''), (''function'', ''omninode_internal'', ''update_cost_by_repo_snapshots_updated_at'', ''''), (''function'', ''omninode_internal'', ''update_instruction_eval_aggregate_snapshots_updated_at'', ''''), (''function'', ''omninode_internal'', ''update_llm_cost_aggregates_updated_at'', ''''), (''function'', ''omninode_internal'', ''update_skill_execution_snapshots_updated_at'', ''''), (''function'', ''omninode_internal'', ''update_traces_updated_at'', ''''), (''function'', ''tenant'', ''refresh_delegation_budget_state_updated_at'', ''''), (''function'', ''tenant'', ''refresh_savings_estimates_updated_at'', ''''), (''sequence'', ''omninode_internal'', ''baselines_breakdown_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''baselines_comparisons_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''baselines_trend_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''capability_scores_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''contract_registry_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''dep_health_findings_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''gate_activity_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''intent_classification_events_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''merge_state_transitions_projection_cursor_seq'', ''''), (''sequence'', ''omninode_internal'', ''migrations_log_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''omnimarket_schema_migrations_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''overnight_session_phases_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''pr_lifecycle_ledger_entries_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''pr_merged_events_projection_cursor_seq'', ''''), (''sequence'', ''omninode_internal'', ''receipt_gate_rows_id_seq'', ''''), (''sequence'', ''omninode_internal'', ''stripe_webhook_events_id_seq'', ''''), (''sequence'', ''tenant'', ''bootstrap_tokens_id_seq'', ''''), (''sequence'', ''tenant'', ''usage_events_id_seq'', ''''), (''table'', ''omninode_internal'', ''agent_routing_decisions'', ''''), (''table'', ''omninode_internal'', ''baselines_breakdown'', ''''), (''table'', ''omninode_internal'', ''baselines_comparisons'', ''''), (''table'', ''omninode_internal'', ''baselines_quality_snapshots'', ''''), (''table'', ''omninode_internal'', ''baselines_roi_snapshots'', ''''), (''table'', ''omninode_internal'', ''baselines_snapshots'', ''''), (''table'', ''omninode_internal'', ''baselines_trend'', ''''), (''table'', ''omninode_internal'', ''capability_scores'', ''''), (''table'', ''omninode_internal'', ''capsule_store'', ''''), (''table'', ''omninode_internal'', ''context_roi_scores'', ''''), (''table'', ''omninode_internal'', ''contract_registry'', ''''), (''table'', ''omninode_internal'', ''cost_by_repo_snapshots'', ''''), (''table'', ''omninode_internal'', ''dep_health_findings'', ''''), (''table'', ''omninode_internal'', ''deployment_evidence_projection'', ''''), (''table'', ''omninode_internal'', ''deployment_readiness_projection'', ''''), (''table'', ''omninode_internal'', ''event_chain'', ''''), (''table'', ''omninode_internal'', ''evidence_correlation_trace_projection'', ''''), (''table'', ''omninode_internal'', ''evidence_dashboard_projection'', ''''), (''table'', ''omninode_internal'', ''evidence_readiness_aggregate_projection'', ''''), (''table'', ''omninode_internal'', ''gate_activity'', ''''), (''table'', ''omninode_internal'', ''gate_metrics'', ''''), (''table'', ''omninode_internal'', ''generation_events'', ''''), (''table'', ''omninode_internal'', ''instruction_eval_aggregate_snapshots'', ''''), (''table'', ''omninode_internal'', ''intent_classification_events'', ''''), (''table'', ''omninode_internal'', ''live_events'', ''''), (''table'', ''omninode_internal'', ''llm_call_metrics'', ''''), (''table'', ''omninode_internal'', ''llm_cost_aggregates'', ''''), (''table'', ''omninode_internal'', ''llm_delegation_daily_projection'', ''''), (''table'', ''omninode_internal'', ''llm_routing_decisions'', ''''), (''table'', ''omninode_internal'', ''mcp_tools'', ''''), (''table'', ''omninode_internal'', ''merge_state_transitions'', ''''), (''table'', ''omninode_internal'', ''migrations_log'', ''''), (''table'', ''omninode_internal'', ''nightly_loop_decisions'', ''''), (''table'', ''omninode_internal'', ''nightly_loop_iterations'', ''''), (''table'', ''omninode_internal'', ''node_schema_migrations'', ''''), (''table'', ''omninode_internal'', ''node_service_registry'', ''''), (''table'', ''omninode_internal'', ''omnimarket_schema_migrations'', ''''), (''table'', ''omninode_internal'', ''overnight_session_phases'', ''''), (''table'', ''omninode_internal'', ''overnight_sessions'', ''''), (''table'', ''omninode_internal'', ''pattern_learning_artifacts'', ''''), (''table'', ''omninode_internal'', ''pr_lifecycle_ledger_entries'', ''''), (''table'', ''omninode_internal'', ''pr_merged_events'', ''''), (''table'', ''omninode_internal'', ''receipt_gate_rows'', ''''), (''table'', ''omninode_internal'', ''renderer_capability_projection'', ''''), (''table'', ''omninode_internal'', ''sandbox_decisions'', ''''), (''table'', ''omninode_internal'', ''schema_migrations'', ''''), (''table'', ''omninode_internal'', ''session_outcomes'', ''''), (''table'', ''omninode_internal'', ''session_replay_snapshots'', ''''), (''table'', ''omninode_internal'', ''skill_execution_snapshots'', ''''), (''table'', ''omninode_internal'', ''stripe_webhook_events'', ''''), (''table'', ''omninode_internal'', ''swarm_runs'', ''''), (''table'', ''omninode_internal'', ''traces'', ''''), (''table'', ''omninode_internal'', ''voice_sessions'', ''''), (''table'', ''platform_catalog'', ''plan_entitlements'', ''''), (''table'', ''platform_catalog'', ''plan_tiers'', ''''), (''table'', ''platform_catalog'', ''usage_privileges'', ''''), (''table'', ''tenant'', ''bootstrap_tokens'', ''''), (''table'', ''tenant'', ''delegation_budget_state'', ''''), (''table'', ''tenant'', ''delegation_events'', ''''), (''table'', ''tenant'', ''gateway_rate_limit_events'', ''''), (''table'', ''tenant'', ''gateway_workflows'', ''''), (''table'', ''tenant'', ''projection_delegation_inference_response_text'', ''''), (''table'', ''tenant'', ''savings_estimates'', ''''), (''table'', ''tenant'', ''tenant_api_keys'', ''''), (''table'', ''tenant'', ''tenant_billing'', ''''), (''table'', ''tenant'', ''tenant_credentials'', ''''), (''table'', ''tenant'', ''tenant_plans'', ''''), (''table'', ''tenant'', ''tenants'', ''''), (''table'', ''tenant'', ''usage_events'', ''''), (''table'', ''tenant'', ''workflow_results'', ''''), (''view'', ''omninode_internal'', ''projection_capsule_effectiveness'', ''''), (''view'', ''omninode_internal'', ''projection_overnight_readiness'', ''''), (''view'', ''omninode_internal'', ''projection_routing_decision'', ''''), (''view'', ''tenant'', ''projection_cost_savings_overview'', ''''), (''view'', ''tenant'', ''projection_delegation_model_routing'', ''''), (''view'', ''tenant'', ''projection_delegation_quality_gate'', ''''), (''view'', ''tenant'', ''projection_delegation_savings'', ''''), (''view'', ''tenant'', ''projection_delegation_savings_series'', ''''), (''view'', ''tenant'', ''projection_delegation_summary'', ''''), (''view'', ''tenant'', ''projection_delegation_token_usage'', '''')] extra=[]' + - matrix requires a typed principal_inventory source for every database + - matrix requires an independent typed acl_policy source for every database + - 'node_migration_service_ownership: blocked table ''schema_migrations'': Retained live evidence identifies an omnidash-owned ledger shaped (filename TEXT PRIMARY KEY, applied_at TIMESTAMPTZ), but no authoritative repository DDL or current owner declaration was found. It must not be conflated with node_schema_migrations or another service''s incompatible schema_migrations shape.' + - 'node_migration_service_ownership: completion_status is missing' + - 'node_migration_service_ownership: object ''node_schema_migrations'' current_schemas=() do not prove target schema ''omninode_internal''; full object ACL rendering is gated' + - 'node_migration_service_ownership: retained_live_census is missing' + - 'node_migration_service_ownership: runtime_evidence.full_day_datname_usename_activity=''blocked'': live database access was outside this build lane''s authorization' + - 'node_migration_service_ownership: runtime_evidence.live_catalog_parity=''blocked'': a fresh authorized catalog read is required for current parity' + - 'node_migration_service_ownership: target physical database ''omnidash_analytics'' is absent from materialized_physical_databases; full object ACL rendering is gated until the additive target is inventoried' + - 'omnibase_infra: requires exactly one CONNECT policy, got []' + - 'omniclaude: requires exactly one CONNECT policy, got []' + - 'omnidash_analytics: requires exactly one CONNECT policy, got []' + - 'omniintelligence: requires exactly one CONNECT policy, got []' + - 'omnimarket_relation_inventory: blocked table ''delegation_judge_verdict_events'': schema ''unresolved'' does not resolve through deployment topology' + - 'omnimarket_relation_inventory: blocked table ''delegation_shadow_comparisons'': no authoritative CREATE TABLE migration found' + - 'omnimarket_relation_inventory: blocked table ''delegation_workflow_state'': producer, consumers, and customer-ownership semantics are unresolved' + - 'omnimarket_relation_inventory: blocked table ''event_bus_events'': observed in the retained live census but no authoritative repository DDL was found' + - 'omnimarket_relation_inventory: blocked table ''schema_migrations'': retained live evidence identifies the omnidash filename ledger, but no authoritative repository DDL or owner declaration was found' + - 'omnimarket_relation_inventory: completion_status=''blocked_pending_live_catalog_and_activity_evidence''' + - 'omnimarket_relation_inventory: object ''agent_routing_decisions'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''baselines_breakdown'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''baselines_breakdown_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''baselines_comparisons'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''baselines_comparisons_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''baselines_quality_snapshots'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''baselines_roi_snapshots'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''baselines_snapshots'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''baselines_trend'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''baselines_trend_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''capability_scores'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''capability_scores_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''capsule_store'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''context_roi_scores'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''contract_registry'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''contract_registry_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''cost_by_repo_snapshots'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''delegation_budget_state'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''delegation_events'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''delegation_judge_verdict_events'' uses unknown schema ''unresolved''' + - 'omnimarket_relation_inventory: object ''delegation_shadow_comparisons'' blocked: no authoritative CREATE TABLE migration found' + - 'omnimarket_relation_inventory: object ''delegation_shadow_comparisons'' classification_status=''blocked''' + - 'omnimarket_relation_inventory: object ''delegation_shadow_comparisons'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''delegation_shadow_comparisons'' lacks an exact owner_declaration' + - 'omnimarket_relation_inventory: object ''dep_health_findings'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''dep_health_findings_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''deployment_evidence_projection'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''deployment_readiness_projection'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''event_chain'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''evidence_correlation_trace_projection'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''evidence_dashboard_projection'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''evidence_readiness_aggregate_projection'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''gate_activity'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''gate_activity_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''gate_metrics'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''generation_events'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''instruction_eval_aggregate_snapshots'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''intent_classification_events'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''intent_classification_events_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''live_events'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''llm_call_metrics'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''llm_cost_aggregates'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''llm_delegation_daily_projection'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''llm_routing_decisions'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''mcp_tools'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''merge_state_transitions'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''merge_state_transitions_projection_cursor_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''nightly_loop_decisions'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''nightly_loop_iterations'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''node_service_registry'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''omnimarket_schema_migrations'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''omnimarket_schema_migrations_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''overnight_session_phases'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''overnight_session_phases_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''overnight_sessions'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''pattern_learning_artifacts'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''pr_lifecycle_ledger_entries'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''pr_lifecycle_ledger_entries_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''pr_merged_events'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''pr_merged_events_projection_cursor_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''projection_capsule_effectiveness'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''projection_cost_savings_overview'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''projection_delegation_inference_response_text'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''projection_delegation_model_routing'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''projection_delegation_quality_gate'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''projection_delegation_savings'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''projection_delegation_savings_series'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''projection_delegation_summary'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''projection_delegation_token_usage'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''projection_overnight_readiness'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''projection_routing_decision'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''receipt_gate_rows'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''receipt_gate_rows_id_seq'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''refresh_baselines_quality_snapshots_projected_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''refresh_baselines_roi_snapshots_projected_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''refresh_capsule_store_updated_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''refresh_context_roi_scores_updated_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''refresh_delegation_budget_state_updated_at'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''refresh_intent_classification_events_updated_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''refresh_savings_estimates_updated_at'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''refresh_session_outcomes_updated_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''refresh_voice_sessions_updated_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''renderer_capability_projection'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''sandbox_decisions'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''savings_estimates'' is currently in schemas [''public''], not the target schema ''tenant''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''session_outcomes'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''session_replay_snapshots'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''skill_execution_snapshots'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''swarm_runs'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''traces'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''update_cost_by_repo_snapshots_updated_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''update_instruction_eval_aggregate_snapshots_updated_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''update_llm_cost_aggregates_updated_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''update_skill_execution_snapshots_updated_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''update_traces_updated_at'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: object ''voice_sessions'' is currently in schemas [''public''], not the target schema ''omninode_internal''; full object ACL rendering is gated until the additive target object is materialized and inventoried' + - 'omnimarket_relation_inventory: relation_counts.procedure is not inventoried; zero procedures cannot be inferred' + - 'omnimarket_relation_inventory: relation_counts.type is not inventoried; zero user-defined types cannot be inferred' + - 'omnimarket_relation_inventory: retained_live_census.observed_base_tables=86 does not match exact typed rows=55' + - 'omnimarket_relation_inventory: retained_live_census.observed_extensions is not inventoried' + - 'omnimarket_relation_inventory: retained_live_census.observed_functions is not inventoried' + - 'omnimarket_relation_inventory: retained_live_census.observed_procedures is not inventoried' + - 'omnimarket_relation_inventory: retained_live_census.observed_sequences is not inventoried' + - 'omnimarket_relation_inventory: retained_live_census.observed_types is not inventoried' + - 'omnimarket_relation_inventory: retained_live_census.observed_views_and_materialized_views=9 does not match exact typed rows=10' + - 'omnimarket_relation_inventory: retained_live_census=''blocked'': the retained census contains counts but not a complete object-name export; repository migrations therefore cannot prove name-for-name live parity without a fresh authorized catalog read' + - 'omnimarket_relation_inventory: runtime_evidence.full_day_datname_usename_activity=''blocked'': live database access was outside this build lane''s authorization' + - 'omnimarket_relation_inventory: runtime_evidence.live_catalog_parity=''blocked'': repository sources contain 54 CREATE TABLE objects while the retained live census reports 86 base tables; exact overlap and the at-least-32-table gap require an authorized catalog read' + - 'omnimemory: requires exactly one CONNECT policy, got []' + - 'omninode_cloud: requires exactly one CONNECT policy, got []' + - 'omninode_internal.refresh_baselines_quality_snapshots_projected_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'omninode_internal.refresh_baselines_roi_snapshots_projected_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'omninode_internal.refresh_capsule_store_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'omninode_internal.refresh_context_roi_scores_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'omninode_internal.refresh_intent_classification_events_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'omninode_internal.refresh_session_outcomes_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'omninode_internal.refresh_voice_sessions_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'omninode_internal.update_cost_by_repo_snapshots_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'omninode_internal.update_instruction_eval_aggregate_snapshots_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'omninode_internal.update_llm_cost_aggregates_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'omninode_internal.update_skill_execution_snapshots_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'omninode_internal.update_traces_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'onex_api_service_ownership: completion_status=''blocked_pending_live_catalog_and_activity_evidence''' + - 'onex_api_service_ownership: object ''bootstrap_tokens'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''bootstrap_tokens_id_seq'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''gateway_rate_limit_events'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''gateway_workflows'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''migrations_log'' current_schemas=() do not prove target schema ''omninode_internal''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''migrations_log_id_seq'' current_schemas=() do not prove target schema ''omninode_internal''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''plan_entitlements'' current_schemas=() do not prove target schema ''platform_catalog''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''plan_tiers'' current_schemas=() do not prove target schema ''platform_catalog''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''schema_migrations'' current_schemas=() do not prove target schema ''omninode_internal''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''stripe_webhook_events'' current_schemas=() do not prove target schema ''omninode_internal''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''stripe_webhook_events_id_seq'' current_schemas=() do not prove target schema ''omninode_internal''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''tenant_api_keys'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''tenant_billing'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''tenant_credentials'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''tenant_plans'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''tenants'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''usage_events'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''usage_events_id_seq'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''usage_privileges'' current_schemas=() do not prove target schema ''platform_catalog''; full object ACL rendering is gated' + - 'onex_api_service_ownership: object ''workflow_results'' current_schemas=() do not prove target schema ''tenant''; full object ACL rendering is gated' + - 'onex_api_service_ownership: retained_live_census.observed_extensions is missing' + - 'onex_api_service_ownership: retained_live_census.observed_functions is missing' + - 'onex_api_service_ownership: retained_live_census.observed_procedures is missing' + - 'onex_api_service_ownership: retained_live_census.observed_sequences is missing' + - 'onex_api_service_ownership: retained_live_census.observed_types is missing' + - 'onex_api_service_ownership: retained_live_census=''blocked'': Source names match the retained 16-table list, but a fresh authorized catalog read is required to prove that the retained census is still current.' + - 'onex_api_service_ownership: runtime_evidence.full_day_datname_usename_activity=''blocked'': live database access was outside this build lane''s authorization' + - 'onex_api_service_ownership: runtime_evidence.live_catalog_parity=''blocked'': source inventory cannot prove the retained live 16-table census is current' + - 'onex_api_service_ownership: target physical database ''omnidash_analytics'' is absent from materialized_physical_databases; full object ACL rendering is gated until the additive target is inventoried' + - 'tenant.refresh_delegation_budget_state_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - 'tenant.refresh_savings_estimates_updated_at: ownership and ACL rendering require an explicit function_signature to avoid overload widening' + - topology declares zero explicit object grants for 112 typed ownership objects + - 'umami: requires exactly one CONNECT policy, got []' +database_owners: {} +declared_principals: + application: + - app_dashboard + - omninode_runtime + - onex_api + - tenant_projection_writer +default_privileges: + - database_ref: application + grantee: PUBLIC + object_type: FUNCTION + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: app_dashboard + object_type: FUNCTION + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: omninode_runtime + object_type: FUNCTION + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: onex_api + object_type: FUNCTION + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: tenant_projection_writer + object_type: FUNCTION + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: PUBLIC + object_type: SEQUENCE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: app_dashboard + object_type: SEQUENCE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: omninode_runtime + object_type: SEQUENCE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: onex_api + object_type: SEQUENCE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: tenant_projection_writer + object_type: SEQUENCE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: PUBLIC + object_type: TABLE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: app_dashboard + object_type: TABLE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: omninode_runtime + object_type: TABLE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: onex_api + object_type: TABLE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: tenant_projection_writer + object_type: TABLE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: PUBLIC + object_type: TYPE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: app_dashboard + object_type: TYPE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: omninode_runtime + object_type: TYPE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: onex_api + object_type: TYPE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: tenant_projection_writer + object_type: TYPE + owner: owner_omninode_internal + physical_database: omnidash_analytics + privileges: [] + schema_ref: omninode_internal + - database_ref: application + grantee: PUBLIC + object_type: FUNCTION + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: app_dashboard + object_type: FUNCTION + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: omninode_runtime + object_type: FUNCTION + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: onex_api + object_type: FUNCTION + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: tenant_projection_writer + object_type: FUNCTION + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: PUBLIC + object_type: SEQUENCE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: app_dashboard + object_type: SEQUENCE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: omninode_runtime + object_type: SEQUENCE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: onex_api + object_type: SEQUENCE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: tenant_projection_writer + object_type: SEQUENCE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: PUBLIC + object_type: TABLE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: app_dashboard + object_type: TABLE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: omninode_runtime + object_type: TABLE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: onex_api + object_type: TABLE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: tenant_projection_writer + object_type: TABLE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: PUBLIC + object_type: TYPE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: app_dashboard + object_type: TYPE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: omninode_runtime + object_type: TYPE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: onex_api + object_type: TYPE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: tenant_projection_writer + object_type: TYPE + owner: owner_platform_catalog + physical_database: omnidash_analytics + privileges: [] + schema_ref: platform_catalog + - database_ref: application + grantee: PUBLIC + object_type: FUNCTION + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: app_dashboard + object_type: FUNCTION + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: omninode_runtime + object_type: FUNCTION + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: onex_api + object_type: FUNCTION + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: tenant_projection_writer + object_type: FUNCTION + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: PUBLIC + object_type: SEQUENCE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: app_dashboard + object_type: SEQUENCE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: omninode_runtime + object_type: SEQUENCE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: onex_api + object_type: SEQUENCE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: tenant_projection_writer + object_type: SEQUENCE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: PUBLIC + object_type: TABLE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: app_dashboard + object_type: TABLE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: omninode_runtime + object_type: TABLE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: onex_api + object_type: TABLE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: tenant_projection_writer + object_type: TABLE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: PUBLIC + object_type: TYPE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: app_dashboard + object_type: TYPE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: omninode_runtime + object_type: TYPE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: onex_api + object_type: TYPE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant + - database_ref: application + grantee: tenant_projection_writer + object_type: TYPE + owner: owner_onex_tenant + physical_database: omnidash_analytics + privileges: [] + schema_ref: tenant +excluded_objects: + - omnimarket_relation_inventory:platform_catalog.pgcrypto:extension + - onex_api_service_ownership:pgcrypto:extension +governed_role_states: [] +objects: + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: refresh_baselines_quality_snapshots_projected_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_baselines_quality + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: refresh_baselines_roi_snapshots_projected_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_baselines_roi + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: refresh_capsule_store_updated_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_capsule_store + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: refresh_context_roi_scores_updated_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_context_roi + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: refresh_intent_classification_events_updated_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_intent_classification + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: refresh_session_outcomes_updated_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_session_outcome + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: refresh_voice_sessions_updated_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_voice_sessions + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: update_cost_by_repo_snapshots_updated_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_cost_by_repo + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: update_instruction_eval_aggregate_snapshots_updated_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_instruction_eval + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: update_llm_cost_aggregates_updated_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_cost_summary + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: update_skill_execution_snapshots_updated_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_skill_executions + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: update_traces_updated_at + object_type: FUNCTION + owner: owner_omninode_internal + owner_declaration: node_projection_traces + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: baselines_breakdown_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_projection_baselines + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: baselines_comparisons_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_projection_baselines + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: baselines_trend_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_projection_baselines + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: capability_scores_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_canary_score_reducer + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: contract_registry_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_contract_registry + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: dep_health_findings_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_projection_dep_health + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: gate_activity_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_omnigate_projection + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: intent_classification_events_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_projection_intent_classification + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: merge_state_transitions_projection_cursor_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_merge_state_projection + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: migrations_log_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: omnimarket_schema_migrations_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: service:omnimarket_projection_migration_runner + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: overnight_session_phases_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_projection_overnight + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: pr_lifecycle_ledger_entries_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_pr_lifecycle_state_reducer + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: pr_merged_events_projection_cursor_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_pr_merged_projection + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: receipt_gate_rows_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: node_projection_receipt_gate + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: stripe_webhook_events_id_seq + object_type: SEQUENCE + owner: owner_omninode_internal + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: agent_routing_decisions + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_routing_decision + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: baselines_breakdown + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_baselines + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: baselines_comparisons + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_baselines + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: baselines_quality_snapshots + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_baselines_quality + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: baselines_roi_snapshots + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_baselines_roi + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: baselines_snapshots + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_baselines + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: baselines_trend + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_baselines + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: capability_scores + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_canary_score_reducer + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: capsule_store + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_capsule_store + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: context_roi_scores + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_context_roi + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: contract_registry + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_contract_registry + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: cost_by_repo_snapshots + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_cost_by_repo + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: dep_health_findings + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_dep_health + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: deployment_evidence_projection + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_deployment_evidence_reducer + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: deployment_readiness_projection + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_deployment_evidence_reducer + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: event_chain + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_event_chain + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: evidence_correlation_trace_projection + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_evidence_dashboard_reducer + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: evidence_dashboard_projection + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_evidence_dashboard_reducer + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: evidence_readiness_aggregate_projection + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_evidence_dashboard_reducer + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: gate_activity + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_omnigate_projection + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: gate_metrics + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_omnigate_projection + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: generation_events + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_delegation + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: instruction_eval_aggregate_snapshots + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_instruction_eval + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: intent_classification_events + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_intent_classification + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: live_events + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_live_events + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: llm_call_metrics + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_llm_cost + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: llm_cost_aggregates + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_cost_summary + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: llm_delegation_daily_projection + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_llm_delegation_projection + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: llm_routing_decisions + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_llm_routing + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: mcp_tools + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_mcp_tools + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: merge_state_transitions + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_merge_state_projection + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: migrations_log + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: nightly_loop_decisions + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_nightly_loop_controller + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: nightly_loop_iterations + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_nightly_loop_controller + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: node_schema_migrations + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: service:omnibase_infra_node_migration_runner + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - node_migration_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: node_service_registry + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_registration + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: omnimarket_schema_migrations + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: service:omnimarket_projection_migration_runner + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: overnight_session_phases + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_overnight + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: overnight_sessions + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_overnight + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: pattern_learning_artifacts + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_pattern_learning + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: pr_lifecycle_ledger_entries + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_pr_lifecycle_state_reducer + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: pr_merged_events + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_pr_merged_projection + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: view + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: projection_capsule_effectiveness + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_capsule_store + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: view + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: projection_overnight_readiness + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_overnight + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: view + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: projection_routing_decision + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_llm_routing + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: receipt_gate_rows + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_receipt_gate + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: renderer_capability_projection + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_renderer_capability_projection + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: sandbox_decisions + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_sandbox_decisions + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: schema_migrations + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: session_outcomes + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_session_outcome + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: session_replay_snapshots + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_session_replay + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: skill_execution_snapshots + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_skill_executions + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: stripe_webhook_events + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: swarm_runs + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_swarm + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: traces + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_traces + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: OMNINODE_INTERNAL + function_signature: null + object_ref: voice_sessions + object_type: TABLE + owner: owner_omninode_internal + owner_declaration: node_projection_voice_sessions + physical_database: omnidash_analytics + schema_ref: omninode_internal + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: PLATFORM_CATALOG + function_signature: null + object_ref: plan_entitlements + object_type: TABLE + owner: owner_platform_catalog + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: platform_catalog + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: PLATFORM_CATALOG + function_signature: null + object_ref: plan_tiers + object_type: TABLE + owner: owner_platform_catalog + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: platform_catalog + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: PLATFORM_CATALOG + function_signature: null + object_ref: usage_privileges + object_type: TABLE + owner: owner_platform_catalog + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: platform_catalog + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: function + database_ref: application + domain: TENANT + function_signature: null + object_ref: refresh_delegation_budget_state_updated_at + object_type: FUNCTION + owner: owner_onex_tenant + owner_declaration: node_projection_delegation + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: function + database_ref: application + domain: TENANT + function_signature: null + object_ref: refresh_savings_estimates_updated_at + object_type: FUNCTION + owner: owner_onex_tenant + owner_declaration: node_projection_savings + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: TENANT + function_signature: null + object_ref: bootstrap_tokens_id_seq + object_type: SEQUENCE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: sequence + database_ref: application + domain: TENANT + function_signature: null + object_ref: usage_events_id_seq + object_type: SEQUENCE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: bootstrap_tokens + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: delegation_budget_state + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: node_projection_delegation + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: delegation_events + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: node_projection_delegation + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: gateway_rate_limit_events + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: gateway_workflows + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: view + database_ref: application + domain: TENANT + function_signature: null + object_ref: projection_cost_savings_overview + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: node_projection_savings + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: projection_delegation_inference_response_text + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: node_projection_delegation_inference_response + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: view + database_ref: application + domain: TENANT + function_signature: null + object_ref: projection_delegation_model_routing + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: node_projection_delegation + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: view + database_ref: application + domain: TENANT + function_signature: null + object_ref: projection_delegation_quality_gate + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: node_projection_delegation + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: view + database_ref: application + domain: TENANT + function_signature: null + object_ref: projection_delegation_savings + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: node_projection_savings + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: view + database_ref: application + domain: TENANT + function_signature: null + object_ref: projection_delegation_savings_series + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: node_projection_savings + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: view + database_ref: application + domain: TENANT + function_signature: null + object_ref: projection_delegation_summary + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: node_projection_delegation + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: view + database_ref: application + domain: TENANT + function_signature: null + object_ref: projection_delegation_token_usage + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: node_projection_delegation + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: savings_estimates + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: node_projection_savings + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - omnimarket_relation_inventory + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: tenant_api_keys + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: tenant_billing + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: tenant_credentials + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: tenant_plans + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: tenants + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: usage_events + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false + - catalog_kind: table + database_ref: application + domain: TENANT + function_signature: null + object_ref: workflow_results + object_type: TABLE + owner: owner_onex_tenant + owner_declaration: service:onex_api + physical_database: omnidash_analytics + schema_ref: tenant + source_keys: + - onex_api_service_ownership + target_materialized: false +observed_connect_database_owners: {} +observed_connect_principals: {} +observed_objects: [] +observed_owner_roles: [] +observed_principals: + application: [] +observed_role_states: [] +observed_schema_owners: + application: {} +principal_domains: {} +required_connect_databases: + - keycloak + - omnibase_infra + - omniclaude + - omnidash_analytics + - omniintelligence + - omnimemory + - omninode_cloud + - umami +retained_administrative_principals: [] +rows: + - database_ref: application + function_signature: null + object_ref: null + object_type: DATABASE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: null + - database_ref: application + function_signature: null + object_ref: refresh_baselines_quality_snapshots_projected_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_baselines_roi_snapshots_projected_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_capsule_store_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_context_roi_scores_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_intent_classification_events_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_session_outcomes_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_voice_sessions_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_cost_by_repo_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_instruction_eval_aggregate_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_llm_cost_aggregates_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_skill_execution_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_traces_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_delegation_budget_state_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: refresh_savings_estimates_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: baselines_breakdown_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_comparisons_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_trend_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capability_scores_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: contract_registry_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: dep_health_findings_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_activity_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: intent_classification_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: merge_state_transitions_projection_cursor_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: migrations_log_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: omnimarket_schema_migrations_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_session_phases_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_lifecycle_ledger_entries_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_merged_events_projection_cursor_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: receipt_gate_rows_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: stripe_webhook_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: bootstrap_tokens_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: usage_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: agent_routing_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_breakdown + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_comparisons + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_quality_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_roi_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_trend + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capability_scores + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capsule_store + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: context_roi_scores + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: contract_registry + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: cost_by_repo_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: dep_health_findings + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: deployment_evidence_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: deployment_readiness_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: event_chain + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_correlation_trace_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_dashboard_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_readiness_aggregate_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_activity + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_metrics + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: generation_events + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: instruction_eval_aggregate_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: intent_classification_events + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: live_events + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_call_metrics + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_cost_aggregates + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_delegation_daily_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_routing_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: mcp_tools + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: merge_state_transitions + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: migrations_log + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: nightly_loop_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: nightly_loop_iterations + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: node_schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: node_service_registry + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: omnimarket_schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_session_phases + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_sessions + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pattern_learning_artifacts + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_lifecycle_ledger_entries + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_merged_events + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_capsule_effectiveness + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_overnight_readiness + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_routing_decision + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: receipt_gate_rows + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: renderer_capability_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: sandbox_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: session_outcomes + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: session_replay_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: skill_execution_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: stripe_webhook_events + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: swarm_runs + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: traces + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: voice_sessions + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: plan_entitlements + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: plan_tiers + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: usage_privileges + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: bootstrap_tokens + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: delegation_budget_state + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: delegation_events + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: gateway_rate_limit_events + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: gateway_workflows + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_cost_savings_overview + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_inference_response_text + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_model_routing + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_quality_gate + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_savings + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_savings_series + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_summary + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_token_usage + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: savings_estimates + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_api_keys + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_billing + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_credentials + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_plans + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenants + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: usage_events + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: workflow_results + object_type: TABLE + physical_database: omnidash_analytics + principal: PUBLIC + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: null + object_type: DATABASE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: + - CONNECT + schema_ref: null + - database_ref: application + function_signature: null + object_ref: refresh_baselines_quality_snapshots_projected_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_baselines_roi_snapshots_projected_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_capsule_store_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_context_roi_scores_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_intent_classification_events_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_session_outcomes_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_voice_sessions_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_cost_by_repo_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_instruction_eval_aggregate_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_llm_cost_aggregates_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_skill_execution_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_traces_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_delegation_budget_state_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: refresh_savings_estimates_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: app_dashboard + privileges: + - USAGE + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: app_dashboard + privileges: + - USAGE + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: baselines_breakdown_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_comparisons_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_trend_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capability_scores_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: contract_registry_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: dep_health_findings_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_activity_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: intent_classification_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: merge_state_transitions_projection_cursor_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: migrations_log_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: omnimarket_schema_migrations_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_session_phases_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_lifecycle_ledger_entries_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_merged_events_projection_cursor_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: receipt_gate_rows_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: stripe_webhook_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: bootstrap_tokens_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: usage_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: agent_routing_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_breakdown + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_comparisons + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_quality_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_roi_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_trend + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capability_scores + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capsule_store + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: context_roi_scores + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: contract_registry + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: cost_by_repo_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: dep_health_findings + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: deployment_evidence_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: deployment_readiness_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: event_chain + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_correlation_trace_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_dashboard_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_readiness_aggregate_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_activity + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_metrics + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: generation_events + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: instruction_eval_aggregate_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: intent_classification_events + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: live_events + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_call_metrics + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_cost_aggregates + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_delegation_daily_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_routing_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: mcp_tools + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: merge_state_transitions + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: migrations_log + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: nightly_loop_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: nightly_loop_iterations + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: node_schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: node_service_registry + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: omnimarket_schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_session_phases + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_sessions + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pattern_learning_artifacts + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_lifecycle_ledger_entries + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_merged_events + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_capsule_effectiveness + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_overnight_readiness + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_routing_decision + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: receipt_gate_rows + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: renderer_capability_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: sandbox_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: session_outcomes + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: session_replay_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: skill_execution_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: stripe_webhook_events + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: swarm_runs + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: traces + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: voice_sessions + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: plan_entitlements + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: plan_tiers + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: usage_privileges + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: bootstrap_tokens + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: delegation_budget_state + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: delegation_events + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: gateway_rate_limit_events + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: gateway_workflows + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_cost_savings_overview + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_inference_response_text + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_model_routing + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_quality_gate + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_savings + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_savings_series + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_summary + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_token_usage + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: savings_estimates + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_api_keys + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_billing + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_credentials + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_plans + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenants + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: usage_events + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: workflow_results + object_type: TABLE + physical_database: omnidash_analytics + principal: app_dashboard + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: null + object_type: DATABASE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: + - CONNECT + schema_ref: null + - database_ref: application + function_signature: null + object_ref: refresh_baselines_quality_snapshots_projected_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_baselines_roi_snapshots_projected_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_capsule_store_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_context_roi_scores_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_intent_classification_events_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_session_outcomes_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_voice_sessions_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_cost_by_repo_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_instruction_eval_aggregate_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_llm_cost_aggregates_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_skill_execution_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_traces_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_delegation_budget_state_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: refresh_savings_estimates_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: + - USAGE + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: baselines_breakdown_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_comparisons_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_trend_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capability_scores_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: contract_registry_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: dep_health_findings_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_activity_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: intent_classification_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: merge_state_transitions_projection_cursor_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: migrations_log_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: omnimarket_schema_migrations_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_session_phases_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_lifecycle_ledger_entries_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_merged_events_projection_cursor_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: receipt_gate_rows_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: stripe_webhook_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: bootstrap_tokens_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: usage_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: agent_routing_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_breakdown + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_comparisons + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_quality_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_roi_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_trend + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capability_scores + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capsule_store + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: context_roi_scores + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: contract_registry + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: cost_by_repo_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: dep_health_findings + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: deployment_evidence_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: deployment_readiness_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: event_chain + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_correlation_trace_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_dashboard_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_readiness_aggregate_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_activity + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_metrics + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: generation_events + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: instruction_eval_aggregate_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: intent_classification_events + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: live_events + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_call_metrics + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_cost_aggregates + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_delegation_daily_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_routing_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: mcp_tools + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: merge_state_transitions + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: migrations_log + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: nightly_loop_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: nightly_loop_iterations + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: node_schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: node_service_registry + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: omnimarket_schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_session_phases + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_sessions + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pattern_learning_artifacts + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_lifecycle_ledger_entries + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_merged_events + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_capsule_effectiveness + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_overnight_readiness + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_routing_decision + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: receipt_gate_rows + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: renderer_capability_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: sandbox_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: session_outcomes + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: session_replay_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: skill_execution_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: stripe_webhook_events + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: swarm_runs + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: traces + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: voice_sessions + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: plan_entitlements + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: plan_tiers + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: usage_privileges + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: bootstrap_tokens + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: delegation_budget_state + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: delegation_events + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: gateway_rate_limit_events + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: gateway_workflows + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_cost_savings_overview + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_inference_response_text + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_model_routing + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_quality_gate + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_savings + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_savings_series + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_summary + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_token_usage + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: savings_estimates + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_api_keys + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_billing + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_credentials + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_plans + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenants + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: usage_events + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: workflow_results + object_type: TABLE + physical_database: omnidash_analytics + principal: omninode_runtime + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: null + object_type: DATABASE + physical_database: omnidash_analytics + principal: onex_api + privileges: + - CONNECT + schema_ref: null + - database_ref: application + function_signature: null + object_ref: refresh_baselines_quality_snapshots_projected_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_baselines_roi_snapshots_projected_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_capsule_store_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_context_roi_scores_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_intent_classification_events_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_session_outcomes_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_voice_sessions_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_cost_by_repo_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_instruction_eval_aggregate_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_llm_cost_aggregates_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_skill_execution_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_traces_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_delegation_budget_state_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: refresh_savings_estimates_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: onex_api + privileges: + - USAGE + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: onex_api + privileges: + - USAGE + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: baselines_breakdown_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_comparisons_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_trend_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capability_scores_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: contract_registry_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: dep_health_findings_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_activity_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: intent_classification_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: merge_state_transitions_projection_cursor_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: migrations_log_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: omnimarket_schema_migrations_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_session_phases_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_lifecycle_ledger_entries_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_merged_events_projection_cursor_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: receipt_gate_rows_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: stripe_webhook_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: bootstrap_tokens_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: usage_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: agent_routing_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_breakdown + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_comparisons + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_quality_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_roi_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_trend + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capability_scores + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capsule_store + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: context_roi_scores + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: contract_registry + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: cost_by_repo_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: dep_health_findings + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: deployment_evidence_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: deployment_readiness_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: event_chain + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_correlation_trace_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_dashboard_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_readiness_aggregate_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_activity + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_metrics + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: generation_events + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: instruction_eval_aggregate_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: intent_classification_events + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: live_events + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_call_metrics + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_cost_aggregates + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_delegation_daily_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_routing_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: mcp_tools + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: merge_state_transitions + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: migrations_log + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: nightly_loop_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: nightly_loop_iterations + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: node_schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: node_service_registry + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: omnimarket_schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_session_phases + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_sessions + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pattern_learning_artifacts + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_lifecycle_ledger_entries + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_merged_events + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_capsule_effectiveness + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_overnight_readiness + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_routing_decision + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: receipt_gate_rows + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: renderer_capability_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: sandbox_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: session_outcomes + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: session_replay_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: skill_execution_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: stripe_webhook_events + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: swarm_runs + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: traces + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: voice_sessions + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: plan_entitlements + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: plan_tiers + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: usage_privileges + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: bootstrap_tokens + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: delegation_budget_state + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: delegation_events + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: gateway_rate_limit_events + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: gateway_workflows + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_cost_savings_overview + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_inference_response_text + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_model_routing + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_quality_gate + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_savings + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_savings_series + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_summary + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_token_usage + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: savings_estimates + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_api_keys + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_billing + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_credentials + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_plans + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenants + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: usage_events + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: workflow_results + object_type: TABLE + physical_database: omnidash_analytics + principal: onex_api + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: null + object_type: DATABASE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: + - CONNECT + schema_ref: null + - database_ref: application + function_signature: null + object_ref: refresh_baselines_quality_snapshots_projected_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_baselines_roi_snapshots_projected_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_capsule_store_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_context_roi_scores_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_intent_classification_events_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_session_outcomes_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_voice_sessions_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_cost_by_repo_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_instruction_eval_aggregate_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_llm_cost_aggregates_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_skill_execution_snapshots_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: update_traces_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: refresh_delegation_budget_state_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: refresh_savings_estimates_updated_at + object_type: FUNCTION + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: null + object_type: SCHEMA + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: + - USAGE + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: baselines_breakdown_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_comparisons_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_trend_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capability_scores_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: contract_registry_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: dep_health_findings_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_activity_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: intent_classification_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: merge_state_transitions_projection_cursor_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: migrations_log_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: omnimarket_schema_migrations_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_session_phases_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_lifecycle_ledger_entries_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_merged_events_projection_cursor_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: receipt_gate_rows_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: stripe_webhook_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: bootstrap_tokens_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: usage_events_id_seq + object_type: SEQUENCE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: agent_routing_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_breakdown + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_comparisons + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_quality_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_roi_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: baselines_trend + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capability_scores + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: capsule_store + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: context_roi_scores + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: contract_registry + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: cost_by_repo_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: dep_health_findings + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: deployment_evidence_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: deployment_readiness_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: event_chain + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_correlation_trace_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_dashboard_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: evidence_readiness_aggregate_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_activity + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: gate_metrics + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: generation_events + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: instruction_eval_aggregate_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: intent_classification_events + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: live_events + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_call_metrics + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_cost_aggregates + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_delegation_daily_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: llm_routing_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: mcp_tools + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: merge_state_transitions + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: migrations_log + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: nightly_loop_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: nightly_loop_iterations + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: node_schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: node_service_registry + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: omnimarket_schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_session_phases + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: overnight_sessions + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pattern_learning_artifacts + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_lifecycle_ledger_entries + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: pr_merged_events + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_capsule_effectiveness + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_overnight_readiness + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: projection_routing_decision + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: receipt_gate_rows + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: renderer_capability_projection + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: sandbox_decisions + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: schema_migrations + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: session_outcomes + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: session_replay_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: skill_execution_snapshots + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: stripe_webhook_events + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: swarm_runs + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: traces + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: voice_sessions + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: omninode_internal + - database_ref: application + function_signature: null + object_ref: plan_entitlements + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: plan_tiers + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: usage_privileges + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: platform_catalog + - database_ref: application + function_signature: null + object_ref: bootstrap_tokens + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: delegation_budget_state + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: delegation_events + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: gateway_rate_limit_events + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: gateway_workflows + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_cost_savings_overview + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_inference_response_text + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_model_routing + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_quality_gate + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_savings + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_savings_series + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_summary + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: projection_delegation_token_usage + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: savings_estimates + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_api_keys + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_billing + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_credentials + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenant_plans + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: tenants + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: usage_events + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant + - database_ref: application + function_signature: null + object_ref: workflow_results + object_type: TABLE + physical_database: omnidash_analytics + principal: tenant_projection_writer + privileges: [] + schema_ref: tenant +scaffold_blockers: + - 'ACL scaffold policy violation: additive scaffold intended roles are not already safe or proven absent: [''app_dashboard'', ''omninode_runtime'', ''onex_api'', ''owner_omninode_internal'', ''owner_onex_tenant'', ''owner_platform_catalog'', ''tenant_projection_writer'']' + - 'ACL scaffold policy violation: allowed CONNECT database keys do not cover required CONNECT set' + - 'ACL scaffold policy violation: application: scaffold lacks exact target schema evidence' + - 'ACL scaffold policy violation: application: scaffold lacks presence/absence evidence for declared principals' + - 'ACL scaffold policy violation: keycloak: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: observed CONNECT database keys do not cover required CONNECT set' + - 'ACL scaffold policy violation: observed CONNECT owner database keys do not cover required CONNECT set' + - 'ACL scaffold policy violation: omnibase_infra: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: omniclaude: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: omnidash_analytics: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: omniintelligence: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: omnimemory: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: omninode_cloud: CONNECT allowlist is empty' + - 'ACL scaffold policy violation: principal ''app_dashboard'' has cross-domain scaffold access' + - 'ACL scaffold policy violation: principal ''omninode_runtime'' has cross-domain scaffold access' + - 'ACL scaffold policy violation: principal ''onex_api'' has cross-domain scaffold access' + - 'ACL scaffold policy violation: principal ''tenant_projection_writer'' has cross-domain scaffold access' + - 'ACL scaffold policy violation: scaffold database owner map is incomplete' + - 'ACL scaffold policy violation: scaffold managed owners lack governed owner-role state' + - 'ACL scaffold policy violation: scaffold managed owners lack presence/absence evidence' + - 'ACL scaffold policy violation: scaffold workloads lack governed LOGIN role state' + - 'ACL scaffold policy violation: umami: CONNECT allowlist is empty' + - 'application: requires exactly one independent ACL policy, got []' + - 'application: requires exactly one principal inventory, got []' + - 'keycloak: requires exactly one CONNECT policy, got []' + - matrix requires a typed principal_inventory source for every database + - matrix requires an independent typed acl_policy source for every database + - 'omnibase_infra: requires exactly one CONNECT policy, got []' + - 'omniclaude: requires exactly one CONNECT policy, got []' + - 'omnidash_analytics: requires exactly one CONNECT policy, got []' + - 'omniintelligence: requires exactly one CONNECT policy, got []' + - 'omnimemory: requires exactly one CONNECT policy, got []' + - 'omninode_cloud: requires exactly one CONNECT policy, got []' + - 'umami: requires exactly one CONNECT policy, got []' +scaffold_status: BLOCKED +schema_domains: + application: + omninode_internal: OMNINODE_INTERNAL + platform_catalog: PLATFORM_CATALOG + tenant: TENANT +schema_version: '1.0' +sources: + - path: topology/kubernetes/source-lock.yaml + purpose: rendered_topology + repository: OmniNode-ai/omninode_infra + revision: 1a1fff92cbf8e02e488ac6e6e89b1087ca396657 + sha256: aa6b4502f79244160f07769da788f1826f2b9d5e756b477508eddf5af929226f + source_key: kubernetes_topology_projection + - path: k8s/migrations/application-relation-ownership.yaml + purpose: service_ownership + repository: OmniNode-ai/omninode_infra + revision: 39033d55147ef22a061b665345b506246d3aa543 + sha256: 8ca701c4804d2f7d749dc1ef2a26fc383dcd3806bc42c004ed8851a5ff5d25e0 + source_key: node_migration_service_ownership + - path: docs/evidence/OMN-15423-relation-inventory.json + purpose: relation_inventory + repository: OmniNode-ai/omnimarket + revision: a0f65e9471da06b88eecb91bbc1c847989cb10d8 + sha256: c7ff16080dc553ae204da405519f51b5b631847b129f5876db136c048a4c63da + source_key: omnimarket_relation_inventory + - path: db/migrations/application-relation-ownership.yaml + purpose: service_ownership + repository: OmniNode-ai/omninode_infra + revision: 39033d55147ef22a061b665345b506246d3aa543 + sha256: 4a47bb42a8e724ffc44453cb649b754568f3ffa83dabaf831ccfaebaa0723712 + source_key: onex_api_service_ownership + - path: docker/legacy-rds-fixture/fixture-manifest.json + purpose: legacy_fixture + repository: OmniNode-ai/omnibase_infra + revision: 18df1728bdbcc7a773c790f64b6569f58b524841 + sha256: 1cdc1db7d707bc2daa72ad8f1521bb589cf87594e5b48f6480893b08875e3f26 + source_key: sanitized_legacy_fixture + - path: src/omnibase_infra/topology/instances/local.yaml + purpose: topology + repository: OmniNode-ai/omnibase_infra + revision: d7f1ac464a54696ac9ed05fd8f3766463a0a7eea + sha256: e4170e97e26fb9c63a53f5f660cedacd15bf7e99609309c54bff15c3d026ac8d + source_key: topology_owner + - path: src/omnibase_infra/validation/application_relation_ownership.py + purpose: typed_loader + repository: OmniNode-ai/omnibase_infra + revision: 9e9e62c3177f568b7e5110f653ee4be6dcda965c + sha256: be712ed5d6372dc9bc0720742291b2739f7b815c61abbab3d87157dfb80216c4 + source_key: typed_ownership_loader +status: BLOCKED +verified_evidence_source_keys: [] diff --git a/docker/application-acl-proof/generated/prechange-fixture-acl.json b/docker/application-acl-proof/generated/prechange-fixture-acl.json new file mode 100644 index 0000000000..2bb246673f --- /dev/null +++ b/docker/application-acl-proof/generated/prechange-fixture-acl.json @@ -0,0 +1,1361 @@ +{ + "column_acl": [ + { + "catalog_kind": "table", + "column_name": "display_name", + "grantee": "rls_admin", + "grantor": "onex_api", + "is_grantable": true, + "object_name": "tenant_accounts", + "privilege_type": "UPDATE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "column_name": "display_name", + "grantee": "untrusted_login", + "grantor": "onex_api", + "is_grantable": true, + "object_name": "tenant_accounts", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "column_name": "display_name", + "grantee": "untrusted_login", + "grantor": "rls_admin", + "is_grantable": false, + "object_name": "tenant_accounts", + "privilege_type": "UPDATE", + "schema_name": "tenant" + } + ], + "database_acl": [ + { + "database_name": "keycloak", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "keycloak", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "keycloak", + "grantee": "app_dashboard", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CREATE" + }, + { + "database_name": "keycloak", + "grantee": "external_connect_parent", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnibase_infra", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnibase_infra", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omnibase_infra", + "grantee": "keycloak_service", + "grantor": "rls_admin", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnibase_infra", + "grantee": "onex_api", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CREATE" + }, + { + "database_name": "omnibase_infra", + "grantee": "onex_api", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omnibase_infra", + "grantee": "rls_admin", + "grantor": "postgres", + "is_grantable": true, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnibase_infra", + "grantee": "rls_admin", + "grantor": "postgres", + "is_grantable": true, + "privilege_type": "CREATE" + }, + { + "database_name": "omniclaude", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omniclaude", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omnidash_analytics", + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnidash_analytics", + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omnidash_analytics", + "grantee": "untrusted_login", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnidash_analytics", + "grantee": "untrusted_login", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omniintelligence", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omniintelligence", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omnimemory", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnimemory", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omninode_cloud", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omninode_cloud", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "umami", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "umami", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + } + ], + "database_owner": [ + { + "database_name": "keycloak", + "owner": "postgres" + }, + { + "database_name": "omnibase_infra", + "owner": "postgres" + }, + { + "database_name": "omniclaude", + "owner": "postgres" + }, + { + "database_name": "omnidash_analytics", + "owner": "app_dashboard" + }, + { + "database_name": "omniintelligence", + "owner": "postgres" + }, + { + "database_name": "omnimemory", + "owner": "postgres" + }, + { + "database_name": "omninode_cloud", + "owner": "postgres" + }, + { + "database_name": "umami", + "owner": "postgres" + } + ], + "default_acl": [ + { + "grantee": "PUBLIC", + "grantor": "owner_onex_tenant", + "is_grantable": false, + "object_type": "FUNCTION", + "owner": "owner_onex_tenant", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "grantee": "app_dashboard", + "grantor": "owner_onex_tenant", + "is_grantable": false, + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "privilege_type": "SELECT", + "schema_name": null + }, + { + "grantee": "app_dashboard", + "grantor": "owner_onex_tenant", + "is_grantable": true, + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "grantee": "keycloak_service", + "grantor": "owner_onex_tenant", + "is_grantable": true, + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "grantee": "untrusted_login", + "grantor": "owner_onex_tenant", + "is_grantable": false, + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "privilege_type": "SELECT", + "schema_name": null + } + ], + "default_acl_catalog_rows": [ + { + "object_type": "FUNCTION", + "owner": "owner_onex_tenant", + "raw_acl": "=X/owner_onex_tenant", + "schema_name": "tenant" + }, + { + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "raw_acl": "app_dashboard=r*/owner_onex_tenant,keycloak_service=r*/owner_onex_tenant", + "schema_name": "tenant" + }, + { + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "raw_acl": "app_dashboard=r/owner_onex_tenant,owner_onex_tenant=arwdDxt/owner_onex_tenant,untrusted_login=r/owner_onex_tenant", + "schema_name": null + } + ], + "memberships": [ + { + "admin_option": false, + "grantor": "postgres", + "inherit_option": false, + "member_role": "owner_onex_tenant", + "parent_role": "rls_admin", + "set_option": true + }, + { + "admin_option": false, + "grantor": "postgres", + "inherit_option": true, + "member_role": "omninode_runtime", + "parent_role": "owner_omninode_internal", + "set_option": true + }, + { + "admin_option": false, + "grantor": "postgres", + "inherit_option": true, + "member_role": "shadow_login", + "parent_role": "keycloak_service", + "set_option": true + }, + { + "admin_option": false, + "grantor": "postgres", + "inherit_option": true, + "member_role": "tenant_projection_writer", + "parent_role": "owner_onex_tenant", + "set_option": true + }, + { + "admin_option": false, + "grantor": "rls_admin", + "inherit_option": false, + "member_role": "keycloak_service", + "parent_role": "external_connect_parent", + "set_option": true + }, + { + "admin_option": true, + "grantor": "postgres", + "inherit_option": true, + "member_role": "app_dashboard", + "parent_role": "rls_admin", + "set_option": true + }, + { + "admin_option": true, + "grantor": "postgres", + "inherit_option": true, + "member_role": "db_migrator", + "parent_role": "owner_onex_tenant", + "set_option": false + } + ], + "object_acl": [ + { + "catalog_kind": "function", + "function_signature": "()", + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "object_name": "delegation_event_count", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "()", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(VARIADIC tenant.account_id_span[])", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(\"arg'name%\" integer)", + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "object_name": "hostile_signature", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(bigint, bigint)", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(bigint, bigint, text)", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(tenant.account_id_span)", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "DELETE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "INSERT", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "REFERENCES", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "TRIGGER", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "TRUNCATE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "UPDATE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "multirange_type", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span_set", + "object_type": "TYPE", + "privilege_type": "USAGE", + "schema_name": "tenant" + }, + { + "catalog_kind": "procedure", + "function_signature": "(IN p_payload text)", + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "object_name": "record_delegation", + "object_type": "PROCEDURE", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "range_type", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span", + "object_type": "TYPE", + "privilege_type": "USAGE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "DELETE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "INSERT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "REFERENCES", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "TRIGGER", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "TRUNCATE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "UPDATE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "DELETE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "INSERT", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "REFERENCES", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "TRIGGER", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "TRUNCATE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "UPDATE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "DELETE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "INSERT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "REFERENCES", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "TRIGGER", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "TRUNCATE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "UPDATE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": true, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "DELETE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "INSERT", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "REFERENCES", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "TRIGGER", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "TRUNCATE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "UPDATE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "rls_admin", + "grantor": "onex_api", + "is_grantable": true, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "rls_admin", + "grantor": "onex_api", + "is_grantable": true, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "untrusted_login", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "untrusted_login", + "grantor": "rls_admin", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "untrusted_login", + "grantor": "rls_admin", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "type", + "grantee": "PUBLIC", + "grantor": "omninode_runtime", + "is_grantable": false, + "object_name": "runtime_code", + "object_type": "TYPE", + "privilege_type": "USAGE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "type", + "grantee": "PUBLIC", + "grantor": "omninode_runtime", + "is_grantable": false, + "object_name": "runtime_status", + "object_type": "TYPE", + "privilege_type": "USAGE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "type", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_ref", + "object_type": "TYPE", + "privilege_type": "USAGE", + "schema_name": "tenant" + } + ], + "object_owners": [ + { + "catalog_kind": "function", + "function_signature": "()", + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "owner": "onex_api", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "()", + "object_name": "delegation_event_count", + "object_type": "FUNCTION", + "owner": "app_dashboard", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(VARIADIC tenant.account_id_span[])", + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "owner": "onex_api", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(\"arg'name%\" integer)", + "object_name": "hostile_signature", + "object_type": "FUNCTION", + "owner": "app_dashboard", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(bigint, bigint)", + "object_name": "account_id_span", + "object_type": "FUNCTION", + "owner": "onex_api", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(bigint, bigint, text)", + "object_name": "account_id_span", + "object_type": "FUNCTION", + "owner": "onex_api", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(tenant.account_id_span)", + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "owner": "onex_api", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "materialized_view", + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "owner": "onex_api", + "owner_keyword": "MATERIALIZED VIEW", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "multirange_type", + "object_name": "account_id_span_set", + "object_type": "TYPE", + "owner": "onex_api", + "owner_keyword": "TYPE", + "schema_name": "tenant" + }, + { + "catalog_kind": "procedure", + "function_signature": "(IN p_payload text)", + "object_name": "record_delegation", + "object_type": "PROCEDURE", + "owner": "app_dashboard", + "owner_keyword": "PROCEDURE", + "schema_name": "tenant" + }, + { + "catalog_kind": "range_type", + "object_name": "account_id_span", + "object_type": "TYPE", + "owner": "onex_api", + "owner_keyword": "TYPE", + "schema_name": "tenant" + }, + { + "catalog_kind": "sequence", + "object_name": "delegation_events_id_seq", + "object_type": "SEQUENCE", + "owner": "app_dashboard", + "owner_keyword": "SEQUENCE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "object_name": "delegation_events", + "object_type": "TABLE", + "owner": "app_dashboard", + "owner_keyword": "TABLE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "object_name": "partitioned_events", + "object_type": "TABLE", + "owner": "onex_api", + "owner_keyword": "TABLE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "object_name": "plan_tiers", + "object_type": "TABLE", + "owner": "onex_api", + "owner_keyword": "TABLE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "object_name": "runtime_state", + "object_type": "TABLE", + "owner": "tenant_projection_writer", + "owner_keyword": "TABLE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "object_name": "tenant_accounts", + "object_type": "TABLE", + "owner": "onex_api", + "owner_keyword": "TABLE", + "schema_name": "tenant" + }, + { + "catalog_kind": "type", + "object_name": "account_ref", + "object_type": "TYPE", + "owner": "onex_api", + "owner_keyword": "TYPE", + "schema_name": "tenant" + }, + { + "catalog_kind": "type", + "object_name": "runtime_code", + "object_type": "TYPE", + "owner": "omninode_runtime", + "owner_keyword": "TYPE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "type", + "object_name": "runtime_status", + "object_type": "TYPE", + "owner": "omninode_runtime", + "owner_keyword": "TYPE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "view", + "object_name": "tenant_account_names", + "object_type": "TABLE", + "owner": "app_dashboard", + "owner_keyword": "VIEW", + "schema_name": "tenant" + } + ], + "provenance": { + "authorization_scope": "synthetic_proof", + "dump_derived": false, + "live_database_read": false, + "postgres_major": 16, + "source": "sanitized_postgresql_16_fixture" + }, + "roles": [ + { + "rolbypassrls": false, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "db_migrator", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "owner_omninode_internal", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "owner_onex_tenant", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "owner_platform_catalog", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "keycloak_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "omnibase_infra_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "omniclaude_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "omniintelligence_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "omnimemory_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "omninode_cloud_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "umami_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "omninode_runtime", + "rolreplication": true, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "untrusted_login", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": true, + "rolinherit": true, + "rolname": "app_dashboard", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": true, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "onex_api", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": true, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "external_connect_parent", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": true, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "rls_admin", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": true, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "tenant_projection_writer", + "rolreplication": false, + "rolsuper": false + } + ], + "schema_acl": [ + { + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "CREATE", + "schema_name": "tenant" + }, + { + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "tenant" + }, + { + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "privilege_type": "CREATE", + "schema_name": "platform_catalog" + }, + { + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "platform_catalog" + }, + { + "grantee": "PUBLIC", + "grantor": "pg_database_owner", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "public" + }, + { + "grantee": "PUBLIC", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "privilege_type": "CREATE", + "schema_name": "omninode_internal" + }, + { + "grantee": "PUBLIC", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "omninode_internal" + }, + { + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "omninode_internal" + }, + { + "grantee": "untrusted_login", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "tenant" + } + ], + "schema_owners": [ + { + "owner": "app_dashboard", + "schema_name": "tenant" + }, + { + "owner": "onex_api", + "schema_name": "platform_catalog" + }, + { + "owner": "pg_database_owner", + "schema_name": "public" + }, + { + "owner": "tenant_projection_writer", + "schema_name": "omninode_internal" + } + ], + "schema_version": "3.0" +} diff --git a/docker/application-acl-proof/seed.sql b/docker/application-acl-proof/seed.sql new file mode 100644 index 0000000000..e698fb8e27 --- /dev/null +++ b/docker/application-acl-proof/seed.sql @@ -0,0 +1,187 @@ +-- SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +-- SPDX-License-Identifier: MIT +-- Synthetic unsafe ACL baseline. No dump, live identifier, or credential occurs here. + +\set ON_ERROR_STOP on + +CREATE ROLE owner_onex_tenant NOLOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; +CREATE ROLE owner_omninode_internal NOLOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; +CREATE ROLE owner_platform_catalog NOLOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; +CREATE ROLE onex_api LOGIN PASSWORD 'acl-proof-only' CREATEDB NOSUPERUSER NOBYPASSRLS NOCREATEROLE NOREPLICATION; -- pragma: allowlist secret +CREATE ROLE tenant_projection_writer LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER BYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; -- pragma: allowlist secret +CREATE ROLE app_dashboard LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB CREATEROLE NOREPLICATION; -- pragma: allowlist secret +CREATE ROLE omninode_runtime LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE REPLICATION; -- pragma: allowlist secret +CREATE ROLE untrusted_login LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; -- pragma: allowlist secret +CREATE ROLE db_migrator NOLOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; +CREATE ROLE probe_migrator NOLOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION NOINHERIT; +CREATE ROLE legacy_probe_login LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION NOINHERIT; -- pragma: allowlist secret +CREATE ROLE rls_admin NOLOGIN NOSUPERUSER BYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; +CREATE ROLE external_connect_parent NOLOGIN NOSUPERUSER BYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; +CREATE ROLE keycloak_service LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION NOINHERIT; -- pragma: allowlist secret +CREATE ROLE omnibase_infra_service LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION NOINHERIT; -- pragma: allowlist secret +CREATE ROLE omninode_cloud_service LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION NOINHERIT; -- pragma: allowlist secret +CREATE ROLE omniclaude_service LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION NOINHERIT; -- pragma: allowlist secret +CREATE ROLE omniintelligence_service LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION NOINHERIT; -- pragma: allowlist secret +CREATE ROLE omnimemory_service LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION NOINHERIT; -- pragma: allowlist secret +CREATE ROLE umami_service LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION NOINHERIT; -- pragma: allowlist secret +CREATE ROLE shadow_login LOGIN PASSWORD 'acl-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; -- pragma: allowlist secret + +CREATE DATABASE keycloak; +CREATE DATABASE omnibase_infra; +CREATE DATABASE omninode_cloud; +CREATE DATABASE omniclaude; +CREATE DATABASE omniintelligence; +CREATE DATABASE omnimemory; +CREATE DATABASE umami; +CREATE DATABASE acl_scaffold_probe; + +GRANT CREATE ON DATABASE keycloak TO app_dashboard; +GRANT CREATE, TEMPORARY ON DATABASE omnibase_infra TO onex_api; +GRANT CONNECT ON DATABASE keycloak TO external_connect_parent; +GRANT external_connect_parent TO rls_admin WITH ADMIN OPTION; +SET ROLE rls_admin; +GRANT external_connect_parent TO keycloak_service; +RESET ROLE; +GRANT keycloak_service TO shadow_login; +GRANT CONNECT, CREATE ON DATABASE omnibase_infra TO rls_admin WITH GRANT OPTION; +SET ROLE rls_admin; +GRANT CONNECT ON DATABASE omnibase_infra TO keycloak_service; +RESET ROLE; + +ALTER DATABASE omnidash_analytics OWNER TO app_dashboard; + +GRANT owner_onex_tenant TO tenant_projection_writer; +GRANT owner_omninode_internal TO omninode_runtime; +GRANT rls_admin TO app_dashboard WITH ADMIN TRUE, INHERIT TRUE, SET TRUE; +GRANT rls_admin TO owner_onex_tenant WITH ADMIN FALSE, INHERIT FALSE, SET TRUE; +GRANT owner_onex_tenant TO db_migrator + WITH ADMIN TRUE, INHERIT TRUE, SET FALSE; + +CREATE SCHEMA tenant AUTHORIZATION app_dashboard; +CREATE SCHEMA omninode_internal AUTHORIZATION tenant_projection_writer; +CREATE SCHEMA platform_catalog AUTHORIZATION onex_api; +CREATE SCHEMA legacy_acl_sentinel AUTHORIZATION owner_onex_tenant; + +CREATE TABLE tenant.tenant_accounts ( + account_id BIGINT PRIMARY KEY, + display_name TEXT NOT NULL +); +CREATE SEQUENCE tenant.delegation_events_id_seq; +CREATE TABLE tenant.delegation_events ( + id BIGINT PRIMARY KEY DEFAULT nextval('tenant.delegation_events_id_seq'), + payload TEXT NOT NULL +); +CREATE TABLE tenant.partitioned_events ( + event_id BIGINT NOT NULL, + occurred_at DATE NOT NULL +) PARTITION BY RANGE (occurred_at); +ALTER SEQUENCE tenant.delegation_events_id_seq OWNED BY tenant.delegation_events.id; +CREATE FUNCTION tenant.delegation_event_count() +RETURNS BIGINT +LANGUAGE sql +AS $$SELECT count(*) FROM tenant.delegation_events$$; +CREATE FUNCTION tenant.hostile_signature(IN "arg'name%" INTEGER) +RETURNS INTEGER +LANGUAGE sql +AS $$SELECT "arg'name%"$$; +CREATE PROCEDURE tenant.record_delegation(p_payload TEXT) +LANGUAGE sql +AS $$INSERT INTO tenant.delegation_events (payload) VALUES (p_payload)$$; +CREATE VIEW tenant.tenant_account_names AS +SELECT account_id, display_name FROM tenant.tenant_accounts; +CREATE TYPE tenant.account_ref AS ( + account_id BIGINT, + display_name TEXT +); +CREATE TYPE tenant.account_id_span AS RANGE ( + subtype = BIGINT, + multirange_type_name = tenant.account_id_span_set +); + +CREATE TYPE omninode_internal.runtime_status AS ENUM ('ready', 'blocked'); +CREATE DOMAIN omninode_internal.runtime_code AS TEXT + CHECK (VALUE <> ''); +CREATE TABLE omninode_internal.runtime_state ( + state_id BIGINT PRIMARY KEY, + status omninode_internal.runtime_status NOT NULL +); +CREATE TABLE platform_catalog.plan_tiers ( + code TEXT PRIMARY KEY, + display_name TEXT NOT NULL +); +CREATE MATERIALIZED VIEW platform_catalog.plan_tier_snapshot AS +SELECT code, display_name FROM platform_catalog.plan_tiers; + +ALTER TABLE tenant.tenant_accounts OWNER TO onex_api; +ALTER TABLE tenant.delegation_events OWNER TO app_dashboard; +ALTER TABLE tenant.partitioned_events OWNER TO onex_api; +ALTER SEQUENCE tenant.delegation_events_id_seq OWNER TO app_dashboard; +ALTER FUNCTION tenant.delegation_event_count() OWNER TO app_dashboard; +ALTER FUNCTION tenant.hostile_signature(IN "arg'name%" INTEGER) OWNER TO app_dashboard; +ALTER PROCEDURE tenant.record_delegation(TEXT) OWNER TO app_dashboard; +ALTER VIEW tenant.tenant_account_names OWNER TO app_dashboard; +ALTER TYPE tenant.account_ref OWNER TO onex_api; +ALTER TYPE tenant.account_id_span OWNER TO onex_api; +ALTER TYPE tenant.account_id_span_set OWNER TO onex_api; +ALTER FUNCTION tenant.account_id_span(BIGINT, BIGINT) OWNER TO onex_api; +ALTER FUNCTION tenant.account_id_span(BIGINT, BIGINT, TEXT) OWNER TO onex_api; +ALTER FUNCTION tenant.account_id_span_set() OWNER TO onex_api; +ALTER FUNCTION tenant.account_id_span_set(VARIADIC tenant.account_id_span[]) OWNER TO onex_api; +ALTER FUNCTION tenant.account_id_span_set(tenant.account_id_span) OWNER TO onex_api; +ALTER TYPE omninode_internal.runtime_status OWNER TO omninode_runtime; +ALTER DOMAIN omninode_internal.runtime_code OWNER TO omninode_runtime; +ALTER TABLE omninode_internal.runtime_state OWNER TO tenant_projection_writer; +ALTER TABLE platform_catalog.plan_tiers OWNER TO onex_api; +ALTER MATERIALIZED VIEW platform_catalog.plan_tier_snapshot OWNER TO onex_api; + +GRANT ALL PRIVILEGES ON DATABASE omnidash_analytics TO app_dashboard; +GRANT CONNECT, TEMPORARY ON DATABASE omnidash_analytics TO untrusted_login; +GRANT CREATE, USAGE ON SCHEMA tenant, omninode_internal, platform_catalog TO PUBLIC; +GRANT USAGE ON SCHEMA tenant TO untrusted_login; +GRANT USAGE ON SCHEMA omninode_internal TO app_dashboard; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA tenant, omninode_internal, platform_catalog TO app_dashboard; +GRANT SELECT ON tenant.tenant_accounts TO untrusted_login; +GRANT SELECT (display_name) ON tenant.tenant_accounts TO untrusted_login WITH GRANT OPTION; +GRANT UPDATE (display_name) ON tenant.tenant_accounts TO rls_admin WITH GRANT OPTION; +SET ROLE rls_admin; +GRANT UPDATE (display_name) ON tenant.tenant_accounts TO untrusted_login; +RESET ROLE; +GRANT SELECT ON tenant.tenant_accounts TO app_dashboard WITH GRANT OPTION; +GRANT SELECT ON tenant.tenant_accounts TO rls_admin WITH GRANT OPTION; +SET ROLE rls_admin; +GRANT SELECT ON tenant.tenant_accounts TO untrusted_login; +RESET ROLE; +GRANT SELECT ON tenant.partitioned_events TO rls_admin WITH GRANT OPTION; +SET ROLE rls_admin; +GRANT SELECT ON tenant.partitioned_events TO untrusted_login; +RESET ROLE; +GRANT TRIGGER ON tenant.tenant_accounts TO onex_api; +GRANT USAGE ON SCHEMA omninode_internal TO tenant_projection_writer; +GRANT SELECT ON omninode_internal.runtime_state TO tenant_projection_writer; +GRANT EXECUTE ON FUNCTION tenant.delegation_event_count() TO PUBLIC; +GRANT USAGE ON TYPE omninode_internal.runtime_status TO PUBLIC; + +ALTER DEFAULT PRIVILEGES FOR ROLE owner_onex_tenant IN SCHEMA tenant + GRANT SELECT ON TABLES TO app_dashboard WITH GRANT OPTION; +ALTER DEFAULT PRIVILEGES FOR ROLE owner_onex_tenant IN SCHEMA tenant + GRANT EXECUTE ON FUNCTIONS TO PUBLIC; +ALTER DEFAULT PRIVILEGES FOR ROLE owner_onex_tenant + GRANT SELECT ON TABLES TO app_dashboard; +ALTER DEFAULT PRIVILEGES FOR ROLE owner_onex_tenant + GRANT SELECT ON TABLES TO untrusted_login; +ALTER DEFAULT PRIVILEGES FOR ROLE owner_onex_tenant IN SCHEMA tenant + GRANT SELECT ON TABLES TO keycloak_service WITH GRANT OPTION; +ALTER DEFAULT PRIVILEGES FOR ROLE owner_onex_tenant IN SCHEMA legacy_acl_sentinel + GRANT SELECT ON TABLES TO untrusted_login; + +INSERT INTO tenant.tenant_accounts VALUES (1, 'Synthetic'); +INSERT INTO tenant.delegation_events (payload) VALUES ('before-matrix'); +INSERT INTO omninode_internal.runtime_state VALUES (1, 'ready'); +INSERT INTO platform_catalog.plan_tiers VALUES ('beta', 'Beta'); +REFRESH MATERIALIZED VIEW platform_catalog.plan_tier_snapshot; + +\connect acl_scaffold_probe +CREATE TABLE public.legacy_scaffold_data(id integer PRIMARY KEY); +INSERT INTO public.legacy_scaffold_data VALUES (1); +GRANT SELECT, INSERT ON public.legacy_scaffold_data TO legacy_probe_login; +\connect omnidash_analytics diff --git a/docker/application-acl-proof/source-lock.yaml b/docker/application-acl-proof/source-lock.yaml new file mode 100644 index 0000000000..8ed72bfacd --- /dev/null +++ b/docker/application-acl-proof/source-lock.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +schema_version: "1.0" +required_connect_databases: + - keycloak + - omnibase_infra + - omnidash_analytics + - omninode_cloud + - omniclaude + - omniintelligence + - omnimemory + - umami +sources: + - source_key: topology_owner + repository: OmniNode-ai/omnibase_infra + revision: d7f1ac464a54696ac9ed05fd8f3766463a0a7eea + path: src/omnibase_infra/topology/instances/local.yaml + sha256: e4170e97e26fb9c63a53f5f660cedacd15bf7e99609309c54bff15c3d026ac8d + purpose: topology + - source_key: typed_ownership_loader + repository: OmniNode-ai/omnibase_infra + revision: 9e9e62c3177f568b7e5110f653ee4be6dcda965c + path: src/omnibase_infra/validation/application_relation_ownership.py + sha256: be712ed5d6372dc9bc0720742291b2739f7b815c61abbab3d87157dfb80216c4 + purpose: typed_loader + - source_key: kubernetes_topology_projection + repository: OmniNode-ai/omninode_infra + revision: 1a1fff92cbf8e02e488ac6e6e89b1087ca396657 + path: topology/kubernetes/source-lock.yaml + sha256: aa6b4502f79244160f07769da788f1826f2b9d5e756b477508eddf5af929226f + purpose: rendered_topology + - source_key: omnimarket_relation_inventory + repository: OmniNode-ai/omnimarket + revision: a0f65e9471da06b88eecb91bbc1c847989cb10d8 + path: docs/evidence/OMN-15423-relation-inventory.json + sha256: c7ff16080dc553ae204da405519f51b5b631847b129f5876db136c048a4c63da + purpose: relation_inventory + - source_key: onex_api_service_ownership + repository: OmniNode-ai/omninode_infra + revision: 39033d55147ef22a061b665345b506246d3aa543 + path: db/migrations/application-relation-ownership.yaml + sha256: 4a47bb42a8e724ffc44453cb649b754568f3ffa83dabaf831ccfaebaa0723712 + purpose: service_ownership + - source_key: node_migration_service_ownership + repository: OmniNode-ai/omninode_infra + revision: 39033d55147ef22a061b665345b506246d3aa543 + path: k8s/migrations/application-relation-ownership.yaml + sha256: 8ca701c4804d2f7d749dc1ef2a26fc383dcd3806bc42c004ed8851a5ff5d25e0 + purpose: service_ownership + - source_key: sanitized_legacy_fixture + repository: OmniNode-ai/omnibase_infra + revision: 18df1728bdbcc7a773c790f64b6569f58b524841 + path: docker/legacy-rds-fixture/fixture-manifest.json + sha256: 1cdc1db7d707bc2daa72ad8f1521bb589cf87594e5b48f6480893b08875e3f26 + purpose: legacy_fixture diff --git a/docker/application-domain-enforcement/Dockerfile b/docker/application-domain-enforcement/Dockerfile new file mode 100644 index 0000000000..75b06d2b4e --- /dev/null +++ b/docker/application-domain-enforcement/Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile:1.7 +# SPDX-License-Identifier: MIT + +ARG UV_VERSION=0.11.8 + +FROM postgres:16-alpine AS postgres +COPY docker/application-domain-enforcement/seed.sql /docker-entrypoint-initdb.d/00-domain-seed.sql + +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv-bin + +FROM python:3.12-slim AS proof + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src \ + UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_HTTP_TIMEOUT=600 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential git libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=uv-bin /uv /uvx /usr/local/bin/ +WORKDIR /app +COPY pyproject.toml uv.lock README.md LICENSE ./ +RUN uv sync --frozen --no-dev --no-install-project + +COPY src/ ./src/ +COPY scripts/ci/prove_application_database_domain_enforcement.py ./proof/prove.py +COPY config/application_database_domain_proof_ownership.yaml ./proof/ownership.yaml + +CMD ["/app/.venv/bin/python", "/app/proof/prove.py"] diff --git a/docker/application-domain-enforcement/compose.yml b/docker/application-domain-enforcement/compose.yml new file mode 100644 index 0000000000..06a7b2861d --- /dev/null +++ b/docker/application-domain-enforcement/compose.yml @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: MIT +services: + postgres: + build: + context: ../.. + dockerfile: docker/application-domain-enforcement/Dockerfile + target: postgres + environment: + POSTGRES_DB: omnidash_analytics + POSTGRES_PASSWORD: domain-proof-admin-only # pragma: allowlist secret + tmpfs: + - /var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d omnidash_analytics"] + interval: 1s + timeout: 3s + retries: 30 + proof: + build: + context: ../.. + dockerfile: docker/application-domain-enforcement/Dockerfile + target: proof + depends_on: + postgres: + condition: service_healthy + environment: + ADMIN_DSN: postgresql://postgres:domain-proof-admin-only@postgres:5432/omnidash_analytics # pragma: allowlist secret + OWNERSHIP_MANIFEST: /app/proof/ownership.yaml + POOL_DSN_ONEX_API: postgresql://onex_api:domain-proof-only@postgres:5432/omnidash_analytics # pragma: allowlist secret + POOL_DSN_TENANT_PROJECTION: postgresql://tenant_projection_writer:domain-proof-only@postgres:5432/omnidash_analytics # pragma: allowlist secret + POOL_DSN_APP_DASHBOARD: postgresql://app_dashboard:domain-proof-only@postgres:5432/omnidash_analytics # pragma: allowlist secret + POOL_DSN_OMNINODE_RUNTIME_SERVICE: postgresql://omninode_runtime:domain-proof-only@postgres:5432/omnidash_analytics # pragma: allowlist secret diff --git a/docker/application-domain-enforcement/seed.sql b/docker/application-domain-enforcement/seed.sql new file mode 100644 index 0000000000..522ef42906 --- /dev/null +++ b/docker/application-domain-enforcement/seed.sql @@ -0,0 +1,87 @@ +-- SPDX-License-Identifier: MIT + +CREATE ROLE owner_onex_tenant NOLOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; +CREATE ROLE owner_omninode_internal NOLOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; +CREATE ROLE owner_platform_catalog NOLOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; +CREATE ROLE tenant_control_admin NOLOGIN NOSUPERUSER BYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; + +CREATE ROLE onex_api LOGIN PASSWORD 'domain-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; -- pragma: allowlist secret +CREATE ROLE tenant_projection_writer LOGIN PASSWORD 'domain-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; -- pragma: allowlist secret +CREATE ROLE app_dashboard LOGIN PASSWORD 'domain-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; -- pragma: allowlist secret +CREATE ROLE omninode_runtime LOGIN PASSWORD 'domain-proof-only' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; -- pragma: allowlist secret + +CREATE SCHEMA tenant AUTHORIZATION owner_onex_tenant; +CREATE SCHEMA omninode_internal AUTHORIZATION owner_omninode_internal; +CREATE SCHEMA platform_catalog AUTHORIZATION owner_platform_catalog; + +CREATE TABLE tenant.tenants ( + id UUID PRIMARY KEY, + tenant_name TEXT NOT NULL +); +ALTER TABLE tenant.tenants OWNER TO owner_onex_tenant; +ALTER TABLE tenant.tenants ENABLE ROW LEVEL SECURITY; +ALTER TABLE tenant.tenants FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_identity_isolation ON tenant.tenants + AS PERMISSIVE FOR ALL TO PUBLIC + USING (id = current_setting('app.tenant_id', true)::uuid) + WITH CHECK (id = current_setting('app.tenant_id', true)::uuid); + +INSERT INTO tenant.tenants (id, tenant_name) VALUES + ('11111111-1111-1111-1111-111111111111', 'tenant-a'), + ('22222222-2222-2222-2222-222222222222', 'tenant-b'); + +CREATE TABLE tenant.events ( + event_id UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + payload TEXT NOT NULL +); +ALTER TABLE tenant.events OWNER TO owner_onex_tenant; +ALTER TABLE tenant.events ENABLE ROW LEVEL SECURITY; +ALTER TABLE tenant.events FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON tenant.events + AS PERMISSIVE FOR ALL TO PUBLIC + USING (tenant_id = current_setting('app.tenant_id', true)::uuid) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)::uuid); + +INSERT INTO tenant.events (event_id, tenant_id, payload) VALUES + ('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1', '11111111-1111-1111-1111-111111111111', 'tenant-a-first'), + ('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa2', '11111111-1111-1111-1111-111111111111', 'tenant-a-second'), + ('bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbb1', '22222222-2222-2222-2222-222222222222', 'tenant-b-only'); + +CREATE VIEW tenant.events_view WITH (security_invoker = true) AS + SELECT event_id, tenant_id, payload FROM tenant.events; +ALTER VIEW tenant.events_view OWNER TO owner_onex_tenant; + +CREATE FUNCTION tenant.safe_report() RETURNS INTEGER + LANGUAGE sql + SECURITY DEFINER + SET search_path = pg_catalog, tenant, pg_temp + AS 'SELECT count(*)::integer FROM tenant.events'; +ALTER FUNCTION tenant.safe_report() OWNER TO owner_onex_tenant; +REVOKE ALL ON FUNCTION tenant.safe_report() FROM PUBLIC; + +CREATE TABLE omninode_internal.runtime_state ( + state_id UUID PRIMARY KEY, + source_tenant_id UUID NULL, + payload TEXT NOT NULL +); +ALTER TABLE omninode_internal.runtime_state OWNER TO owner_omninode_internal; + +CREATE TABLE platform_catalog.feature_flags ( + flag_id UUID PRIMARY KEY, + flag_name TEXT NOT NULL UNIQUE +); +ALTER TABLE platform_catalog.feature_flags OWNER TO owner_platform_catalog; + +GRANT CONNECT ON DATABASE omnidash_analytics TO onex_api, tenant_projection_writer, app_dashboard, omninode_runtime; +GRANT USAGE ON SCHEMA tenant TO onex_api, tenant_projection_writer, app_dashboard; +GRANT USAGE ON SCHEMA tenant TO tenant_control_admin; +GRANT SELECT, INSERT ON tenant.tenants TO onex_api, tenant_control_admin; +GRANT SELECT ON tenant.events TO onex_api, tenant_projection_writer, app_dashboard; +GRANT SELECT ON tenant.events_view TO onex_api, tenant_projection_writer, app_dashboard; +GRANT INSERT, UPDATE, DELETE ON tenant.events TO onex_api, tenant_projection_writer; +GRANT EXECUTE ON FUNCTION tenant.safe_report() TO app_dashboard; +GRANT USAGE ON SCHEMA omninode_internal TO omninode_runtime; +GRANT SELECT, INSERT, UPDATE, DELETE ON omninode_internal.runtime_state TO omninode_runtime; +GRANT USAGE ON SCHEMA platform_catalog TO onex_api, app_dashboard, omninode_runtime; +GRANT SELECT ON platform_catalog.feature_flags TO onex_api, app_dashboard, omninode_runtime; diff --git a/docker/catalog/bundles.yaml b/docker/catalog/bundles.yaml index 32f531b2a4..372e539f31 100644 --- a/docker/catalog/bundles.yaml +++ b/docker/catalog/bundles.yaml @@ -142,6 +142,7 @@ omnimarket-projections: - omnimarket-projection-delegation - omnimarket-projection-baselines - omnimarket-projection-registration + - omnimarket-projection-live-events includes: - core inject_required_env: diff --git a/docker/catalog/database-consumers.yaml b/docker/catalog/database-consumers.yaml new file mode 100644 index 0000000000..1aa6fabe5f --- /dev/null +++ b/docker/catalog/database-consumers.yaml @@ -0,0 +1,47 @@ +schema_version: "1.0" +environment: local +consumers: + forward-migration: + physical_database_envs: + - NODE_POSTGRES_DB + migration-gate: + physical_database_envs: + - NODE_POSTGRES_DB + omnidash: + bindings: + - app_dashboard + omnimarket-projection-delegation: + bindings: + - tenant_projection + omnimarket-projection-llm-cost: + bindings: + - tenant_projection + omnimarket-projection-savings: + bindings: + - tenant_projection + omnimarket-projection-session-outcome: + bindings: + - tenant_projection + projection-api: + bindings: + - app_dashboard +deferred_consumers: + omnimarket-projection-baselines: + tracking_ticket: OMN-15360 + reason: Baseline-family domain ownership is resolved by the dedicated P6 ticket. + omnimarket-projection-cost-by-repo: + tracking_ticket: OMN-15423 + reason: Repository-cost ownership remains blocked on total relation classification. + omnimarket-projection-live-events: + tracking_ticket: OMN-15423 + reason: >- + live_events is a global, platform-wide operational event stream with no tenant_id column (omninode_internal domain per the OMN-15423 relation inventory), not per-tenant business data -- same internal-domain classification block as omnimarket-projection-registration. + omnimarket-projection-registration: + tracking_ticket: OMN-15423 + reason: Registry tables require the source-backed internal-domain classification. + omninode-runtime: + tracking_ticket: OMN-15421 + reason: The mixed-domain runtime must split tenant and internal adapters before binding. + runtime-canary: + tracking_ticket: OMN-15421 + reason: The canary inherits the mixed-domain runtime adapter split. diff --git a/docker/catalog/database-topology/judge.yaml b/docker/catalog/database-topology/judge.yaml new file mode 100644 index 0000000000..65e7677e17 --- /dev/null +++ b/docker/catalog/database-topology/judge.yaml @@ -0,0 +1,312 @@ +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: omninode_internal + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - live_events + - log_entries + privileges: + - INSERT + - SELECT + - UPDATE + schema: omninode_internal + - object_type: TABLE + objects: + - baselines_breakdown + - baselines_comparisons + - baselines_quality_snapshots + - baselines_roi_snapshots + - baselines_snapshots + - baselines_trend + - capsule_store + - contract_registry + - cost_by_repo_snapshots + - deployment_evidence_projection + - deployment_readiness_projection + - event_chain + - evidence_correlation_trace_projection + - evidence_dashboard_projection + - evidence_readiness_aggregate_projection + - gate_activity + - gate_metrics + - generation_events + - intent_classification_events + - llm_call_metrics + - llm_delegation_daily_projection + - llm_routing_decisions + - mcp_tools + - merge_state_transitions + - nightly_loop_decisions + - nightly_loop_iterations + - node_service_registry + - overnight_session_phases + - overnight_sessions + - pr_lifecycle_ledger_entries + - pr_merged_events + - receipt_gate_rows + - renderer_capability_projection + - sandbox_decisions + - session_outcomes + - session_replay_snapshots + - swarm_runs + - traces + - voice_sessions + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + - object_type: TABLE + objects: + - nightly_loop_configs + privileges: + - SELECT + schema: public + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - agent_routing_decisions + - capability_scores + - context_roi_scores + - delegation_budget_state + - delegation_events + - delegation_judge_verdict_events + - delegation_shadow_comparisons + - dep_health_findings + - instruction_eval_aggregate_snapshots + - llm_cost_aggregates + - pattern_learning_artifacts + - projection_delegation_inference_response_text + - savings_estimates + - skill_execution_snapshots + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omnibase_infra: + bindings: + omninode_runtime_service: + database_ref: omnibase_infra + dsn_env: OMNIBASE_INFRA_DB_URL + principal: role_omnibase_infra + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.service + owners: + owner_omnibase_infra: + login: false + physical_name: omnibase_infra + principals: + role_omnibase_infra: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - delegation_workflow_state + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omnibase_infra + omniintelligence: + bindings: + omninode_runtime_service: + database_ref: omniintelligence + dsn_env: OMNIINTELLIGENCE_DB_URL + principal: role_omniintelligence + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omniintelligence.service + owners: + owner_omniintelligence: + login: false + physical_name: omniintelligence + principals: + role_omniintelligence: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - dispatch_eval_results + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omniintelligence +environment: judge +schema_version: '1.0' +source: + path: src/omnibase_infra/topology/instances/local.yaml + profile_catalog_path: src/omnibase_infra/topology/application_database_profiles.yaml + profile_catalog_sha256: 759f32da77ad41b874fbb05d2824c44f9de4c64e9d269d99da21445225a90ca1 + repository: OmniNode-ai/omnibase_infra + sha256: 0994b80a5eb48f72c08d393d3120a2807f47dd26dc85ade47fc7c880b4a9228f +topology_instance: local diff --git a/docker/catalog/database-topology/local.yaml b/docker/catalog/database-topology/local.yaml new file mode 100644 index 0000000000..ece9450161 --- /dev/null +++ b/docker/catalog/database-topology/local.yaml @@ -0,0 +1,312 @@ +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: omninode_internal + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - live_events + - log_entries + privileges: + - INSERT + - SELECT + - UPDATE + schema: omninode_internal + - object_type: TABLE + objects: + - baselines_breakdown + - baselines_comparisons + - baselines_quality_snapshots + - baselines_roi_snapshots + - baselines_snapshots + - baselines_trend + - capsule_store + - contract_registry + - cost_by_repo_snapshots + - deployment_evidence_projection + - deployment_readiness_projection + - event_chain + - evidence_correlation_trace_projection + - evidence_dashboard_projection + - evidence_readiness_aggregate_projection + - gate_activity + - gate_metrics + - generation_events + - intent_classification_events + - llm_call_metrics + - llm_delegation_daily_projection + - llm_routing_decisions + - mcp_tools + - merge_state_transitions + - nightly_loop_decisions + - nightly_loop_iterations + - node_service_registry + - overnight_session_phases + - overnight_sessions + - pr_lifecycle_ledger_entries + - pr_merged_events + - receipt_gate_rows + - renderer_capability_projection + - sandbox_decisions + - session_outcomes + - session_replay_snapshots + - swarm_runs + - traces + - voice_sessions + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + - object_type: TABLE + objects: + - nightly_loop_configs + privileges: + - SELECT + schema: public + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - agent_routing_decisions + - capability_scores + - context_roi_scores + - delegation_budget_state + - delegation_events + - delegation_judge_verdict_events + - delegation_shadow_comparisons + - dep_health_findings + - instruction_eval_aggregate_snapshots + - llm_cost_aggregates + - pattern_learning_artifacts + - projection_delegation_inference_response_text + - savings_estimates + - skill_execution_snapshots + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omnibase_infra: + bindings: + omninode_runtime_service: + database_ref: omnibase_infra + dsn_env: OMNIBASE_INFRA_DB_URL + principal: role_omnibase_infra + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.service + owners: + owner_omnibase_infra: + login: false + physical_name: omnibase_infra + principals: + role_omnibase_infra: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - delegation_workflow_state + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omnibase_infra + omniintelligence: + bindings: + omninode_runtime_service: + database_ref: omniintelligence + dsn_env: OMNIINTELLIGENCE_DB_URL + principal: role_omniintelligence + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omniintelligence.service + owners: + owner_omniintelligence: + login: false + physical_name: omniintelligence + principals: + role_omniintelligence: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - dispatch_eval_results + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omniintelligence +environment: local +schema_version: '1.0' +source: + path: src/omnibase_infra/topology/instances/local.yaml + profile_catalog_path: src/omnibase_infra/topology/application_database_profiles.yaml + profile_catalog_sha256: 759f32da77ad41b874fbb05d2824c44f9de4c64e9d269d99da21445225a90ca1 + repository: OmniNode-ai/omnibase_infra + sha256: 0994b80a5eb48f72c08d393d3120a2807f47dd26dc85ade47fc7c880b4a9228f +topology_instance: local diff --git a/docker/catalog/database-topology/onex-dev.yaml b/docker/catalog/database-topology/onex-dev.yaml new file mode 100644 index 0000000000..b272a13dbb --- /dev/null +++ b/docker/catalog/database-topology/onex-dev.yaml @@ -0,0 +1,312 @@ +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: DATABASE_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: omninode_internal + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - live_events + - log_entries + privileges: + - INSERT + - SELECT + - UPDATE + schema: omninode_internal + - object_type: TABLE + objects: + - baselines_breakdown + - baselines_comparisons + - baselines_quality_snapshots + - baselines_roi_snapshots + - baselines_snapshots + - baselines_trend + - capsule_store + - contract_registry + - cost_by_repo_snapshots + - deployment_evidence_projection + - deployment_readiness_projection + - event_chain + - evidence_correlation_trace_projection + - evidence_dashboard_projection + - evidence_readiness_aggregate_projection + - gate_activity + - gate_metrics + - generation_events + - intent_classification_events + - llm_call_metrics + - llm_delegation_daily_projection + - llm_routing_decisions + - mcp_tools + - merge_state_transitions + - nightly_loop_decisions + - nightly_loop_iterations + - node_service_registry + - overnight_session_phases + - overnight_sessions + - pr_lifecycle_ledger_entries + - pr_merged_events + - receipt_gate_rows + - renderer_capability_projection + - sandbox_decisions + - session_outcomes + - session_replay_snapshots + - swarm_runs + - traces + - voice_sessions + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + - object_type: TABLE + objects: + - nightly_loop_configs + privileges: + - SELECT + schema: public + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - agent_routing_decisions + - capability_scores + - context_roi_scores + - delegation_budget_state + - delegation_events + - delegation_judge_verdict_events + - delegation_shadow_comparisons + - dep_health_findings + - instruction_eval_aggregate_snapshots + - llm_cost_aggregates + - pattern_learning_artifacts + - projection_delegation_inference_response_text + - savings_estimates + - skill_execution_snapshots + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omnibase_infra: + bindings: + omninode_runtime_service: + database_ref: omnibase_infra + dsn_env: OMNIBASE_INFRA_DB_URL + principal: role_omnibase_infra + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.service + owners: + owner_omnibase_infra: + login: false + physical_name: omnibase_infra + principals: + role_omnibase_infra: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - delegation_workflow_state + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omnibase_infra + omniintelligence: + bindings: + omninode_runtime_service: + database_ref: omniintelligence + dsn_env: OMNIINTELLIGENCE_DB_URL + principal: role_omniintelligence + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omniintelligence.service + owners: + owner_omniintelligence: + login: false + physical_name: omniintelligence + principals: + role_omniintelligence: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - dispatch_eval_results + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omniintelligence +environment: onex-dev +schema_version: '1.0' +source: + path: src/omnibase_infra/topology/instances/onex-dev.yaml + profile_catalog_path: src/omnibase_infra/topology/application_database_profiles.yaml + profile_catalog_sha256: 759f32da77ad41b874fbb05d2824c44f9de4c64e9d269d99da21445225a90ca1 + repository: OmniNode-ai/omnibase_infra + sha256: c856d2032bf585f93a71d2353002ec14128da4bbd4328ca1012a6a3aaa7b8da3 +topology_instance: onex-dev diff --git a/docker/catalog/database-topology/onex-prod.yaml b/docker/catalog/database-topology/onex-prod.yaml new file mode 100644 index 0000000000..0b147d9739 --- /dev/null +++ b/docker/catalog/database-topology/onex-prod.yaml @@ -0,0 +1,312 @@ +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: DATABASE_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: omninode_internal + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - live_events + - log_entries + privileges: + - INSERT + - SELECT + - UPDATE + schema: omninode_internal + - object_type: TABLE + objects: + - baselines_breakdown + - baselines_comparisons + - baselines_quality_snapshots + - baselines_roi_snapshots + - baselines_snapshots + - baselines_trend + - capsule_store + - contract_registry + - cost_by_repo_snapshots + - deployment_evidence_projection + - deployment_readiness_projection + - event_chain + - evidence_correlation_trace_projection + - evidence_dashboard_projection + - evidence_readiness_aggregate_projection + - gate_activity + - gate_metrics + - generation_events + - intent_classification_events + - llm_call_metrics + - llm_delegation_daily_projection + - llm_routing_decisions + - mcp_tools + - merge_state_transitions + - nightly_loop_decisions + - nightly_loop_iterations + - node_service_registry + - overnight_session_phases + - overnight_sessions + - pr_lifecycle_ledger_entries + - pr_merged_events + - receipt_gate_rows + - renderer_capability_projection + - sandbox_decisions + - session_outcomes + - session_replay_snapshots + - swarm_runs + - traces + - voice_sessions + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + - object_type: TABLE + objects: + - nightly_loop_configs + privileges: + - SELECT + schema: public + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - agent_routing_decisions + - capability_scores + - context_roi_scores + - delegation_budget_state + - delegation_events + - delegation_judge_verdict_events + - delegation_shadow_comparisons + - dep_health_findings + - instruction_eval_aggregate_snapshots + - llm_cost_aggregates + - pattern_learning_artifacts + - projection_delegation_inference_response_text + - savings_estimates + - skill_execution_snapshots + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omnibase_infra: + bindings: + omninode_runtime_service: + database_ref: omnibase_infra + dsn_env: OMNIBASE_INFRA_DB_URL + principal: role_omnibase_infra + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.service + owners: + owner_omnibase_infra: + login: false + physical_name: omnibase_infra + principals: + role_omnibase_infra: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - delegation_workflow_state + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omnibase_infra + omniintelligence: + bindings: + omninode_runtime_service: + database_ref: omniintelligence + dsn_env: OMNIINTELLIGENCE_DB_URL + principal: role_omniintelligence + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omniintelligence.service + owners: + owner_omniintelligence: + login: false + physical_name: omniintelligence + principals: + role_omniintelligence: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - dispatch_eval_results + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omniintelligence +environment: onex-prod +schema_version: '1.0' +source: + path: src/omnibase_infra/topology/instances/onex-prod.yaml + profile_catalog_path: src/omnibase_infra/topology/application_database_profiles.yaml + profile_catalog_sha256: 759f32da77ad41b874fbb05d2824c44f9de4c64e9d269d99da21445225a90ca1 + repository: OmniNode-ai/omnibase_infra + sha256: c856d2032bf585f93a71d2353002ec14128da4bbd4328ca1012a6a3aaa7b8da3 +topology_instance: onex-prod diff --git a/docker/catalog/database-topology/prod.yaml b/docker/catalog/database-topology/prod.yaml new file mode 100644 index 0000000000..a1ff9ba7c0 --- /dev/null +++ b/docker/catalog/database-topology/prod.yaml @@ -0,0 +1,312 @@ +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: omninode_internal + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - live_events + - log_entries + privileges: + - INSERT + - SELECT + - UPDATE + schema: omninode_internal + - object_type: TABLE + objects: + - baselines_breakdown + - baselines_comparisons + - baselines_quality_snapshots + - baselines_roi_snapshots + - baselines_snapshots + - baselines_trend + - capsule_store + - contract_registry + - cost_by_repo_snapshots + - deployment_evidence_projection + - deployment_readiness_projection + - event_chain + - evidence_correlation_trace_projection + - evidence_dashboard_projection + - evidence_readiness_aggregate_projection + - gate_activity + - gate_metrics + - generation_events + - intent_classification_events + - llm_call_metrics + - llm_delegation_daily_projection + - llm_routing_decisions + - mcp_tools + - merge_state_transitions + - nightly_loop_decisions + - nightly_loop_iterations + - node_service_registry + - overnight_session_phases + - overnight_sessions + - pr_lifecycle_ledger_entries + - pr_merged_events + - receipt_gate_rows + - renderer_capability_projection + - sandbox_decisions + - session_outcomes + - session_replay_snapshots + - swarm_runs + - traces + - voice_sessions + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + - object_type: TABLE + objects: + - nightly_loop_configs + privileges: + - SELECT + schema: public + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - agent_routing_decisions + - capability_scores + - context_roi_scores + - delegation_budget_state + - delegation_events + - delegation_judge_verdict_events + - delegation_shadow_comparisons + - dep_health_findings + - instruction_eval_aggregate_snapshots + - llm_cost_aggregates + - pattern_learning_artifacts + - projection_delegation_inference_response_text + - savings_estimates + - skill_execution_snapshots + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omnibase_infra: + bindings: + omninode_runtime_service: + database_ref: omnibase_infra + dsn_env: OMNIBASE_INFRA_DB_URL + principal: role_omnibase_infra + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.service + owners: + owner_omnibase_infra: + login: false + physical_name: omnibase_infra + principals: + role_omnibase_infra: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - delegation_workflow_state + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omnibase_infra + omniintelligence: + bindings: + omninode_runtime_service: + database_ref: omniintelligence + dsn_env: OMNIINTELLIGENCE_DB_URL + principal: role_omniintelligence + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omniintelligence.service + owners: + owner_omniintelligence: + login: false + physical_name: omniintelligence + principals: + role_omniintelligence: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - dispatch_eval_results + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omniintelligence +environment: prod +schema_version: '1.0' +source: + path: src/omnibase_infra/topology/instances/local.yaml + profile_catalog_path: src/omnibase_infra/topology/application_database_profiles.yaml + profile_catalog_sha256: 759f32da77ad41b874fbb05d2824c44f9de4c64e9d269d99da21445225a90ca1 + repository: OmniNode-ai/omnibase_infra + sha256: 0994b80a5eb48f72c08d393d3120a2807f47dd26dc85ade47fc7c880b4a9228f +topology_instance: local diff --git a/docker/catalog/database-topology/stability-test.yaml b/docker/catalog/database-topology/stability-test.yaml new file mode 100644 index 0000000000..bbcbfaf73e --- /dev/null +++ b/docker/catalog/database-topology/stability-test.yaml @@ -0,0 +1,312 @@ +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: omninode_internal + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - live_events + - log_entries + privileges: + - INSERT + - SELECT + - UPDATE + schema: omninode_internal + - object_type: TABLE + objects: + - baselines_breakdown + - baselines_comparisons + - baselines_quality_snapshots + - baselines_roi_snapshots + - baselines_snapshots + - baselines_trend + - capsule_store + - contract_registry + - cost_by_repo_snapshots + - deployment_evidence_projection + - deployment_readiness_projection + - event_chain + - evidence_correlation_trace_projection + - evidence_dashboard_projection + - evidence_readiness_aggregate_projection + - gate_activity + - gate_metrics + - generation_events + - intent_classification_events + - llm_call_metrics + - llm_delegation_daily_projection + - llm_routing_decisions + - mcp_tools + - merge_state_transitions + - nightly_loop_decisions + - nightly_loop_iterations + - node_service_registry + - overnight_session_phases + - overnight_sessions + - pr_lifecycle_ledger_entries + - pr_merged_events + - receipt_gate_rows + - renderer_capability_projection + - sandbox_decisions + - session_outcomes + - session_replay_snapshots + - swarm_runs + - traces + - voice_sessions + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + - object_type: TABLE + objects: + - nightly_loop_configs + privileges: + - SELECT + schema: public + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - agent_routing_decisions + - capability_scores + - context_roi_scores + - delegation_budget_state + - delegation_events + - delegation_judge_verdict_events + - delegation_shadow_comparisons + - dep_health_findings + - instruction_eval_aggregate_snapshots + - llm_cost_aggregates + - pattern_learning_artifacts + - projection_delegation_inference_response_text + - savings_estimates + - skill_execution_snapshots + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omnibase_infra: + bindings: + omninode_runtime_service: + database_ref: omnibase_infra + dsn_env: OMNIBASE_INFRA_DB_URL + principal: role_omnibase_infra + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.service + owners: + owner_omnibase_infra: + login: false + physical_name: omnibase_infra + principals: + role_omnibase_infra: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - delegation_workflow_state + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omnibase_infra + omniintelligence: + bindings: + omninode_runtime_service: + database_ref: omniintelligence + dsn_env: OMNIINTELLIGENCE_DB_URL + principal: role_omniintelligence + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omniintelligence.service + owners: + owner_omniintelligence: + login: false + physical_name: omniintelligence + principals: + role_omniintelligence: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - dispatch_eval_results + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omniintelligence +environment: stability-test +schema_version: '1.0' +source: + path: src/omnibase_infra/topology/instances/local.yaml + profile_catalog_path: src/omnibase_infra/topology/application_database_profiles.yaml + profile_catalog_sha256: 759f32da77ad41b874fbb05d2824c44f9de4c64e9d269d99da21445225a90ca1 + repository: OmniNode-ai/omnibase_infra + sha256: 0994b80a5eb48f72c08d393d3120a2807f47dd26dc85ade47fc7c880b4a9228f +topology_instance: local diff --git a/docker/catalog/database-topology/test.yaml b/docker/catalog/database-topology/test.yaml new file mode 100644 index 0000000000..d76d6f669f --- /dev/null +++ b/docker/catalog/database-topology/test.yaml @@ -0,0 +1,312 @@ +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: omninode_internal + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - live_events + - log_entries + privileges: + - INSERT + - SELECT + - UPDATE + schema: omninode_internal + - object_type: TABLE + objects: + - baselines_breakdown + - baselines_comparisons + - baselines_quality_snapshots + - baselines_roi_snapshots + - baselines_snapshots + - baselines_trend + - capsule_store + - contract_registry + - cost_by_repo_snapshots + - deployment_evidence_projection + - deployment_readiness_projection + - event_chain + - evidence_correlation_trace_projection + - evidence_dashboard_projection + - evidence_readiness_aggregate_projection + - gate_activity + - gate_metrics + - generation_events + - intent_classification_events + - llm_call_metrics + - llm_delegation_daily_projection + - llm_routing_decisions + - mcp_tools + - merge_state_transitions + - nightly_loop_decisions + - nightly_loop_iterations + - node_service_registry + - overnight_session_phases + - overnight_sessions + - pr_lifecycle_ledger_entries + - pr_merged_events + - receipt_gate_rows + - renderer_capability_projection + - sandbox_decisions + - session_outcomes + - session_replay_snapshots + - swarm_runs + - traces + - voice_sessions + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + - object_type: TABLE + objects: + - nightly_loop_configs + privileges: + - SELECT + schema: public + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: tenant + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - agent_routing_decisions + - capability_scores + - context_roi_scores + - delegation_budget_state + - delegation_events + - delegation_judge_verdict_events + - delegation_shadow_comparisons + - dep_health_findings + - instruction_eval_aggregate_snapshots + - llm_cost_aggregates + - pattern_learning_artifacts + - projection_delegation_inference_response_text + - savings_estimates + - skill_execution_snapshots + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omnibase_infra: + bindings: + omninode_runtime_service: + database_ref: omnibase_infra + dsn_env: OMNIBASE_INFRA_DB_URL + principal: role_omnibase_infra + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.service + owners: + owner_omnibase_infra: + login: false + physical_name: omnibase_infra + principals: + role_omnibase_infra: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - delegation_workflow_state + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omnibase_infra + omniintelligence: + bindings: + omninode_runtime_service: + database_ref: omniintelligence + dsn_env: OMNIINTELLIGENCE_DB_URL + principal: role_omniintelligence + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omniintelligence.service + owners: + owner_omniintelligence: + login: false + physical_name: omniintelligence + principals: + role_omniintelligence: + bypass_rls: false + grants: + - object_type: DATABASE + objects: [] + privileges: + - CONNECT + schema: null + - object_type: SCHEMA + objects: [] + privileges: + - USAGE + schema: public + - object_type: TABLE + objects: + - dispatch_eval_results + privileges: + - INSERT + - SELECT + - UPDATE + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omniintelligence +environment: test +schema_version: '1.0' +source: + path: src/omnibase_infra/topology/instances/local.yaml + profile_catalog_path: src/omnibase_infra/topology/application_database_profiles.yaml + profile_catalog_sha256: 759f32da77ad41b874fbb05d2824c44f9de4c64e9d269d99da21445225a90ca1 + repository: OmniNode-ai/omnibase_infra + sha256: 0994b80a5eb48f72c08d393d3120a2807f47dd26dc85ade47fc7c880b4a9228f +topology_instance: local diff --git a/docker/catalog/services/omnimarket-projection-live-events.yaml b/docker/catalog/services/omnimarket-projection-live-events.yaml new file mode 100644 index 0000000000..8f0d97f3b7 --- /dev/null +++ b/docker/catalog/services/omnimarket-projection-live-events.yaml @@ -0,0 +1,35 @@ +name: omnimarket-projection-live-events +description: Kafka-to-DB projection consumer for live-events (OMN-13079); additionally publishes bus-fed snapshot deltas (OMN-15800) +image: omnimarket-projection:latest +layer: runtime +container_name: omnimarket-projection-live-events +required_env: + - OMNIDASH_ANALYTICS_DB_URL +hardcoded_env: + KAFKA_BROKERS: redpanda:9092 + PYTHONUNBUFFERED: "1" +operational_defaults: + KAFKA_CONSUMER_GROUP: omnimarket-projections-v1 + ONEX_LOG_LEVEL: INFO +ports: null +healthcheck: null +volumes: [] +depends_on: + - service: redpanda + condition: service_healthy + - service: postgres + condition: service_healthy +restart: unless-stopped +resources: + cpus: "0.5" + memory: 256M + cpus_reservation: "0.1" + memory_reservation: 64M +stop_grace_period: 30s +command: + - python + - -m + - omnimarket.nodes.node_projection_live_events.handlers.handler_live_events +labels: + com.omninode.service: omnimarket-projection-live-events + com.omninode.layer: runtime diff --git a/docker/catalog/services/omninode-runtime.yaml b/docker/catalog/services/omninode-runtime.yaml index e80b662c0b..ade0a86640 100644 --- a/docker/catalog/services/omninode-runtime.yaml +++ b/docker/catalog/services/omninode-runtime.yaml @@ -41,6 +41,7 @@ operational_defaults: POSTGRES_USER: postgres ONEX_LOG_LEVEL: INFO ONEX_ENVIRONMENT: local + ONEX_DATABASE_TOPOLOGY_PROFILE: local KEYCLOAK_REALM: omninode KEYCLOAK_ADMIN_CLIENT_ID: onex-admin KEYCLOAK_ISSUER: http://localhost:28080/realms/omninode diff --git a/docker/catalog/services/runtime-canary.yaml b/docker/catalog/services/runtime-canary.yaml index a84f36b03e..e3a555d4fa 100644 --- a/docker/catalog/services/runtime-canary.yaml +++ b/docker/catalog/services/runtime-canary.yaml @@ -40,6 +40,7 @@ operational_defaults: POSTGRES_USER: postgres ONEX_LOG_LEVEL: INFO ONEX_ENVIRONMENT: local + ONEX_DATABASE_TOPOLOGY_PROFILE: local KEYCLOAK_REALM: omninode KEYCLOAK_ADMIN_CLIENT_ID: onex-admin KEYCLOAK_ISSUER: http://localhost:28080/realms/omninode diff --git a/docker/catalog/services/runtime-effects.yaml b/docker/catalog/services/runtime-effects.yaml index aa2f0452f5..3d08ad1f4a 100644 --- a/docker/catalog/services/runtime-effects.yaml +++ b/docker/catalog/services/runtime-effects.yaml @@ -17,6 +17,7 @@ required_env: - LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST - LOCAL_LLM_SHARED_SECRET - GITHUB_TOKEN + - DEPLOY_AGENT_HMAC_SECRET hardcoded_env: OMNIBASE_INFRA_DB_URL: 'postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/omnibase_infra' OMNIINTELLIGENCE_DB_URL: 'postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/omniintelligence' @@ -60,6 +61,7 @@ operational_defaults: POSTGRES_USER: postgres ONEX_LOG_LEVEL: INFO ONEX_ENVIRONMENT: local + ONEX_DATABASE_TOPOLOGY_PROFILE: local KEYCLOAK_REALM: omninode KEYCLOAK_ADMIN_CLIENT_ID: onex-admin KEYCLOAK_ISSUER: http://localhost:28080/realms/omninode diff --git a/docker/catalog/services/runtime-worker.yaml b/docker/catalog/services/runtime-worker.yaml index 51a083aceb..99338d728b 100644 --- a/docker/catalog/services/runtime-worker.yaml +++ b/docker/catalog/services/runtime-worker.yaml @@ -33,6 +33,7 @@ operational_defaults: POSTGRES_USER: postgres ONEX_LOG_LEVEL: INFO ONEX_ENVIRONMENT: local + ONEX_DATABASE_TOPOLOGY_PROFILE: local KEYCLOAK_REALM: omninode KEYCLOAK_ADMIN_CLIENT_ID: onex-admin KEYCLOAK_ISSUER: http://localhost:28080/realms/omninode diff --git a/docker/docker-compose.dev-lane.yml b/docker/docker-compose.dev-lane.yml new file mode 100644 index 0000000000..ecf938f040 --- /dev/null +++ b/docker/docker-compose.dev-lane.yml @@ -0,0 +1,139 @@ +# docker-compose.dev-lane.yml +# OMN-15379 — dev/lab lane overlay. Operator ruling 15 (2026-07-29). +# +# WHAT THIS IS FOR +# ---------------- +# It carries exactly ONE thing: the lane indicator that lets the compose +# forward-migration runner apply the node_projection_registration trio +# (0000 CREATE / 0001 heartbeat columns / 0002 ENABLE + FORCE ROW LEVEL +# SECURITY) on the lab lane, which operator ruling 15 designates as the FORCE +# proving ground. Every other fenced node migration stays fenced here too — the +# delegation 0023-0026 tenant-RLS hold is a different, still-pending ruling. +# +# WHY IT IS A SEPARATE FILE AND NOT A LINE IN docker-compose.infra.yml +# -------------------------------------------------------------------- +# This is the whole fail-closed mechanism, so it is worth being explicit. +# +# docker-compose.infra.yml is the BASE that every lane merges: stability-test, +# prod and judge each layer their overlay ON TOP of it and override only what +# they need (stability-test's forward-migration override is a single +# `container_name:` line — it inherits the base `environment:` block wholesale). +# So `ONEX_MIGRATION_LANE: dev` written into the base would be INHERITED by +# stability-test, by prod, by judge, and — the part that matters — by any lane +# added in future by someone who has never heard of this fence. That is +# fail-OPEN, and silently so. +# +# Inverting it makes the default safe: the base says nothing, the runner treats +# "nothing" as the FULL fence, and the dev lane opts in by loading this file. +# A lane that does not load it (stability-test, prod, judge, CI, a raw +# `docker compose -f docker/docker-compose.infra.yml up` against a fresh volume, +# and every future lane) is fully fenced without having to know this file +# exists. +# +# WHO LOADS IT +# ------------ +# The dev lane's `-f` list, in the two places that own lane -> compose-file +# mapping and are kept matched: +# * scripts/deploy-runtime.sh resolve_compose_file_args() +# * scripts/deploy-agent/deploy_agent/executor.py _LANE_CONFIGS[DEV] +# Manually: +# docker compose -p omnibase-infra \ +# -f docker/docker-compose.infra.yml \ +# -f docker/docker-compose.dev-lane.yml \ +# --profile runtime up -d +# +# Deliberately NO `name:` key: the project comes from `-p omnibase-infra` (or +# from the base file), and pinning it here would make this overlay unusable in +# any context that names the project itself. +# +# Ticket: OMN-15379 (cold compose lane blocked by the registration trio) +# Ruling: operator ruling 15, 2026-07-29 — FORCE on node_service_registry is +# lab-lane-only; the omninode_infra k8s fence is unchanged at 7 ids. +# ============================================================================= +# LAB-LANE ANALYTICS CONNECTION IDENTITY (OMN-15363) +# ============================================================================= +# WHAT THIS IS FOR +# ---------------- +# The base file gives every runtime service the same analytics DSN: +# +# docker-compose.infra.yml: +# OMNIDASH_ANALYTICS_DB_URL: +# "postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@postgres:5432/omnidash_analytics" +# +# `postgres` is the table OWNER and carries `rolsuper`+`rolbypassrls`. Postgres +# exempts owners and BYPASSRLS roles from row-level security UNCONDITIONALLY — +# FORCE included. So while this lane connects as `postgres`, the FORCE ROW LEVEL +# SECURITY that operator ruling 15 designates this lane to prove is INERT against +# every connection the database actually sees, and any "no RLS errors" reading +# taken here is a false clean rather than evidence. +# +# Live readback that established this, .201 lab lane, 2026-07-31T03:33Z: 400 +# consecutive pg_stat_activity samples on `omnidash_analytics` returned exactly +# one client — `postgres`, from the projection-api container. There was no +# non-exempt connection to enforce anything against. +# +# These overrides connect the lane's analytics consumers as `role_omnidash` +# instead: non-owner, NOSUPERUSER, NOBYPASSRLS. Authorization for that role is +# provisioned by docker/migrations/forward/096_grant_role_omnidash_omnidash_analytics.sql; +# its LOGIN + password stay deployment-owned (ROLE_OMNIDASH_PASSWORD, consumed by +# docker/migrations/forward/000_create_multiple_databases.sh — no credential +# material lives in a migration). +# +# WHY THE ROLE IS SPELLED OUT PER SERVICE INSTEAD OF PARAMETERISED IN THE BASE +# --------------------------------------------------------------------------- +# Same fail-closed inversion the lane indicator above uses, and for the same +# reason. A `${OMNIDASH_ANALYTICS_DB_USER:-postgres}` seam in the base would be +# resolved from the HOST environment — and `.201` runs the lab, stability-test, +# judge and prod lanes from one `~/.omnibase/.env`, so setting that variable +# would repoint every lane's analytics connection at the next recreate. Writing +# the identity here, in the file only the dev/lab project loads, makes leaking it +# to another lane structurally impossible rather than merely unlikely. +# +# It is also the readable form: an operator reading this deploy config sees WHICH +# ROLE CONNECTS, not a variable name that has to be resolved against a host env +# file to find out. +# +# `:?` and not `:-`: an unset ROLE_OMNIDASH_PASSWORD must fail at compose render, +# not silently fall back to a DSN with an empty password (or, worse, to the +# superuser). This lane is the proving ground; a silent fallback here is the +# false-clean this whole block exists to remove. +# +# SCOPE +# ----- +# Every service in the base file that merges the `x-runtime-env` anchor, i.e. +# every service that carries OMNIDASH_ANALYTICS_DB_URL at all. Overriding only +# the one service observed connecting today would leave the other four able to +# open a BYPASSRLS session against the same tables the moment their dispatch path +# changes — a partial cutover proves nothing. +# +# Ticket: OMN-15363. Gate this enables: OMN-15416 (prove FORCE-RLS with real +# non-owner pools). Unchanged by this block: no FORCE/RLS state on any table, no +# migration fence, no other lane. +x-dev-lane-analytics-env: &dev_lane_analytics_env + OMNIDASH_ANALYTICS_DB_URL: "postgresql://role_omnidash:${ROLE_OMNIDASH_PASSWORD:?ROLE_OMNIDASH_PASSWORD must be set — the lab lane connects to omnidash_analytics as the non-BYPASSRLS role_omnidash (OMN-15363), never as the postgres superuser}@postgres:5432/omnidash_analytics" +services: + forward-migration: + environment: + # Consumed by scripts/run-forward-migrations.sh. `dev` is the only value + # that releases anything; unset/unknown = full fence. The release SET is + # committed in the runner, not supplied here — this names a policy, it + # does not carry migration ids. + ONEX_MIGRATION_LANE: dev + labels: + - "com.omninode.lane=dev" + - "com.omninode.ticket=OMN-15379" + omninode-runtime: + environment: + !!merge <<: *dev_lane_analytics_env + runtime-effects: + environment: + !!merge <<: *dev_lane_analytics_env + projection-api: + environment: + !!merge <<: *dev_lane_analytics_env + runtime-worker: + environment: + !!merge <<: *dev_lane_analytics_env + omninode-contract-resolver: + environment: + !!merge <<: *dev_lane_analytics_env diff --git a/docker/docker-compose.e2e.yml b/docker/docker-compose.e2e.yml index 96f8f55176..be9eadcfd0 100644 --- a/docker/docker-compose.e2e.yml +++ b/docker/docker-compose.e2e.yml @@ -19,7 +19,7 @@ # docker compose -f docker/docker-compose.e2e.yml down -v # # Network: -# Uses omnibase-infra-network (created automatically). +# Uses omnibase-infra-e2e-network (created automatically). # All services communicate over this internal Docker network. # # Ports Exposed (for host access during testing): @@ -30,7 +30,30 @@ # - Infisical: 8880 # - Valkey: 6379 # - Runtime Health: 8085 (when runtime profile enabled) -name: ${OMNIBASE_INFRA_COMPOSE_PROJECT:-omnibase-infra} +# +# NOTE: these HOST port defaults still overlap live .201 lane bindings +# (19092/18082/8880/8085). Any caller that runs this stack on a shared host +# MUST override POSTGRES_PORT / KAFKA_PORT / REDPANDA_ADMIN_PORT / +# REDPANDA_SCHEMA_REGISTRY_PORT / REDPANDA_PANDAPROXY_PORT / VALKEY_PORT with +# free ports -- see .github/workflows/nightly-integration.yml and +# .github/workflows/reusable-runtime-boot.yml, which both allocate them +# dynamically. A port clash is a loud bind failure, not silent lane damage. +# ========================================================================== +# OMN-15565 -- LANE ISOLATION. Every default below is namespaced under +# `omnibase-infra-e2e`, NOT `omnibase-infra`. The previous default resolved +# this ephemeral test stack into the *live lab/dev lane's* compose project, so +# `docker compose -f docker/docker-compose.e2e.yml down -v --remove-orphans` +# (nightly-integration.yml) deleted the lab lane's containers AND its Postgres/ +# Redpanda/Valkey data volumes every night. The project name, container_names, +# volume names and network name are ALL part of that aliasing -- container, +# volume and network names are global to the Docker daemon and are not scoped +# by the compose project -- so all four are re-namespaced together. +# +# Guarded by tests/unit/docker/test_e2e_compose_lane_isolation.py, which +# derives the protected lane namespaces from deploy/lane-census/lane-manifest.yaml +# and the lane compose files rather than hardcoding them. +# ========================================================================== +name: ${OMNIBASE_INFRA_COMPOSE_PROJECT:-omnibase-infra-e2e} # ========================================================================== # IMPORTANT: Requires Docker Compose v2.20+ (nested variable expansion). # Older versions will SILENTLY produce malformed DSNs. @@ -52,7 +75,7 @@ services: # ========================================================================== postgres: image: postgres:16-alpine - container_name: ${OMNIBASE_INFRA_POSTGRES_CONTAINER:-omnibase-infra-postgres} + container_name: ${OMNIBASE_INFRA_POSTGRES_CONTAINER:-omnibase-infra-e2e-postgres} environment: POSTGRES_USER: postgres # NOTE: Unlike docker-compose.infra.yml which fails fast when POSTGRES_PASSWORD @@ -87,7 +110,7 @@ services: # ========================================================================== redpanda: image: redpandadata/redpanda:v24.2.7 - container_name: ${OMNIBASE_INFRA_REDPANDA_CONTAINER:-omnibase-infra-redpanda} + container_name: ${OMNIBASE_INFRA_REDPANDA_CONTAINER:-omnibase-infra-e2e-redpanda} command: - redpanda - start @@ -137,7 +160,7 @@ services: # ========================================================================== redpanda-topic-manager: image: redpandadata/redpanda:v24.2.7 - container_name: ${OMNIBASE_INFRA_TOPIC_MANAGER_CONTAINER:-omnibase-infra-topic-manager} + container_name: ${OMNIBASE_INFRA_TOPIC_MANAGER_CONTAINER:-omnibase-infra-e2e-topic-manager} depends_on: redpanda: condition: service_healthy @@ -204,7 +227,7 @@ services: # ========================================================================== valkey: image: valkey/valkey:8.0-alpine - container_name: ${OMNIBASE_INFRA_VALKEY_CONTAINER:-omnibase-infra-valkey} + container_name: ${OMNIBASE_INFRA_VALKEY_CONTAINER:-omnibase-infra-e2e-valkey} ports: - "${VALKEY_PORT:-6379}:6379" networks: @@ -224,7 +247,7 @@ services: # ========================================================================== infisical: image: infisical/infisical:v0.146.0-postgres - container_name: ${OMNIBASE_INFRA_INFISICAL_CONTAINER:-omnibase-infra-infisical} + container_name: ${OMNIBASE_INFRA_INFISICAL_CONTAINER:-omnibase-infra-e2e-infisical} depends_on: postgres: condition: service_healthy @@ -259,7 +282,7 @@ services: build: context: .. dockerfile: docker/Dockerfile.runtime - container_name: ${OMNIBASE_INFRA_RUNTIME_CONTAINER:-omnibase-infra-runtime} + container_name: ${OMNIBASE_INFRA_RUNTIME_CONTAINER:-omnibase-infra-e2e-runtime} profiles: ["runtime"] depends_on: postgres: @@ -291,9 +314,16 @@ services: VALKEY_PORT: 6379 # Runtime configuration ONEX_ENVIRONMENT: test + ONEX_DATABASE_TOPOLOGY_PROFILE: test ONEX_LOG_LEVEL: ${ONEX_LOG_LEVEL:-DEBUG} ONEX_GROUP_ID: onex-runtime-e2e ONEX_CONTRACTS_DIR: /app/contracts + # OMN-15628: standalone e2e lane — it does not layer + # docker-compose.infra.yml, so it inherits nothing from that file's + # x-runtime-env anchor and needs its own routing-tiers pin. Same image + # (docker/Dockerfile.runtime), same in-image path, same fail-closed + # consumer in omnimarket's routing reducer. + DELEGATION_ROUTING_TIERS_PATH: /app/config/delegation/routing_tiers.yaml ports: - "${RUNTIME_PORT:-8085}:8085" networks: @@ -313,17 +343,17 @@ services: # ========================================================================== networks: omnibase-infra-network: - name: ${OMNIBASE_INFRA_NETWORK:-omnibase-infra-network} + name: ${OMNIBASE_INFRA_NETWORK:-omnibase-infra-e2e-network} driver: bridge # ========================================================================== # Volumes # ========================================================================== volumes: postgres_data: - name: ${OMNIBASE_INFRA_POSTGRES_VOLUME:-omnibase-infra-postgres-data} + name: ${OMNIBASE_INFRA_POSTGRES_VOLUME:-omnibase-infra-e2e-postgres-data} redpanda_data: - name: ${OMNIBASE_INFRA_REDPANDA_VOLUME:-omnibase-infra-redpanda-data} + name: ${OMNIBASE_INFRA_REDPANDA_VOLUME:-omnibase-infra-e2e-redpanda-data} valkey_data: - name: ${OMNIBASE_INFRA_VALKEY_VOLUME:-omnibase-infra-valkey-data} + name: ${OMNIBASE_INFRA_VALKEY_VOLUME:-omnibase-infra-e2e-valkey-data} runtime_logs: - name: ${OMNIBASE_INFRA_RUNTIME_LOG_VOLUME:-omnibase-infra-runtime-logs} + name: ${OMNIBASE_INFRA_RUNTIME_LOG_VOLUME:-omnibase-infra-e2e-runtime-logs} diff --git a/docker/docker-compose.gateway-attach-test-lane.yml b/docker/docker-compose.gateway-attach-test-lane.yml new file mode 100644 index 0000000000..e4b4dd4a4b --- /dev/null +++ b/docker/docker-compose.gateway-attach-test-lane.yml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# Edge-side test-lane attach connector (OMN-15752). +# +# Parallel to, and independent of, docker/docker-compose.gateway.yml (the live +# bastion-path forwarder -- untouched by this file, per the operator's "do not +# touch the bastion" directive). This overlay is the .201-side client half of +# the OMN-15750 attach control plane: it dials OUT to +# node_gateway_attach_effect's command topics with a per-tenant Keycloak +# client-credentials token and never learns a broker hostname or holds an MSK +# credential -- unlike the bastion path, this connector has NO extra_hosts, +# NO TPM device, and NO AWS_* environment, because attach/session validation +# never touches Kafka directly (candidate A's whole point: customers, and our +# own house tenant proving the same path, never see brokers). +# +# Usage: +# docker compose -f docker/docker-compose.gateway-attach-test-lane.yml \ +# --profile test-lane up +# +# This does not start by default (no `up -d` without --profile) so it can +# never collide with, or be mistaken for, the live gateway-forwarder service. +services: + gateway-attach-test-lane: + image: "${GATEWAY_ATTACH_TEST_LANE_IMAGE:?GATEWAY_ATTACH_TEST_LANE_IMAGE is required}" + pull_policy: never + build: + context: .. + dockerfile: docker/Dockerfile.runtime + args: + BUILD_SOURCE: "${BUILD_SOURCE:-release}" + EXPECTED_BUILD_SOURCE: "${EXPECTED_BUILD_SOURCE:-release}" + container_name: omninode-gateway-attach-test-lane + profiles: + - test-lane + command: + - python + - -m + - scripts.proof.gateway_attach_e2e_proof + - "--live" + - "--edge-instance-id=${GATEWAY_ATTACH_EDGE_INSTANCE_ID:?GATEWAY_ATTACH_EDGE_INSTANCE_ID is required}" + - "--bootstrap-servers=${GATEWAY_ATTACH_TEST_LANE_BOOTSTRAP_SERVERS:?GATEWAY_ATTACH_TEST_LANE_BOOTSTRAP_SERVERS is required}" + - "--attach-request-topic=onex.cmd.omnibase-infra.gateway-attach-request.v1" + - "--heartbeat-request-topic=onex.cmd.omnibase-infra.gateway-heartbeat-request.v1" + - "--detach-request-topic=onex.cmd.omnibase-infra.gateway-detach-request.v1" + - "--session-event-topic=onex.evt.omnibase-infra.gateway-session.v1" + environment: + # Tenant client-credentials secret -- ref only, resolved at container + # start from Infisical by the shared secret-resolver bootstrap. Never a + # literal secret in this file or in an env_file checked into git. + GATEWAY_ATTACH_TENANT_CLIENT_CREDENTIALS_REF: "${GATEWAY_ATTACH_TENANT_CLIENT_CREDENTIALS_REF:?GATEWAY_ATTACH_TENANT_CLIENT_CREDENTIALS_REF is required}" + ONEX_TOPIC_ENFORCEMENT_MODE: "reject" + networks: + - gateway-runtime + restart: "no" + labels: + - "com.omninode.service=gateway-attach-test-lane" + - "com.omninode.layer=gateway" + - "com.omninode.ticket=OMN-15752" +networks: + gateway-runtime: + external: true + name: omnibase-infra-network diff --git a/docker/docker-compose.gateway.yml b/docker/docker-compose.gateway.yml new file mode 100644 index 0000000000..a193a9ab81 --- /dev/null +++ b/docker/docker-compose.gateway.yml @@ -0,0 +1,80 @@ +# Standalone operator-edge gateway overlay. This process connects outbound to +# staging MSK and to the .201 local Redpanda listener; it opens no inbound port. +services: + gateway-forwarder: + image: "${GATEWAY_IMAGE:?GATEWAY_IMAGE is required}" + pull_policy: never + build: + context: .. + dockerfile: docker/Dockerfile.runtime + args: + BUILD_SOURCE: "${BUILD_SOURCE:-release}" + EXPECTED_BUILD_SOURCE: "${EXPECTED_BUILD_SOURCE:-release}" + container_name: omninode-gateway-forwarder + # Starting as the image's non-root runtime user avoids the generic + # entrypoint's root-to-user transition dropping the TPM supplemental group. + user: "${GATEWAY_CONTAINER_UID:?GATEWAY_CONTAINER_UID is required}:${GATEWAY_CONTAINER_GID:?GATEWAY_CONTAINER_GID is required}" + group_add: + - "${GATEWAY_TPM_GROUP_ID:?GATEWAY_TPM_GROUP_ID is required}" + command: + - onex-gateway-forwarder + - --config + - /app/config/gateway-forwarder.yaml + - --ready-file + - /tmp/gateway-forwarder-ready + environment: + AWS_PROFILE: "${GATEWAY_AWS_PROFILE:-gateway}" + AWS_CONFIG_FILE: "/run/aws/config" + AWS_REGION: "us-east-1" + AWS_DEFAULT_REGION: "us-east-1" + AWS_SDK_LOAD_CONFIG: "1" + ONEX_TOPIC_ENFORCEMENT_MODE: "reject" + volumes: + - ./gateway/beta-gateway-canary.yaml:/app/config/gateway-forwarder.yaml:ro + # /app/data is created in the runtime image as omniinfra:omniinfra. A + # fresh named volume inherits that ownership, which matters because this + # service deliberately starts as the non-root runtime user. + - gateway-delivery-state:/app/data + # IAM Roles Anywhere supplies short-lived session credentials through + # credential_process. No AWS access key is stored on the edge host. + - ${GATEWAY_AWS_CONFIG_FILE:?GATEWAY_AWS_CONFIG_FILE is required}:/run/aws/config:ro + - ${GATEWAY_AWS_CERTIFICATE_FILE:?GATEWAY_AWS_CERTIFICATE_FILE is required}:/run/aws/certificate.pem:ro + - ${GATEWAY_AWS_PRIVATE_KEY_FILE:?GATEWAY_AWS_PRIVATE_KEY_FILE is required}:/run/aws/private-key.tss:ro + - ${GATEWAY_AWS_SIGNING_HELPER_FILE:?GATEWAY_AWS_SIGNING_HELPER_FILE is required}:/usr/local/bin/aws_signing_helper:ro + devices: + # The wrapped private key is usable only through the .201 TPM. The raw + # signing key never exists in a host file or inside the container. + - ${GATEWAY_TPM_DEVICE:?GATEWAY_TPM_DEVICE is required}:/dev/tpmrm0 + networks: + - gateway-runtime + extra_hosts: + - "b-1.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com:100.53.215.198" + - "b-2.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com:100.53.215.198" + restart: unless-stopped + stop_grace_period: 30s + healthcheck: + # OMN-15741 (G1): path-verifying probe, not a sentinel-file check. Dials + # both broker legs directly with the same transport/credentials as real + # traffic and produces+reads-back a dedicated canary record; exits + # non-zero when either leg's path is dead even though the process is + # still up and `test -f` would have reported healthy (the exact failure + # mode of the 2026-08-04 outage -- ready-file present, forwarding 0% + # for 4 days). Real checks are cadence-bound (contract + # `gateway_forwarder.canary.cadence_seconds`, default 30s) via + # --state-file so this does not hit the brokers on every 15s tick. + test: ["CMD", "onex-gateway-canary-probe", "--config", "/app/config/gateway-forwarder.yaml", "--state-file", "/tmp/gateway-canary-probe-state.json"] + interval: 15s + timeout: 25s + retries: 4 + start_period: 90s + labels: + - "com.omninode.service=gateway-forwarder" + - "com.omninode.layer=gateway" + - "com.omninode.ticket=OMN-12912" +networks: + gateway-runtime: + external: true + name: omnibase-infra-network +volumes: + gateway-delivery-state: + name: omninode-gateway-delivery-state diff --git a/docker/docker-compose.infra.yml b/docker/docker-compose.infra.yml index 6aa3a039cd..876c29ff10 100644 --- a/docker/docker-compose.infra.yml +++ b/docker/docker-compose.infra.yml @@ -166,6 +166,25 @@ x-runtime-env: &runtime-env BIFROST_CONTRACT_PATH: /app/data/delegation/bifrost_delegation.yaml BIFROST_SOURCE_CONTRACT_PATH: ${BIFROST_SOURCE_CONTRACT_PATH:-} BIFROST_VERIFY_ENDPOINTS: ${BIFROST_VERIFY_ENDPOINTS:?runtime policy contract must set BIFROST_VERIFY_ENDPOINTS} + # OMN-15645: Hardcoded default — same locus/reason as BIFROST_CONTRACT_PATH + # above (OMN-12864: no ${VAR:-} ambient-override footgun). omnimarket#2000 + # (OMN-15628) removed the packaged-default fallback in the delegation routing + # reducer, so an unbound key now fails boot with ProtocolConfigurationError + # instead of silently defaulting. The value is NOT a literal python3.X + # site-packages path (that trap is exactly what OMN-15628's runtime + # entrypoint self-heal exists to correct for a *stale* pin) — it is a fixed, + # version-independent location that docker/Dockerfile.runtime bakes into the + # image at build time from the installed omnimarket package's own configs/ + # directory (glob-derived, never a hardcoded interpreter minor version). + # /app/config/ (not /app/contracts/) deliberately: the runtime services + # bind-mount ../contracts:/app/contracts:ro (host content that would shadow + # anything baked into that path at image-build time); /app/config/ carries + # no volume/bind-mount entry anywhere in these compose lane files. + # Services that have no delegation-routing surface (projection-api, + # contract-resolver) explicitly set DELEGATION_ROUTING_TIERS_PATH: "" in + # their own environment blocks below, mirroring the BIFROST_CONTRACT_PATH + # opt-out. + DELEGATION_ROUTING_TIERS_PATH: /app/config/delegation/routing_tiers.yaml ONEX_SECRET_RESOLVER_CONFIG_PATH: ${DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_PATH:?runtime policy contract must set DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_PATH} ONEX_SECRET_RESOLVER_CONFIG_JSON: ${DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_JSON:?runtime policy contract must set DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_JSON} # Restrict the runtime surface to first-party infra + marketplace packages. @@ -174,10 +193,15 @@ x-runtime-env: &runtime-env ONEX_RUNTIME_CONTRACTS_DIR: /app/contracts/runtime ONEX_LOG_LEVEL: ${ONEX_LOG_LEVEL:-INFO} ONEX_ENVIRONMENT: ${ONEX_ENVIRONMENT:-local} + ONEX_DATABASE_TOPOLOGY_PROFILE: local # Local Docker Redpanda uses a bounded single-shard broker. Cap default topic # creation to one partition unless an operator explicitly opts into wider # contract-declared partition counts. ONEX_TOPIC_PROVISIONER_MAX_PARTITIONS: ${ONEX_TOPIC_PROVISIONER_MAX_PARTITIONS:-1} + ONEX_BOUNDARY_DLQ_ENABLED: ${DEV_BOUNDARY_DLQ_ENABLED:?runtime policy contract must set DEV_BOUNDARY_DLQ_ENABLED} + ONEX_SINGLE_OWNER_COMMAND_TOPICS: ${ONEX_SINGLE_OWNER_COMMAND_TOPICS:-0} + ONEX_TOPIC_ENFORCEMENT_MODE: ${ONEX_TOPIC_ENFORCEMENT_MODE:-warn} + ONEX_WIRING_STRICT_MODE: ${ONEX_WIRING_STRICT_MODE:-0} # Linear API for build loop ticket fetching LINEAR_API_KEY: ${LINEAR_API_KEY:?LINEAR_API_KEY must be set} # GitHub API/CLI auth for runtime-owned PR lifecycle inventory and merge gates. @@ -185,6 +209,30 @@ x-runtime-env: &runtime-env # read GITHUB_TOKEN. GITHUB_TOKEN: ${GITHUB_TOKEN:?GITHUB_TOKEN must be set} GH_TOKEN: ${GITHUB_TOKEN:?GITHUB_TOKEN must be set} + # OMN-15529 (OMN-15362 blocker 3 cutover): OnexBot-OCC-Writer App identity for + # the OCC companion producer (node_occ_companion_effect / OccCompanionEmitter). + # This anchor is an ALLOWLIST, not a host-env pass-through — a credential in + # ~/.omnibase/.env is invisible to the container unless it is named here. + # + # Optional by design, NOT a silent fallback: the App key is provisioned only on + # hosts the operator has minted it onto (.201 today), and every other lane must + # render and run unchanged without it. Empty is the safe, inert state on both + # consumers, verified field-by-field against omnimarket@dev: + # * handler_occ_companion_effect.py:187 and occ_companion_emitter.py:208 read + # `os.environ.get("OMNI_OCC_GITHUB_AUTH_MODE", "pat").strip().lower() or "pat"` + # — "" coalesces to PAT mode, the current behaviour, unchanged. + # * github_app_auth.py resolves both credentials through + # `resolve_api_key(..., required=False)`, which treats an empty value as + # absent and raises GitHubAppCredentialMissingError in app mode rather than + # falling back to the shared PAT. + # `:?` is therefore wrong here — it would wedge every lane that has no App key. + ONEXBOT_OCC_APP_ID: ${ONEXBOT_OCC_APP_ID:-} + ONEXBOT_OCC_PRIVATE_KEY: ${ONEXBOT_OCC_PRIVATE_KEY:-} + OMNI_OCC_GITHUB_AUTH_MODE: ${OMNI_OCC_GITHUB_AUTH_MODE:-} + # OMN-15009: node_redeploy_deploy_effect signs rebuild commands before they + # cross the runtime/deploy-agent trust boundary. The deploy agent rejects + # unsigned commands, so an absent secret must fail at compose render time. + DEPLOY_AGENT_HMAC_SECRET: ${DEPLOY_AGENT_HMAC_SECRET:?DEPLOY_AGENT_HMAC_SECRET must be set in ~/.omnibase/.env} # --- Infisical config (OMN-5382: removed from base env — injected per-bundle) --- # Infisical vars are only present when the secrets bundle is selected. # --- Container-internal service addresses (OMN-5381: same invariant as OMN-3431 for Kafka) --- @@ -359,9 +407,17 @@ services: - redpanda - start - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092 - - --advertise-kafka-addr internal://redpanda:9092,external://${DEV_REDPANDA_ADVERTISE_HOST:-localhost}:${REDPANDA_EXTERNAL_PORT:-19092} + # OMN-15173: DEV_REDPANDA_ADVERTISE_HOST has NO silent default. A silent + # ${VAR:-localhost} default rendered a broker address unreachable by any + # off-host client (CI runner, another machine) whenever the operator + # forgot to export the var — the client-side symptom (connection refused + # / timeout against a "localhost" the client cannot resolve to the actual + # host) surfaces far from this file. Fail fast instead: set it explicitly + # to localhost for pure single-host dev, or to the reachable host/IP for + # off-host clients. + - --advertise-kafka-addr internal://redpanda:9092,external://${DEV_REDPANDA_ADVERTISE_HOST:?DEV_REDPANDA_ADVERTISE_HOST is required - use localhost for single-host dev or the reachable host/IP for off-host clients}:${REDPANDA_EXTERNAL_PORT:-19092} - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082 - - --advertise-pandaproxy-addr internal://redpanda:8082,external://${DEV_REDPANDA_ADVERTISE_HOST:-localhost}:${REDPANDA_PANDAPROXY_PORT:-18082} + - --advertise-pandaproxy-addr internal://redpanda:8082,external://${DEV_REDPANDA_ADVERTISE_HOST:?DEV_REDPANDA_ADVERTISE_HOST is required - use localhost for single-host dev or the reachable host/IP for off-host clients}:${REDPANDA_PANDAPROXY_PORT:-18082} - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081 - --rpc-addr redpanda:33145 - --advertise-rpc-addr redpanda:33145 @@ -952,6 +1008,11 @@ services: # Projection API serves materialized read models and has no runtime data # volume for the rendered Bifrost delegation contract. BIFROST_CONTRACT_PATH: "" + # OMN-15645: Projection API is a read-only materialized-view server; it + # never invokes the delegation routing reducer, so it deliberately has no + # delegation-routing surface (mirrors the BIFROST_CONTRACT_PATH opt-out + # above). + DELEGATION_ROUTING_TIERS_PATH: "" ports: - "${PROJECTION_API_PORT:-3002}:3002" healthcheck: @@ -973,9 +1034,25 @@ services: # retries (5) because Kafka consumer groups are joined sequentially at # startup. Autoheal uses a workspace-wide 2400s grace period so main, # effects, and workers can all finish contract wiring before restart checks. - # OMN-7569: Default replicas=0 — workers join the same per-topic consumer groups - # as the main runtime on 1-partition topics, causing a constant rebalance storm. - # Scale to 0 until the kernel supports profile-based subscription filtering. + # OMN-7569 (SUPERSEDED by OMN-14968): this block used to default replicas to 0 + # via a bare ${WORKER_REPLICAS:-0}, because workers joined the same per-topic + # consumer groups as the main runtime on 1-partition topics and caused a constant + # rebalance storm — "scale to 0 UNTIL the kernel supports profile-based + # subscription filtering". That filtering shipped: each lane process now carries a + # disjoint capability set (this service resolves DEV_RUNTIME_WORKER_CAPABILITIES = + # workflow.dispatch,contract.update,runtime.worker, disjoint from the main + # runtime's market.skill-proof,workflow.orchestration,runtime.main), and OMN-12990 + # pinned worker replicas to 1 for EVERY lane in + # contracts/services/runtime_policy.contract.yaml. + # + # OMN-12988 / OMN-12990 converted the stability-test and prod overlays to the + # lane-prefixed fail-closed form but left this base line bare, so the dev lane + # (which has no overlay — this file IS the dev lane) still resolved to 0: a plain + # `up -d --no-deps runtime-worker` exited 0 creating nothing, while + # deploy-runtime.sh's RUNTIME_SERVICES / RT-6 deploy readback requires a running + # container, aborting and auto-restoring every dev-lane deploy. deploy.replicas + # below is now lane-prefixed and fail-closed, exactly like the sibling + # DEV_RUNTIME_WORKER_* vars in this same service block. runtime-worker: !!merge <<: *runtime-base profiles: ["runtime", "full"] @@ -1014,7 +1091,7 @@ services: start_period: 1200s deploy: mode: replicated - replicas: ${WORKER_REPLICAS:-0} + replicas: ${DEV_WORKER_REPLICAS:?runtime policy contract must set DEV_WORKER_REPLICAS (source docker/runtime-policy.env)} resources: limits: cpus: '0.5' @@ -1315,6 +1392,12 @@ services: # Contract resolver does not mount /app/data, so it must not render the # Bifrost delegation contract into the shared runtime data path. BIFROST_CONTRACT_PATH: "" + # OMN-15645: Contract resolver is a synchronous HTTP bridge for + # NodeContractResolveCompute (contract *resolution*, not delegation + # *routing*); it never invokes the delegation routing reducer, so it + # deliberately has no delegation-routing surface (mirrors the + # BIFROST_CONTRACT_PATH opt-out above). + DELEGATION_ROUTING_TIERS_PATH: "" CORS_ORIGINS: ${CONTRACT_RESOLVER_CORS_ORIGINS:-http://localhost:3000,http://localhost:3001,http://localhost:8085} CONTRACT_RESOLVER_PORT: "8091" ONEX_LOG_LEVEL: ${ONEX_LOG_LEVEL:-INFO} diff --git a/docker/docker-compose.judge.yml b/docker/docker-compose.judge.yml index 987afdd781..49f30d216f 100644 --- a/docker/docker-compose.judge.yml +++ b/docker/docker-compose.judge.yml @@ -86,11 +86,22 @@ x-judge-runtime-env: &judge-runtime-env ONEX_LOG_LEVEL: ${ONEX_LOG_LEVEL:-INFO} ONEX_ENVIRONMENT: judge KAFKA_ENVIRONMENT: judge + ONEX_DATABASE_TOPOLOGY_PROFILE: judge ONEX_TOPIC_PROVISIONER_MAX_PARTITIONS: ${JUDGE_TOPIC_PROVISIONER_MAX_PARTITIONS:?runtime policy contract must set JUDGE_TOPIC_PROVISIONER_MAX_PARTITIONS} ONEX_STATE_ROOT: /app/data/.onex_state_judge ONEX_STATE_DIR: /app/data/.onex_state_judge ONEX_BOX_ID: judge-local BIFROST_CONTRACT_PATH: ${BIFROST_CONTRACT_PATH:-/app/data/delegation/bifrost_delegation.yaml} + # OMN-15628: judge redefines its own x-runtime-env anchor, so it must carry + # the routing-tiers pin explicitly alongside BIFROST_CONTRACT_PATH. Same + # in-image path and same fail-closed consumer as the base infra anchor + # (docker/docker-compose.infra.yml); see that comment for the full rationale. + # OMN-15628/OMN-15645: same canonical value as docker-compose.infra.yml's + # x-runtime-env. Judge owns a separate anchor, so it needs the binding + # explicitly; it builds from docker/Dockerfile.runtime, which bakes + # /app/config/delegation/routing_tiers.yaml from the installed omnimarket + # package (glob-derived, no interpreter minor version in the literal). + DELEGATION_ROUTING_TIERS_PATH: /app/config/delegation/routing_tiers.yaml BIFROST_SOURCE_CONTRACT_PATH: ${BIFROST_SOURCE_CONTRACT_PATH:-} LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST: ${LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST:?runtime policy contract must set LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST} LLM_CODER_URL: ${LLM_CODER_URL:-https://generativelanguage.googleapis.com} @@ -417,6 +428,11 @@ services: RUNTIME_PROFILE: projection-api OTEL_SERVICE_NAME: omnimarket-judge-projection-api BIFROST_CONTRACT_PATH: "" + # Projection API serves materialized read models and does not invoke the + # delegation routing reducer. Keep the judge overlay opt-out explicit so + # layered base+judge renders cannot inherit the runtime anchor's routing + # tiers path. + DELEGATION_ROUTING_TIERS_PATH: "" ports: !override - "${JUDGE_PROJECTION_API_PORT:-43002}:3002" volumes: diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 49467ea105..ad9900c4ab 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -134,6 +134,7 @@ services: OMNIMEMORY_MEMGRAPH_HOST: ${PROD_RUNTIME_MAIN_OMNIMEMORY_MEMGRAPH_HOST?runtime policy contract must set PROD_RUNTIME_MAIN_OMNIMEMORY_MEMGRAPH_HOST} ONEX_ENVIRONMENT: prod KAFKA_ENVIRONMENT: prod + ONEX_DATABASE_TOPOLOGY_PROFILE: prod ONEX_STATE_ROOT: /app/data/.onex_state_prod ONEX_BOX_ID: omninode-pc ONEX_RUNTIME_ID: prod-main @@ -170,6 +171,7 @@ services: OMNIMEMORY_MEMGRAPH_HOST: ${PROD_RUNTIME_EFFECTS_OMNIMEMORY_MEMGRAPH_HOST?runtime policy contract must set PROD_RUNTIME_EFFECTS_OMNIMEMORY_MEMGRAPH_HOST} ONEX_ENVIRONMENT: prod KAFKA_ENVIRONMENT: prod + ONEX_DATABASE_TOPOLOGY_PROFILE: prod ONEX_STATE_ROOT: /app/data/.onex_state_prod ONEX_BOX_ID: omninode-pc ONEX_RUNTIME_ID: prod-effects @@ -197,6 +199,7 @@ services: environment: ONEX_ENVIRONMENT: prod KAFKA_ENVIRONMENT: prod + ONEX_DATABASE_TOPOLOGY_PROFILE: prod OTEL_SERVICE_NAME: omnimarket-prod-projection-api ports: !override - "${PROD_PROJECTION_API_PORT:-23002}:3002" @@ -236,8 +239,9 @@ services: - "com.omninode.runtime.id=prod-worker" deploy: # OMN-12990: fail-fast on the ledgered config value. Without this override - # the prod worker inherited the base ${WORKER_REPLICAS:-0} default and a - # plain recreate would silently run zero workers. Sourcing + # the prod worker inherited the base compose's bare ${WORKER_REPLICAS:-0} + # default (lane-prefixed and fail-closed since OMN-14968) and a plain + # recreate would silently run zero workers. Sourcing # docker/runtime-policy.env supplies PROD_WORKER_REPLICAS=1; a recreate # without it now fails loudly instead of dropping the worker with no signal. replicas: ${PROD_WORKER_REPLICAS:?runtime policy contract must set PROD_WORKER_REPLICAS (source docker/runtime-policy.env)} diff --git a/docker/docker-compose.runners.yml b/docker/docker-compose.runners.yml index dc949b2bc0..c6ede1bc16 100644 --- a/docker/docker-compose.runners.yml +++ b/docker/docker-compose.runners.yml @@ -1,6 +1,6 @@ # docker-compose.runners.yml # OmniNode self-hosted GitHub Actions runners -# Ticket: OMN-3276 / Epic: OMN-3273 | Scaled: OMN-3714 | Reconciled to live 48: OMN-12582 | Phase-A scale to 64: OMN-14029 +# Ticket: OMN-3276 / Epic: OMN-3273 | Scaled: OMN-3714 | Reconciled to live 48: OMN-12582 | Phase-A scale to 64: OMN-14029 | Saturation scale to 72: OMN-15978 # # Runs the full steady-state runner fleet. The fleet count is owned by # config/runner_fleet.yaml (expected_count). Each runner registers with GitHub @@ -34,10 +34,10 @@ # Verify: cat /proc/sys/fs/inotify/max_user_watches # # Resource allocation per runner (RTX 5090 host with ample RAM): -# mem_limit: 6g - per-runner memory limit (not a reservation); 64 runners +# mem_limit: 6g - per-runner memory limit (not a reservation); 72 runners # share the host's 91 GB, peak usage ~24 GB observed live # memswap_limit: 12g - bounded swap headroom for large CI spikes -# cpus: "2.0" - per-runner CFS quota (not a reservation); 64 runners +# cpus: "2.0" - per-runner CFS quota (not a reservation); 72 runners # share the host's 32 vCPUs (over-subscription tolerated) # pids_limit: 4096 - prevents fork bombs from runaway test processes # @@ -57,7 +57,7 @@ x-runner-base: &runner-base RUNNER_GROUP: omnibase-ci ACTIONS_RUNNER_HOOK_JOB_STARTED: /usr/local/bin/runner-job-started.sh # OMN-14027 C3 — fleet-wide uv download stampede cap. - # 64 runners NAT through one home uplink and share one anonymous egress + # 72 runners NAT through one home uplink and share one anonymous egress # budget. The hardened setup-python-uv composite serializes downloads to 1 in # its OWN step, but the 18 workflows that bypass it (OMN-14193) run raw # `uv sync` at uv's built-in concurrency default — 64x in parallel — and @@ -84,7 +84,7 @@ x-runner-base: &runner-base # fleet at a cache host that is not yet up would force every runner onto the # PyPI fallback. Per the OMN-14027 rollout, devpi is stood up first # (docker/docker-compose.pypi-cache.yml) and ONE canary runner is wired via - # its own container env, leaving the other 63 on direct egress until the + # its own container env, leaving the other 71 on direct egress until the # canary is measured green. PyPI stays as the fallback index so a cache # miss/outage degrades, not fails-closed. Cache endpoint + activation steps: # config/runner_fleet.yaml (pypi_cache:) and @@ -886,12 +886,108 @@ services: - ./runners/runner-job-started.sh:/usr/local/bin/runner-job-started.sh:ro - ./runners/healthcheck.sh:/usr/local/bin/healthcheck.sh:ro - runner-64-creds:/home/runner/.runner-creds + omninode-runner-65: + !!merge <<: *runner-base + container_name: omninode-runner-65 + environment: + !!merge <<: *runner-env + RUNNER_NAME: omninode-runner-65 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./runners/entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + - ./runners/runner-job-started.sh:/usr/local/bin/runner-job-started.sh:ro + - ./runners/healthcheck.sh:/usr/local/bin/healthcheck.sh:ro + - runner-65-creds:/home/runner/.runner-creds + omninode-runner-66: + !!merge <<: *runner-base + container_name: omninode-runner-66 + environment: + !!merge <<: *runner-env + RUNNER_NAME: omninode-runner-66 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./runners/entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + - ./runners/runner-job-started.sh:/usr/local/bin/runner-job-started.sh:ro + - ./runners/healthcheck.sh:/usr/local/bin/healthcheck.sh:ro + - runner-66-creds:/home/runner/.runner-creds + omninode-runner-67: + !!merge <<: *runner-base + container_name: omninode-runner-67 + environment: + !!merge <<: *runner-env + RUNNER_NAME: omninode-runner-67 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./runners/entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + - ./runners/runner-job-started.sh:/usr/local/bin/runner-job-started.sh:ro + - ./runners/healthcheck.sh:/usr/local/bin/healthcheck.sh:ro + - runner-67-creds:/home/runner/.runner-creds + omninode-runner-68: + !!merge <<: *runner-base + container_name: omninode-runner-68 + environment: + !!merge <<: *runner-env + RUNNER_NAME: omninode-runner-68 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./runners/entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + - ./runners/runner-job-started.sh:/usr/local/bin/runner-job-started.sh:ro + - ./runners/healthcheck.sh:/usr/local/bin/healthcheck.sh:ro + - runner-68-creds:/home/runner/.runner-creds + omninode-runner-69: + !!merge <<: *runner-base + container_name: omninode-runner-69 + environment: + !!merge <<: *runner-env + RUNNER_NAME: omninode-runner-69 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./runners/entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + - ./runners/runner-job-started.sh:/usr/local/bin/runner-job-started.sh:ro + - ./runners/healthcheck.sh:/usr/local/bin/healthcheck.sh:ro + - runner-69-creds:/home/runner/.runner-creds + omninode-runner-70: + !!merge <<: *runner-base + container_name: omninode-runner-70 + environment: + !!merge <<: *runner-env + RUNNER_NAME: omninode-runner-70 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./runners/entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + - ./runners/runner-job-started.sh:/usr/local/bin/runner-job-started.sh:ro + - ./runners/healthcheck.sh:/usr/local/bin/healthcheck.sh:ro + - runner-70-creds:/home/runner/.runner-creds + omninode-runner-71: + !!merge <<: *runner-base + container_name: omninode-runner-71 + environment: + !!merge <<: *runner-env + RUNNER_NAME: omninode-runner-71 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./runners/entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + - ./runners/runner-job-started.sh:/usr/local/bin/runner-job-started.sh:ro + - ./runners/healthcheck.sh:/usr/local/bin/healthcheck.sh:ro + - runner-71-creds:/home/runner/.runner-creds + omninode-runner-72: + !!merge <<: *runner-base + container_name: omninode-runner-72 + environment: + !!merge <<: *runner-env + RUNNER_NAME: omninode-runner-72 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./runners/entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + - ./runners/runner-job-started.sh:/usr/local/bin/runner-job-started.sh:ro + - ./runners/healthcheck.sh:/usr/local/bin/healthcheck.sh:ro + - runner-72-creds:/home/runner/.runner-creds # omninode-deploy-runner -- OMN-14889: dedicated runner for the # release-train-lab.yml tag-cut/deploy jobs (cut_release_train_tag.sh, # refresh_stability_lane.sh, refresh_dev_lane.sh). These scripts need real # host-path git access to OMNI_HOME sibling clones (they `git fetch`/ # `checkout --detach`/tag the clones directly and run deploy-runtime.sh's - # BUILD_SOURCE=workspace path against them) -- the shared 48-64 omnibase-ci + # BUILD_SOURCE=workspace path against them) -- the shared 72-runner omnibase-ci # fleet above deliberately does NOT have this (docker.sock only, no # OMNI_HOME bind mount) so that a compromised/buggy CI job from the general # fleet cannot mutate host git state or lane containers. Fork 1 (OMN-14889 @@ -992,7 +1088,7 @@ services: # gid 1001 belongs to a DIFFERENT real user -- daniyal -- so that # would have handed him direct read access to operator secrets), and # does not touch the runner uid/Dockerfile shared with the other - # 64-runner fleet. Re-copied on every container start/recreate, so a + # 72-runner fleet. Re-copied on every container start/recreate, so a # host-side env rotation is picked up on next restart. mkdir -p /home/runner/.runner-creds chown "$${runner_uid}:$${runner_gid}" /home/runner/.runner-creds @@ -1218,3 +1314,19 @@ volumes: name: omninode-runner-63-creds runner-64-creds: name: omninode-runner-64-creds + runner-65-creds: + name: omninode-runner-65-creds + runner-66-creds: + name: omninode-runner-66-creds + runner-67-creds: + name: omninode-runner-67-creds + runner-68-creds: + name: omninode-runner-68-creds + runner-69-creds: + name: omninode-runner-69-creds + runner-70-creds: + name: omninode-runner-70-creds + runner-71-creds: + name: omninode-runner-71-creds + runner-72-creds: + name: omninode-runner-72-creds diff --git a/docker/docker-compose.stability-test.yml b/docker/docker-compose.stability-test.yml index 0cfc7d90be..c4c50b878c 100644 --- a/docker/docker-compose.stability-test.yml +++ b/docker/docker-compose.stability-test.yml @@ -58,6 +58,16 @@ services: - redpanda:33145 - --advertise-rpc-addr - redpanda:33145 + # OMN-14013: this lane's `command:` fully replaces the base redpanda.yaml + # command (Compose Docker merge semantics for a scalar-list key), which + # silently dropped the base's `--set topic_partitions_per_shard=7000` + # startup flag for this lane -- belt #2 of the base's documented 3-belt + # partition-cap design (redpanda/.bootstrap.yaml) was never actually + # applied here. Restore it explicitly, pinned to the lane's raised cap so + # a cold start (fresh volume, before redpanda-partition-cap's post-start + # rpk call completes) never bootstraps at the un-raised 7000 default. + - --set + - topic_partitions_per_shard=15000 ports: !override - *stability_test_redpanda_kafka_port_publish - *stability_test_redpanda_admin_port_publish @@ -68,7 +78,20 @@ services: command: - | set -eu - /usr/bin/rpk -X brokers=redpanda:9092 -X admin.hosts=redpanda:9644 cluster config set topic_partitions_per_shard 7000 + # OMN-14013: this is the belt that ACTUALLY governs the durable cap for + # this lane. warm_broker_topic_provisioning() in deploy-runtime.sh + # force-recreates this one-shot service on EVERY redeploy of the lane + # (refresh_stability_lane.sh -> deploy-runtime.sh --restart), including + # a restart-only deploy that never touches the volume. A raised cap + # applied only via a live `rpk cluster config set` (not committed here) + # is therefore NOT durable -- the next redeploy silently resets it back + # to whatever literal value is hardcoded on this line. Confirmed live + # regression: 2026-07-06 raised to 15000 dynamically -> reset to 7000 + # by a later redeploy -> re-raised to 8000 dynamically on 2026-07-26 as + # a stopgap (see OMN-14013 ticket comments). Pin the value HERE so it + # survives every redeploy; edit this value (and the --set flag on the + # `redpanda` service's command above) together if it must change again. + /usr/bin/rpk -X brokers=redpanda:9092 -X admin.hosts=redpanda:9644 cluster config set topic_partitions_per_shard 15000 /usr/bin/rpk -X brokers=redpanda:9092 -X admin.hosts=redpanda:9644 cluster config set topic_memory_per_partition 1048576 depends_on: redpanda: @@ -109,6 +132,7 @@ services: OMNIMEMORY_MEMGRAPH_HOST: ${STABILITY_TEST_RUNTIME_MAIN_OMNIMEMORY_MEMGRAPH_HOST?runtime policy contract must set STABILITY_TEST_RUNTIME_MAIN_OMNIMEMORY_MEMGRAPH_HOST} ONEX_ENVIRONMENT: stability-test KAFKA_ENVIRONMENT: stability-test + ONEX_DATABASE_TOPOLOGY_PROFILE: stability-test ONEX_TOPIC_PROVISIONER_MAX_PARTITIONS: ${STABILITY_TEST_TOPIC_PROVISIONER_MAX_PARTITIONS:?runtime policy contract must set STABILITY_TEST_TOPIC_PROVISIONER_MAX_PARTITIONS} ONEX_STATE_ROOT: /app/data/.onex_state_stability_test ONEX_STATE_DIR: /app/data/.onex_state_stability_test @@ -139,7 +163,43 @@ services: - runtime_data:/app/data - ${OMNICLAUDE_SKILLS_DIR:-./skills}:/app/skills:ro - "${OMNI_HOME:?OMNI_HOME required for stability session health probes}:${OMNI_HOME:?OMNI_HOME required for stability session health probes}:ro" - labels: + # OMN-15217: strict semantic healthcheck — this lane's green is cited as + # stability-proof for prod promotion (OMN-13418), so it must not be able to + # report healthy while the runtime reports DEGRADED. Verified mask on + # 2026-07-27T12:58Z: `Up 3 hours (healthy)` alongside + # `Runtime health check: status=DEGRADED contracts=296 errors=4`. + # + # --degraded-policy fail flips Docker health when the runtime's own monitor + # reports DEGRADED/CRITICAL. Timings are inherited from the base service + # (interval 30s / timeout 10s / retries 5 / start_period 1800s) and restated + # here so the flap budget is readable at the strict call site: the monitor's + # first verdict lands ~one RUNTIME_HEALTH_CHECK_INTERVAL (300s) after boot + # and an absent verdict passes, so startup cannot flap on it. + # + # Fail-closed on image skew is intentional: if the container runs an image + # built before this check existed, the interpreter exits non-zero and the + # container reads unhealthy. A stability lane whose image predates the + # honest check must not be citable as proof. + healthcheck: + test: + - CMD + - python + - /usr/local/bin/onex-container-healthcheck + - --degraded-policy + - fail + interval: 30s + timeout: 10s + retries: 5 + start_period: 1800s + # OMN-15217: autoheal is deliberately NOT enabled on this lane's runtime + # containers. Compose appends label sequences, so the base service's + # `autoheal=true` would survive this block without `!override`. With the + # strict healthcheck above, "unhealthy" now means "semantically degraded", + # and semantic degradation is usually restart-immune — four contracts that + # fail to import will fail to import again — so leaving autoheal armed would + # convert an honest signal into a restart loop and destroy the forensic + # state this lane exists to preserve. Restore by removing `!override`. + labels: !override - "com.omninode.lane=stability-test" - "com.omninode.ticket=OMN-10281" - "com.omninode.runtime.address=runtime://omninode-pc/stability-test/main" @@ -155,6 +215,7 @@ services: OMNIMEMORY_MEMGRAPH_HOST: ${STABILITY_TEST_RUNTIME_EFFECTS_OMNIMEMORY_MEMGRAPH_HOST?runtime policy contract must set STABILITY_TEST_RUNTIME_EFFECTS_OMNIMEMORY_MEMGRAPH_HOST} ONEX_ENVIRONMENT: stability-test KAFKA_ENVIRONMENT: stability-test + ONEX_DATABASE_TOPOLOGY_PROFILE: stability-test ONEX_TOPIC_PROVISIONER_MAX_PARTITIONS: ${STABILITY_TEST_TOPIC_PROVISIONER_MAX_PARTITIONS:?runtime policy contract must set STABILITY_TEST_TOPIC_PROVISIONER_MAX_PARTITIONS} ONEX_STATE_ROOT: /app/data/.onex_state_stability_test ONEX_STATE_DIR: /app/data/.onex_state_stability_test @@ -182,7 +243,43 @@ services: - effects_data:/app/data - ${OMNICLAUDE_SKILLS_DIR:-./skills}:/app/skills:ro - "${OMNI_HOME:?OMNI_HOME required for stability session health probes}:${OMNI_HOME:?OMNI_HOME required for stability session health probes}:ro" - labels: + # OMN-15217: strict semantic healthcheck — this lane's green is cited as + # stability-proof for prod promotion (OMN-13418), so it must not be able to + # report healthy while the runtime reports DEGRADED. Verified mask on + # 2026-07-27T12:58Z: `Up 3 hours (healthy)` alongside + # `Runtime health check: status=DEGRADED contracts=296 errors=4`. + # + # --degraded-policy fail flips Docker health when the runtime's own monitor + # reports DEGRADED/CRITICAL. Timings are inherited from the base service + # (interval 30s / timeout 10s / retries 5 / start_period 1800s) and restated + # here so the flap budget is readable at the strict call site: the monitor's + # first verdict lands ~one RUNTIME_HEALTH_CHECK_INTERVAL (300s) after boot + # and an absent verdict passes, so startup cannot flap on it. + # + # Fail-closed on image skew is intentional: if the container runs an image + # built before this check existed, the interpreter exits non-zero and the + # container reads unhealthy. A stability lane whose image predates the + # honest check must not be citable as proof. + healthcheck: + test: + - CMD + - python + - /usr/local/bin/onex-container-healthcheck + - --degraded-policy + - fail + interval: 30s + timeout: 10s + retries: 5 + start_period: 1800s + # OMN-15217: autoheal is deliberately NOT enabled on this lane's runtime + # containers. Compose appends label sequences, so the base service's + # `autoheal=true` would survive this block without `!override`. With the + # strict healthcheck above, "unhealthy" now means "semantically degraded", + # and semantic degradation is usually restart-immune — four contracts that + # fail to import will fail to import again — so leaving autoheal armed would + # convert an honest signal into a restart loop and destroy the forensic + # state this lane exists to preserve. Restore by removing `!override`. + labels: !override - "com.omninode.lane=stability-test" - "com.omninode.ticket=OMN-10281" - "com.omninode.runtime.address=runtime://omninode-pc/stability-test/effects" @@ -193,6 +290,7 @@ services: environment: ONEX_ENVIRONMENT: stability-test KAFKA_ENVIRONMENT: stability-test + ONEX_DATABASE_TOPOLOGY_PROFILE: stability-test OTEL_SERVICE_NAME: omnimarket-stability-test-projection-api ports: !override - "${STABILITY_TEST_PROJECTION_API_PORT:-13002}:3002" @@ -236,14 +334,44 @@ services: - stability_test_worker_data:/app/data - ${OMNICLAUDE_SKILLS_DIR:-./skills}:/app/skills:ro - "${OMNI_HOME:?OMNI_HOME required for stability session health probes}:${OMNI_HOME:?OMNI_HOME required for stability session health probes}:ro" - labels: + # OMN-15217: strict semantic healthcheck — see omninode-runtime above for the + # full rationale. The worker is not an afterthought here: it reproduces the + # masking defect independently. Verified live 2026-07-27T14:18Z on this lane, + # `omninode-stability-test-runtime-worker`: `Up 4 hours (healthy)` with + # healthcheck `["CMD","curl","-sf","http://localhost:8085/health"]` while its + # own monitor logged `Runtime health check: status=DEGRADED contracts=5 + # errors=4`. Leaving one of the lane's three runtime containers on the + # shallow probe would leave the lane able to lie by exactly the mechanism + # this ticket closes. + # + # Base runtime-worker inherits interval 30s / timeout 10s / retries 5, but + # start_period 1200s (not 1800s) — restated here so the flap budget is + # readable at the strict call site rather than inferred from the base. + healthcheck: + test: + - CMD + - python + - /usr/local/bin/onex-container-healthcheck + - --degraded-policy + - fail + interval: 30s + timeout: 10s + retries: 5 + start_period: 1200s + # OMN-15217: autoheal off for the same reason as the other two runtime + # containers. `autoheal=true` was confirmed live on this container's + # resolved label set (docker inspect, 2026-07-27T14:18Z), so a plain + # `labels:` block here would keep it armed — compose appends label + # sequences. Restore by removing `!override`. + labels: !override - "com.omninode.lane=stability-test" - "com.omninode.ticket=OMN-10281" - "com.omninode.runtime.address=runtime://omninode-pc/stability-test/worker" - "com.omninode.runtime.id=stability-test-worker" deploy: # OMN-12990: fail-fast on the ledgered config value, NOT a soft :-1 - # default. The base compose sets ${WORKER_REPLICAS:-0}; a plain recreate + # default (the base compose kept a bare ${WORKER_REPLICAS:-0} until + # OMN-14968 made it lane-prefixed and fail-closed too); a plain recreate # that omits the policy env would silently drop the worker. Sourcing # docker/runtime-policy.env (rendered from runtime_policy.contract.yaml) # supplies STABILITY_TEST_WORKER_REPLICAS=1; a recreate without it now diff --git a/docker/domain-adapter-proof/Dockerfile b/docker/domain-adapter-proof/Dockerfile new file mode 100644 index 0000000000..646e29b643 --- /dev/null +++ b/docker/domain-adapter-proof/Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile:1.7 +# SPDX-License-Identifier: MIT + +ARG UV_VERSION=0.11.8 +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv-bin + +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src \ + UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_HTTP_TIMEOUT=600 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential git libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=uv-bin /uv /uvx /usr/local/bin/ +WORKDIR /app + +# Dependency layer is rebuilt only when the exact lock surface changes. +COPY pyproject.toml uv.lock README.md LICENSE ./ +RUN uv sync --frozen --no-dev --no-install-project + +# Copy the actual runtime source and typed topology fixture after dependencies. +COPY src/ ./src/ +COPY tests/fixtures/application_relation_ownership/topology.yaml ./proof/topology.yaml +COPY docker/domain-adapter-proof/prove.py ./proof/prove.py + +USER 65534:65534 +CMD ["/app/.venv/bin/python", "/app/proof/prove.py"] diff --git a/docker/domain-adapter-proof/compose.yml b/docker/domain-adapter-proof/compose.yml new file mode 100644 index 0000000000..2bf7c44e9e --- /dev/null +++ b/docker/domain-adapter-proof/compose.yml @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: MIT +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_DB: omnidash_analytics + POSTGRES_PASSWORD: postgres-proof-only + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d omnidash_analytics"] + interval: 1s + timeout: 3s + retries: 30 + proof: + build: + context: ../.. + dockerfile: docker/domain-adapter-proof/Dockerfile + depends_on: + postgres: + condition: service_healthy + command: + - /app/.venv/bin/python + - /app/proof/prove.py + - --admin-dsn + - postgresql://postgres:postgres-proof-only@postgres:5432/omnidash_analytics + - --tenant-dsn + - postgresql://tenant_projection_writer:domain-adapter-proof-only@postgres:5432/omnidash_analytics + - --internal-dsn + - postgresql://omninode_runtime:domain-adapter-proof-only@postgres:5432/omnidash_analytics + - --catalog-dsn + - postgresql://app_dashboard:domain-adapter-proof-only@postgres:5432/omnidash_analytics diff --git a/docker/domain-adapter-proof/prove.py b/docker/domain-adapter-proof/prove.py new file mode 100644 index 0000000000..b1e7c8bd7b --- /dev/null +++ b/docker/domain-adapter-proof/prove.py @@ -0,0 +1,582 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Rebuilt-image PostgreSQL 16 domain-adapter proof for OMN-15421.""" + +from __future__ import annotations + +import argparse +import sys +from collections.abc import Callable +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Literal +from unittest.mock import patch +from uuid import UUID, uuid4 + +import psycopg2 +import psycopg2.extras + +from omnibase_core.crypto.crypto_ed25519_signer import generate_keypair +from omnibase_core.models.contracts.subcontracts.model_db_table_declaration import ( + ModelDbTableDeclaration, +) +from omnibase_core.models.core.model_deployment_topology import ModelDeploymentTopology +from omnibase_core.models.envelope.model_message_envelope import ModelMessageEnvelope +from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope +from omnibase_infra.errors.error_projection import ProjectionTenantContextError +from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + ProjectionDatabaseTarget, + _build_projection_db_adapter, + _resolve_projection_database_target, +) +from omnibase_infra.runtime.projection_tenant_authority import ( + VerifiedProjectionTenantAuthority, + verify_signed_projection_tenant_authority, +) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--admin-dsn", required=True) + parser.add_argument("--tenant-dsn", required=True) + parser.add_argument("--internal-dsn", required=True) + parser.add_argument("--catalog-dsn", required=True) + return parser.parse_args() + + +_ARGS = _parse_args() +ADMIN_DSN = _ARGS.admin_dsn +TENANT_DSN = _ARGS.tenant_dsn +INTERNAL_DSN = _ARGS.internal_dsn +CATALOG_DSN = _ARGS.catalog_dsn +DATABASE = "omnidash_analytics" +TENANT_ROLE = "tenant_projection_writer" +INTERNAL_ROLE = "omninode_runtime" +CATALOG_ROLE = "app_dashboard" +ROLE_PASSWORD = "domain-adapter-proof-only" # pragma: allowlist secret +TENANT_A = UUID("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa") +TENANT_B = UUID("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb") +TENANT_TABLE = "delegation_events" +INTERNAL_TABLE = "future_internal_projection" +CATALOG_TABLE = "plan_tiers" + +psycopg2.extras.register_uuid() + +TOPOLOGY = ModelDeploymentTopology.from_yaml(Path(__file__).with_name("topology.yaml")) + + +class _KeyProvider: + def __init__(self, runtime_id: str, public_key: bytes) -> None: + self._keys = {runtime_id: public_key} + + def get_public_key(self, runtime_id: str) -> bytes | None: + return self._keys.get(runtime_id) + + def register_key(self, runtime_id: str, public_key: bytes) -> None: + self._keys[runtime_id] = public_key + + def has_key(self, runtime_id: str) -> bool: + return runtime_id in self._keys + + def list_runtime_ids(self) -> list[str]: + return sorted(self._keys) + + +@dataclass(frozen=True) +class _TenantBindingResolver: + runtime_id: str + realm: str + bus_id: str + tenant_id: UUID + + def resolve_tenant_id( + self, + *, + runtime_id: str, + realm: str, + bus_id: str, + ) -> UUID | None: + if (runtime_id, realm, bus_id) != ( + self.runtime_id, + self.realm, + self.bus_id, + ): + return None + return self.tenant_id + + +@dataclass(frozen=True) +class _SignedFixture: + envelope: ModelMessageEnvelope[ModelEventEnvelope[dict[str, object]]] + key_provider: _KeyProvider + resolver: _TenantBindingResolver + + def verify(self) -> VerifiedProjectionTenantAuthority: + return verify_signed_projection_tenant_authority( + self.envelope, + self.key_provider, + self.resolver, + ) + + +def _signed_fixture( + tenant_value: str, + *, + bound_tenant: UUID, +) -> _SignedFixture: + runtime_id = "tenant-gateway-proof" + realm = "docker-proof" + bus_id = "domain-adapter-proof" + event = ModelEventEnvelope[dict[str, object]]( + payload={"proof": True}, + correlation_id=uuid4(), + ) + keypair = generate_keypair() + envelope = ModelMessageEnvelope[ + ModelEventEnvelope[dict[str, object]] + ].create_signed( + realm=realm, + runtime_id=runtime_id, + bus_id=bus_id, + tenant_id=tenant_value, + payload=event, + trace_id=event.correlation_id, + private_key=keypair.private_key_bytes, + emitted_at=datetime.now(UTC), + ) + return _SignedFixture( + envelope=envelope, + key_provider=_KeyProvider(runtime_id, keypair.public_key_bytes), + resolver=_TenantBindingResolver(runtime_id, realm, bus_id, bound_tenant), + ) + + +def _verified_dispatch( + tenant_id: UUID, +) -> tuple[VerifiedProjectionTenantAuthority, ModelEventEnvelope[dict[str, object]]]: + fixture = _signed_fixture(str(tenant_id), bound_tenant=tenant_id) + return fixture.verify(), fixture.envelope.payload + + +def _target( + table: str, + schema: str, + *, + access: Literal["read", "write", "read_write"] = "read_write", + catalog_read_binding: str | None = None, + catalog_write_binding: str | None = None, +) -> ProjectionDatabaseTarget: + declaration = ModelDbTableDeclaration( + name=table, + database_ref="application", + schema=schema, + migration=f"proof/{schema}/{table}.sql", + access=access, + role=f"{table}_proof", + ) + return _resolve_projection_database_target( + (declaration,), + TOPOLOGY, + catalog_read_binding=catalog_read_binding, + catalog_write_binding=catalog_write_binding, + ) + + +def _adapter( + target: ProjectionDatabaseTarget, + authority: VerifiedProjectionTenantAuthority | None = None, + event: ModelEventEnvelope[dict[str, object]] | None = None, +) -> object: + urls = { + "tenant_projection": TENANT_DSN, + "omninode_runtime_service": INTERNAL_DSN, + "app_dashboard": CATALOG_DSN, + } + return _build_projection_db_adapter( + {binding.binding_ref: urls[binding.binding_ref] for binding in target.bindings}, + target, + authority, + event, + ) + + +def _raises( + error_type: type[BaseException], + action: Callable[[], object], +) -> BaseException: + try: + action() + except error_type as exc: + return exc + raise AssertionError(f"Expected {error_type.__name__}") + + +def _admin_rows( + sql: str, + params: tuple[object, ...] | None = None, +) -> list[tuple[object, ...]]: + conn = psycopg2.connect(ADMIN_DSN) + conn.autocommit = True + try: + with conn.cursor() as cursor: + cursor.execute(sql, params) + return list(cursor.fetchall()) + finally: + conn.close() + + +def _initialize_database() -> None: + conn = psycopg2.connect(ADMIN_DSN) + conn.autocommit = True + with conn.cursor() as cursor: + for role in (TENANT_ROLE, INTERNAL_ROLE, CATALOG_ROLE): + cursor.execute( + f"CREATE ROLE {role} LOGIN PASSWORD %s NOSUPERUSER NOBYPASSRLS", + (ROLE_PASSWORD,), + ) + cursor.execute(f"GRANT CONNECT ON DATABASE {DATABASE} TO {role}") + cursor.execute("CREATE SCHEMA tenant") + cursor.execute("CREATE SCHEMA omninode_internal") + cursor.execute("CREATE SCHEMA platform_catalog") + cursor.execute( + f""" + CREATE TABLE tenant.{TENANT_TABLE} ( + correlation_id UUID PRIMARY KEY, + task_type TEXT NOT NULL, + tenant_id UUID NOT NULL + ); + ALTER TABLE tenant.{TENANT_TABLE} ENABLE ROW LEVEL SECURITY; + ALTER TABLE tenant.{TENANT_TABLE} FORCE ROW LEVEL SECURITY; + CREATE POLICY tenant_isolation ON tenant.{TENANT_TABLE} + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)::uuid) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)::uuid); + CREATE TABLE omninode_internal.{INTERNAL_TABLE} ( + correlation_id UUID PRIMARY KEY, + source_tenant_id UUID NULL, + status TEXT NOT NULL + ); + CREATE TABLE platform_catalog.{CATALOG_TABLE} ( + tier_id TEXT PRIMARY KEY, + display_name TEXT NOT NULL + ); + """ + ) + cursor.execute(f"GRANT USAGE ON SCHEMA tenant TO {TENANT_ROLE}") + cursor.execute( + f"GRANT SELECT, INSERT, UPDATE ON tenant.{TENANT_TABLE} TO {TENANT_ROLE}" + ) + cursor.execute(f"GRANT USAGE ON SCHEMA omninode_internal TO {INTERNAL_ROLE}") + cursor.execute( + "GRANT SELECT, INSERT, UPDATE ON " + f"omninode_internal.{INTERNAL_TABLE} TO {INTERNAL_ROLE}" + ) + cursor.execute(f"GRANT USAGE ON SCHEMA platform_catalog TO {CATALOG_ROLE}") + cursor.execute( + f"GRANT SELECT ON platform_catalog.{CATALOG_TABLE} TO {CATALOG_ROLE}" + ) + cursor.execute( + "INSERT INTO platform_catalog.plan_tiers VALUES ('beta', 'Beta')" + ) + conn.close() + + +def _assert_database_identities() -> None: + for dsn, principal in ( + (TENANT_DSN, TENANT_ROLE), + (INTERNAL_DSN, INTERNAL_ROLE), + (CATALOG_DSN, CATALOG_ROLE), + ): + conn = psycopg2.connect(dsn) + try: + with conn.cursor() as cursor: + cursor.execute("SELECT current_user, current_database()") + assert cursor.fetchone() == (principal, DATABASE) + finally: + conn.close() + + +def _prove_real_rls_with_check() -> None: + """Prove the database itself rejects tenant-B rows under tenant-A GUC.""" + conn = psycopg2.connect(TENANT_DSN) + try: + with conn.cursor() as cursor: + _raises( + psycopg2.errors.InsufficientPrivilege, + lambda: cursor.execute( + "INSERT INTO tenant.delegation_events VALUES (%s, %s, %s)", + (uuid4(), "unset-context", TENANT_A), + ), + ) + conn.rollback() + + with conn.cursor() as cursor: + cursor.execute("SET LOCAL app.tenant_id = %s", (str(TENANT_A),)) + _raises( + psycopg2.errors.InsufficientPrivilege, + lambda: cursor.execute( + "INSERT INTO tenant.delegation_events VALUES (%s, %s, %s)", + (uuid4(), "wrong-insert", TENANT_B), + ), + ) + conn.rollback() + + correlation_id = uuid4() + with conn.cursor() as cursor: + cursor.execute("SET LOCAL app.tenant_id = %s", (str(TENANT_A),)) + cursor.execute( + "INSERT INTO tenant.delegation_events VALUES (%s, %s, %s)", + (correlation_id, "valid-a", TENANT_A), + ) + conn.commit() + with conn.cursor() as cursor: + cursor.execute("SET LOCAL app.tenant_id = %s", (str(TENANT_A),)) + _raises( + psycopg2.errors.InsufficientPrivilege, + lambda: cursor.execute( + "UPDATE tenant.delegation_events SET tenant_id = %s " + "WHERE correlation_id = %s", + (TENANT_B, correlation_id), + ), + ) + conn.rollback() + finally: + conn.close() + + +def _prove_rollback_clears_reused_connection( + tenant_target: ProjectionDatabaseTarget, +) -> None: + """Force SQL failure after SET LOCAL, then reuse the connection for B.""" + shared = psycopg2.connect(TENANT_DSN) + authority_a, event_a = _verified_dispatch(TENANT_A) + authority_b, event_b = _verified_dispatch(TENANT_B) + try: + with patch("psycopg2.connect", return_value=shared): + adapter_a = _adapter(tenant_target, authority_a, event_a) + _raises( + psycopg2.errors.NotNullViolation, + lambda: adapter_a.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": uuid4()}, + ), + ) + with shared.cursor() as cursor: + cursor.execute("SELECT current_setting('app.tenant_id', true)") + assert cursor.fetchone()[0] in (None, "") + adapter_b = _adapter(tenant_target, authority_b, event_b) + assert adapter_b.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": uuid4(), "task_type": "after-rollback"}, + ) + with shared.cursor() as cursor: + cursor.execute("SELECT current_setting('app.tenant_id', true)") + assert cursor.fetchone()[0] in (None, "") + finally: + shared.close() + + +def _prove_signature_failures() -> None: + malformed = _signed_fixture("not-a-uuid", bound_tenant=TENANT_A) + sentinel = _signed_fixture(str(UUID(int=0)), bound_tenant=UUID(int=0)) + _raises(ProjectionTenantContextError, malformed.verify) + _raises(ProjectionTenantContextError, sentinel.verify) + + fixture = _signed_fixture(str(TENANT_A), bound_tenant=TENANT_A) + tampered = fixture.envelope.model_copy(update={"tenant_id": str(TENANT_B)}) + _raises( + ProjectionTenantContextError, + lambda: verify_signed_projection_tenant_authority( + tampered, + fixture.key_provider, + _TenantBindingResolver( + fixture.envelope.runtime_id, + fixture.envelope.realm, + fixture.envelope.bus_id, + TENANT_B, + ), + ), + ) + wrong_binding = _TenantBindingResolver( + fixture.envelope.runtime_id, + fixture.envelope.realm, + fixture.envelope.bus_id, + TENANT_B, + ) + _raises( + ProjectionTenantContextError, + lambda: verify_signed_projection_tenant_authority( + fixture.envelope, + fixture.key_provider, + wrong_binding, + ), + ) + + +def main() -> None: + _initialize_database() + _assert_database_identities() + tenant_target = _target(TENANT_TABLE, "tenant") + internal_target = _target(INTERNAL_TABLE, "omninode_internal") + catalog_target = _target( + CATALOG_TABLE, + "platform_catalog", + access="read", + catalog_read_binding="app_dashboard", + ) + assert [domain.value for domain in tenant_target.domains] == ["TENANT"] + assert [domain.value for domain in internal_target.domains] == ["OMNINODE_INTERNAL"] + assert [domain.value for domain in catalog_target.domains] == ["PLATFORM_CATALOG"] + + _prove_signature_failures() + _prove_real_rls_with_check() + + authority_a, event_a = _verified_dispatch(TENANT_A) + tenant_a = _adapter(tenant_target, authority_a, event_a) + correlation_a = uuid4() + try: + assert tenant_a.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": correlation_a, "task_type": "signed-a"}, + ) + found = tenant_a.query(TENANT_TABLE, {"correlation_id": correlation_a}) + assert found[0]["correlation_id"] == correlation_a + assert isinstance(found[0]["correlation_id"], UUID) + assert found[0]["tenant_id"] == TENANT_A + finally: + tenant_a.close() + + authority_b, event_b = _verified_dispatch(TENANT_B) + tenant_b = _adapter(tenant_target, authority_b, event_b) + try: + assert tenant_b.query(TENANT_TABLE, {"correlation_id": correlation_a}) == [] + finally: + tenant_b.close() + + with patch("psycopg2.connect") as connect: + missing = _adapter(tenant_target) + _raises( + ProjectionTenantContextError, + lambda: missing.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": uuid4(), "task_type": "missing"}, + ), + ) + mismatch = _adapter(tenant_target, authority_a, event_a) + _raises( + ProjectionTenantContextError, + lambda: mismatch.upsert( + TENANT_TABLE, + "correlation_id", + { + "correlation_id": uuid4(), + "task_type": "wrong-row-tenant", + "tenant_id": TENANT_B, + }, + ), + ) + connect.assert_not_called() + + _prove_rollback_clears_reused_connection(tenant_target) + + internal = _adapter(internal_target) + internal_id = uuid4() + try: + assert internal.upsert( + INTERNAL_TABLE, + "correlation_id", + { + "correlation_id": internal_id, + "source_tenant_id": TENANT_A, + "status": "complete", + }, + ) + assert ( + internal.query(INTERNAL_TABLE, {"correlation_id": internal_id})[0]["status"] + == "complete" + ) + internal_connection = next(iter(internal._connections.values())) + with internal_connection.cursor() as cursor: + cursor.execute("SELECT current_setting('app.tenant_id', true)") + assert cursor.fetchone()[0] in (None, "") + _raises( + ValueError, + lambda: internal.upsert( + INTERNAL_TABLE, + "correlation_id", + { + "correlation_id": uuid4(), + "tenant_id": TENANT_A, + "status": "invalid", + }, + ), + ) + finally: + internal.close() + + catalog = _adapter(catalog_target) + try: + assert catalog.query(CATALOG_TABLE)[0]["tier_id"] == "beta" + _raises( + PermissionError, + lambda: catalog.upsert( + CATALOG_TABLE, + "tier_id", + {"tier_id": "pro", "display_name": "Pro"}, + ), + ) + finally: + catalog.close() + _raises( + ValueError, + lambda: _target(CATALOG_TABLE, "platform_catalog", access="write"), + ) + + for dsn, sql in ( + (TENANT_DSN, f"SELECT * FROM omninode_internal.{INTERNAL_TABLE}"), + (INTERNAL_DSN, "SELECT * FROM tenant.delegation_events"), + ( + CATALOG_DSN, + "INSERT INTO platform_catalog.plan_tiers VALUES ('x', 'X')", + ), + ): + conn = psycopg2.connect(dsn) + conn.autocommit = True + try: + _raises( + psycopg2.errors.InsufficientPrivilege, + lambda conn=conn, sql=sql: conn.cursor().execute(sql), + ) + finally: + conn.close() + + miswired = _build_projection_db_adapter( + {"tenant_projection": INTERNAL_DSN}, + tenant_target, + authority_a, + event_a, + ) + _raises( + PermissionError, + lambda: miswired.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": uuid4(), "task_type": "miswired"}, + ), + ) + + rows = _admin_rows( + "SELECT tenant_id FROM tenant.delegation_events ORDER BY tenant_id" + ) + assert TENANT_A in {row[0] for row in rows} + assert TENANT_B in {row[0] for row in rows} + sys.stdout.write("OMN-15421 PostgreSQL 16 rebuilt-container proof: PASS\n") + + +if __name__ == "__main__": + main() diff --git a/docker/entrypoint-runtime.sh b/docker/entrypoint-runtime.sh index 0fc6291027..bf3d0af27e 100755 --- a/docker/entrypoint-runtime.sh +++ b/docker/entrypoint-runtime.sh @@ -161,6 +161,29 @@ if [ -n "${ONEX_SECRET_RESOLVER_CONFIG_PATH:-}" ]; then python -m omnibase_infra.runtime.render_secret_resolver_config fi +if [ -n "${DELEGATION_ROUTING_TIERS_PATH:-}" ] && [ ! -f "${DELEGATION_ROUTING_TIERS_PATH}" ]; then + # OMN-15628 remediation: DELEGATION_ROUTING_TIERS_PATH is pinned in the k8s + # manifest as a literal string that embeds the venv's Python minor version + # (e.g. .../python3.12/site-packages/omnimarket/configs/routing_tiers.yaml). + # A base-image Python version bump silently breaks that pin with no signal + # until the routing reducer fails closed at first use. Self-heal here by + # re-deriving the path from the installed omnimarket package's OWN + # location, which always matches whatever Python is actually running in + # this image -- never a hardcoded guess. This is a best-effort correction, + # not a silent-fallback: if re-derivation also fails to find a real file, + # the original (possibly-stale) pinned value is left untouched and the + # routing reducer still fails closed with an attributable error, per + # CLAUDE.md rule 8. + echo "[entrypoint] WARNING: DELEGATION_ROUTING_TIERS_PATH=${DELEGATION_ROUTING_TIERS_PATH} does not exist -- attempting to re-derive from the installed omnimarket package" + RESOLVED_TIERS_PATH=$(python -c "import pathlib, omnimarket; print(pathlib.Path(omnimarket.__file__).resolve().parent / 'configs' / 'routing_tiers.yaml')" 2>/dev/null) || RESOLVED_TIERS_PATH="" + if [ -n "${RESOLVED_TIERS_PATH}" ] && [ -f "${RESOLVED_TIERS_PATH}" ]; then + echo "[entrypoint] Re-derived DELEGATION_ROUTING_TIERS_PATH=${RESOLVED_TIERS_PATH}" + export DELEGATION_ROUTING_TIERS_PATH="${RESOLVED_TIERS_PATH}" + else + echo "[entrypoint] WARNING: could not re-derive a valid routing_tiers.yaml path -- leaving DELEGATION_ROUTING_TIERS_PATH as pinned; the routing reducer fails closed with an attributable error if it truly does not exist (OMN-15628)" + fi +fi + echo "[entrypoint] Starting runtime kernel..." exec "$@" diff --git a/docker/env-example-full.txt b/docker/env-example-full.txt index dc27f52554..25f69a8b80 100644 --- a/docker/env-example-full.txt +++ b/docker/env-example-full.txt @@ -180,11 +180,20 @@ POSTGRES_PASSWORD=__REPLACE_WITH_SECURE_PASSWORD__ # Consumer group ID for effects runtime # ONEX_EFFECTS_GROUP_ID=onex-runtime-effects +# HMAC-SHA256 key shared by runtime-effects and deploy-agent. Required for +# signed rebuild commands; generate once and store in ~/.omnibase/.env. +# DEPLOY_AGENT_HMAC_SECRET=__REPLACE_WITH_SECURE_RANDOM_SECRET__ + # ============================================================================= # Worker Runtime Service Configuration # ============================================================================= -# Number of worker replicas (for horizontal scaling) -# WORKER_REPLICAS=2 +# Number of worker replicas (for horizontal scaling). OMN-14968: this is the +# lane-prefixed DEV_WORKER_REPLICAS and it is `:?`-required in +# docker-compose.infra.yml -- the compose render FAILS when it is unset rather +# than silently scaling the worker to zero. The ledgered value is rendered into +# docker/runtime-policy.env from contracts/services/runtime_policy.contract.yaml; +# set it here only to override that pin. +# DEV_WORKER_REPLICAS=2 # Kafka topics for worker runtime # ONEX_WORKER_INPUT_TOPIC=worker-requests @@ -300,6 +309,9 @@ POSTGRES_PASSWORD=__REPLACE_WITH_SECURE_PASSWORD__ # External port mappings for infrastructure services # These map host ports to internal container ports # POSTGRES_EXTERNAL_PORT=5436 +# OMN-15173: DEV_REDPANDA_ADVERTISE_HOST is REQUIRED (no silent default) - +# compose config render fails fast when unset. 'localhost' is only correct +# for pure single-host dev; set the reachable host/IP for off-host clients. # DEV_REDPANDA_ADVERTISE_HOST=localhost # REDPANDA_EXTERNAL_PORT=19092 # REDPANDA_PANDAPROXY_PORT=18082 diff --git a/docker/gateway/beta-gateway-canary.yaml b/docker/gateway/beta-gateway-canary.yaml new file mode 100644 index 0000000000..e973a6adac --- /dev/null +++ b/docker/gateway/beta-gateway-canary.yaml @@ -0,0 +1,51 @@ +# Resolved materialization of node_bus_forwarder_effect/contract.yaml for the +# one approved staging canary tenant. No secret values belong in this file. +forwarder: + tenant_identity: + tenant_id: "79afa726-3852-464f-b7a4-d4b8b9c75ee7" + tenant_slug: "beta-gateway-canary-79afa7263852" + principal_id: "t-79afa7263852464fb7a4d4b8b9c75ee7" + cloud_bus: + broker_provider_id: "22222222-2222-2222-2222-222222222222" + cloud_broker_ref: "gateway.cloud.kafka.broker" + cloud_auth_ref: "gateway.cloud.kafka.msk_iam" + acl_provisioner_ref: "gateway.cloud.kafka.authorization" + msk_region_ref: "gateway.cloud.kafka.msk_region" + security_protocol: "SASL_SSL" + sasl_mechanism: "AWS_MSK_IAM" + local_transport_flavor: "containerized" + mirror_topic_set: "node_bus_forwarder_effect" + canary_topic_set: "node_bus_forwarder_effect" + heartbeat_interval_seconds: 15 + max_silence_window_seconds: 60 + lag_threshold_messages: 500 + lag_threshold_seconds: 120 + drain_deadline_seconds: 30 + dedupe_store_path: "/app/data/gateway/delivery.sqlite3" + dedupe_retention_hours: 24 + forward_retry_initial_seconds: 1 + forward_retry_max_seconds: 30 +local_bus: + bootstrap_servers: "redpanda:9092" + environment: "beta-gateway-local" + timeout_seconds: 30 + # OMN-15781: "latest" silently drops any backlog produced while the + # consumer group was unjoined (crash/LeaveGroup/cold restart before + # rejoin); enable_auto_commit=false alone does not protect against this + # since it only preserves offsets already inside the read window. The + # runtime-config model_validator (ModelGatewayForwarderRuntimeConfig) + # fails closed on anything other than "earliest" on either leg. + auto_offset_reset: "earliest" + enable_auto_commit: false + security_protocol: "PLAINTEXT" +cloud_bus: + bootstrap_servers: "b-1.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com:9098,b-2.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com:9098" + environment: "beta-gateway-cloud" + timeout_seconds: 60 + # OMN-15781: see local_bus.auto_offset_reset comment above -- same hazard, + # both legs are pull-based at-least-once consumers. + auto_offset_reset: "earliest" + enable_auto_commit: false + security_protocol: "SASL_SSL" + sasl_mechanism: "AWS_MSK_IAM" + msk_region: "us-east-1" diff --git a/docker/gateway/onex-gateway-forwarder.service b/docker/gateway/onex-gateway-forwarder.service new file mode 100644 index 0000000000..f7202c50a7 --- /dev/null +++ b/docker/gateway/onex-gateway-forwarder.service @@ -0,0 +1,22 @@ +[Unit] +Description=OmniNode dev hybrid gateway forwarder +Documentation=https://github.com/OmniNode-ai/omnibase_infra/tree/dev/docker/gateway +Wants=network-online.target +After=network-online.target docker.service tailscaled.service +Requires=docker.service + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/opt/omninode/gateway +EnvironmentFile=/etc/omninode/gateway/gateway.env +ExecStartPre=/usr/bin/grep -Eq "^GATEWAY_IMAGE=sha256:[0-9a-f]{64}$" /etc/omninode/gateway/gateway.env +ExecStartPre=/usr/bin/docker image inspect ${GATEWAY_IMAGE} +ExecStart=/usr/bin/docker compose --project-name omninode-gateway --env-file /etc/omninode/gateway/gateway.env -f /opt/omninode/gateway/docker-compose.gateway.yml up -d --no-build --wait --wait-timeout 120 gateway-forwarder +ExecReload=/usr/bin/docker compose --project-name omninode-gateway --env-file /etc/omninode/gateway/gateway.env -f /opt/omninode/gateway/docker-compose.gateway.yml up -d --no-build --force-recreate --wait --wait-timeout 120 gateway-forwarder +ExecStop=/usr/bin/docker compose --project-name omninode-gateway --env-file /etc/omninode/gateway/gateway.env -f /opt/omninode/gateway/docker-compose.gateway.yml stop --timeout 30 gateway-forwarder +TimeoutStartSec=180 +TimeoutStopSec=60 + +[Install] +WantedBy=multi-user.target diff --git a/docker/keycloak/desired-clients.json b/docker/keycloak/desired-clients.json index 97f290b0e0..9b34912ade 100644 --- a/docker/keycloak/desired-clients.json +++ b/docker/keycloak/desired-clients.json @@ -50,6 +50,18 @@ "jsonType.label": "String" } }, + { + "name": "principal_id", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "config": { + "user.attribute": "principal_id", + "claim.name": "principal_id", + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true", + "jsonType.label": "String" + } + }, { "name": "onex-api-audience", "protocolMapper": "oidc-audience-mapper", diff --git a/docker/legacy-rds-fixture/Dockerfile b/docker/legacy-rds-fixture/Dockerfile new file mode 100644 index 0000000000..ac17a8360c --- /dev/null +++ b/docker/legacy-rds-fixture/Dockerfile @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +FROM postgres:16-alpine AS fresh + +COPY docker/legacy-rds-fixture/cluster-seed.sql /docker-entrypoint-initdb.d/00-cluster-seed.sql + +FROM postgres:16-alpine AS legacy + +COPY docker/legacy-rds-fixture/cluster-seed.sql /docker-entrypoint-initdb.d/00-cluster-seed.sql +COPY docker/legacy-rds-fixture/legacy-seed.sql /docker-entrypoint-initdb.d/10-legacy-seed.sql + +FROM postgres:16-alpine AS proof + +COPY scripts/run-forward-migrations.sh /opt/omn15422/run-forward-migrations.sh +COPY docker/migrations/ /migrations/ +COPY docker/legacy-rds-fixture/ledger-control/ /opt/omn15422/ledger-control/ +COPY docker/migrations/forward/fenced-node-migrations.yaml /opt/omn15422/ledger-control/forward/fenced-node-migrations.yaml +COPY docker/migrations/forward/grandfathered-force-rls-migrations.yaml /opt/omn15422/ledger-control/forward/grandfathered-force-rls-migrations.yaml +COPY docker/migrations/forward/_ledger/bootstrap.sql /opt/omn15422/ledger-control/forward/_ledger/bootstrap.sql +COPY docker/legacy-rds-fixture/fixture-manifest.json /opt/omn15422/fixture-manifest.json +COPY docker/legacy-rds-fixture/prove.sh /opt/omn15422/prove.sh +COPY docker/legacy-rds-fixture/cutover-proof/prove.sh /opt/omn15422/cutover-proof/prove.sh +COPY src/omnibase_infra/migration/cutover/sql/bootstrap.sql /opt/omn15422/cutover-proof/bootstrap.sql + +ENTRYPOINT ["sh", "/opt/omn15422/prove.sh"] diff --git a/docker/legacy-rds-fixture/README.md b/docker/legacy-rds-fixture/README.md new file mode 100644 index 0000000000..1664b7903b --- /dev/null +++ b/docker/legacy-rds-fixture/README.md @@ -0,0 +1,29 @@ +# Sanitized legacy-RDS fixture (OMN-15422) + +This directory is a wholly synthetic PostgreSQL 16 reproduction surface for the +one-application-database program. It contains no dump-derived value, customer data, +credential, secret, or live database observation. Its provenance is limited to the +committed/static evidence named in `fixture-manifest.json`. + +Run the proof with rebuilt images: + +```bash +docker compose -f docker/legacy-rds-fixture/compose.yml \ + up --build --abort-on-container-exit --exit-code-from proof +docker compose -f docker/legacy-rds-fixture/compose.yml \ + down --volumes --remove-orphans +``` + +The harness runs each detector against a safe control and a seeded RED control. It +then applies the real vendored shape-reconciliation migrations twice, runs the real +forward runner twice on a clean install, and executes the real legacy upgrade twice. +The OMN-15420 extension also initializes the explicit cutover proof repository and +exercises family-local mismatch isolation, pre-checkpoint DSN rollback, bounded +dual-write detection, post-checkpoint refusal, complete reverse-delta coverage, an +explicit forward-fix-only family, and durable hash-chain readback. + +The real fresh and legacy runners are expected to stop at the unresolved OMN-15423 +domain preflight before ledger or DDL mutation. The harness succeeds only when that +exact blocker is observed; any other failure or an unreviewed surprise success fails +the proof. The OMN-15420 controls exercise proof mechanics only: they do not activate +a live writer, grants, RLS, routing, dual-write, cutover, or destructive cleanup. diff --git a/docker/legacy-rds-fixture/cluster-seed.sql b/docker/legacy-rds-fixture/cluster-seed.sql new file mode 100644 index 0000000000..2a439f4c78 --- /dev/null +++ b/docker/legacy-rds-fixture/cluster-seed.sql @@ -0,0 +1,58 @@ +-- SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +-- SPDX-License-Identifier: MIT +-- Synthetic cluster bootstrap shared by the fresh and legacy fixtures. +-- No password, secret, dump, customer identifier, or live catalog value occurs here. + +\set ON_ERROR_STOP on + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'omninodeadmin') THEN + CREATE ROLE omninodeadmin NOLOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'role_omnibase') THEN + CREATE ROLE role_omnibase LOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'role_omnidash') THEN + CREATE ROLE role_omnidash LOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'role_omniweb') THEN + CREATE ROLE role_omniweb LOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app_dashboard') THEN + CREATE ROLE app_dashboard LOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'onex_api') THEN + CREATE ROLE onex_api LOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; +END +$$; + +SELECT 'CREATE DATABASE omnibase_infra' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'omnibase_infra')\gexec +SELECT 'CREATE DATABASE omnidash_analytics' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'omnidash_analytics')\gexec +SELECT 'CREATE DATABASE omninode_cloud' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'omninode_cloud')\gexec +SELECT 'CREATE DATABASE omniintelligence' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'omniintelligence')\gexec +SELECT 'CREATE DATABASE omniclaude' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'omniclaude')\gexec +SELECT 'CREATE DATABASE omnimemory' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'omnimemory')\gexec +SELECT 'CREATE DATABASE omniweb' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'omniweb')\gexec +SELECT 'CREATE DATABASE infisical_db' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'infisical_db')\gexec +SELECT 'CREATE DATABASE keycloak' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'keycloak')\gexec + +GRANT CONNECT ON DATABASE omnibase_infra TO role_omnibase; +GRANT CONNECT ON DATABASE omnidash_analytics TO role_omnidash, app_dashboard; +GRANT CONNECT ON DATABASE omninode_cloud TO onex_api; + +\connect omnidash_analytics +GRANT USAGE, CREATE ON SCHEMA public TO role_omnidash; + +\connect omniweb +GRANT USAGE, CREATE ON SCHEMA public TO role_omniweb; diff --git a/docker/legacy-rds-fixture/compose.yml b/docker/legacy-rds-fixture/compose.yml new file mode 100644 index 0000000000..a98e318f0e --- /dev/null +++ b/docker/legacy-rds-fixture/compose.yml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +services: + fresh-postgres: + build: + context: ../.. + dockerfile: docker/legacy-rds-fixture/Dockerfile + target: fresh + environment: + POSTGRES_HOST_AUTH_METHOD: trust + tmpfs: + - /var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 2s + timeout: 2s + retries: 30 + legacy-postgres: + build: + context: ../.. + dockerfile: docker/legacy-rds-fixture/Dockerfile + target: legacy + environment: + POSTGRES_HOST_AUTH_METHOD: trust + tmpfs: + - /var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 2s + timeout: 2s + retries: 30 + proof: + build: + context: ../.. + dockerfile: docker/legacy-rds-fixture/Dockerfile + target: proof + depends_on: + fresh-postgres: + condition: service_healthy + legacy-postgres: + condition: service_healthy diff --git a/docker/legacy-rds-fixture/cutover-proof/prove.sh b/docker/legacy-rds-fixture/cutover-proof/prove.sh new file mode 100644 index 0000000000..6606b20a64 --- /dev/null +++ b/docker/legacy-rds-fixture/cutover-proof/prove.sh @@ -0,0 +1,209 @@ +#!/bin/sh +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# Real PostgreSQL 16 persistence/RED controls for OMN-15420. The Python +# integration suite drives the production repository and state machine; this +# rebuilt-image proof independently verifies the durable schema and rollback +# predicates inside the same sanitized legacy fixture. + +set -eu + +host="${LEGACY_HOST:-legacy-postgres}" +port="${LEGACY_PORT:-5432}" +database="omn15420_cutover_proof" +bootstrap="/opt/omn15422/cutover-proof/bootstrap.sql" + +fail_cutover() { + echo "fixture_status=FAIL detail=cutover_receipts:$1" >&2 + exit 1 +} + +sql_value_cutover() { + statement="$1" + psql -X -qAt -h "$host" -p "$port" -U postgres -d "$database" \ + -v ON_ERROR_STOP=1 -c "$statement" +} + +exists="$(psql -X -qAt -h "$host" -p "$port" -U postgres -d postgres \ + -v ON_ERROR_STOP=1 -c "SELECT count(*) FROM pg_database WHERE datname='$database'")" +[ "$exists" = "0" ] || fail_cutover "proof database unexpectedly exists" +psql -X -q -h "$host" -p "$port" -U postgres -d postgres \ + -v ON_ERROR_STOP=1 -c "CREATE DATABASE $database" +psql -X -q -h "$host" -p "$port" -U postgres -d "$database" \ + -v ON_ERROR_STOP=1 -f "$bootstrap" + +psql -X -q -h "$host" -p "$port" -U postgres -d "$database" \ + -v ON_ERROR_STOP=1 <<'EOSQL' +INSERT INTO omninode_internal.cutover_family_contracts + (family_id, contract_json, contract_hash) +VALUES + ( + '30000000-0000-0000-0000-000000000001', + '{"family_id":"30000000-0000-0000-0000-000000000001","family_key":"tenant.usage","family_kind":"projection","source_binding_ref":"application.legacy","target_binding_ref":"application.target","source_evidence_contract_hash":"3333333333333333333333333333333333333333333333333333333333333333","target_evidence_contract_hash":"4444444444444444444444444444444444444444444444444444444444444444","post_checkpoint_mode":"reverse_delta","reverse_delta_contract_ref":"contracts/reverse-delta/usage.yaml","forward_fix_runbook_ref":"","dual_write_max_seconds":30,"observation_window_seconds":60}', + repeat('1', 64) + ), + ( + '30000000-0000-0000-0000-000000000002', + '{"family_id":"30000000-0000-0000-0000-000000000002","family_key":"tenant.control-plane","family_kind":"control_plane","source_binding_ref":"application.legacy","target_binding_ref":"application.target","source_evidence_contract_hash":"3333333333333333333333333333333333333333333333333333333333333333","target_evidence_contract_hash":"4444444444444444444444444444444444444444444444444444444444444444","post_checkpoint_mode":"forward_fix_only","reverse_delta_contract_ref":"","forward_fix_runbook_ref":"runbooks/control-plane-forward-fix.md","dual_write_max_seconds":0,"observation_window_seconds":60}', + repeat('2', 64) + ); + +INSERT INTO omninode_internal.transformation_receipts + (receipt_id, family_id, status, receipt_hash, receipt_json, generated_at) +VALUES + ('10000000-0000-0000-0000-000000000001', '30000000-0000-0000-0000-000000000001', 'pass', + repeat('a', 64), '{"status":"pass","dimensions":14}', clock_timestamp()), + ('20000000-0000-0000-0000-000000000001', '30000000-0000-0000-0000-000000000002', 'fail', + repeat('b', 64), '{"status":"fail","red":"owner_mismatch"}', clock_timestamp()), + ('20000000-0000-0000-0000-000000000002', '30000000-0000-0000-0000-000000000002', 'pass', + repeat('c', 64), '{"status":"pass","dimensions":14}', clock_timestamp()); + +UPDATE omninode_internal.cutover_family_contracts +SET status = 'blocked', + blocked_receipt_id = '20000000-0000-0000-0000-000000000001' +WHERE family_id = '30000000-0000-0000-0000-000000000002'; +UPDATE omninode_internal.cutover_family_contracts +SET last_known_good_receipt_id = '10000000-0000-0000-0000-000000000001' +WHERE family_id = '30000000-0000-0000-0000-000000000001'; +EOSQL + +[ "$(sql_value_cutover "SELECT count(*) FROM omninode_internal.cutover_family_contracts WHERE family_id='30000000-0000-0000-0000-000000000002' AND status='blocked'")" = "1" ] \ + || fail_cutover "failed receipt did not block its family" +[ "$(sql_value_cutover "SELECT count(*) FROM omninode_internal.cutover_family_contracts WHERE family_id='30000000-0000-0000-0000-000000000001' AND status='ready'")" = "1" ] \ + || fail_cutover "one mismatch leaked into another family" +echo "fixture_case=cutover_family_mismatch_isolation positive=PASS red=DETECTED red_signature=owner_mismatch_family_local" + +# No target-only authoritative write exists yet, so restoring the source DSN is +# the positive pre-checkpoint rollback control. +pre_checkpoint_safe="$(sql_value_cutover "SELECT (first_target_write_event_id IS NULL)::int FROM omninode_internal.cutover_family_contracts WHERE family_id='30000000-0000-0000-0000-000000000001'")" +[ "$pre_checkpoint_safe" = "1" ] || fail_cutover "pre-checkpoint DSN rollback was refused" +echo "fixture_case=cutover_pre_checkpoint_dsn_rollback status=PASS source_authoritative=true" + +# Seed and discriminate an expired blind dual-write window. Production code +# refuses to append it; the SQL control proves the durable detector is non-vacuous. +sql_value_cutover "UPDATE omninode_internal.cutover_family_contracts SET dual_write_expires_at=clock_timestamp()-interval '1 second' WHERE family_id='30000000-0000-0000-0000-000000000001'" >/dev/null +expired_count="$(sql_value_cutover "SELECT count(*) FROM omninode_internal.cutover_family_contracts WHERE dual_write_expires_at < clock_timestamp()")" +[ "$expired_count" = "1" ] || fail_cutover "expired dual-write RED was not detected" +sql_value_cutover "UPDATE omninode_internal.cutover_family_contracts SET dual_write_expires_at=NULL WHERE family_id='30000000-0000-0000-0000-000000000001'" >/dev/null +echo "fixture_case=cutover_blind_dual_write positive=PASS red=DETECTED red_signature=expired_window" + +psql -X -q -h "$host" -p "$port" -U postgres -d "$database" \ + -v ON_ERROR_STOP=1 <<'EOSQL' +INSERT INTO omninode_internal.cutover_journal + (event_id, family_id, sequence, event_kind, request_json, receipt_id, + previous_event_hash, event_hash, occurred_at) +VALUES + ('11000000-0000-0000-0000-000000000001', '30000000-0000-0000-0000-000000000001', 1, + 'writer_checkpoint', '{"checkpoint":"source-to-target"}', + '10000000-0000-0000-0000-000000000001', repeat('0', 64), repeat('3', 64), + clock_timestamp()), + ('11000000-0000-0000-0000-000000000002', '30000000-0000-0000-0000-000000000001', 2, + 'application_path_write_proven', + '{"database_ref":"application","principal":"tenant_projection_writer","schema":"tenant","target_sequence":7}', + NULL, repeat('3', 64), repeat('4', 64), clock_timestamp()), + ('11000000-0000-0000-0000-000000000003', '30000000-0000-0000-0000-000000000001', 3, + 'writer_quiesced', '{"target_sequence":8}', NULL, + repeat('4', 64), repeat('5', 64), clock_timestamp()); +UPDATE omninode_internal.cutover_family_contracts +SET status = 'checkpointed', + checkpoint_event_id = '11000000-0000-0000-0000-000000000001', + first_target_write_event_id = '11000000-0000-0000-0000-000000000002', + first_target_sequence = 7, + quiescence_event_id = '11000000-0000-0000-0000-000000000003', + quiesced_target_sequence = 8, + last_sequence = 3, + last_event_hash = repeat('5', 64), + last_event_kind = 'writer_quiesced' +WHERE family_id = '30000000-0000-0000-0000-000000000001'; +EOSQL + +unsafe_direct="$(sql_value_cutover "SELECT (first_target_write_event_id IS NOT NULL AND verified_reverse_delta_proof_id IS NULL)::int FROM omninode_internal.cutover_family_contracts WHERE family_id='30000000-0000-0000-0000-000000000001'")" +[ "$unsafe_direct" = "1" ] || fail_cutover "post-write rollback refusal RED was not detected" +echo "fixture_case=cutover_post_checkpoint_direct_rollback positive=REFUSED red=DETECTED red_signature=reverse_delta_unproven" + +psql -X -q -h "$host" -p "$port" -U postgres -d "$database" \ + -v ON_ERROR_STOP=1 <<'EOSQL' +INSERT INTO omninode_internal.reverse_delta_proofs + (proof_id, family_id, start_sequence, end_sequence, quiescence_event_id, + reconciliation_receipt_id, proof_json, proven_at) +VALUES + ('12000000-0000-0000-0000-000000000001', '30000000-0000-0000-0000-000000000001', 7, 8, + '11000000-0000-0000-0000-000000000003', + '10000000-0000-0000-0000-000000000001', + '{"behavioral_readback_ref":"proof/reverse-delta/readback"}', clock_timestamp()); +INSERT INTO omninode_internal.reverse_delta_entries + (entry_id, proof_id, family_id, target_sequence, entry_json) +VALUES + ('13000000-0000-0000-0000-000000000007', + '12000000-0000-0000-0000-000000000001', '30000000-0000-0000-0000-000000000001', 7, + '{"inverse_artifact_ref":"proof/reverse/7"}'); +EOSQL + +gap_count="$(sql_value_cutover "SELECT count(*) FROM generate_series(7,8) sequence LEFT JOIN omninode_internal.reverse_delta_entries entry ON entry.family_id='30000000-0000-0000-0000-000000000001' AND entry.target_sequence=sequence WHERE entry.entry_id IS NULL")" +[ "$gap_count" = "1" ] || fail_cutover "incomplete reverse-delta RED was not detected" +echo "fixture_case=cutover_reverse_delta_coverage positive=PENDING red=DETECTED red_signature=sequence_gap gap_count=$gap_count" + +psql -X -q -h "$host" -p "$port" -U postgres -d "$database" \ + -v ON_ERROR_STOP=1 <<'EOSQL' +INSERT INTO omninode_internal.reverse_delta_entries + (entry_id, proof_id, family_id, target_sequence, entry_json) +VALUES + ('13000000-0000-0000-0000-000000000008', + '12000000-0000-0000-0000-000000000001', '30000000-0000-0000-0000-000000000001', 8, + '{"inverse_artifact_ref":"proof/reverse/8"}'); +UPDATE omninode_internal.cutover_family_contracts +SET verified_reverse_delta_proof_id = '12000000-0000-0000-0000-000000000001' +WHERE family_id = '30000000-0000-0000-0000-000000000001' + AND NOT EXISTS ( + SELECT 1 + FROM generate_series(first_target_sequence, quiesced_target_sequence) sequence + LEFT JOIN omninode_internal.reverse_delta_entries entry + ON entry.family_id = cutover_family_contracts.family_id + AND entry.target_sequence = sequence + WHERE entry.entry_id IS NULL + ); +EOSQL + +[ "$(sql_value_cutover "SELECT count(*) FROM omninode_internal.cutover_family_contracts WHERE family_id='30000000-0000-0000-0000-000000000001' AND verified_reverse_delta_proof_id IS NOT NULL")" = "1" ] \ + || fail_cutover "complete reverse delta did not satisfy rollback predicate" +echo "fixture_case=cutover_reverse_delta_complete status=PASS entries=2 reconciliation_receipt=PASS behavioral_readback=RECORDED" + +psql -X -q -h "$host" -p "$port" -U postgres -d "$database" \ + -v ON_ERROR_STOP=1 <<'EOSQL' +INSERT INTO omninode_internal.cutover_journal + (event_id, family_id, sequence, event_kind, request_json, receipt_id, + previous_event_hash, event_hash, occurred_at) +VALUES + ('21000000-0000-0000-0000-000000000001', '30000000-0000-0000-0000-000000000002', 1, + 'writer_checkpoint', '{"checkpoint":"source-to-target"}', + '20000000-0000-0000-0000-000000000002', repeat('0', 64), repeat('6', 64), + clock_timestamp()), + ('21000000-0000-0000-0000-000000000002', '30000000-0000-0000-0000-000000000002', 2, + 'application_path_write_proven', + '{"database_ref":"application","principal":"onex_api","schema":"tenant","target_sequence":1}', + NULL, repeat('6', 64), repeat('7', 64), clock_timestamp()), + ('21000000-0000-0000-0000-000000000003', '30000000-0000-0000-0000-000000000002', 3, + 'forward_fix_recorded', '{"runbook":"runbooks/control-plane-forward-fix.md"}', + '20000000-0000-0000-0000-000000000002', repeat('7', 64), repeat('8', 64), + clock_timestamp()); +UPDATE omninode_internal.cutover_family_contracts +SET status = 'checkpointed', + blocked_receipt_id = NULL, + last_known_good_receipt_id = '20000000-0000-0000-0000-000000000002', + checkpoint_event_id = '21000000-0000-0000-0000-000000000001', + first_target_write_event_id = '21000000-0000-0000-0000-000000000002', + first_target_sequence = 1, + last_sequence = 3, + last_event_hash = repeat('8', 64), + last_event_kind = 'forward_fix_recorded' +WHERE family_id = '30000000-0000-0000-0000-000000000002'; +EOSQL + +forward_denied="$(sql_value_cutover "SELECT count(*) FROM omninode_internal.cutover_family_contracts WHERE family_id='30000000-0000-0000-0000-000000000002' AND contract_json->>'post_checkpoint_mode'='forward_fix_only' AND first_target_write_event_id IS NOT NULL AND verified_reverse_delta_proof_id IS NULL")" +[ "$forward_denied" = "1" ] || fail_cutover "forward-fix-only rollback refusal was not durable" +echo "fixture_case=cutover_forward_fix_only status=PASS direct_dsn_rollback=REFUSED snapshot=RECORDED final_delta=RECORDED" + +[ "$(sql_value_cutover "SELECT count(*) FROM omninode_internal.cutover_journal journal LEFT JOIN omninode_internal.cutover_journal previous ON previous.family_id=journal.family_id AND previous.sequence=journal.sequence-1 WHERE journal.family_id='30000000-0000-0000-0000-000000000001' AND journal.sequence>1 AND journal.previous_event_hash<>previous.event_hash")" = "0" ] \ + || fail_cutover "journal hash chain is broken" +echo "fixture_case=cutover_durable_journal status=PASS hash_chain=true reconnect_readback=true" diff --git a/docker/legacy-rds-fixture/fixture-manifest.json b/docker/legacy-rds-fixture/fixture-manifest.json new file mode 100644 index 0000000000..654321dd6e --- /dev/null +++ b/docker/legacy-rds-fixture/fixture-manifest.json @@ -0,0 +1,114 @@ +{ + "ticket": "OMN-15422", + "extension_tickets": [ + "OMN-15413", + "OMN-15420" + ], + "postgres_major": 16, + "provenance": { + "sources": [ + "OMN-15332 ledger-shape description", + "OMN-15335 two-owner description", + "OMN-15376 committed shape-drift execution test", + "OMN-15384 flat/node overlap inventory", + "OMN-15423 committed source inventory" + ], + "live_database_read": false, + "dump_derived": false + }, + "sanitization": { + "customer_data": false, + "credentials": false, + "secrets": false, + "identifiers": "invented deterministic controls only" + }, + "database_names": [ + "omnibase_infra", + "omnidash_analytics", + "omninode_cloud" + ], + "ledger_shapes": [ + "migration_id_checksum_source_set", + "version_nullable_checksum", + "filename_applied_at" + ], + "cases": [ + { + "id": "mapping_ambiguity", + "positive_fixture": "omn15422_fixture.mapping_positive", + "red_fixture": "omn15422_fixture.mapping_red", + "detector": "one legacy tenant value resolves to exactly one UUID", + "expected_red_signature": "ambiguous mapping group count > 0" + }, + { + "id": "checksum_conflict", + "positive_fixture": "omn15422_fixture.checksum_positive", + "red_fixture": "omn15422_fixture.checksum_red", + "detector": "one migration identity carries exactly one checksum", + "expected_red_signature": "conflicting checksum group count > 0" + }, + { + "id": "owner_drift", + "positive_fixture": "omn15422_fixture.owner_positive", + "red_fixture": "omn15422_fixture.owner_red", + "detector": "catalog owner equals role_omnidash", + "expected_red_signature": "owner mismatch count > 0" + }, + { + "id": "unsafe_rls_policy", + "positive_fixture": "omn15422_fixture.tenant_usage_safe", + "red_fixture": "public.tenant_usage_legacy", + "detector": "UUID NOT NULL plus ENABLE and FORCE RLS", + "expected_red_signature": "legacy varchar and ENABLE-only defects > 0" + }, + { + "id": "unsafe_view", + "positive_fixture": "omn15422_fixture.tenant_usage_safe_view", + "red_fixture": "omn15422_fixture.tenant_usage_red_view", + "detector": "view carries security_invoker=true", + "expected_red_signature": "missing invoker option" + }, + { + "id": "unsafe_function", + "positive_fixture": "omn15422_fixture.tenant_usage_safe_count()", + "red_fixture": "omn15422_fixture.tenant_usage_red_count()", + "detector": "not SECURITY DEFINER and no PUBLIC EXECUTE", + "expected_red_signature": "definer or PUBLIC EXECUTE defect > 0" + }, + { + "id": "transformation_collision", + "positive_fixture": "omn15422_fixture.transform_positive", + "red_fixture": "omn15422_fixture.transform_red", + "detector": "target key remains unique after transformation", + "expected_red_signature": "duplicate target-key group count > 0" + }, + { + "id": "flat_node_shape_collision", + "positive_fixture": "flat_node_parity_control in both application databases", + "red_fixture": "llm_cost_aggregates in both application databases", + "detector": "ordered information_schema column signature equality", + "expected_red_signature": "flat and node signatures differ" + }, + { + "id": "legacy_shape_collision", + "positive_fixture": "omnibase_infra.public.schema_migrations", + "red_fixture": "omnidash_analytics.public.schema_migrations", + "detector": "real run-forward-migrations.sh preflight plus PostgreSQL 16 ledger integration proof", + "expected_red_signature": "legacy shape controls remain discriminated before successful forward migration" + }, + { + "id": "application_migration_ledger", + "positive_fixture": "ledger-control fresh and sanitized legacy databases, each run twice", + "red_fixture": "historical unresolved delegation_judge_verdict_events domain replayed by fixture controls", + "detector": "real runner selects one checksum ledger, imports three source shapes, preserves source OIDs/rows, and completes real fresh/legacy forward migrations twice", + "expected_red_signature": "second real forward pass is idempotent after OMN-15423 blocker ledger is emptied" + }, + { + "id": "cutover_receipts_and_rollback_boundary", + "positive_fixture": "omn15420_cutover_proof.omninode_internal cutover journal", + "red_fixture": "family-local owner mismatch, expired dual-write, reverse-delta sequence gap, and forward-fix-only post-write rollback", + "detector": "durable receipt/journal schema plus per-family rollback predicates in PostgreSQL 16", + "expected_red_signature": "mismatch isolation, direct rollback refusal, bounded dual-write, and incomplete reverse-delta detection" + } + ] +} diff --git a/docker/legacy-rds-fixture/ledger-control/forward/000_db_metadata.sql b/docker/legacy-rds-fixture/ledger-control/forward/000_db_metadata.sql new file mode 100644 index 0000000000..e2133737d7 --- /dev/null +++ b/docker/legacy-rds-fixture/ledger-control/forward/000_db_metadata.sql @@ -0,0 +1,12 @@ +-- SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +-- SPDX-License-Identifier: MIT + +CREATE TABLE IF NOT EXISTS public.db_metadata ( + id BOOLEAN PRIMARY KEY, + migrations_complete BOOLEAN NOT NULL DEFAULT FALSE, + runner_completed_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +INSERT INTO public.db_metadata (id) +VALUES (TRUE) +ON CONFLICT (id) DO NOTHING; diff --git a/docker/legacy-rds-fixture/ledger-control/forward/_ledger/application-migration-blocks.tsv b/docker/legacy-rds-fixture/ledger-control/forward/_ledger/application-migration-blocks.tsv new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/legacy-rds-fixture/ledger-control/forward/_ledger/application-migrations.tsv b/docker/legacy-rds-fixture/ledger-control/forward/_ledger/application-migrations.tsv new file mode 100644 index 0000000000..5b1dfc8ae1 --- /dev/null +++ b/docker/legacy-rds-fixture/ledger-control/forward/_ledger/application-migrations.tsv @@ -0,0 +1 @@ +nodes/node_example/0001_create_example.sql node:node_example node:node_example omninode_internal node:node_example:0001_create_example.sql 1f605f28cc1f4a1a7500862be51c35d01431cacba2d34201150f3ae3deb6c923 diff --git a/docker/legacy-rds-fixture/ledger-control/forward/_ledger/cloud-migration-aliases.tsv b/docker/legacy-rds-fixture/ledger-control/forward/_ledger/cloud-migration-aliases.tsv new file mode 100644 index 0000000000..8bdf440592 --- /dev/null +++ b/docker/legacy-rds-fixture/ledger-control/forward/_ledger/cloud-migration-aliases.tsv @@ -0,0 +1 @@ +20260101_cloud_control 20260101_cloud_control.sql diff --git a/docker/legacy-rds-fixture/ledger-control/forward/_ledger/legacy-node-migrations.tsv b/docker/legacy-rds-fixture/ledger-control/forward/_ledger/legacy-node-migrations.tsv new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/legacy-rds-fixture/ledger-control/forward/nodes/node_example/0001_create_example.sql b/docker/legacy-rds-fixture/ledger-control/forward/nodes/node_example/0001_create_example.sql new file mode 100644 index 0000000000..d1ea8d9880 --- /dev/null +++ b/docker/legacy-rds-fixture/ledger-control/forward/nodes/node_example/0001_create_example.sql @@ -0,0 +1,6 @@ +-- SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +-- SPDX-License-Identifier: MIT + +CREATE TABLE IF NOT EXISTS public.omn15413_ledger_control ( + id INTEGER PRIMARY KEY +); diff --git a/docker/legacy-rds-fixture/legacy-seed.sql b/docker/legacy-rds-fixture/legacy-seed.sql new file mode 100644 index 0000000000..c8f5789a61 --- /dev/null +++ b/docker/legacy-rds-fixture/legacy-seed.sql @@ -0,0 +1,255 @@ +-- SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +-- SPDX-License-Identifier: MIT +-- +-- Wholly synthetic legacy-RDS fixture for OMN-15422. +-- Current application DB names: omnibase_infra, omnidash_analytics, +-- omninode_cloud. Representative roles: omninodeadmin, role_omnidash, +-- app_dashboard, onex_api. Values are invented controls, never dump-derived. + +\set ON_ERROR_STOP on + +-- Checksum-capable Python/compose ledger and the flat half of a dual producer. +\connect omnibase_infra +CREATE TABLE public.schema_migrations ( + migration_id TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now(), + checksum TEXT NOT NULL, + source_set TEXT NOT NULL +); +INSERT INTO public.schema_migrations (migration_id, checksum, source_set) VALUES + ('docker/000_synthetic.sql', 'sha256:synthetic-control', 'docker'); + +CREATE TYPE cost_aggregation_window AS ENUM ('24h', '7d', '30d'); +CREATE TABLE public.llm_cost_aggregates ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + aggregation_key VARCHAR(512) NOT NULL, + "window" cost_aggregation_window NOT NULL, + total_cost_usd NUMERIC(14, 6) NOT NULL DEFAULT 0, + total_tokens BIGINT NOT NULL DEFAULT 0, + call_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +ALTER TABLE public.llm_cost_aggregates OWNER TO omninodeadmin; + +CREATE TABLE public.flat_node_parity_control ( + id UUID PRIMARY KEY, + payload JSONB NOT NULL +); +ALTER TABLE public.flat_node_parity_control OWNER TO omninodeadmin; + +CREATE TABLE public.baselines_comparisons ( + id BIGSERIAL PRIMARY KEY, + cohort TEXT NOT NULL, + measured_at TIMESTAMPTZ NOT NULL +); +ALTER TABLE public.baselines_comparisons OWNER TO omninodeadmin; + +-- Live-legacy omnidash ledger shape plus the post-OMN-15332 node ledger. +\connect omnidash_analytics +CREATE TABLE public.schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +INSERT INTO public.schema_migrations (filename, applied_at) +SELECT format('%04s_synthetic_legacy.sql', sequence), + TIMESTAMPTZ '2026-01-01 00:00:00+00' + sequence * INTERVAL '1 minute' +FROM generate_series(0, 22) AS sequence; +ALTER TABLE public.schema_migrations OWNER TO omninodeadmin; + +CREATE TABLE public.node_schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now(), + checksum TEXT NOT NULL DEFAULT '' +); +INSERT INTO public.node_schema_migrations (version, checksum) VALUES + ( + 'node:node_canary_score_reducer:0001_create_capability_scores.sql', + '7195b07a7fae809f141a136a67b799ad492e79d280f54c325ac14875dcfcc2cd' + ); +ALTER TABLE public.node_schema_migrations OWNER TO role_omnidash; + +-- Deliberately old node-side shapes. These are empty so current reconciling +-- migrations can prove shape convergence without inventing row backfills. +CREATE TABLE public.llm_cost_aggregates ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + model_name TEXT, + total_cost_usd NUMERIC(14, 6), + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +ALTER TABLE public.llm_cost_aggregates OWNER TO omninodeadmin; + +CREATE TABLE public.baselines_comparisons ( + id BIGSERIAL PRIMARY KEY +); +ALTER TABLE public.baselines_comparisons OWNER TO omninodeadmin; + +CREATE TABLE public.flat_node_parity_control ( + id UUID PRIMARY KEY, + payload JSONB NOT NULL +); +ALTER TABLE public.flat_node_parity_control OWNER TO omninodeadmin; + +-- Text/varchar tenants, sentinels, FK/index/view/function dependencies, and +-- legacy ENABLE-only RLS. These rows are synthetic and intentionally invalid +-- under the target UUID/no-sentinel contract. +CREATE TABLE public.tenants_legacy ( + tenant_id VARCHAR(64) PRIMARY KEY, + display_name TEXT NOT NULL +); +INSERT INTO public.tenants_legacy (tenant_id, display_name) VALUES + ('legacy-acme', 'Synthetic tenant A'), + ('omninode', 'Synthetic sentinel'), + ('00000000-0000-0000-0000-000000000000', 'Synthetic zero UUID sentinel'), + ('', 'Synthetic empty sentinel'); +ALTER TABLE public.tenants_legacy OWNER TO omninodeadmin; + +CREATE TABLE public.tenant_usage_legacy ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tenant_id VARCHAR(64) NOT NULL REFERENCES public.tenants_legacy (tenant_id), + external_key TEXT NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + UNIQUE (tenant_id, external_key) +); +CREATE INDEX tenant_usage_legacy_tenant_idx + ON public.tenant_usage_legacy (tenant_id); +ALTER TABLE public.tenant_usage_legacy ENABLE ROW LEVEL SECURITY; +CREATE POLICY tenant_usage_legacy_policy ON public.tenant_usage_legacy + USING (tenant_id = current_setting('app.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)); +INSERT INTO public.tenant_usage_legacy (tenant_id, external_key) VALUES + ('legacy-acme', 'usage-a'), + ('omninode', 'usage-sentinel'), + ('00000000-0000-0000-0000-000000000000', 'usage-zero'); +ALTER TABLE public.tenant_usage_legacy OWNER TO omninodeadmin; + +CREATE VIEW public.tenant_usage_legacy_view AS +SELECT id, tenant_id, external_key FROM public.tenant_usage_legacy; +ALTER VIEW public.tenant_usage_legacy_view OWNER TO omninodeadmin; + +CREATE FUNCTION public.tenant_usage_legacy_count() +RETURNS BIGINT +LANGUAGE sql +SECURITY DEFINER +AS $$SELECT count(*) FROM public.tenant_usage_legacy$$; +ALTER FUNCTION public.tenant_usage_legacy_count() OWNER TO omninodeadmin; + +GRANT SELECT, INSERT, UPDATE ON public.tenant_usage_legacy TO role_omnidash; +GRANT SELECT ON public.tenant_usage_legacy_view TO app_dashboard; +SET ROLE omninodeadmin; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT ON TABLES TO app_dashboard; +RESET ROLE; + +-- Safe and RED controls. Every catalog detector in prove.sh is executed once +-- against each side of the pair; a detector that cannot distinguish them fails. +CREATE SCHEMA omn15422_fixture AUTHORIZATION omninodeadmin; + +CREATE TABLE omn15422_fixture.mapping_positive ( + legacy_tenant_value TEXT NOT NULL, + tenant_uuid UUID NOT NULL +); +INSERT INTO omn15422_fixture.mapping_positive VALUES + ('legacy-a', '11111111-1111-4111-8111-111111111111'), + ('legacy-b', '22222222-2222-4222-8222-222222222222'); + +CREATE TABLE omn15422_fixture.mapping_red ( + legacy_tenant_value TEXT NOT NULL, + tenant_uuid UUID NOT NULL +); +INSERT INTO omn15422_fixture.mapping_red VALUES + ('ambiguous', '33333333-3333-4333-8333-333333333333'), + ('ambiguous', '44444444-4444-4444-8444-444444444444'); + +CREATE TABLE omn15422_fixture.checksum_positive ( + migration_id TEXT NOT NULL, + checksum TEXT NOT NULL +); +INSERT INTO omn15422_fixture.checksum_positive VALUES + ('stream:0001', 'sha256:positive'); + +CREATE TABLE omn15422_fixture.checksum_red ( + migration_id TEXT NOT NULL, + checksum TEXT NOT NULL +); +INSERT INTO omn15422_fixture.checksum_red VALUES + ('stream:0001', 'sha256:first'), + ('stream:0001', 'sha256:conflict'); + +CREATE TABLE omn15422_fixture.owner_positive (id INTEGER PRIMARY KEY); +ALTER TABLE omn15422_fixture.owner_positive OWNER TO role_omnidash; +CREATE TABLE omn15422_fixture.owner_red (id INTEGER PRIMARY KEY); +ALTER TABLE omn15422_fixture.owner_red OWNER TO omninodeadmin; + +CREATE TABLE omn15422_fixture.tenant_usage_safe ( + id UUID PRIMARY KEY, + tenant_id UUID NOT NULL +); +ALTER TABLE omn15422_fixture.tenant_usage_safe ENABLE ROW LEVEL SECURITY; +ALTER TABLE omn15422_fixture.tenant_usage_safe FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_usage_safe_policy + ON omn15422_fixture.tenant_usage_safe + USING (tenant_id = current_setting('app.tenant_id', true)::uuid) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)::uuid); + +CREATE VIEW omn15422_fixture.tenant_usage_safe_view +WITH (security_invoker = true) AS +SELECT id, tenant_id FROM omn15422_fixture.tenant_usage_safe; +CREATE VIEW omn15422_fixture.tenant_usage_red_view AS +SELECT id, tenant_id FROM omn15422_fixture.tenant_usage_safe; + +CREATE FUNCTION omn15422_fixture.tenant_usage_safe_count() +RETURNS BIGINT +LANGUAGE sql +SECURITY INVOKER +SET search_path = pg_catalog, omn15422_fixture +AS $$SELECT count(*) FROM tenant_usage_safe$$; +REVOKE ALL ON FUNCTION omn15422_fixture.tenant_usage_safe_count() FROM PUBLIC; +GRANT EXECUTE ON FUNCTION omn15422_fixture.tenant_usage_safe_count() TO app_dashboard; + +CREATE FUNCTION omn15422_fixture.tenant_usage_red_count() +RETURNS BIGINT +LANGUAGE sql +SECURITY DEFINER +AS $$SELECT count(*) FROM omn15422_fixture.tenant_usage_safe$$; + +CREATE TABLE omn15422_fixture.transform_positive ( + source_id INTEGER PRIMARY KEY, + target_key TEXT NOT NULL +); +INSERT INTO omn15422_fixture.transform_positive VALUES (1, 'target-a'), (2, 'target-b'); +CREATE TABLE omn15422_fixture.transform_red ( + source_id INTEGER PRIMARY KEY, + target_key TEXT NOT NULL +); +INSERT INTO omn15422_fixture.transform_red VALUES (1, 'collision'), (2, 'collision'); + +-- Third legacy ledger shape: version key plus nullable checksum. +\connect omninode_cloud +CREATE TABLE public.schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now(), + checksum TEXT +); +INSERT INTO public.schema_migrations (version, checksum) VALUES + ('20251209_m4_usage_schema.sql', NULL); +ALTER TABLE public.schema_migrations OWNER TO omninodeadmin; + +CREATE TABLE public.migrations_log ( + id SERIAL PRIMARY KEY, + migration_name TEXT NOT NULL, + direction TEXT NOT NULL CHECK (direction IN ('forward', 'rollback')), + executed_at TIMESTAMPTZ NOT NULL, + notes TEXT, + UNIQUE (migration_name, direction) +); +INSERT INTO public.migrations_log + (migration_name, direction, executed_at, notes) +VALUES + ( + '20251209_m4_usage_schema', + 'forward', + TIMESTAMPTZ '2026-01-02 00:00:00+00', + 'Synthetic legacy audit row' + ); +ALTER TABLE public.migrations_log OWNER TO omninodeadmin; diff --git a/docker/legacy-rds-fixture/prove.sh b/docker/legacy-rds-fixture/prove.sh new file mode 100644 index 0000000000..0d53dc456e --- /dev/null +++ b/docker/legacy-rds-fixture/prove.sh @@ -0,0 +1,390 @@ +#!/bin/sh +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +set -eu + +FRESH_HOST="${FRESH_HOST:-fresh-postgres}" +LEGACY_HOST="${LEGACY_HOST:-legacy-postgres}" +FRESH_PORT="${FRESH_PORT:-5432}" +LEGACY_PORT="${LEGACY_PORT:-5432}" +MIGRATIONS_DIR="${MIGRATIONS_DIR:-/migrations/forward}" +RUNNER="${RUNNER:-/opt/omn15422/run-forward-migrations.sh}" +CONTROL_MIGRATIONS_DIR="${CONTROL_MIGRATIONS_DIR:-/opt/omn15422/ledger-control/forward}" + +fail() { + echo "fixture_status=FAIL detail=$1" >&2 + exit 1 +} + +sql_value() { + host="$1" + database="$2" + statement="$3" + if [ "$host" = "$FRESH_HOST" ]; then + port="$FRESH_PORT" + else + port="$LEGACY_PORT" + fi + psql -X -qAt -h "$host" -p "$port" -U postgres -d "$database" \ + -v ON_ERROR_STOP=1 -c "$statement" +} + +assert_pair() { + case_id="$1" + positive_sql="$2" + red_sql="$3" + red_signature="$4" + + positive_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "$positive_sql")" + [ "$positive_count" = "0" ] || fail "$case_id positive control reported $positive_count defect(s)" + + red_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "$red_sql")" + [ "$red_count" -gt 0 ] || fail "$case_id RED control was not discriminated" + echo "fixture_case=$case_id positive=PASS red=DETECTED red_signature=$red_signature red_count=$red_count" +} + +assert_pair \ + mapping_ambiguity \ + "SELECT count(*) FROM (SELECT legacy_tenant_value FROM omn15422_fixture.mapping_positive GROUP BY legacy_tenant_value HAVING count(DISTINCT tenant_uuid) <> 1) AS defects" \ + "SELECT count(*) FROM (SELECT legacy_tenant_value FROM omn15422_fixture.mapping_red GROUP BY legacy_tenant_value HAVING count(DISTINCT tenant_uuid) <> 1) AS defects" \ + ambiguous_mapping + +assert_pair \ + checksum_conflict \ + "SELECT count(*) FROM (SELECT migration_id FROM omn15422_fixture.checksum_positive GROUP BY migration_id HAVING count(DISTINCT checksum) <> 1) AS defects" \ + "SELECT count(*) FROM (SELECT migration_id FROM omn15422_fixture.checksum_red GROUP BY migration_id HAVING count(DISTINCT checksum) <> 1) AS defects" \ + checksum_conflict + +assert_pair \ + owner_drift \ + "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_roles r ON r.oid=c.relowner WHERE n.nspname='omn15422_fixture' AND c.relname='owner_positive' AND r.rolname <> 'role_omnidash'" \ + "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_roles r ON r.oid=c.relowner WHERE n.nspname='omn15422_fixture' AND c.relname='owner_red' AND r.rolname <> 'role_omnidash'" \ + owner_mismatch + +assert_pair \ + unsafe_rls_policy \ + "SELECT (CASE WHEN c.relrowsecurity AND c.relforcerowsecurity THEN 0 ELSE 1 END) + (SELECT count(*) FROM information_schema.columns WHERE table_schema='omn15422_fixture' AND table_name='tenant_usage_safe' AND column_name='tenant_id' AND (data_type <> 'uuid' OR is_nullable <> 'NO')) FROM pg_class c WHERE c.oid='omn15422_fixture.tenant_usage_safe'::regclass" \ + "SELECT (CASE WHEN c.relrowsecurity AND c.relforcerowsecurity THEN 0 ELSE 1 END) + (SELECT count(*) FROM information_schema.columns WHERE table_schema='public' AND table_name='tenant_usage_legacy' AND column_name='tenant_id' AND (data_type <> 'uuid' OR is_nullable <> 'NO')) FROM pg_class c WHERE c.oid='public.tenant_usage_legacy'::regclass" \ + legacy_varchar_enable_only + +assert_pair \ + unsafe_view \ + "SELECT CASE WHEN coalesce(reloptions, ARRAY[]::text[]) @> ARRAY['security_invoker=true'] THEN 0 ELSE 1 END FROM pg_class WHERE oid='omn15422_fixture.tenant_usage_safe_view'::regclass" \ + "SELECT CASE WHEN coalesce(reloptions, ARRAY[]::text[]) @> ARRAY['security_invoker=true'] THEN 0 ELSE 1 END FROM pg_class WHERE oid='omn15422_fixture.tenant_usage_red_view'::regclass" \ + missing_security_invoker + +assert_pair \ + unsafe_function \ + "SELECT (CASE WHEN p.prosecdef THEN 1 ELSE 0 END) + (CASE WHEN EXISTS (SELECT 1 FROM aclexplode(coalesce(p.proacl, acldefault('f', p.proowner))) acl WHERE acl.grantee=0 AND acl.privilege_type='EXECUTE') THEN 1 ELSE 0 END) FROM pg_proc p WHERE p.oid='omn15422_fixture.tenant_usage_safe_count()'::regprocedure" \ + "SELECT (CASE WHEN p.prosecdef THEN 1 ELSE 0 END) + (CASE WHEN EXISTS (SELECT 1 FROM aclexplode(coalesce(p.proacl, acldefault('f', p.proowner))) acl WHERE acl.grantee=0 AND acl.privilege_type='EXECUTE') THEN 1 ELSE 0 END) FROM pg_proc p WHERE p.oid='omn15422_fixture.tenant_usage_red_count()'::regprocedure" \ + definer_or_public_execute + +assert_pair \ + transformation_collision \ + "SELECT count(*) FROM (SELECT target_key FROM omn15422_fixture.transform_positive GROUP BY target_key HAVING count(*) > 1) AS defects" \ + "SELECT count(*) FROM (SELECT target_key FROM omn15422_fixture.transform_red GROUP BY target_key HAVING count(*) > 1) AS defects" \ + duplicate_target_key + +signature() { + host="$1" + database="$2" + relation="$3" + sql_value "$host" "$database" "SELECT string_agg(column_name || ':' || data_type || ':' || is_nullable, ',' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema='public' AND table_name='$relation'" +} + +flat_control="$(signature "$LEGACY_HOST" omnibase_infra flat_node_parity_control)" +node_control="$(signature "$LEGACY_HOST" omnidash_analytics flat_node_parity_control)" +[ "$flat_control" = "$node_control" ] || fail "flat/node positive shape control diverged" +flat_red="$(signature "$LEGACY_HOST" omnibase_infra llm_cost_aggregates)" +node_red="$(signature "$LEGACY_HOST" omnidash_analytics llm_cost_aggregates)" +[ "$flat_red" != "$node_red" ] || fail "flat/node RED shape collision was not discriminated" +echo "fixture_case=flat_node_shape_collision positive=PASS red=DETECTED red_signature=column_signature_mismatch" + +ledger_positive="$(signature "$LEGACY_HOST" omnibase_infra schema_migrations)" +ledger_red="$(signature "$LEGACY_HOST" omnidash_analytics schema_migrations)" +ledger_version="$(signature "$LEGACY_HOST" omninode_cloud schema_migrations)" +node_ledger="$(signature "$LEGACY_HOST" omnidash_analytics node_schema_migrations)" +case "$ledger_positive" in + migration_id:*checksum:*source_set:*) ;; + *) fail "checksum-capable positive ledger shape missing: $ledger_positive" ;; +esac +case "$ledger_red" in + filename:*applied_at:*) ;; + *) fail "filename/applied_at legacy ledger shape missing: $ledger_red" ;; +esac +case "$ledger_version" in + version:*applied_at:*checksum:*) ;; + *) fail "version/nullable-checksum legacy ledger shape missing: $ledger_version" ;; +esac +case "$node_ledger" in + version:*applied_at:*checksum:*) ;; + *) fail "node ledger shape missing: $node_ledger" ;; +esac +echo "fixture_case=legacy_shape_collision positive=PASS red=SEE_REAL_RUNNER" + +dependency_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT (SELECT count(*) FROM pg_constraint WHERE conrelid='public.tenant_usage_legacy'::regclass AND contype='f') + (SELECT count(*) FROM pg_indexes WHERE schemaname='public' AND tablename='tenant_usage_legacy') + (SELECT count(*) FROM pg_views WHERE schemaname='public' AND viewname='tenant_usage_legacy_view') + (SELECT count(*) FROM pg_proc WHERE oid='public.tenant_usage_legacy_count()'::regprocedure)")" +[ "$dependency_count" -ge 4 ] || fail "dependent FK/index/view/function catalog is incomplete" +sentinel_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT count(*) FROM public.tenants_legacy WHERE tenant_id IN ('', 'omninode', '00000000-0000-0000-0000-000000000000')")" +[ "$sentinel_count" = "3" ] || fail "synthetic sentinel corpus is incomplete" +acl_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT (SELECT count(*) FROM information_schema.role_table_grants WHERE table_schema='public' AND table_name='tenant_usage_legacy' AND grantee='role_omnidash') + (SELECT count(*) FROM pg_default_acl d CROSS JOIN LATERAL aclexplode(d.defaclacl) acl JOIN pg_roles r ON r.oid=acl.grantee WHERE r.rolname='app_dashboard' AND acl.privilege_type='SELECT')")" +[ "$acl_count" -ge 2 ] || fail "legacy grants/default privileges are incomplete" +policy_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT count(*) FROM pg_policies WHERE schemaname='public' AND tablename='tenant_usage_legacy' AND policyname='tenant_usage_legacy_policy'")" +[ "$policy_count" = "1" ] || fail "legacy RLS policy is missing" +echo "fixture_case=dependencies_acl_and_sentinels status=PASS dependency_count=$dependency_count acl_count=$acl_count policy_count=$policy_count sentinel_count=$sentinel_count" + +# Reproduce OMN-15335 with the real migration as the non-owner role. The same +# file then runs twice as postgres to prove the OMN-15376 shape reconciliation +# is idempotent independently of the earlier ledger wall. +owner_log="$(mktemp)" +if psql -X -h "$LEGACY_HOST" -p "$LEGACY_PORT" -U role_omnidash -d omnidash_analytics \ + -v ON_ERROR_STOP=1 \ + -f "$MIGRATIONS_DIR/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql" \ + >"$owner_log" 2>&1; then + fail "owner-drift RED migration unexpectedly succeeded" +fi +grep -F 'must be owner of table llm_cost_aggregates' "$owner_log" >/dev/null \ + || { sed -n '1,160p' "$owner_log"; fail "owner-drift failure signature moved"; } +echo "fixture_case=owner_drift_real_migration red=DETECTED red_signature=must_be_owner" + +for pass in 1 2; do + psql -X -q -h "$LEGACY_HOST" -p "$LEGACY_PORT" -U postgres -d omnidash_analytics \ + -v ON_ERROR_STOP=1 \ + -f "$MIGRATIONS_DIR/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql" + psql -X -q -h "$LEGACY_HOST" -p "$LEGACY_PORT" -U postgres -d omnidash_analytics \ + -v ON_ERROR_STOP=1 \ + -f "$MIGRATIONS_DIR/nodes/node_projection_baselines/0001_create_baselines_tables.sql" + echo "fixture_case=legacy_shape_reconciliation pass=$pass status=PASS" +done +echo "fixture_case=owner_drift_real_migration positive=PASS red=DETECTED passes=2" + +run_forward() { + host="$1" + port="$2" + database="$3" + log="$4" + POSTGRES_HOST="$host" \ + POSTGRES_PORT="$port" \ + POSTGRES_USER=postgres \ + POSTGRES_PASSWORD='' \ + POSTGRES_DB="$database" \ + NODE_POSTGRES_DB=omnidash_analytics \ + MIGRATIONS_DIR="$MIGRATIONS_DIR" \ + sh "$RUNNER" >"$log" 2>&1 +} + +create_fixture_database() { + host="$1" + port="$2" + database="$3" + exists="$(psql -X -qAt -h "$host" -p "$port" -U postgres -d postgres \ + -v ON_ERROR_STOP=1 -v database="$database" -f - <<'EOSQL' +SELECT count(*) FROM pg_database WHERE datname = :'database'; +EOSQL +)" + if [ "$exists" = "0" ]; then + psql -X -q -h "$host" -p "$port" -U postgres -d postgres \ + -v ON_ERROR_STOP=1 -c "CREATE DATABASE ${database}" + fi +} + +run_control_forward() { + host="$1" + port="$2" + service_database="$3" + application_database="$4" + cloud_database="$5" + log="$6" + POSTGRES_HOST="$host" \ + POSTGRES_PORT="$port" \ + POSTGRES_USER=postgres \ + POSTGRES_PASSWORD='' \ + POSTGRES_DB="$service_database" \ + NODE_POSTGRES_DB="$application_database" \ + OMNINODE_CLOUD_HISTORY_DB="$cloud_database" \ + MIGRATIONS_DIR="$CONTROL_MIGRATIONS_DIR" \ + sh "$RUNNER" >"$log" 2>&1 +} + +# Positive ledger controls use separate synthetic databases and the same real +# runner/bootstrap artifact. They cross the OMN-15423 preflight hold without +# weakening it: the control tree contains one fully classified migration and +# an empty committed block set. Both fresh and legacy histories run twice. +for database in omn15413_control_service omn15413_control_app omn15413_control_cloud; do + create_fixture_database "$FRESH_HOST" "$FRESH_PORT" "$database" +done +for database in omn15413_control_service omn15413_control_app omn15413_control_cloud; do + create_fixture_database "$LEGACY_HOST" "$LEGACY_PORT" "$database" +done + +psql -X -q -h "$LEGACY_HOST" -p "$LEGACY_PORT" -U postgres \ + -d omn15413_control_app -v ON_ERROR_STOP=1 <<'EOSQL' +CREATE TABLE public.schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL +); +INSERT INTO public.schema_migrations VALUES + ('0001_legacy_control.sql', TIMESTAMPTZ '2026-01-01 00:00:00+00'); +CREATE TABLE public.node_schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL, + checksum TEXT NOT NULL +); +INSERT INTO public.node_schema_migrations VALUES ( + 'node:node_example:0001_create_example.sql', + TIMESTAMPTZ '2026-01-02 00:00:00+00', + '1f605f28cc1f4a1a7500862be51c35d01431cacba2d34201150f3ae3deb6c923' +); +EOSQL +psql -X -q -h "$LEGACY_HOST" -p "$LEGACY_PORT" -U postgres \ + -d omn15413_control_cloud -v ON_ERROR_STOP=1 <<'EOSQL' +CREATE TABLE public.schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL, + checksum TEXT +); +INSERT INTO public.schema_migrations VALUES + ('20260101_cloud_control.sql', TIMESTAMPTZ '2026-01-03 00:00:00+00', NULL); +CREATE TABLE public.migrations_log ( + migration_name TEXT NOT NULL, + direction TEXT NOT NULL, + executed_at TIMESTAMPTZ NOT NULL, + UNIQUE (migration_name, direction) +); +INSERT INTO public.migrations_log VALUES + ('20260101_cloud_control', 'forward', TIMESTAMPTZ '2026-01-03 00:00:00+00'); +EOSQL + +legacy_control_node_oid="$(sql_value "$LEGACY_HOST" omn15413_control_app \ + "SELECT 'public.node_schema_migrations'::regclass::oid")" +legacy_control_cloud_source="$(sql_value "$LEGACY_HOST" omn15413_control_cloud \ + "SELECT version || '|' || coalesce(checksum, '') || '|' || applied_at::text FROM public.schema_migrations")" + +for pass in 1 2; do + fresh_control_log="$(mktemp)" + run_control_forward "$FRESH_HOST" "$FRESH_PORT" \ + omn15413_control_service omn15413_control_app omn15413_control_cloud \ + "$fresh_control_log" \ + || { sed -n '1,240p' "$fresh_control_log"; fail "fresh ledger control pass $pass failed"; } + grep -F 'Sentinel set. Migration gate will report HEALTHY.' "$fresh_control_log" >/dev/null \ + || fail "fresh ledger control pass $pass omitted terminal sentinel proof" + if [ "$pass" = "2" ]; then + grep -F 'Complete: 0 infra applied, 1 infra skipped; 0 node applied, 1 node skipped' "$fresh_control_log" >/dev/null \ + || { tail -n 80 "$fresh_control_log"; fail "fresh ledger control second pass was not idempotent"; } + fi + echo "fixture_case=application_ledger_fresh pass=$pass status=PASS" + + legacy_control_log="$(mktemp)" + run_control_forward "$LEGACY_HOST" "$LEGACY_PORT" \ + omn15413_control_service omn15413_control_app omn15413_control_cloud \ + "$legacy_control_log" \ + || { sed -n '1,240p' "$legacy_control_log"; fail "legacy ledger control pass $pass failed"; } + grep -F 'Sentinel set. Migration gate will report HEALTHY.' "$legacy_control_log" >/dev/null \ + || fail "legacy ledger control pass $pass omitted terminal sentinel proof" + echo "fixture_case=application_ledger_legacy pass=$pass status=PASS" +done + +[ "$(sql_value "$FRESH_HOST" omn15413_control_app "SELECT count(*) FROM platform_catalog.schema_migrations")" = "1" ] \ + || fail "fresh ledger control canonical row count drifted" +[ "$(sql_value "$LEGACY_HOST" omn15413_control_app "SELECT count(*) FROM platform_catalog.schema_migrations")" = "3" ] \ + || fail "legacy ledger control did not import all three source shapes" +[ "$(sql_value "$LEGACY_HOST" omn15413_control_app "SELECT 'platform_catalog.schema_migrations'::regclass::oid")" = "$legacy_control_node_oid" ] \ + || fail "selected node ledger was copied instead of moved in place" +[ "$(sql_value "$LEGACY_HOST" omn15413_control_app "SELECT count(*) FROM public.schema_migrations")" = "1" ] \ + || fail "filename-only source ledger was rewritten" +[ "$(sql_value "$LEGACY_HOST" omn15413_control_cloud "SELECT version || '|' || coalesce(checksum, '') || '|' || applied_at::text FROM public.schema_migrations")" = "$legacy_control_cloud_source" ] \ + || fail "cloud applied-set source ledger was rewritten" +echo "fixture_case=application_ledger_sources status=PASS selected_oid_preserved=true sources_immutable=true" + +# OMN-15695: the pre-OMN-15413 runner's migration_id node ledger is adopted in +# place of being refused. The service-owned row in the same relation must stay +# ignored, the source relation must survive untouched, and the second pass must +# apply nothing. +for database in omn15695_adopt_service omn15695_adopt_app omn15695_adopt_cloud; do + create_fixture_database "$LEGACY_HOST" "$LEGACY_PORT" "$database" +done +psql -X -q -h "$LEGACY_HOST" -p "$LEGACY_PORT" -U postgres \ + -d omn15695_adopt_app -v ON_ERROR_STOP=1 <<'EOSQL' +CREATE TABLE public.schema_migrations ( + migration_id TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + checksum TEXT NOT NULL, + source_set TEXT NOT NULL +); +INSERT INTO public.schema_migrations (migration_id, applied_at, checksum, source_set) +VALUES + ('node:node_example:0001_create_example.sql', + TIMESTAMPTZ '2026-07-31 06:44:27.696803+00', 'applied-by-runner', 'node'), + ('docker/000_db_metadata.sql', + TIMESTAMPTZ '2026-07-31 06:44:27.696803+00', 'applied-by-runner', 'docker'); +EOSQL +adopt_source_oid="$(sql_value "$LEGACY_HOST" omn15695_adopt_app \ + "SELECT 'public.schema_migrations'::regclass::oid")" + +for pass in 1 2; do + adopt_log="$(mktemp)" + run_control_forward "$LEGACY_HOST" "$LEGACY_PORT" \ + omn15695_adopt_service omn15695_adopt_app omn15695_adopt_cloud \ + "$adopt_log" \ + || { sed -n '1,240p' "$adopt_log"; fail "migration_id adoption pass $pass failed"; } + grep -F 'Sentinel set. Migration gate will report HEALTHY.' "$adopt_log" >/dev/null \ + || fail "migration_id adoption pass $pass omitted terminal sentinel proof" + grep -F '0 node applied, 1 node skipped' "$adopt_log" >/dev/null \ + || { tail -n 80 "$adopt_log"; fail "migration_id adoption pass $pass re-applied an adopted migration"; } + echo "fixture_case=application_ledger_migration_id_adoption pass=$pass status=PASS" +done + +[ "$(sql_value "$LEGACY_HOST" omn15695_adopt_app "SELECT count(*) FROM platform_catalog.schema_migrations")" = "1" ] \ + || fail "migration_id adoption imported the service-owned row or duplicated the node row" +[ "$(sql_value "$LEGACY_HOST" omn15695_adopt_app "SELECT checksum || '|' || checksum_kind FROM platform_catalog.schema_migrations")" \ + = "1f605f28cc1f4a1a7500862be51c35d01431cacba2d34201150f3ae3deb6c923|content_sha256" ] \ + || fail "adopted row did not carry the checked-in manifest checksum" +[ "$(sql_value "$LEGACY_HOST" omn15695_adopt_app "SELECT provenance FROM platform_catalog.schema_migrations")" \ + = "adopted:omn15695_adopt_app:public.schema_migrations:migration_id:node:node_example:0001_create_example.sql:raw-checksum=applied-by-runner" ] \ + || fail "adopted row did not record its raw source checksum in provenance" +[ "$(sql_value "$LEGACY_HOST" omn15695_adopt_app "SELECT count(*) FROM platform_catalog.schema_migrations WHERE applied_at = TIMESTAMPTZ '2026-07-31 06:44:27.696803+00'")" = "1" ] \ + || fail "adopted row did not preserve its original applied_at" +[ "$(sql_value "$LEGACY_HOST" omn15695_adopt_app "SELECT count(*) FROM public.schema_migrations")" = "2" ] \ + || fail "migration_id adoption source ledger was rewritten" +[ "$(sql_value "$LEGACY_HOST" omn15695_adopt_app "SELECT 'public.schema_migrations'::regclass::oid")" = "$adopt_source_oid" ] \ + || fail "migration_id adoption source ledger was moved instead of preserved" +echo "fixture_case=application_ledger_migration_id_sources status=PASS source_preserved=true service_row_ignored=true" + +for pass in 1 2; do + fresh_log="$(mktemp)" + run_forward "$FRESH_HOST" "$FRESH_PORT" omnibase_infra "$fresh_log" \ + || { sed -n '1,240p' "$fresh_log"; fail "fresh real migration pass $pass failed"; } + grep -F 'Sentinel set. Migration gate will report HEALTHY.' "$fresh_log" >/dev/null \ + || fail "fresh real migration pass $pass omitted terminal sentinel proof" + if [ "$pass" = "2" ]; then + grep -E 'Complete: 0 infra applied, [0-9]+ infra skipped; 0 node applied, [0-9]+ node skipped' "$fresh_log" >/dev/null \ + || { tail -n 80 "$fresh_log"; fail "fresh real migration second pass was not idempotent"; } + fi + echo "fixture_case=fresh_install status=PASS pass=$pass blocker=none" +done + +# The fixture executes the real legacy-upgrade entry point twice. The +# OMN-15423 blocker ledger is now empty, so the real runner must complete and +# then prove idempotence instead of stopping at the historical unresolved-domain +# preflight hold. +for pass in 1 2; do + legacy_log="$(mktemp)" + run_forward "$LEGACY_HOST" "$LEGACY_PORT" omnibase_infra "$legacy_log" \ + || { sed -n '1,240p' "$legacy_log"; fail "legacy upgrade pass $pass failed"; } + grep -F 'Sentinel set. Migration gate will report HEALTHY.' "$legacy_log" >/dev/null \ + || fail "legacy upgrade pass $pass omitted terminal sentinel proof" + if [ "$pass" = "2" ]; then + grep -E 'Complete: 0 infra applied, [0-9]+ infra skipped; 0 node applied, [0-9]+ node skipped' "$legacy_log" >/dev/null \ + || { tail -n 80 "$legacy_log"; fail "legacy upgrade second pass was not idempotent"; } + fi + echo "fixture_case=legacy_upgrade status=PASS pass=$pass blocker=none" +done + +[ "$(sql_value "$FRESH_HOST" omnidash_analytics "SELECT to_regclass('platform_catalog.schema_migrations') IS NOT NULL")" = "t" ] \ + || fail "fresh real migration omitted the canonical application ledger" +[ "$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT to_regclass('platform_catalog.schema_migrations') IS NOT NULL")" = "t" ] \ + || fail "legacy upgrade omitted the canonical application ledger" + +sh /opt/omn15422/cutover-proof/prove.sh + +echo "fixture_status=PASS blocker=none" diff --git a/docker/migrations/forward/031_create_llm_call_metrics_and_cost_aggregates.sql b/docker/migrations/forward/031_create_llm_call_metrics_and_cost_aggregates.sql index 7a9602473e..f8625027a9 100644 --- a/docker/migrations/forward/031_create_llm_call_metrics_and_cost_aggregates.sql +++ b/docker/migrations/forward/031_create_llm_call_metrics_and_cost_aggregates.sql @@ -167,6 +167,142 @@ CREATE TABLE IF NOT EXISTS llm_cost_aggregates ( ) ); +-- ---- BEGIN OMN-15655 legacy shape reconciliation: llm_cost_aggregates ---- +-- CREATE TABLE IF NOT EXISTS no-ops when a legacy table with this name already +-- exists. Converge the table before column-dependent indexes/comments run, while +-- preserving pre-existing rows and failing loudly if required columns cannot be +-- made NOT NULL without inventing data. +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS id UUID; +ALTER TABLE llm_cost_aggregates ALTER COLUMN id SET DEFAULT gen_random_uuid(); +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS aggregation_key VARCHAR(512); +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS "window" cost_aggregation_window; +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS total_cost_usd NUMERIC(14, 6) DEFAULT 0; +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS total_tokens BIGINT DEFAULT 0; +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS call_count INTEGER DEFAULT 0; +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS estimated_coverage_pct NUMERIC(5, 2); +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_rows BIGINT; +BEGIN + LOOP + WITH batch AS ( + SELECT ctid + FROM llm_cost_aggregates + WHERE id IS NULL + LIMIT 10000 + ) + UPDATE llm_cost_aggregates AS target + SET id = gen_random_uuid() + FROM batch + WHERE target.ctid = batch.ctid; + + GET DIAGNOSTICS v_rows = ROW_COUNT; + EXIT WHEN v_rows = 0; + END LOOP; +END$$; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'aggregation_key', 'window', 'total_cost_usd', 'total_tokens', 'call_count', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'llm_cost_aggregates'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'llm_cost_aggregates'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15655: cannot converge llm_cost_aggregates.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling; the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +DECLARE + v_pk_columns TEXT[]; +BEGIN + SELECT array_agg(a.attname::text ORDER BY k.ordinality) INTO v_pk_columns + FROM pg_constraint c + CROSS JOIN LATERAL unnest(c.conkey) WITH ORDINALITY AS k(attnum, ordinality) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + WHERE c.conrelid = 'llm_cost_aggregates'::regclass + AND c.contype = 'p'; + + IF v_pk_columns IS NULL THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT llm_cost_aggregates_pkey PRIMARY KEY (id); + ELSIF v_pk_columns <> ARRAY['id']::text[] THEN + RAISE EXCEPTION + 'OMN-15655: llm_cost_aggregates primary key covers %, expected {id}; operator schema ruling required.', + v_pk_columns; + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'llm_cost_aggregates'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['aggregation_key', 'window']::text[] + ) THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT unique_aggregation_key_window UNIQUE (aggregation_key, "window"); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_cost_aggregates'::regclass AND conname = 'non_negative_total_cost_usd' + ) THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT non_negative_total_cost_usd CHECK (total_cost_usd >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_cost_aggregates'::regclass AND conname = 'non_negative_agg_total_tokens' + ) THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT non_negative_agg_total_tokens CHECK (total_tokens >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_cost_aggregates'::regclass AND conname = 'non_negative_call_count' + ) THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT non_negative_call_count CHECK (call_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_cost_aggregates'::regclass AND conname = 'valid_estimated_coverage_pct' + ) THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT valid_estimated_coverage_pct CHECK (estimated_coverage_pct IS NULL OR (estimated_coverage_pct >= 0.00 AND estimated_coverage_pct <= 100.00)); + END IF; +END$$; +-- ---- END OMN-15655 legacy shape reconciliation: llm_cost_aggregates ---- + -- ============================================================================ -- INDEXES: llm_call_metrics -- ============================================================================ diff --git a/docker/migrations/forward/050_create_baselines_tables.sql b/docker/migrations/forward/050_create_baselines_tables.sql index 105fe5cbde..31a0ad26f0 100644 --- a/docker/migrations/forward/050_create_baselines_tables.sql +++ b/docker/migrations/forward/050_create_baselines_tables.sql @@ -77,6 +77,117 @@ CREATE TABLE IF NOT EXISTS baselines_comparisons ( CONSTRAINT uk_baselines_comparisons_date UNIQUE (comparison_date) ); +-- ---- BEGIN OMN-15655 legacy shape reconciliation: baselines_comparisons ---- +-- Legacy application databases can already contain a node-shaped +-- baselines_comparisons table. CREATE TABLE IF NOT EXISTS then no-ops, and the +-- first index below fails on missing comparison_date. Converge empty drift +-- tables to the declared root shape; refuse non-empty incompatible tables so +-- the migration never invents historical A/B data. +DO $$ +DECLARE + v_rows BIGINT; + v_id_type TEXT; + v_pk_columns TEXT[]; +BEGIN + SELECT count(*) INTO v_rows FROM baselines_comparisons; + + SELECT data_type INTO v_id_type + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'baselines_comparisons' + AND column_name = 'id'; + + IF v_id_type IS DISTINCT FROM 'uuid' THEN + IF v_rows = 0 THEN + ALTER TABLE baselines_comparisons DROP COLUMN IF EXISTS id CASCADE; + ALTER TABLE baselines_comparisons ADD COLUMN id UUID DEFAULT gen_random_uuid(); + ALTER TABLE baselines_comparisons ADD CONSTRAINT baselines_comparisons_pkey PRIMARY KEY (id); + ELSE + RAISE EXCEPTION + 'OMN-15655: cannot converge baselines_comparisons.id from % to uuid with % pre-existing row(s); operator data mapping required.', + coalesce(v_id_type, ''), v_rows; + END IF; + ELSE + SELECT array_agg(a.attname::text ORDER BY k.ordinality) INTO v_pk_columns + FROM pg_constraint c + CROSS JOIN LATERAL unnest(c.conkey) WITH ORDINALITY AS k(attnum, ordinality) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + WHERE c.conrelid = 'baselines_comparisons'::regclass + AND c.contype = 'p'; + + IF v_pk_columns IS NULL THEN + ALTER TABLE baselines_comparisons ADD CONSTRAINT baselines_comparisons_pkey PRIMARY KEY (id); + ELSIF v_pk_columns <> ARRAY['id']::text[] THEN + RAISE EXCEPTION + 'OMN-15655: baselines_comparisons primary key covers %, expected {id}; operator schema ruling required.', + v_pk_columns; + END IF; + END IF; +END$$; + +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS comparison_date DATE; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS period_label TEXT; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS treatment_sessions BIGINT DEFAULT 0; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS treatment_success_rate REAL; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS treatment_avg_latency_ms REAL; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS treatment_avg_cost_tokens REAL; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS treatment_total_tokens BIGINT DEFAULT 0; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS control_sessions BIGINT DEFAULT 0; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS control_success_rate REAL; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS control_avg_latency_ms REAL; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS control_avg_cost_tokens REAL; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS control_total_tokens BIGINT DEFAULT 0; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS roi_pct REAL; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS latency_improvement_pct REAL; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS cost_improvement_pct REAL; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS sample_size BIGINT DEFAULT 0; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS computed_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY[ + 'id', 'comparison_date', 'treatment_sessions', 'treatment_total_tokens', + 'control_sessions', 'control_total_tokens', 'sample_size', + 'computed_at', 'created_at', 'updated_at' + ] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', + 'baselines_comparisons'::regclass, + v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', + 'baselines_comparisons'::regclass, + v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15655: cannot converge baselines_comparisons.% to NOT NULL -- % pre-existing row(s) hold NULL; operator data mapping required.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_comparisons'::regclass + AND conname = 'uk_baselines_comparisons_date' + ) THEN + ALTER TABLE baselines_comparisons + ADD CONSTRAINT uk_baselines_comparisons_date UNIQUE (comparison_date); + END IF; +END$$; +-- ---- END OMN-15655 legacy shape reconciliation: baselines_comparisons ---- + -- ============================================================================= -- INDEXES: baselines_comparisons -- ============================================================================= diff --git a/docker/migrations/forward/083_create_log_entries.sql b/docker/migrations/forward/083_create_log_entries.sql index eab67c8ff9..9d9baf37ad 100644 --- a/docker/migrations/forward/083_create_log_entries.sql +++ b/docker/migrations/forward/083_create_log_entries.sql @@ -1,4 +1,29 @@ -- ============================================================================= +-- TOMBSTONE (OMN-15846, 2026-08-10): this file is UNDELIVERABLE via the k8s +-- Job that applies docker/migrations/forward/*.sql +-- (omninode_infra/k8s/migrations/omnibase-infra-migrate.yaml). That Job owns +-- only the omnibase_infra database; its flat loop's `psql -f` apply is +-- gated on `directive_db == $DB_NAME` and is UNREACHABLE for this file's +-- `\connect omnidash_analytics` below, in that loop or any other in the +-- runner. It has never executed anywhere -- live-confirmed 2026-08-10 +-- (role_omnidash / omninode_runtime, omninode-dev-postgres RDS): +-- `to_regclass('public.log_entries')` is NULL in BOTH omnibase_infra and +-- omnidash_analytics. omnibase_infra's own schema_migrations carried a +-- false "applied" row for this file (applied_at 2026-07-28T22:53:47Z, +-- checksum byte-identical to this file's live content) -- the same +-- OMN-15819 masking class OMN-15846's classification-ordering fix +-- unmasked for 098/099. Kept in place, byte-unchanged below this header, +-- as ledgered history (migration files are append-only) -- do NOT delete +-- it and do NOT try to make it deliverable in place; the fix is the +-- node-owned migration under +-- docker/migrations/forward/nodes/node_log_persistence_effect/, which +-- CREATEs (this file's original intent, unchanged content) the table via +-- the node-owned loop's role_omnidash connection to omnidash_analytics -- +-- the one code path in the runner that can actually reach this database. +-- Static pre-merge enforcement in THIS repo: +-- tests/ci/test_flat_migration_no_foreign_connect_gate.py / +-- docker/migrations/forward/cross-database-flat-migrations.yaml. +-- ============================================================================= -- MIGRATION: Create log_entries table in omnidash_analytics -- ============================================================================= -- Ticket: OMN-12131 (Event Bus Observability — log_entries table) diff --git a/docker/migrations/forward/094_create_app_dashboard_role.sql b/docker/migrations/forward/094_create_app_dashboard_role.sql index 4515eb0df0..20547de5f9 100644 --- a/docker/migrations/forward/094_create_app_dashboard_role.sql +++ b/docker/migrations/forward/094_create_app_dashboard_role.sql @@ -2,7 +2,8 @@ -- MIGRATION: Create app_dashboard role (NOSUPERUSER, NOBYPASSRLS, non-owner) -- ============================================================================= -- Ticket: OMN-14899 (blocks OMN-14894 — RLS across the projection tables) --- Version: 1.0.0 +-- OMN-15343 (must apply under an ordinary, non-CREATEROLE role on RDS) +-- Version: 1.1.0 -- -- PURPOSE: -- app_dashboard is the RUNTIME connection role for the dashboard/projection @@ -18,25 +19,42 @@ -- after the guarded CREATE), not just requested at create time. A -- pre-existing app_dashboard role with either flag set is corrected, -- never trusted. --- * The SUPERUSER/REPLICATION/BYPASSRLS ALTER is privilege-guarded: only --- the executing role's OWN attributes gate ALTER ROLE's ability to --- change these three (Postgres core behavior, not this migration's --- choice) — even reasserting an already-correct `false` requires the --- executing role to already hold the attribute. RDS's master account --- is CREATEROLE + CREATEDB but explicitly NOSUPERUSER/NOREPLICATION/ --- NOBYPASSRLS, so an unconditional ALTER of those three flags fails --- `permission denied to alter role` on EVERY real RDS apply, not just --- when the role pre-exists with an escalated flag. This migration only --- attempts that ALTER when pg_roles shows one of the three already --- set, and raises an explicit, actionable exception (naming the --- escalated flag) rather than silently failing or silently succeeding --- if the executing role also lacks the privilege to correct it — that --- case needs a true superuser, by design, and must never pass quietly. --- * NOLOGIN here: no credential material ever lives in a migration. The --- LOGIN + password attach is a deployment-owned, operator-gated step --- (AWS Secrets Manager per OMN-14899; local lanes may ALTER ROLE ... --- LOGIN with lane-local credentials). Same convention as the --- omnidash_app role in omnidash/db/migrations/0001_tenant_rls.sql. +-- * EVERY privileged statement is gated on an OBSERVED DIVERGENCE. Postgres +-- requires role-administration rights for ALTER ROLE, and the executing +-- role's OWN attributes additionally gate SUPERUSER / REPLICATION / +-- BYPASSRLS — even reasserting an already-correct `false`. An +-- unconditional ALTER is therefore not "idempotent": it is a privilege +-- demand made on every apply. This file reads pg_roles first and issues a +-- statement only when observed state differs from required state. Two +-- consequences, both proven by execution in +-- tests/integration/migrations/test_094_app_dashboard_role.py: +-- - RDS's master account (CREATEROLE + CREATEDB but explicitly +-- NOSUPERUSER / NOREPLICATION / NOBYPASSRLS) applies this file +-- cleanly on a fresh cluster. +-- - An ORDINARY service role with NO CREATEROLE at all applies it +-- cleanly when the role already exists in the required state. That is +-- the live cloud case (OMN-15343): the k8s migration Job holds no +-- cluster-admin credential on the managed instance by design, and the +-- managed instance has no `postgres` role at all (live readback +-- 2026-07-29: `select count(*) from pg_roles where rolname='postgres'` +-- -> 0), so this file has to be a true no-op there or it can never be +-- recorded — and a migration that cannot be recorded blocks every +-- later migration behind it. +-- When a divergence IS observed and the executing role cannot correct it, +-- this migration raises an explicit, actionable exception naming the flag. +-- It never silently succeeds and never silently leaves an escalation in +-- place. +-- * NOLOGIN is a CREATE-TIME default, deliberately NOT re-asserted on a +-- pre-existing role. No credential material ever lives in a migration: the +-- LOGIN + password attach is a deployment-owned, operator-gated step (AWS +-- Secrets Manager per OMN-14899; local lanes may ALTER ROLE ... LOGIN with +-- lane-local credentials). Re-asserting NOLOGIN would REVOKE that +-- deployment-owned attach — on the cloud instance app_dashboard already +-- carries LOGIN (live readback 2026-07-29: rolcanlogin = t), so a blanket +-- `ALTER ROLE app_dashboard NOLOGIN` here would break the dashboard's +-- runtime connection as a side effect of recording a migration. LOGIN is +-- not the isolation control for this role in any case: NOSUPERUSER, +-- NOBYPASSRLS and non-ownership are, and all three are still enforced. -- * app_dashboard must NEVER own tables. Table creation stays with the -- migration/runtime role (postgres on compose lanes). Ownership would -- silently bypass ENABLE ROW LEVEL SECURITY. @@ -48,40 +66,78 @@ -- exists. -- -- IDEMPOTENCY: --- Safe to re-run: guarded CREATE ROLE (duplicate_object / unique_violation --- both caught — roles are cluster-wide and two migration paths may race, --- see omnidash 0001's OMN-10875 note); the unconditional ALTER is --- idempotent; the privilege-gated ALTER only runs its EXECUTE branch when --- pg_roles shows an actual escalation, so a correct role never touches it. +-- Safe to re-run: the CREATE is skipped when pg_roles already shows the role +-- AND still carries the duplicate_object / unique_violation guard for the +-- genuine race (roles are cluster-wide and two migration paths may race, see +-- omnidash 0001's OMN-10875 note); every ALTER runs only on an observed +-- divergence, so a correct role touches none of them. -- -- ROLLBACK: -- See rollback/rollback_094_create_app_dashboard_role.sql -- ============================================================================= +-- Guarded CREATE. The pg_roles pre-check is what makes this file runnable by a +-- role WITHOUT CREATEROLE: Postgres checks create-role privilege BEFORE it +-- checks whether the name is already taken, so an unconditional CREATE ROLE +-- raises `permission denied to create role` (42501) rather than the +-- duplicate_object the handler below is written for. The handler is retained +-- for the genuine race: two migration paths creating the role concurrently. DO $$ BEGIN - BEGIN - CREATE ROLE app_dashboard WITH - NOLOGIN - NOSUPERUSER - NOBYPASSRLS - NOCREATEDB - NOCREATEROLE - NOREPLICATION; - EXCEPTION - WHEN duplicate_object OR unique_violation THEN - NULL; -- role already exists (possibly created concurrently) - END; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app_dashboard') THEN + BEGIN + CREATE ROLE app_dashboard WITH + NOLOGIN + NOSUPERUSER + NOBYPASSRLS + NOCREATEDB + NOCREATEROLE + NOREPLICATION; + EXCEPTION + WHEN duplicate_object OR unique_violation THEN + NULL; -- role already exists (created concurrently) + END; + END IF; END; $$; --- Enforce the non-privilege-gated flags unconditionally — these never --- require the executing role to already hold them (RDS master account --- compatible: CREATEROLE alone is sufficient). -ALTER ROLE app_dashboard - NOLOGIN - NOCREATEDB - NOCREATEROLE; +-- CREATEDB / CREATEROLE: not gated by the executing role's own attributes the +-- way the three below are, but ALTER ROLE still demands role-administration +-- rights, so this is gated on an observed divergence too. Immediately after the +-- guarded CREATE, and on every re-run against a correct role, both flags are +-- already false and no statement is issued at all. +DO $$ +DECLARE + current_flags RECORD; +BEGIN + SELECT rolcreatedb, rolcreaterole + INTO current_flags + FROM pg_roles + WHERE rolname = 'app_dashboard'; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'app_dashboard role does not exist and could not be created — the ' + 'executing role lacks CREATEROLE. On a managed instance the role is ' + 'provisioned at the provisioning seam (OMN-15343); this migration ' + 'refuses to record itself against a role that is not there.'; + END IF; + + IF current_flags.rolcreatedb OR current_flags.rolcreaterole THEN + BEGIN + EXECUTE 'ALTER ROLE app_dashboard NOCREATEDB NOCREATEROLE'; + EXCEPTION + WHEN insufficient_privilege THEN + RAISE EXCEPTION + 'app_dashboard carries an unexpected privilege (rolcreatedb=%, ' + 'rolcreaterole=%) and the executing role lacks the role-administration ' + 'rights to correct it — fix this role at the provisioning seam before ' + 'the dashboard read path can be trusted', + current_flags.rolcreatedb, current_flags.rolcreaterole; + END; + END IF; +END; +$$; -- SUPERUSER/BYPASSRLS/REPLICATION are privilege-gated by Postgres core: the -- executing role must already hold an attribute to ALTER it, even to diff --git a/docker/migrations/forward/095_add_attach_readiness_to_runtime_manifests.sql b/docker/migrations/forward/095_add_attach_readiness_to_runtime_manifests.sql new file mode 100644 index 0000000000..3e2c3262f5 --- /dev/null +++ b/docker/migrations/forward/095_add_attach_readiness_to_runtime_manifests.sql @@ -0,0 +1,92 @@ +-- SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +-- SPDX-License-Identifier: MIT +-- +-- Migration 095: carry boot attach-readiness on the runtime_manifests row (OMN-15512). +-- +-- WHY: +-- The runtime already publishes what it WIRED (migration 079). What actually +-- ATTACHED was computed at boot into ModelRuntimeAttachReadiness and then +-- discarded into the log stream, so the only way to read the NOT-READY +-- blocker set was `ssh` + `docker logs omninode-runtime | grep NOT-READY`. +-- That is literally how OMN-15508 had to be diagnosed. These columns retire +-- that manual step: the blocker set with its named topics rides the SAME +-- event onto the SAME row — no new topic, no new reducer, no new table. +-- +-- This matters even when /health is green: on 2026-07-30 the dev lane served +-- /health 200 healthy:true with registered_handlers ["db","http"] WHILE +-- NOT-READY warnings were still firing (34 in a trailing 5m window). Green +-- liveness is not evidence that consumers attached. +-- +-- COLUMNS: +-- attach_state — aggregate tri-state from EnumRuntimeReadinessState +-- ('ready' | 'degraded' | 'failed'), plus 'unknown' +-- for a boot where the per-contract interleave never +-- ran. 'unknown' is deliberately NOT 'ready': absence +-- of evidence is not evidence of attachment. +-- attach_required_contracts — contracts the interleave walked. +-- attach_attached_contracts — contracts whose consumer attached. +-- attach_not_ready_contracts — the blocker set: one JSON object per contract that +-- did NOT attach, each a serialized +-- ModelContractAttachResult carrying contract_name, +-- status, topics_subscribed, the readiness confirm +-- outcome (including the failing topic names), and a +-- human-readable detail string. Bounded by the +-- not-attached count, NOT by the 475+ contracts walked +-- at boot: contracts that attached are already +-- enumerated in the `contracts`/`handlers` columns. +-- +-- INVARIANT (holds whenever attach_state <> 'unknown'): +-- jsonb_array_length(attach_not_ready_contracts) +-- = attach_required_contracts - attach_attached_contracts +-- +-- OPERATOR READBACK — replaces `docker logs | grep -c NOT-READY`: +-- SELECT started_at, +-- attach_state, +-- attach_attached_contracts || '/' || attach_required_contracts AS attached, +-- jsonb_array_length(attach_not_ready_contracts) AS not_ready, +-- jsonb_agg(blocker -> 'contract_name') AS contracts, +-- jsonb_agg(blocker -> 'readiness' -> 'failures') AS failing_topics +-- FROM runtime_manifests, +-- LATERAL jsonb_array_elements(attach_not_ready_contracts) AS blocker +-- WHERE runtime_profile = 'main' +-- GROUP BY started_at, attach_state, attach_attached_contracts, +-- attach_required_contracts, attach_not_ready_contracts +-- ORDER BY started_at DESC +-- LIMIT 1; +-- +-- IDEMPOTENCY: +-- ADD COLUMN IF NOT EXISTS is safe to re-run. +-- +-- BACKFILL: +-- None. runtime_manifests is append-only, one row per process boot; historical +-- rows predate the producer and correctly read attach_state='unknown'. +-- +-- ROLLBACK: +-- ALTER TABLE runtime_manifests +-- DROP COLUMN IF EXISTS attach_state, +-- DROP COLUMN IF EXISTS attach_required_contracts, +-- DROP COLUMN IF EXISTS attach_attached_contracts, +-- DROP COLUMN IF EXISTS attach_not_ready_contracts; + +ALTER TABLE runtime_manifests + ADD COLUMN IF NOT EXISTS attach_state TEXT NOT NULL DEFAULT 'unknown', + ADD COLUMN IF NOT EXISTS attach_required_contracts INT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS attach_attached_contracts INT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS attach_not_ready_contracts JSONB NOT NULL DEFAULT '[]'; + +-- Degraded-boot lookup: "which boots did not fully attach, most recent first". +-- Partial index — a fully-attached boot is the common case and is not indexed. +CREATE INDEX IF NOT EXISTS idx_runtime_manifests_attach_degraded + ON runtime_manifests (runtime_profile, started_at DESC) + WHERE attach_state <> 'ready'; + +COMMENT ON COLUMN runtime_manifests.attach_state IS + 'Aggregate boot attach tri-state (OMN-15512): ready | degraded | failed, or ' + 'unknown when the per-contract interleave did not run. Never infer ' + 'attachment from process liveness — a green /health can coexist with ' + 'attach_state=degraded.'; + +COMMENT ON COLUMN runtime_manifests.attach_not_ready_contracts IS + 'Blocker set (OMN-15512): serialized ModelContractAttachResult per contract ' + 'that did NOT attach, including the topics whose readiness confirm failed. ' + 'Replaces `docker logs omninode-runtime | grep NOT-READY`.'; diff --git a/docker/migrations/forward/096_grant_role_omnidash_omnidash_analytics.sql b/docker/migrations/forward/096_grant_role_omnidash_omnidash_analytics.sql new file mode 100644 index 0000000000..95a3fad6d2 --- /dev/null +++ b/docker/migrations/forward/096_grant_role_omnidash_omnidash_analytics.sql @@ -0,0 +1,391 @@ +-- ============================================================================= +-- TOMBSTONE (OMN-15846, 2026-08-10): this file is UNDELIVERABLE via the k8s +-- Job that applies docker/migrations/forward/*.sql +-- (omninode_infra/k8s/migrations/omnibase-infra-migrate.yaml) and, on the +-- one target where it matters (onex-dev RDS), UNNEEDED besides. +-- +-- UNDELIVERABLE: that Job owns only the omnibase_infra database; its flat +-- loop's `psql -f` apply is gated on `directive_db == $DB_NAME` and is +-- UNREACHABLE for this file's `\connect omnidash_analytics` below, in that +-- loop or any other in the runner. It has never executed there -- +-- live-confirmed 2026-08-10 (omninode-dev-postgres RDS): +-- `pg_default_acl` for omnidash_analytics/public is EMPTY (zero rows) -- +-- this file's own unconditional step 6 +-- `ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ... TO role_omnidash` +-- would ALWAYS leave a pg_default_acl entry if it had ever run to +-- completion under ANY executing role; none exists. omnibase_infra's own +-- schema_migrations carried a false "applied" row for this file +-- (applied_at 2026-08-01T22:00:18Z, checksum byte-identical to this file's +-- live content) -- the same OMN-15819 masking class OMN-15846's +-- classification-ordering fix unmasked for 098/099. +-- +-- UNNEEDED ON RDS: this file's OWN header below documents its purpose as a +-- FORCE-RLS non-owner-pool repair for the `.201 lab lane` specifically +-- (compose project `omnibase-infra`) -- read the "PURPOSE" section that +-- follows, unchanged. On onex-dev RDS, role_omnidash is not a marginal +-- non-owner role at all: it is the RDS migration principal (OMN-15335 +-- two-owner split) and OWNS 89 of 90 tables in omnidash_analytics.public +-- (live-verified via pg_tables.tableowner, 2026-08-10) -- an owner is +-- exempt from RLS regardless of rolsuper/rolbypassrls (this file's own +-- step 7 assertion states the same fact), and an owner needs no SELECT/ +-- INSERT/UPDATE grant this file would add. role_omnidash's CREATE on +-- schema public and its one non-owned-table grant (generation_events) are +-- both live-verified as granted BY `pg_database_owner`/omninodeadmin (the +-- provisioning-seam path), not by role_omnibase_infra (the identity this +-- file would have executed under) -- a second, independent confirmation +-- this file never delivered anything on this database. OMN-15355 (P1, "In +-- Review" as of 2026-08-10) is the tracked, systematic successor: it +-- generates the complete ACL/default-privilege matrix from contracts +-- across every database/schema/table, explicitly scoped to include +-- role_omnidash-owned domains and app_dashboard's access boundary -- +-- superseding this file's stopgap intent rather than leaving a gap next +-- to it. +-- +-- Kept in place, byte-unchanged below this header, as ledgered history +-- (migration files are append-only) -- do NOT delete it and do NOT try to +-- make it deliverable in place; it remains applicable, unchanged, to the +-- .201 lab lane it was authored for (that lane's own runner, +-- scripts/run-forward-migrations.sh, is a different code path from the +-- k8s Job this tombstone concerns and is unaffected by it). Static +-- pre-merge enforcement in THIS repo: +-- tests/ci/test_flat_migration_no_foreign_connect_gate.py / +-- docker/migrations/forward/cross-database-flat-migrations.yaml. +-- ============================================================================= +-- MIGRATION: role_omnidash authorization on omnidash_analytics (warm-volume safe) +-- ============================================================================= +-- Ticket: OMN-15363 (open question 1 — "should the compose lane provision +-- role_omnidash, as the k8s/RDS path does?" Answer: yes, and the +-- AUTHORIZATION half belongs in the migration chain.) +-- Related: OMN-15416 (P0/P3 gate — prove FORCE-RLS with real non-owner pools), +-- OMN-15351 (0027's role_omnidash guard), OMN-14899/094 (app_dashboard) +-- Version: 1.0.0 +-- +-- PURPOSE +-- Postgres exempts a table's OWNER and any role with SUPERUSER or BYPASSRLS +-- from row-level security unconditionally — FORCE included. So on a lane +-- whose only connecting role is `postgres` (superuser + rolbypassrls), the +-- RLS/FORCE state on a table is INERT: every policy evaluates against nothing. +-- +-- Live readback, .201 lab lane (compose project `omnibase-infra`), DB +-- `omnidash_analytics`, 2026-07-31T03:33Z — 400 consecutive samples of +-- pg_stat_activity returned exactly ONE client, `postgres` from 172.19.0.10: +-- +-- select rolname, rolsuper, rolbypassrls from pg_roles where rolcanlogin; +-- postgres | t | t +-- role_omniweb | f | f +-- +-- node_service_registry, projection_delegation_inference_response_text and +-- savings_estimates all carry `relrowsecurity` AND `relforcerowsecurity` on +-- that lane and are all owned by `postgres`. The lane has therefore been +-- observed "clean under FORCE" for ~10h47m without ever having exercised the +-- mechanism against a single non-exempt connection. +-- +-- This migration provisions the AUTHORIZATION half of the repair: the +-- non-owner, non-superuser, non-BYPASSRLS role the runtime can connect as. +-- The DSN half (which role the lab lane actually connects with) is +-- `docker/docker-compose.dev-lane.yml`, and the CREDENTIAL half +-- (LOGIN + password) stays deployment-owned — see "WHAT THIS FILE IS NOT". +-- +-- WHY role_omnidash AND NOT A NEW ROLE +-- role_omnidash is already the declared per-service identity for +-- `omnidash_analytics`: `docker/migrations/forward/000_create_multiple_databases.sh` +-- maps `omnidash_analytics:role_omnidash:ROLE_OMNIDASH_PASSWORD`, node +-- migration 0027 grants it the generation_events writer set, and it is the +-- role the cloud RDS path already connects as. Minting a second writer role +-- for the same shape would be the duplicate this repo's one-canonical-model +-- rule exists to prevent. +-- +-- WHY A MIGRATION AND NOT JUST 000 +-- 000_create_multiple_databases.sh is the sanctioned bootstrap, and it is +-- correct — but it is mounted at `/docker-entrypoint-initdb.d` and therefore +-- runs ONLY when the postgres data directory is empty. Every warm lane (the +-- .201 lab lane included, whose volume long predates any ROLE_OMNIDASH_PASSWORD +-- being configured) can never receive those grants. `ROLE_OMNIDASH_PASSWORD` +-- was unset when that volume was initialised, so 000 printed +-- `SKIP: role_omnidash — ROLE_OMNIDASH_PASSWORD not set` and skipped the role, +-- its grants, and its CONNECT privilege in one step. This file re-establishes +-- the grant half on warm volumes through the forward-migration one-shot, which +-- IS part of the sanctioned deploy path. +-- +-- WHAT THIS FILE IS NOT +-- It carries NO credential material and does NOT grant LOGIN. That is the +-- same invariant migration 094 states for app_dashboard: the LOGIN + password +-- attach is deployment-owned (AWS Secrets Manager on the cloud path; +-- ROLE_OMNIDASH_PASSWORD consumed by 000 on compose lanes). Re-asserting +-- NOLOGIN on a pre-existing role would REVOKE that deployment-owned attach, so +-- NOLOGIN here is a CREATE-TIME default only and is never re-asserted. +-- +-- It also does NOT grant CREATE ON SCHEMA public. A role that can create +-- tables OWNS them, and an owner is exempt from RLS — granting CREATE to the +-- application role would reopen, for every future table, exactly the bypass +-- this file exists to close. It equally does NOT *revoke* CREATE: on cloud +-- RDS role_omnidash is the migration principal and performs DDL (OMN-15335), so +-- a blanket revoke here would break the cloud migration path as a side effect. +-- The invariant that is asserted instead — fail-closed, at the bottom of this +-- file — is that role_omnidash owns none of the RLS-covered tables. +-- +-- DATABASE CONTEXT +-- The forward runner applies `docker/*.sql` against POSTGRES_DB +-- (`omnibase_infra`), so this file switches with `\connect omnidash_analytics` +-- partway through — the established in-repo pattern (see migration 083), and +-- the runner's `ensure_directive_database` handles the directive explicitly. +-- Role attributes and GRANT ... ON DATABASE are cluster-wide and are issued +-- BEFORE the switch; every schema/table grant is issued after it. +-- +-- IDEMPOTENCY +-- Safe to re-run. The CREATE is skipped when pg_roles already shows the role +-- (and keeps the duplicate_object/unique_violation handler for the genuine +-- concurrent-creation race). Every ALTER ROLE runs only on an OBSERVED +-- divergence — an unconditional ALTER is not idempotent, it is a privilege +-- demand made on every apply, and Postgres gates SUPERUSER/BYPASSRLS on the +-- EXECUTING role's own attributes even when reasserting an already-correct +-- `false` (094's finding, and the reason the RDS migration principal can apply +-- this file at all). GRANTs are idempotent by definition. +-- +-- ROLLBACK +-- See rollback/rollback_096_grant_role_omnidash_omnidash_analytics.sql +-- ============================================================================= + +-- ----------------------------------------------------------------------------- +-- 1. Role existence. Guarded so a role WITHOUT CREATEROLE can apply this file: +-- Postgres checks create-role privilege BEFORE it checks whether the name is +-- taken, so an unconditional CREATE ROLE raises 42501 rather than the +-- duplicate_object the handler below is written for. +-- ----------------------------------------------------------------------------- +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'role_omnidash') THEN + BEGIN + CREATE ROLE role_omnidash WITH + NOLOGIN + NOSUPERUSER + NOBYPASSRLS + NOREPLICATION; + EXCEPTION + WHEN duplicate_object OR unique_violation THEN + NULL; -- created concurrently by another migration path + END; + END IF; +END; +$$; + +-- ----------------------------------------------------------------------------- +-- 2. The two flags that decide whether RLS applies to this role at all. +-- Deliberately scoped to rolsuper + rolbypassrls: rolcreatedb/rolcreaterole +-- are legitimately held by role_omnidash on cloud RDS, where it is the +-- migration principal (OMN-15335). Correcting those here would break that +-- path; neither affects RLS exemption. +-- +-- Gated on observed divergence. If the flag IS set and the executing role +-- cannot correct it, fail loudly and name it — never silently succeed, never +-- silently leave the escalation in place. +-- ----------------------------------------------------------------------------- +DO $$ +DECLARE + current_flags RECORD; +BEGIN + SELECT rolsuper, rolbypassrls + INTO current_flags + FROM pg_roles + WHERE rolname = 'role_omnidash'; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'role_omnidash does not exist and could not be created — the executing ' + 'role lacks CREATEROLE. On a managed instance the role is provisioned at ' + 'the provisioning seam; this migration refuses to record itself against a ' + 'role that is not there.'; + END IF; + + IF current_flags.rolsuper OR current_flags.rolbypassrls THEN + BEGIN + EXECUTE 'ALTER ROLE role_omnidash NOSUPERUSER NOBYPASSRLS'; + EXCEPTION + WHEN insufficient_privilege THEN + RAISE EXCEPTION + 'role_omnidash carries an RLS-exempting flag (rolsuper=%, ' + 'rolbypassrls=%) and the executing role lacks privilege to correct ' + 'it — a true superuser must fix this role before any FORCE ROW LEVEL ' + 'SECURITY state on omnidash_analytics can be trusted', + current_flags.rolsuper, current_flags.rolbypassrls; + END; + END IF; +END; +$$; + +-- ----------------------------------------------------------------------------- +-- 3. CONNECT on the target database. Issued from the current (omnibase_infra) +-- context because GRANT ... ON DATABASE is cluster-wide, and guarded on the +-- database existing so this file is valid on a cluster that has not been +-- through 000 (e.g. a bare CI Postgres). +-- OMN-15297 is the same defect for app_dashboard: policy + table grants +-- without CONNECT is a role that cannot open a session at all. +-- ----------------------------------------------------------------------------- +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_database WHERE datname = 'omnidash_analytics') THEN + EXECUTE 'GRANT CONNECT ON DATABASE omnidash_analytics TO role_omnidash'; + END IF; +END; +$$; + +\connect omnidash_analytics + +-- ----------------------------------------------------------------------------- +-- 4. Schema resolution. USAGE only — see "WHAT THIS FILE IS NOT" above for why +-- CREATE is neither granted nor revoked here. +-- ----------------------------------------------------------------------------- +GRANT USAGE ON SCHEMA public TO role_omnidash; + +-- ----------------------------------------------------------------------------- +-- 5. Named, least-privilege DML on the tables that are RLS-FORCED today. +-- These three are spelled out rather than left to the blanket grant in step 6 +-- because they are the tables whose isolation this ticket is about: a future +-- narrowing of step 6 must not silently drop the writer's access to them, and +-- a reader of this file should be able to see the exact privilege set the +-- proving ground runs under without deriving it. +-- +-- SELECT, INSERT, UPDATE mirrors node migration 0027's writer set for +-- generation_events. No DELETE, no TRUNCATE, no REFERENCES, no TRIGGER: a +-- projection writer upserts, it does not reshape the table. Each grant is +-- guarded on the table existing, because the registration trio is fenced off +-- every lane except the lab (OMN-15379) and savings/inference-response tables +-- arrive from their own node migration chains. +-- ----------------------------------------------------------------------------- +DO $$ +DECLARE + forced_table text; +BEGIN + FOREACH forced_table IN ARRAY ARRAY[ + 'node_service_registry', + 'projection_delegation_inference_response_text', + 'savings_estimates' + ] LOOP + IF to_regclass('public.' || forced_table) IS NOT NULL THEN + EXECUTE format( + 'GRANT SELECT, INSERT, UPDATE ON public.%I TO role_omnidash', + forced_table + ); + ELSE + RAISE NOTICE + 'skipping grant on public.% — table not present on this lane', forced_table; + END IF; + END LOOP; +END; +$$; + +-- ----------------------------------------------------------------------------- +-- 6. The rest of the analytics surface. +-- Not decoration: `omnidash_analytics` carries ~55 projection tables on the +-- lab lane and the runtime writes across them. Cutting the connection to a +-- non-owner role without these grants would trade a silent RLS bypass for a +-- loud 42501 on every projection that is NOT RLS-covered — which proves +-- nothing about isolation and takes the lane's whole projection path down. +-- This is the same grant set 000_create_multiple_databases.sh applies to +-- every per-service role; it is reproduced here so warm volumes converge on +-- the same state a fresh volume gets. +-- +-- ALTER DEFAULT PRIVILEGES applies only to objects created BY THE EXECUTING +-- ROLE (same caveat 000 carries). On compose lanes that is `postgres`, which +-- is also the migration/table owner, so future projection tables inherit the +-- grant. A lane whose migrations run as a different principal must set its +-- own default privileges at its provisioning seam. +-- ----------------------------------------------------------------------------- +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO role_omnidash; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO role_omnidash; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO role_omnidash; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT USAGE, SELECT ON SEQUENCES TO role_omnidash; + +-- ----------------------------------------------------------------------------- +-- 6b. Re-narrow the RLS-FORCED tables after the blanket grant. +-- Step 5's named grant is SELECT/INSERT/UPDATE, but step 6's +-- `ON ALL TABLES IN SCHEMA public` is a SUPERSET of it and silently re-adds +-- DELETE to the very three tables step 5 was careful to exclude it from. +-- Order matters and this is the only order that works: a REVOKE placed +-- before the blanket GRANT would simply be overwritten. +-- +-- Verified live on the lab lane 2026-07-31T03:56Z: without this block, +-- `information_schema.role_table_grants` reported +-- `DELETE,INSERT,SELECT,UPDATE` on all three — the named grant read as +-- least-privilege while the effective privilege set was not. A claim that is +-- true of a statement but false of the resulting state is the failure mode +-- this block removes. +-- +-- TRUNCATE / REFERENCES / TRIGGER are revoked in the same breath: they are +-- not in the blanket grant today, so these are no-ops that pin the intent +-- against a future widening of step 6. +-- ----------------------------------------------------------------------------- +DO $$ +DECLARE + forced_table text; +BEGIN + FOREACH forced_table IN ARRAY ARRAY[ + 'node_service_registry', + 'projection_delegation_inference_response_text', + 'savings_estimates' + ] LOOP + IF to_regclass('public.' || forced_table) IS NOT NULL THEN + EXECUTE format( + 'REVOKE DELETE, TRUNCATE, REFERENCES, TRIGGER ON public.%I FROM role_omnidash', + forced_table + ); + END IF; + END LOOP; +END; +$$; + +-- ----------------------------------------------------------------------------- +-- 7. Post-conditions. +-- Grants are not the isolation control — ownership and the two role flags +-- are. If role_omnidash owns an RLS-covered table, every policy on that +-- table is inert for it no matter what this file granted, and the lane would +-- report a false clean. Assert both, do not assume either. +-- +-- The two assertions have DELIBERATELY DIFFERENT severities, following the +-- OMN-15351 split: a fact this file itself just established is FATAL when it +-- is wrong, an environment-provisioned fact this file does not own is a +-- WARNING that ENUMERATES what it found. +-- * role flags — set by step 2 of this same file. EXCEPTION. +-- * table ownership — decided at each lane's provisioning seam. On cloud +-- RDS role_omnidash owns part of the schema by design (OMN-15335's +-- two-owner split), so a RAISE here would wedge the entire cloud +-- migration chain behind a lab-lane ticket. WARNING, named table by +-- table, so the state is logged rather than silently tolerated. +-- ----------------------------------------------------------------------------- +DO $$ +DECLARE + owned_rls_tables text; + flags RECORD; +BEGIN + SELECT string_agg(c.relname, ', ' ORDER BY c.relname) + INTO owned_rls_tables + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind = 'r' + AND n.nspname = 'public' + AND c.relrowsecurity + AND pg_get_userbyid(c.relowner) = 'role_omnidash'; + + IF owned_rls_tables IS NOT NULL THEN + RAISE WARNING + 'role_omnidash OWNS RLS-covered table(s) in omnidash_analytics: %. An ' + 'owner is exempt from row-level security (FORCE included), so RLS on ' + 'these tables is INERT against the application role and any "clean under ' + 'RLS" reading taken from them is a false clean. Reassign ownership at the ' + 'provisioning seam before citing RLS evidence from this database.', + owned_rls_tables; + END IF; + + SELECT rolsuper, rolbypassrls INTO flags + FROM pg_roles WHERE rolname = 'role_omnidash'; + + IF flags.rolsuper OR flags.rolbypassrls THEN + RAISE EXCEPTION + 'role_omnidash still carries rolsuper=% / rolbypassrls=% after this ' + 'migration — RLS would be inert for it', + flags.rolsuper, flags.rolbypassrls; + END IF; +END; +$$; diff --git a/docker/migrations/forward/097_grant_app_dashboard_connect_omnidash_analytics.sql b/docker/migrations/forward/097_grant_app_dashboard_connect_omnidash_analytics.sql new file mode 100644 index 0000000000..dd86071553 --- /dev/null +++ b/docker/migrations/forward/097_grant_app_dashboard_connect_omnidash_analytics.sql @@ -0,0 +1,205 @@ +-- onex-create-database: omnidash_analytics +-- ============================================================================= +-- TOMBSTONE (OMN-15846, 2026-08-10): this file is UNDELIVERABLE via the k8s +-- Job that applies docker/migrations/forward/*.sql +-- (omninode_infra/k8s/migrations/omnibase-infra-migrate.yaml) and, on +-- onex-dev RDS today, LATENT rather than live-broken besides. +-- +-- UNDELIVERABLE: that Job owns only the omnibase_infra database; its flat +-- loop's `psql -f` apply is gated on `directive_db == $DB_NAME` and is +-- UNREACHABLE for this file's `\connect omnidash_analytics` below, in that +-- loop or any other in the runner. It has never executed there -- +-- live-confirmed 2026-08-10 (omninode-dev-postgres RDS): app_dashboard's +-- USAGE on schema public is granted BY `pg_database_owner` +-- (`nspacl: app_dashboard=U/pg_database_owner`), never by +-- role_omnibase_infra (the identity this file would have executed under) +-- -- this file's own step 3 `GRANT USAGE ON SCHEMA public TO app_dashboard` +-- left no trace. omnibase_infra's own schema_migrations carried a false +-- "applied" row for this file (applied_at 2026-08-01T22:00:19Z, checksum +-- byte-identical to this file's live content) -- the same OMN-15819 +-- masking class OMN-15846's classification-ordering fix unmasked for +-- 098/099. +-- +-- LATENT, NOT LIVE-BROKEN: this file's own header (the "THE DEFECT" section +-- below, unchanged) documents that the CONNECT gap it repairs is latent +-- until PUBLIC's CONNECT is revoked platform-wide (OMN-15355) -- "not an +-- edge case being defended against ... the declared target state." Live +-- readback 2026-08-10 confirms PUBLIC's CONNECT on omnidash_analytics has +-- NOT been revoked on this RDS instance (`pg_database.datacl` carries +-- `=Tc/omninodeadmin`, i.e. PUBLIC still holds CONNECT), so app_dashboard +-- connects today via that still-live PUBLIC default, not via this file. +-- OMN-15355 (P1, "In Review" as of 2026-08-10, explicit acceptance +-- criterion naming app_dashboard by name) is the tracked, systematic +-- successor that will retire PUBLIC's default CONNECT under its own +-- generated ACL matrix and change-window gate -- it is the correct owner +-- of app_dashboard's post-revocation CONNECT grant, not a flat migration +-- this k8s Job cannot deliver. +-- +-- Kept in place, byte-unchanged below this header, as ledgered history +-- (migration files are append-only) -- do NOT delete it and do NOT try to +-- make it deliverable in place; it remains applicable, unchanged, on any +-- lane whose own runner (scripts/run-forward-migrations.sh, a different +-- code path from the k8s Job this tombstone concerns) can actually reach +-- omnidash_analytics. Static pre-merge enforcement in THIS repo: +-- tests/ci/test_flat_migration_no_foreign_connect_gate.py / +-- docker/migrations/forward/cross-database-flat-migrations.yaml. +-- ============================================================================= +-- MIGRATION: app_dashboard CONNECT + policy-gated read grants on omnidash_analytics +-- ============================================================================= +-- Ticket: OMN-15297 (the grant chain never grants CONNECT — the read role +-- cannot open a session at all) +-- Related: OMN-14899/094 (the role), OMN-14894/0023 (the RLS policies + table +-- grants), OMN-15363/096 (same repair for the writer role_omnidash), +-- OMN-15355 (revoke PUBLIC's CONNECT — the change that makes this +-- latent defect fatal) +-- Version: 1.0.0 +-- +-- THE DEFECT +-- 094 creates app_dashboard. Node migration 0023 grants USAGE ON SCHEMA +-- public and SELECT on the two RLS-covered delegation tables. Nothing in +-- that chain ever grants CONNECT ON DATABASE. On a stock Postgres CONNECT is +-- held by PUBLIC, so the gap is LATENT and every test passes. On a database +-- where PUBLIC's CONNECT has been revoked the role cannot open a session at +-- all and every grant behind it is unreachable: +-- +-- FATAL: permission denied for database "omnidash_analytics" +-- DETAIL: User does not have CONNECT privilege. +-- +-- Live readback, .201 dev lane 2026-07-28 (compose project +-- `omnibase-infra`, db `omnidash_analytics`), with LOGIN attached: +-- +-- rolname | rolcanlogin | rolsuper | rolbypassrls | can_connect +-- app_dashboard | t | f | f | f +-- +-- This reads like a credential problem and is a missing grant. OMN-15355 +-- revokes PUBLIC's CONNECT platform-wide by design, so this is not an +-- edge case being defended against — it is the declared target state. +-- +-- WHY AN ADDITIVE FILE AND NOT AN EDIT TO 0023 +-- 0023 is a VENDORED node migration: the authoritative copy lives in +-- omnimarket (`src/omnimarket/nodes/node_projection_delegation/migrations/`) +-- and is mirrored here by `scripts/sync-node-migrations.sh`. Editing the +-- mirror would be silently reverted by the next sync. 0023 is also on the +-- operator fence (`FENCED_NODE_MIGRATION_IDS`, OMN-15336), so a fix placed +-- inside it would not execute on any fenced lane — the repair has to live +-- where it can actually run. +-- +-- WHY app_dashboard AND NOT role_omnidash +-- Two roles, two paths, both real: role_omnidash is the projection WRITER +-- (096) and app_dashboard is the dashboard READ role (094). This file is the +-- read half only. It grants no DML anywhere. +-- +-- WHAT THIS FILE IS NOT +-- * It carries NO credential material and does NOT grant LOGIN. The +-- LOGIN + password attach is deployment-owned and operator-gated (AWS +-- Secrets Manager, `omninode/staging/rds/app-dashboard`), exactly as 094 +-- states. Re-asserting NOLOGIN would REVOKE that attach. +-- * It does NOT revoke PUBLIC's CONNECT. That is OMN-15355, it has real +-- blast radius across every role and database on the instance, and it is +-- not this ticket's to take unilaterally. +-- * It does NOT grant CREATE ON DATABASE or CREATE ON SCHEMA. A role that +-- can create objects OWNS them, and an owner is exempt from row-level +-- security — FORCE included. Granting CREATE to the read role would +-- reopen, for every future table, precisely the bypass OMN-14894 exists +-- to close. +-- * It does NOT grant SELECT on the projection VIEWS (0010). Postgres +-- evaluates RLS against the VIEW OWNER, so reading through an owner's view +-- silently bypasses tenant isolation until the views are recreated with +-- `security_invoker = true`. Step 3 filters to `relkind = 'r'` for that +-- reason, not by accident. +-- +-- TABLE GRANTS STAY WITH 0023 +-- The table-level SELECT grants stay in vendored node migration 0023, next +-- to the tenant_isolation policies they depend on. Top-level migration 097 +-- is applied before node migrations on fresh asyncpg lanes, so relation +-- grants here would either fail before the tables exist or require dynamic +-- SQL that the OMN-15361 application database gate rightly rejects. +-- +-- DATABASE CONTEXT +-- The forward runner applies `docker/migrations/forward/*.sql` against +-- POSTGRES_DB (`omnibase_infra`), so this file switches with +-- `\connect omnidash_analytics` partway through — the established in-repo +-- pattern (083, 096). GRANT ... ON DATABASE is cluster-wide and is issued +-- BEFORE the switch; every schema/table grant is issued after it. The +-- `onex-create-database` directive on line 1 is honoured by +-- `ensure_directive_database` in `scripts/run-forward-migrations.sh`, so the +-- `\connect` cannot fail on a cluster that has not been through +-- `000_create_multiple_databases.sh`. +-- +-- IDEMPOTENCY +-- Safe to re-run. GRANT/REVOKE are idempotent by definition, and every +-- statement that is not is gated on an observed catalog read. No ALTER ROLE +-- appears anywhere in this file: an unconditional ALTER is not idempotent, +-- it is a privilege demand made on every apply, and it is what stopped 094 +-- applying under the RDS master (OMN-14899) and then under the ordinary +-- service role the k8s Job uses (OMN-15343). +-- +-- EXECUTING-ROLE REQUIREMENTS +-- GRANT requires the executing role to hold the privilege WITH GRANT OPTION +-- or to be the object owner. On compose lanes that is `postgres`; on the +-- managed instance it is the per-database migration principal, which owns +-- the database and the projection tables (OMN-15335). No CREATEROLE and no +-- superuser is required by this file. +-- +-- ROLLBACK +-- See rollback/rollback_097_grant_app_dashboard_connect_omnidash_analytics.sql +-- ============================================================================= + +-- ----------------------------------------------------------------------------- +-- 1. The read role must already exist. It is created by 094, which runs first +-- (filename order). The direct GRANT below fails closed if the role is +-- absent, without using a procedural block that OMN-15361 cannot statically +-- prove. +-- ----------------------------------------------------------------------------- + +-- ----------------------------------------------------------------------------- +-- 2. CONNECT on the target database — the defect this ticket names. +-- Issued from the current (omnibase_infra) context because +-- GRANT ... ON DATABASE is cluster-wide, and guarded on the database +-- existing so this file is valid on a cluster that has not been through +-- 000_create_multiple_databases.sh. +-- +-- The database name is a literal, not current_database(): a plain GRANT has +-- no current_database() shorthand, and the forward runner is connected to +-- omnibase_infra at this point, so resolving it dynamically would grant +-- CONNECT on the WRONG database. +-- ----------------------------------------------------------------------------- +GRANT CONNECT ON DATABASE omnidash_analytics TO app_dashboard; + +\connect omnidash_analytics + +-- ----------------------------------------------------------------------------- +-- 3. Schema resolution. USAGE only — CREATE is neither granted (see "WHAT THIS +-- FILE IS NOT") nor revoked (revoking it from the migration principal would +-- break the cloud migration path as a side effect, OMN-15335). USAGE alone +-- confers no data access; every row still goes through step 3b. +-- ----------------------------------------------------------------------------- +GRANT USAGE ON SCHEMA public TO app_dashboard; + +-- ----------------------------------------------------------------------------- +-- 3b. Table-level SELECT is deliberately absent here. Vendored node migration +-- 0023 owns those grants because it also owns the tenant_isolation policy +-- creation and runs after the delegation tables exist. +-- ----------------------------------------------------------------------------- + +-- ----------------------------------------------------------------------------- +-- 4. Post-conditions. Grants are not the isolation control — the role flags +-- are — so they are asserted, not assumed. +-- +-- These SELECT assertions deliberately avoid DO/RAISE so the deployable SQL +-- remains statically provable by OMN-15361. +-- ----------------------------------------------------------------------------- +SELECT 1 / count(*) AS app_dashboard_connect_assertion + FROM ( + SELECT 1 + WHERE has_database_privilege('app_dashboard', current_database(), 'CONNECT') + ) AS assertion; + +SELECT 1 / count(*) AS app_dashboard_rls_role_flags_assertion + FROM ( + SELECT 1 + FROM pg_catalog.pg_roles + WHERE rolname = 'app_dashboard' + AND NOT rolsuper + AND NOT rolbypassrls + ) AS assertion; diff --git a/docker/migrations/forward/098_create_omninode_internal_schema.sql b/docker/migrations/forward/098_create_omninode_internal_schema.sql new file mode 100644 index 0000000000..28a0e61192 --- /dev/null +++ b/docker/migrations/forward/098_create_omninode_internal_schema.sql @@ -0,0 +1,125 @@ +-- onex-create-database: omnidash_analytics +-- ============================================================================= +-- TOMBSTONE (OMN-15819, 2026-08-10): this file is UNDELIVERABLE via the k8s +-- Job that applies docker/migrations/forward/*.sql +-- (omninode_infra/k8s/migrations/omnibase-infra-migrate.yaml). That Job owns +-- only the omnibase_infra database; its flat loop's `psql -f` apply is +-- gated on `directive_db == $DB_NAME` and is UNREACHABLE for this file's +-- `\connect omnidash_analytics` below, in that loop or any other in the +-- runner. It has never executed anywhere -- live-confirmed 2026-08-10 +-- (omninode_internal schema exists, out-of-band-created, but this file's +-- own CREATE SCHEMA statement is not why). Kept in place, byte-unchanged +-- below this header, as ledgered history (migration files are append-only) +-- -- do NOT delete it and do NOT try to make it deliverable in place; the +-- fix is the node-owned migration under +-- docker/migrations/forward/nodes/node_projection_live_events/, which +-- ASSERTS (does not create) this schema as a guarded precondition of the +-- table it owns (0002_create_omninode_internal_live_events.sql) -- the +-- schema itself is operator-provisioned out-of-band, not created by that +-- replacement or by this file's own CREATE SCHEMA on any path that +-- actually executes. Runner honesty for +-- this class of file (no silent false-"applied" ledger row; fail-closed for +-- any FUTURE cross-DB flat file) is the companion PR in omninode_infra. +-- Static pre-merge enforcement in THIS repo: +-- tests/ci/test_flat_migration_no_foreign_connect_gate.py / +-- docker/migrations/forward/cross-database-flat-migrations.yaml. +-- ============================================================================= +-- MIGRATION: physically create the omninode_internal schema (empty, additive) +-- ============================================================================= +-- Ticket: OMN-15359 (P2-P4 build classified schemas and migrate internal, +-- control-plane, catalog, and tenant targets) +-- Related: OMN-15426 (P5 cut internal projections to the omninode_runtime +-- identity — the consumer this migration unblocks), OMN-15423 (P0 +-- relation inventory/classification, the omninode_runtime TABLE +-- grant list this migration's companion physical-schema-mapping +-- change is derived from), OMN-15355 (one-DB domain separation) +-- Version: 1.0.0 +-- +-- WHAT THIS FILE DOES +-- `CREATE SCHEMA IF NOT EXISTS omninode_internal` inside `omnidash_analytics` +-- (the physical database backing the unified "application" topology +-- database — `docker/catalog/database-topology/*.yaml` `physical_name`). +-- That is the ENTIRE mutation. No table is created, moved, or altered by +-- this file, and no role is created or granted. +-- +-- WHY THIS IS NEEDED NOW (live gap, not speculative) +-- `docker/catalog/database-topology/*.yaml` declares an `omninode_internal` +-- schema (`domain: OMNINODE_INTERNAL`) and a 41-table TABLE-grant list for +-- the `omninode_runtime` principal against it, identical across all 7 +-- shipped profiles. Every one of those 41 tables is created, unqualified, +-- by its own node migration (e.g. +-- `nodes/node_projection_registration/0000_create_node_service_registry.sql`) +-- and therefore physically lands in `public` — the schema those grants +-- target has never existed anywhere in the migration corpus. OMN-15426's +-- live evidence lane (2026-08-03T19:2xZ, rolling ledger) confirmed the +-- consequence directly: `handler_wiring.py` issues schema-qualified SQL +-- against the contract-declared `omninode_internal` target for this table +-- set, and it fails with "relation does not exist" — not a permission +-- error, because the schema itself is absent. A grant was deliberately +-- withheld pending this fix. +-- +-- WHY ADDITIVE-ONLY, NOT A TABLE MOVE, IN THIS FILE +-- This ticket's own scope text is explicit: "Preserve source relations +-- until family-level parity and migration proof complete. Do not use +-- ALTER TABLE ... SET SCHEMA on sources." Moving 41 tables requires a +-- per-family transformation receipt (counts/keys/hashes/FKs/sequences/ +-- grants reconciled) produced through the OMN-15420 cutover-journal +-- machinery (`omnibase_infra.migration.cutover`), which is P5 territory +-- (OMN-15426/OMN-15360), not this P2-P4 schema-build ticket. This file +-- builds the empty target the family-by-family copy will land in; it does +-- not perform that copy. See the companion transformation-receipt note at +-- `docs/migrations/2026-08-06-omninode-internal-schema-transformation-receipt.md` +-- for the full disposition and explicit deferred-work list (physical table +-- copy, `owner_omninode_internal`/`omninode_runtime` role creation, and +-- the full reconciliation proof). +-- +-- CONCURRENT BRIDGE (same PR) +-- `src/omnibase_infra/topology/physical_schema_mapping.py` gains +-- `INTERNAL_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359`, mirroring the +-- already-shipped `TENANT_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359` +-- pattern. Grant/handler-wiring derivation continues to resolve these 41 +-- tables against `public` (their real physical location) until each +-- family's copy lands, instead of failing closed against a schema that +-- exists but holds nothing yet. +-- +-- DATABASE CONTEXT +-- The forward runner applies `docker/migrations/forward/*.sql` against +-- POSTGRES_DB (`omnibase_infra`), so this file switches with +-- `\connect omnidash_analytics` — the established in-repo pattern (083, +-- 096, 097). The `onex-create-database` directive on line 1 is honoured by +-- `ensure_directive_database` in `scripts/run-forward-migrations.sh`, so +-- the `\connect` cannot fail on a cluster that has not been through +-- `000_create_multiple_databases.sh`. +-- +-- IDEMPOTENCY +-- `CREATE SCHEMA IF NOT EXISTS` is idempotent by definition. No role +-- attribute, ownership, or privilege statement appears in this file, so +-- there is nothing here that is a privilege demand on re-apply (contrast +-- 094/097's ALTER ROLE guard discussion — not applicable, this file issues +-- no ALTER of any kind). +-- +-- EXECUTING-ROLE REQUIREMENTS +-- CREATE SCHEMA requires CREATE privilege on the database (or ownership). +-- On compose lanes that is `postgres`; on the managed instance it is the +-- per-database migration principal (OMN-15335). No CREATEROLE and no +-- superuser is required by this file. +-- +-- ROLLBACK +-- See rollback/rollback_098_create_omninode_internal_schema.sql. Rollback +-- is `DROP SCHEMA omninode_internal` and is safe only while the schema +-- remains empty (RESTRICT, not CASCADE) — it must never run once any table +-- has been copied into it. +-- ============================================================================= + +\connect omnidash_analytics + +CREATE SCHEMA IF NOT EXISTS omninode_internal; + +-- Post-condition. Statically provable (no DO/RAISE), matching the +-- OMN-15361 application-database gate's requirement for deployable SQL. +SELECT 1 / count(*) AS omninode_internal_schema_exists_assertion + FROM ( + SELECT 1 + FROM pg_catalog.pg_namespace + WHERE nspname = 'omninode_internal' + ) AS assertion; diff --git a/docker/migrations/forward/099_create_omninode_internal_live_events.sql b/docker/migrations/forward/099_create_omninode_internal_live_events.sql new file mode 100644 index 0000000000..a7269a9a71 --- /dev/null +++ b/docker/migrations/forward/099_create_omninode_internal_live_events.sql @@ -0,0 +1,361 @@ +-- onex-create-database: omnidash_analytics +-- ============================================================================= +-- TOMBSTONE (OMN-15819, 2026-08-10): this file is UNDELIVERABLE via the k8s +-- Job that applies docker/migrations/forward/*.sql +-- (omninode_infra/k8s/migrations/omnibase-infra-migrate.yaml). That Job owns +-- only the omnibase_infra database; its flat loop's `psql -f` apply is +-- gated on `directive_db == $DB_NAME` and is UNREACHABLE for this file's +-- `\connect omnidash_analytics` below, in that loop or any other in the +-- runner. It has never executed anywhere -- live-confirmed 2026-08-10 +-- (role_omnidash, omninode-dev-postgres RDS): `to_regclass('omninode_internal +-- .live_events')` fails closed with `permission denied for schema +-- omninode_internal`, and the runtime write path has been failing every +-- write with UndefinedTable since before this file merged. Kept in place, +-- byte-unchanged below this header, as ledgered history (migration files are +-- append-only) -- do NOT delete it and do NOT try to make it deliverable in +-- place. The table-creation + omninode_runtime-grant half of this file's +-- intent is DELIVERED by the node-owned replacement migration, +-- docker/migrations/forward/nodes/node_projection_live_events/0002_create_omninode_internal_live_events.sql +-- (connects directly to omnidash_analytics as role_omnidash -- the one code +-- path in the runner that can actually reach this database). That +-- replacement does NOT duplicate this file's transform-copy/reconciliation +-- of pre-existing public.live_events rows -- see its own header for scope. +-- Runner honesty for this class of file (no silent false-"applied" ledger +-- row; fail-closed for any FUTURE cross-DB flat file) is the companion PR in +-- omninode_infra. Static pre-merge enforcement in THIS repo: +-- tests/ci/test_flat_migration_no_foreign_connect_gate.py / +-- docker/migrations/forward/cross-database-flat-migrations.yaml. +-- ============================================================================= +-- AMENDMENT (OMN-15838, 2026-08-10): the "byte-unchanged below this header" +-- promise above is superseded for the transform-copy/reconciliation section +-- ONLY. The tombstone above is accurate for its own scope (the k8s Job / +-- managed-RDS path this file cannot reach) but that is not the only path +-- that executes this file: `scripts/run-forward-migrations.sh` (the .201 +-- docker-compose lanes -- dev/stability-test/prod) applies this exact flat +-- file directly via `psql -v ON_ERROR_STOP=1 -f`, tracked one-time per +-- database in that lane's own `public.schema_migrations` (filename-keyed, +-- content-blind -- confirmed by reading the runner script, OMN-15838). On +-- that path this file DOES execute, and its transform-copy + exact-row-count +-- reconciliation (formerly step 2/3 below) is a non-atomic race: it snapshots +-- `public.live_events` via one INSERT...SELECT, then re-counts both tables in +-- separate follow-up statements. Under READ COMMITTED, any row committed to +-- `public.live_events` by the live write path between the INSERT and the +-- recount (observed at ~24 writes/min on the stability lane) makes +-- `v_dst_count <> v_src_count` true by construction, RAISE EXCEPTIONs, and +-- rolls back the INSERT within that DO block -- deterministically failing on +-- any lane with concurrent writers, not intermittently. Because +-- `ON_ERROR_STOP=1` aborts the whole runner script on that RAISE, the +-- migration is never recorded applied and every subsequent refresh attempt +-- retries and fails identically (OMN-15838: 963965 vs 963977 divergence, +-- blocking the OMN-15826/OMN-15837 stability-test refresh outright). +-- +-- Since data delivery for this table is already owned by the node-owned +-- replacement migration this file's own tombstone names above +-- (0002_create_omninode_internal_live_events.sql, which does NOT duplicate +-- the transform-copy), 099's own copy of that logic is now dead weight on +-- top of being racy. This amendment REMOVES the transform-copy + +-- reconciliation DO block entirely; 099 is now schema-shape + grant only +-- (CREATE TABLE, GRANT, ALTER DEFAULT PRIVILEGES, indexes, and the two +-- statically-provable post-condition assertions it already carried) on every +-- path that executes it. See tests/integration/migrations/ +-- test_099_omninode_internal_live_events_omn15359.py for the updated proof +-- set (row-copy/reconciliation tests replaced by a direct regression test +-- that a diverging public/internal row count no longer fails the migration). +-- +-- RESIDUAL NOT FIXED HERE (OMN-15838, noted for the refresh to absorb): the +-- stability-test lane's `omninode_internal.live_events` currently carries +-- ~108MB of index pages built during an earlier successful apply (when the +-- table held ~963965 rows) over what is now an EMPTY table -- the failed +-- re-run's RAISE rolled back that attempt's INSERT (rolling back everything +-- executed inside that DO block) but ran strictly before the CREATE INDEX +-- statements below it in file order, so the indexes from the earlier +-- successful apply were never touched and never shrink on their own +-- (ordinary Postgres MVCC/index bloat, not a bug in the index statements +-- themselves). A REINDEX is a storage-maintenance operation, not a +-- schema-shape assertion, and this migration only ever runs once per +-- database (ledger-gated); adding it here would need to be correct for both +-- the fresh-create and the bloated-reapply case for a benefit that only +-- matters once. Left as an out-of-band step for whichever refresh/recovery +-- procedure next lands the fixed 099 on that lane: `REINDEX TABLE +-- omninode_internal.live_events;` (or DROP + recreate the 4 indexes below) +-- after this migration reports success. +-- ============================================================================= +-- MIGRATION: physically create omninode_internal.live_events (schema shape + +-- grants only -- OMN-15359 original, OMN-15838 removed the +-- transform-copy/reconciliation of public.live_events rows) +-- ============================================================================= +-- Ticket: OMN-15359 (P2-P4 build classified schemas and migrate internal, +-- control-plane, catalog, and tenant targets) +-- Related: OMN-15421 (Projection Domain Adapter Proof), OMN-15423 (relation +-- inventory/ownership), OMN-15425 (tenant projection authority -- +-- separate, unrelated gap), OMN-15819 (node-owned replacement now +-- owns data delivery), OMN-15838 (this amendment -- removed the +-- racy transform-copy/reconciliation block) +-- Version: 1.1.0 +-- +-- WHAT THIS FILE DOES +-- 098_create_omninode_internal_schema.sql created the (empty) omninode_internal +-- schema and a temporary physical bridge (physical_schema_mapping.py's +-- INTERNAL_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359) that let +-- omninode_internal-domain tables keep resolving against `public` where they +-- physically still live. This migration performs the first real family +-- cutover out of that bridge: `live_events`. +-- +-- 1. CREATE TABLE omninode_internal.live_events, identical shape to +-- public.live_events (docker/migrations/forward/nodes/node_projection_live_events/ +-- 0000_create_live_events.sql + 0001's type-classification data fixups). +-- 2. public.live_events is PRESERVED -- not dropped, not ALTER TABLE ... SET +-- SCHEMA'd, and (as of OMN-15838) not read from at all by this file. +-- Backfilling `omninode_internal.live_events` from pre-existing +-- `public.live_events` history is intentionally out of scope here -- see +-- the node-owned replacement migration's own header for the same +-- framing ("a distinct, separately-scoped follow-up if the dashboard +-- needs continuous history in the internal-schema copy"). +-- +-- WHY THIS TABLE, WHY NOW +-- node_projection_live_events/contract.yaml has declared +-- db_io.db_tables[0].schema: omninode_internal since before this migration +-- existed. handler_wiring.py's _resolve_projection_database_target uses that +-- contract-declared schema literally as the SQL write target (it does not +-- consult the physical bridge for query building, only for the grant- +-- privilege check) -- so the runtime has been issuing +-- `INSERT INTO omninode_internal.live_events` since before 098 merged, and +-- failing with "relation does not exist" because no migration had ever +-- physically created that relation. 098 fixed the schema-level half of that +-- gap; this migration fixes the table-level half for this one family. +-- +-- WHY live_events IS ALSO REMOVED FROM THE BRIDGE (companion change, same PR) +-- src/omnibase_infra/topology/physical_schema_mapping.py drops "live_events" +-- from INTERNAL_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359 in this same PR. +-- Post-migration, physical_grant_schema_for_table('omninode_internal', +-- 'live_events') must agree with the INSERT-target schema +-- (omninode_internal) -- leaving it in the bridge after the physical table +-- exists would make the grant-privilege check assert against `public` while +-- every write already lands in `omninode_internal`, silently reintroducing +-- the same class of drift this migration exists to close. The shipped +-- topology TABLE grants (src/omnibase_infra/topology/instances/*.yaml, +-- docker/catalog/database-topology/*.yaml) are regenerated in the same PR +-- via scripts/generate_application_database_table_grants.py --write so the +-- omninode_runtime principal's live_events grant moves from schema: public +-- to schema: omninode_internal, matching physical reality. +-- +-- DATABASE CONTEXT +-- Same pattern as 083/096/097/098: the forward runner applies this against +-- POSTGRES_DB, so this file switches with `\connect omnidash_analytics`. +-- +-- IDEMPOTENCY +-- CREATE TABLE IF NOT EXISTS, the GRANT statements, and CREATE INDEX IF NOT +-- EXISTS are all safe to re-run. +-- +-- ROLLBACK +-- See rollback/rollback_099_create_omninode_internal_live_events.sql. +-- RESTRICT, not CASCADE, and refuses (by construction -- DROP TABLE with no +-- CASCADE on a table nothing else yet depends on) once any downstream +-- object has been built against omninode_internal.live_events. Because +-- public.live_events is untouched, rollback of this migration alone never +-- loses data: the source of truth remains intact throughout. +-- +-- WHY THIS FILE ALSO ISSUES THE omninode_runtime GRANT (grant-gap repair, +-- same PR, projplane-slice-verify) +-- 098 physically created the omninode_internal SCHEMA but deliberately +-- granted nothing (its own docstring lists "owner_omninode_internal/ +-- omninode_runtime role creation" as explicit deferred P5 work). This file +-- physically creates the FIRST TABLE in that schema and simultaneously +-- removes live_events from INTERNAL_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359 +-- -- so, unlike the other ~40 still-bridged families, live_events is live +-- TODAY: handler_wiring._resolve_projection_database_target already issues +-- `INSERT INTO omninode_internal.live_events` against the omninode_runtime +-- binding in production. A migration that creates the table but never +-- grants the write-path role converts the prior UndefinedTable failure into +-- InsufficientPrivilege on first deploy -- proven live against a real RDS +-- snapshot (4 independent checks: the table is owned by a role +-- omninode_runtime is not; `information_schema.role_table_grants` returns +-- zero rows for omninode_runtime against this table; omninode_runtime has +-- no CREATE on omninode_internal; `pg_default_acl` carries no entry for the +-- schema). The GRANT below is derived from, and must stay byte-for-byte +-- consistent with, the single declared source of truth: +-- `src/omnibase_infra/topology/instances/*.yaml` `principals.omninode_runtime +-- .grants[schema: omninode_internal]` (regenerated into +-- `docker/catalog/database-topology/*.yaml` by +-- `scripts/generate_application_database_table_grants.py --write`) -- +-- SELECT, INSERT, UPDATE, no DELETE (a projection writer upserts, it does +-- not reshape the table -- same invariant 096 states for role_omnidash). +-- No other principal declares a live_events grant in that topology today, +-- so no read-side principal needs anything here. +-- +-- WHY THE ROLE IS ALSO GUARD-CREATED HERE (not just granted) +-- Unlike 098's deferred table-privilege work, `omninode_runtime` itself has +-- never been created by ANY migration in this repo (grepped the full +-- `docker/migrations/forward/*.sql` corpus: zero `CREATE ROLE omninode_runtime` +-- hits before this file). The standalone "Migration Integration Test" CI +-- gate applies only `docker/migrations/forward/*.sql` via +-- `scripts/run-migrations.py` against a bare `postgres:16-alpine` service -- +-- it never runs `000_create_multiple_databases.sh` (that script's own +-- SERVICE_DB_MAP does not even list `omninode_runtime`; it is a distinct, +-- newer per-domain principal, not one of the legacy per-microservice +-- `role_*` names). Without a guarded CREATE here, the GRANT below would +-- fail closed with `role "omninode_runtime" does not exist` (42704) in +-- that CI scope on every PR, not just this one. The guard follows the +-- exact precedent 094 (app_dashboard) and 096 (role_omnidash) already +-- established in this file set: CREATE only when absent (so a role +-- provisioned out-of-band on a managed instance is never re-created or +-- clobbered), NOLOGIN at create time (LOGIN + password attach stays +-- deployment-owned -- AWS Secrets Manager on the cloud path, lane-local on +-- compose -- never re-asserted here even though the topology declares +-- `login: true`), and RAISE EXCEPTION rather than silently skip if the +-- executing role lacks CREATEROLE and the role is not already there. +-- +-- WHY ALTER DEFAULT PRIVILEGES IN SCHEMA omninode_internal (decision, not a +-- given) +-- 098's own docstring names ~40 more OMNINODE_INTERNAL-domain families +-- still physically bridged to `public`, each of which will need this exact +-- same GRANT boilerplate the day its own family-cutover migration lands. +-- Repeating steps 1-4 by hand in each of those ~40 files is exactly the +-- defect class this repair exists to close -- a future author who copies +-- 099's CREATE TABLE but forgets the GRANT reintroduces the identical +-- UndefinedTable->InsufficientPrivilege bug this file fixes. `ALTER DEFAULT +-- PRIVILEGES IN SCHEMA omninode_internal GRANT ... TO omninode_runtime` +-- makes every future table created BY THE SAME EXECUTING ROLE in this +-- schema inherit the grant automatically, with no per-migration GRANT +-- statement required. The scope is correct because it is bounded to the +-- caveat 096 already documents for `public`: default privileges apply only +-- to objects the CURRENT (migration-executing) role creates -- exactly the +-- role every future omninode_internal family-cutover migration will run +-- as, on every lane (compose: `postgres`; managed: the OMN-15335 migration +-- principal). If a lane's migrations ever ran as a different principal than +-- the one that issued this ALTER DEFAULT PRIVILEGES, that principal would +-- need its own default-privileges assertion -- the same fact 096 already +-- states for `public` and not a new caveat introduced here. +-- ============================================================================= + +-- ----------------------------------------------------------------------------- +-- 1. Role existence (cluster-wide, issued before \connect). Guarded exactly +-- like 094/096: Postgres checks create-role privilege BEFORE it checks +-- whether the name is taken, so an unconditional CREATE ROLE would raise +-- `permission denied to create role` (42501) instead of the +-- duplicate_object this handler is written for. +-- ----------------------------------------------------------------------------- +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'omninode_runtime') THEN + BEGIN + CREATE ROLE omninode_runtime WITH + NOLOGIN + NOSUPERUSER + NOBYPASSRLS + NOCREATEDB + NOCREATEROLE + NOREPLICATION; + EXCEPTION + WHEN duplicate_object OR unique_violation THEN + NULL; -- created concurrently by another migration path + END; + END IF; +END; +$$; + +-- ----------------------------------------------------------------------------- +-- 2. Fail loud, name the problem, if the role could not be created and is +-- still not there -- never silently record this migration against a role +-- that does not exist. +-- ----------------------------------------------------------------------------- +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'omninode_runtime') THEN + RAISE EXCEPTION + 'omninode_runtime role does not exist and could not be created -- the ' + 'executing role lacks CREATEROLE. On a managed instance the role is ' + 'provisioned at the provisioning seam; this migration refuses to record ' + 'itself against a role that is not there.'; + END IF; +END; +$$; + +-- ----------------------------------------------------------------------------- +-- 3. CONNECT on the target database, issued from the current (omnibase_infra) +-- context because GRANT ... ON DATABASE is cluster-wide. Guarded on the +-- database existing so this file stays valid on a cluster that has not +-- been through 000 -- mirrors 096 step 3 exactly. +-- ----------------------------------------------------------------------------- +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_database WHERE datname = 'omnidash_analytics') THEN + EXECUTE 'GRANT CONNECT ON DATABASE omnidash_analytics TO omninode_runtime'; + END IF; +END; +$$; + +\connect omnidash_analytics + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE IF NOT EXISTS omninode_internal.live_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + event_id TEXT UNIQUE NOT NULL, + type TEXT NOT NULL DEFAULT 'ACTION', + timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), + source TEXT NOT NULL DEFAULT 'platform', + topic TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + payload TEXT NOT NULL DEFAULT '{}', + correlation_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- ----------------------------------------------------------------------------- +-- 4. The actual grant-gap repair. Derived from, and must stay consistent +-- with, `src/omnibase_infra/topology/instances/*.yaml` +-- `principals.omninode_runtime.grants[schema: omninode_internal]`. +-- SELECT/INSERT/UPDATE only -- no DELETE, matching the topology +-- declaration and 096's projection-writer invariant. +-- ----------------------------------------------------------------------------- +GRANT USAGE ON SCHEMA omninode_internal TO omninode_runtime; +GRANT SELECT, INSERT, UPDATE ON omninode_internal.live_events TO omninode_runtime; + +-- ----------------------------------------------------------------------------- +-- 5. Forward-looking default privileges for the ~40 remaining +-- OMNINODE_INTERNAL-domain families still bridged to `public` (098's +-- deferred-work list). See the file-header rationale above for scope and +-- the "same executing role" caveat this inherits from 096. +-- ----------------------------------------------------------------------------- +ALTER DEFAULT PRIVILEGES IN SCHEMA omninode_internal + GRANT SELECT, INSERT, UPDATE ON TABLES TO omninode_runtime; + +-- No transform-copy / reconciliation block here (OMN-15838 removed it -- see +-- the AMENDMENT in the file header for why: it was a non-atomic race against +-- a live `public.live_events` writer, and data delivery for this table is +-- already owned by the node-owned replacement migration, not this file). +-- omninode_internal.live_events is therefore created empty by this file on +-- every path that executes it -- schema shape and grants only. + +-- Indexes matching public.live_events's shape +-- (docker/migrations/forward/nodes/node_projection_live_events/0000_create_live_events.sql). +CREATE INDEX IF NOT EXISTS idx_omninode_internal_live_events_created_at + ON omninode_internal.live_events (created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_omninode_internal_live_events_topic + ON omninode_internal.live_events (topic); + +CREATE INDEX IF NOT EXISTS idx_omninode_internal_live_events_source + ON omninode_internal.live_events (source); + +CREATE INDEX IF NOT EXISTS idx_omninode_internal_live_events_correlation_id + ON omninode_internal.live_events (correlation_id) + WHERE correlation_id IS NOT NULL; + +-- Post-condition. Statically provable (no DO/RAISE), matching the +-- OMN-15361 application-database gate's requirement for deployable SQL. +SELECT 1 / count(*) AS omninode_internal_live_events_exists_assertion + FROM information_schema.tables + WHERE table_schema = 'omninode_internal' AND table_name = 'live_events'; + +-- Post-condition on the grant-gap repair itself: the write-path role must +-- actually carry INSERT on the physical table this migration just created, +-- or the migration Job succeeded while leaving the runtime write path +-- broken. Statically provable, same pattern as the assertion above. +SELECT 1 / count(*) AS omninode_runtime_live_events_insert_grant_assertion + FROM information_schema.role_table_grants + WHERE table_schema = 'omninode_internal' + AND table_name = 'live_events' + AND grantee = 'omninode_runtime' + AND privilege_type = 'INSERT'; diff --git a/docker/migrations/forward/_ledger/application-migration-blocks.tsv b/docker/migrations/forward/_ledger/application-migration-blocks.tsv new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/migrations/forward/_ledger/application-migrations.tsv b/docker/migrations/forward/_ledger/application-migrations.tsv new file mode 100644 index 0000000000..95bb647a6e --- /dev/null +++ b/docker/migrations/forward/_ledger/application-migrations.tsv @@ -0,0 +1,99 @@ +nodes/node_canary_score_reducer/0001_create_capability_scores.sql node:node_canary_score_reducer node:node_canary_score_reducer omninode_internal node:node_canary_score_reducer:0001_create_capability_scores.sql 7195b07a7fae809f141a136a67b799ad492e79d280f54c325ac14875dcfcc2cd +nodes/node_canary_score_reducer/0002_capability_scores_tenant_id_and_rls.sql node:node_canary_score_reducer node:node_canary_score_reducer tenant node:node_canary_score_reducer:0002_capability_scores_tenant_id_and_rls.sql 74d38cca4463ed4f4d20169164d788c5afc79eea58b7f2abd02ef23093388ba4 +nodes/node_canary_score_reducer/0003_capability_scores_tenant_id_to_uuid.sql node:node_canary_score_reducer node:node_canary_score_reducer tenant node:node_canary_score_reducer:0003_capability_scores_tenant_id_to_uuid.sql d1eeefff11fbec255216154ba26aa6f0f29602a0b6fd8fd89e8de5d8d3282b2c +nodes/node_contract_registry/0000_create_contract_registry.sql node:node_contract_registry node:node_contract_registry omninode_internal node:node_contract_registry:0000_create_contract_registry.sql bff49882c3c724de24f75b97c847ab78581064a72b77429a2251a02448fad668 +nodes/node_deployment_evidence_reducer/0001_create_deployment_evidence_projection_tables.sql node:node_deployment_evidence_reducer node:node_deployment_evidence_reducer omninode_internal node:node_deployment_evidence_reducer:0001_create_deployment_evidence_projection_tables.sql 1f3d9bcc7b90bfd9bed76bac2f0b815d742c11b3deadbf5b24a682350ccbbbcd +nodes/node_evidence_dashboard_reducer/0001_create_evidence_dashboard_projection_tables.sql node:node_evidence_dashboard_reducer node:node_evidence_dashboard_reducer omninode_internal node:node_evidence_dashboard_reducer:0001_create_evidence_dashboard_projection_tables.sql b3c4be68802a73d5f865b6730b191a15fa9d2729d3b8a5e75ed82ce3c94b70e8 +nodes/node_llm_delegation_projection/0001_create_llm_delegation_daily_projection.sql node:node_llm_delegation_projection node:node_llm_delegation_projection omninode_internal node:node_llm_delegation_projection:0001_create_llm_delegation_daily_projection.sql f1235bc94144eb1b380911b96752fd9910ce55de5e5db444d424f486b08bf31c +nodes/node_log_persistence_effect/0000_create_log_entries.sql node:node_log_persistence_effect node:node_log_persistence_effect omninode_internal node:node_log_persistence_effect:0000_create_log_entries.sql 80aef2841b22cac674eab3799fea3e35e4680cfc690e22d032863c0787cea76d +nodes/node_merge_state_projection/0001_create_merge_state_transitions.sql node:node_merge_state_projection node:node_merge_state_projection omninode_internal node:node_merge_state_projection:0001_create_merge_state_transitions.sql f61dbb2a31cb1974b52a484b69263e821e93c3404fc6f763ca31abea21b77f4e +nodes/node_nightly_loop_controller/001_create_nightly_loop_tables.sql node:node_nightly_loop_controller node:node_nightly_loop_controller omninode_internal node:node_nightly_loop_controller:001_create_nightly_loop_tables.sql dbe147357ae6e34ce35dcd97e7652f521346e6cb915775ed3d2f9a548af68ead +nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql node:node_omnigate_projection node:node_omnigate_projection omninode_internal node:node_omnigate_projection:0000_create_gate_projection_tables.sql e8409dc137dfecc4dc044632348e8a329f893fc971a9e3e536b81e71cee25747 +nodes/node_pr_lifecycle_state_reducer/0001_create_pr_lifecycle_ledger_entries.sql node:node_pr_lifecycle_state_reducer node:node_pr_lifecycle_state_reducer omninode_internal node:node_pr_lifecycle_state_reducer:0001_create_pr_lifecycle_ledger_entries.sql 1eef35f54647148625bed3e64e80ad91491c75666ae388eedb701f88d6538892 +nodes/node_pr_merged_projection/0001_create_pr_merged_events.sql node:node_pr_merged_projection node:node_pr_merged_projection omninode_internal node:node_pr_merged_projection:0001_create_pr_merged_events.sql 2835e23571dee7bec83c4ff75ada048929e1efb771a1d3be2bfb7b759a995070 +nodes/node_pr_review_bot/001_create_review_bot_bypass_log.sql node:node_pr_review_bot node:node_pr_review_bot omninode_internal node:node_pr_review_bot:001_create_review_bot_bypass_log.sql 63e2646a7f8767fad9ec969b224982e00b83aacb665107ab5b103964d5616e00 +nodes/node_projection_baselines/0001_create_baselines_tables.sql node:node_projection_baselines node:node_projection_baselines omninode_internal node:node_projection_baselines:0001_create_baselines_tables.sql f94dae78957a2890ea80655f3d2280331504d6637f9bdbd9ef014e2c33583f9d +nodes/node_projection_baselines/0002_realign_child_tables_to_producer_schema.sql node:node_projection_baselines node:node_projection_baselines omninode_internal node:node_projection_baselines:0002_realign_child_tables_to_producer_schema.sql b41a161c5a9e6325ced130c4d0217c72d2a2bd258effb4af5349401f3415ec30 +nodes/node_projection_baselines_quality/001_create_baselines_quality_snapshots.sql node:node_projection_baselines_quality node:node_projection_baselines_quality omninode_internal node:node_projection_baselines_quality:001_create_baselines_quality_snapshots.sql a96acb2a46e1a4914cbedafa6779d0c0610dd11cbfd3384aa9261efcd7cd6bd5 +nodes/node_projection_baselines_quality/002_realign_quality_snapshots_to_producer_schema.sql node:node_projection_baselines_quality node:node_projection_baselines_quality omninode_internal node:node_projection_baselines_quality:002_realign_quality_snapshots_to_producer_schema.sql aefec84151b6fa2e07469f1052a224c4f69ed09ac998c9f75373d81b4fff0c42 +nodes/node_projection_baselines_roi/001_create_baselines_roi_snapshots.sql node:node_projection_baselines_roi node:node_projection_baselines_roi omninode_internal node:node_projection_baselines_roi:001_create_baselines_roi_snapshots.sql 1ac7a552d9e65f1961caa4fcfb22aed0361614ff7b9b7c2633b8c214039619e8 +nodes/node_projection_baselines_roi/002_realign_roi_snapshots_to_producer_schema.sql node:node_projection_baselines_roi node:node_projection_baselines_roi omninode_internal node:node_projection_baselines_roi:002_realign_roi_snapshots_to_producer_schema.sql 4cfcb5a1326cf0072f26bf0670de0c39de6f044b1510185d72eb05c25431b9ff +nodes/node_projection_capsule_store/078_create_capsule_store.sql node:node_projection_capsule_store node:node_projection_capsule_store omninode_internal node:node_projection_capsule_store:078_create_capsule_store.sql 401ea38a974834d3d1fbb7792060427d8175d27bec7a8561028cdfa33b6fb06d +nodes/node_projection_capsule_store/079_create_capsule_effectiveness_view.sql node:node_projection_capsule_store node:node_projection_capsule_store omninode_internal node:node_projection_capsule_store:079_create_capsule_effectiveness_view.sql 91b14dec0459aed8b22e75e23e0a67977db0b37134f422187ed53319f6bafe37 +nodes/node_projection_context_roi/001_create_context_roi_scores.sql node:node_projection_context_roi node:node_projection_context_roi omninode_internal node:node_projection_context_roi:001_create_context_roi_scores.sql eabc473fc285af484cb194b93015837d7174464524549e638eaeae5fa9b2f69a +nodes/node_projection_context_roi/002_add_factor_subset_hash_and_routing_source.sql node:node_projection_context_roi node:node_projection_context_roi omninode_internal node:node_projection_context_roi:002_add_factor_subset_hash_and_routing_source.sql 925fc427e0e3365ecfee14233016ba771b0de1d8dd1ea80273db66aa016a9031 +nodes/node_projection_context_roi/003_context_roi_scores_tenant_id_and_rls.sql node:node_projection_context_roi node:node_projection_context_roi tenant node:node_projection_context_roi:003_context_roi_scores_tenant_id_and_rls.sql 78216762ada92a10248226cd77e9fceebcbbc188aab51810667c4e7735f76d19 +nodes/node_projection_cost_by_repo/0001_create_cost_by_repo_snapshots.sql node:node_projection_cost_by_repo node:node_projection_cost_by_repo omninode_internal node:node_projection_cost_by_repo:0001_create_cost_by_repo_snapshots.sql d362c71c1fed114414bf6f7c617dc68c05619ba0c30b17c34676b7a0d583e13e +nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql node:node_projection_cost_summary node:node_projection_cost_summary omninode_internal node:node_projection_cost_summary:0001_create_llm_cost_aggregates.sql 60cad6737c615c1cd6275d5818f0b45e9d8e57bbf1d2f265b1ebba3f6dc2ea4d +nodes/node_projection_cost_summary/0002_llm_cost_aggregates_tenant_id_and_rls.sql node:node_projection_cost_summary node:node_projection_cost_summary tenant node:node_projection_cost_summary:0002_llm_cost_aggregates_tenant_id_and_rls.sql a7e34a87eca0bcbf69c9590b79b59ffaa3bad97497d0778a1b8ce53ccb64c1ec +nodes/node_projection_delegation/0007_delegation_events.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0007_delegation_events.sql 92e614f5e5bf63cd7aa07511f4575925f8e11729d568fe24cf36266fc0505f07 +nodes/node_projection_delegation/0008_generation_events.sql node:node_projection_delegation node:node_projection_delegation omninode_internal node:node_projection_delegation:0008_generation_events.sql d4563a744d4e925a5afd6e176b9284ff08c07b770ee6d49d25b4816b5a785101 +nodes/node_projection_delegation/0009_delegate_skill_projection_metrics.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0009_delegate_skill_projection_metrics.sql ba2f5e8428084d2f7e3386372864aed7b8d47f31d4048e12878f8799060fb03d +nodes/node_projection_delegation/0009a_delegation_events_legacy_schema_reconcile.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0009a_delegation_events_legacy_schema_reconcile.sql b2e66b31070e676933f89b72876f05fdd3e812ad6a87d426c95290c6d0b9a489 +nodes/node_projection_delegation/0010_create_delegation_dashboard_projection_views.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0010_create_delegation_dashboard_projection_views.sql 5756805995f260d47a81fbacd16957c31c68ead6e34cb75a9d729a8af25a10a9 +nodes/node_projection_delegation/0011_delegation_event_projection_versions.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0011_delegation_event_projection_versions.sql b574eef907058ec1a3bd06d2da043750cabc41c9fcfb89429fcced21731e16a7 +nodes/node_projection_delegation/0012_generation_output_columns.sql node:node_projection_delegation node:node_projection_delegation omninode_internal node:node_projection_delegation:0012_generation_output_columns.sql 2c2f0a18ec3baf3ed28755d42fe6e6dcca5f350309df8e0554d9ae8330eb8017 +nodes/node_projection_delegation/0013_generation_proof_fields.sql node:node_projection_delegation node:node_projection_delegation omninode_internal node:node_projection_delegation:0013_generation_proof_fields.sql 500a2497ff8c61803f6b25c473805e2c297890a908959f9ba839d068c46183e2 +nodes/node_projection_delegation/0014_generation_semantic_pass.sql node:node_projection_delegation node:node_projection_delegation omninode_internal node:node_projection_delegation:0014_generation_semantic_pass.sql 655938bafba9b13a409361da63589e839a2327bd5b7938d9c10403e6edd13ae9 +nodes/node_projection_delegation/0015_generation_corpus_acceptance.sql node:node_projection_delegation node:node_projection_delegation omninode_internal node:node_projection_delegation:0015_generation_corpus_acceptance.sql f75c53356965a632ec273aa98795711a29726422d4db999a6d2b7b0cdf1b9734 +nodes/node_projection_delegation/0016_delegation_judge_verdict_events.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0016_delegation_judge_verdict_events.sql b0493d9356dc4a70fac4e8aae98c71adfbd32820f5eaea73e1a3031cc88ccdc3 +nodes/node_projection_delegation/0016_delegation_quality_bar_evidence.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0016_delegation_quality_bar_evidence.sql 29d8c4a5dcf9a85fc9edea0c0996df197a8054263ce7e048dc249a07341a92b2 +nodes/node_projection_delegation/0017_premium_counterfactual.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0017_premium_counterfactual.sql 18045c900787c1984d8a8bd51ee12d99af6c2e9e512fe59907e97c1bbb913549 +nodes/node_projection_delegation/0018_delegation_cost_measurements.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0018_delegation_cost_measurements.sql f10798ce3382ab582482831bb0bd87c5c9831e2dfc28c9031b8bca214308fbf2 +nodes/node_projection_delegation/0019_delegation_budget_state.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0019_delegation_budget_state.sql 161142a604726ad6c0a6b5e881f5b82c2b4a37066ca3fc2327961814a3149e7c +nodes/node_projection_delegation/0020_delegation_context_pack_hash.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0020_delegation_context_pack_hash.sql 1de228eead0e695c0cc14b9824a436fa54ef339739ad8ef278518ef137d79798 +nodes/node_projection_delegation/0021_delegation_tier_distribution_not_tier_routed.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0021_delegation_tier_distribution_not_tier_routed.sql 112dc502e55bff20125d2acc187668ba71f8d6ee37db1b36e78ac8e7c9a19070 +nodes/node_projection_delegation/0022_delegation_events_tenant_id.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0022_delegation_events_tenant_id.sql dc824dd6926e07c58f95ef1313d4ec3afd6e0a0af54bbec9fba20ec2c2424bed +nodes/node_projection_delegation/0023_delegation_rls_tenant_isolation.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0023_delegation_rls_tenant_isolation.sql 5a4f6e83cde6e9fde17ff1786965dda2d8482bec6e16de8d6a354de7853ca4de +nodes/node_projection_delegation/0024_drop_unwired_routing_columns.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0024_drop_unwired_routing_columns.sql 70f75b762938d93e5828dc69d49efe76f4272c3c1c722678d10570daf393ca35 +nodes/node_projection_delegation/0025_delegation_judge_verdict_events_tenant_id.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0025_delegation_judge_verdict_events_tenant_id.sql 4c8c3d4bddd9e1885d5d1fc7fea09a8ef2ad8d62e73b5e5bd7743ee93a14adee +nodes/node_projection_delegation/0026_delegation_judge_verdict_events_rls_tenant_isolation.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0026_delegation_judge_verdict_events_rls_tenant_isolation.sql 3b37d7cf23ce0de237f9f78658058a55d709bb0505647a455231c06fee38bcdb +nodes/node_projection_delegation/0027_generation_events_tenant_rls.sql node:node_projection_delegation node:node_projection_delegation omninode_internal node:node_projection_delegation:0027_generation_events_tenant_rls.sql 9f5ffade664f1e1aaf86ea990c7b66edd99e076f099f00791ecddacaef6ebd7f +nodes/node_projection_delegation/0028_reconcile_delegation_observability_views.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0028_reconcile_delegation_observability_views.sql 16c8e6392e344f017afe14012bac0822e7d290507e14c3351cfe170cf0df0b58 +nodes/node_projection_delegation/0029_delegation_terminal_failure_cause.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0029_delegation_terminal_failure_cause.sql 11a08d7da4834f34d900cac9b3351d04c8d92d40c87d65be6996ef9b40eb498f +nodes/node_projection_delegation/0030_delegation_budget_state_house_tenant_rekey.sql node:node_projection_delegation node:node_projection_delegation tenant node:node_projection_delegation:0030_delegation_budget_state_house_tenant_rekey.sql d12fa41576c40a98323ef44f778e5c49a804fbcb5b59ee69da54192a665515cf +nodes/node_projection_delegation_inference_response/0001_create_projection_delegation_inference_response_text.sql node:node_projection_delegation_inference_response node:node_projection_delegation_inference_response tenant node:node_projection_delegation_inference_response:0001_create_projection_delegation_inference_response_text.sql 17b3290b9cac74bbab65bdf1b814f7ad7ff318b7967263b43ecb82c647fbbf02 +nodes/node_projection_delegation_inference_response/0002_inference_response_text_tenant_rekey.sql node:node_projection_delegation_inference_response node:node_projection_delegation_inference_response tenant node:node_projection_delegation_inference_response:0002_inference_response_text_tenant_rekey.sql 8c5abf5bcea9d228b2053d39214969259cc0e6f4d0d26f69b91579fe22a2d72d +nodes/node_projection_delegation_inference_response/0003_inference_response_text_rls_tenant_isolation.sql node:node_projection_delegation_inference_response node:node_projection_delegation_inference_response tenant node:node_projection_delegation_inference_response:0003_inference_response_text_rls_tenant_isolation.sql 1ea1225f7977510fdb59d58c7917a8f5669234004d5749fd9f3a84cc7fb47208 +nodes/node_projection_dep_health/001_create_dep_health_findings.sql node:node_projection_dep_health node:node_projection_dep_health omninode_internal node:node_projection_dep_health:001_create_dep_health_findings.sql 81e097ced9e5c386183ca2563f786a297263576637703c57ca5ab913b58e08bc +nodes/node_projection_dep_health/002_dep_health_findings_tenant_id_and_rls.sql node:node_projection_dep_health node:node_projection_dep_health tenant node:node_projection_dep_health:002_dep_health_findings_tenant_id_and_rls.sql 22a1f4eb8fd25717ca81e2af34db8585d193f24a824c16339a0c5f9796681b28 +nodes/node_projection_event_chain/0001_create_event_chain.sql node:node_projection_event_chain node:node_projection_event_chain omninode_internal node:node_projection_event_chain:0001_create_event_chain.sql 413f1a55989e337c98b5248723ec8ecaa4643fc363dfbd01b2496efaef200fc3 +nodes/node_projection_instruction_eval/0001_create_instruction_eval_aggregate_snapshots.sql node:node_projection_instruction_eval node:node_projection_instruction_eval omninode_internal node:node_projection_instruction_eval:0001_create_instruction_eval_aggregate_snapshots.sql a75059367f26e95dccfffdf455a3dbdc94c9453cbc3290f5004f38783e77c049 +nodes/node_projection_instruction_eval/0002_instruction_eval_aggregate_snapshots_tenant_id_and_rls.sql node:node_projection_instruction_eval node:node_projection_instruction_eval tenant node:node_projection_instruction_eval:0002_instruction_eval_aggregate_snapshots_tenant_id_and_rls.sql 616bbcd1dc6d1a670f50843e4e5f7acd68c105293a9866950720b2b855727c31 +nodes/node_projection_intent_classification/0000_create_intent_classification_events.sql node:node_projection_intent_classification node:node_projection_intent_classification omninode_internal node:node_projection_intent_classification:0000_create_intent_classification_events.sql 53b37bc8a4544260a61d8f71dcbcd918822cf116c18e1df2f87a53f483c79f48 +nodes/node_projection_live_events/0000_create_live_events.sql node:node_projection_live_events node:node_projection_live_events omninode_internal node:node_projection_live_events:0000_create_live_events.sql 025cef6adfad109adca24183f07d597bd47694574dcee871ccb333f0490ccbf6 +nodes/node_projection_live_events/0001_reclassify_event_lifecycle_types.sql node:node_projection_live_events node:node_projection_live_events omninode_internal node:node_projection_live_events:0001_reclassify_event_lifecycle_types.sql b723afcc935ad53d6517cdccd39b41077e2646eb56d80defc62aa188a312e7f4 +nodes/node_projection_live_events/0002_create_omninode_internal_live_events.sql node:node_projection_live_events node:node_projection_live_events omninode_internal node:node_projection_live_events:0002_create_omninode_internal_live_events.sql b0009d57f9849c975e8fa5bda4b27457cb16ac7acdce000e66b46eabfa9f6355 +nodes/node_projection_llm_cost/0001_create_llm_call_metrics.sql node:node_projection_llm_cost node:node_projection_llm_cost omninode_internal node:node_projection_llm_cost:0001_create_llm_call_metrics.sql 43e896b82c4060f65c3ff98776ece5b567dd452b8842d20c6cf49574bcc4dc7b +nodes/node_projection_llm_routing/0000_create_llm_routing_decisions.sql node:node_projection_llm_routing node:node_projection_llm_routing omninode_internal node:node_projection_llm_routing:0000_create_llm_routing_decisions.sql 0b0cf074733223af0dfb935a64a381bdd39aa9988301e9c06a98ce0e0d81d474 +nodes/node_projection_llm_routing/0001_create_routing_dashboard_projection_view.sql node:node_projection_llm_routing node:node_projection_llm_routing omninode_internal node:node_projection_llm_routing:0001_create_routing_dashboard_projection_view.sql ada776fd7cc10ac28be44ecd3844410af537bbc180ce36404375d30fec0ad9f9 +nodes/node_projection_mcp_tools/0001_create_mcp_tools.sql node:node_projection_mcp_tools node:node_projection_mcp_tools omninode_internal node:node_projection_mcp_tools:0001_create_mcp_tools.sql 97fed831309a0ddedb682ebd0ee2e62d930a55759999bc43d3d27e523a88562c +nodes/node_projection_overnight/0000_create_overnight_sessions_tables.sql node:node_projection_overnight node:node_projection_overnight omninode_internal node:node_projection_overnight:0000_create_overnight_sessions_tables.sql afd9a8bd9be2a9628d27fa7e6fbd918014fa285bf33970d208e183aa67c3a591 +nodes/node_projection_overnight/0001_create_overnight_readiness_projection_view.sql node:node_projection_overnight node:node_projection_overnight omninode_internal node:node_projection_overnight:0001_create_overnight_readiness_projection_view.sql 770aaf52c91e1227dba5cc20c295ed0ab1100368e0ba03a4475f3417c5fed43a +nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql node:node_projection_pattern_learning node:node_projection_pattern_learning omninode_internal node:node_projection_pattern_learning:0000_create_pattern_learning_artifacts.sql 356ece28cb2f5b9fc7ab155bc84cf00e7124e2aba5b9ead28ca99370698b695b +nodes/node_projection_pattern_learning/0001_pattern_learning_artifacts_tenant_id_and_rls.sql node:node_projection_pattern_learning node:node_projection_pattern_learning tenant node:node_projection_pattern_learning:0001_pattern_learning_artifacts_tenant_id_and_rls.sql 80c199b287b8499201b124043388b4b0db22d41a3323e0af24349e32344c89e5 +nodes/node_projection_receipt_gate/0000_create_receipt_gate_projection_table.sql node:node_projection_receipt_gate node:node_projection_receipt_gate omninode_internal node:node_projection_receipt_gate:0000_create_receipt_gate_projection_table.sql bb08e06d443e74816b412306bcf47269061218b63a17ee6ceae5f3a7d24c722b +nodes/node_projection_registration/0000_create_node_service_registry.sql node:node_projection_registration node:node_projection_registration omninode_internal node:node_projection_registration:0000_create_node_service_registry.sql 5c320392190e6b9a1e06ab7177b7560181ca00282e107aebfccc452441e4d478 +nodes/node_projection_registration/0001_add_heartbeat_columns.sql node:node_projection_registration node:node_projection_registration omninode_internal node:node_projection_registration:0001_add_heartbeat_columns.sql 2cda779401d041ae1383937df3f4c614df74d09c80c78f24f7993685ba7a6aed +nodes/node_projection_registration/0002_node_service_registry_tenant_rls.sql node:node_projection_registration node:node_projection_registration omninode_internal node:node_projection_registration:0002_node_service_registry_tenant_rls.sql 3e8a3c12dbefc4432262a1c2533667f4a5f342c39149915bd80b45241c182e98 +nodes/node_projection_registration/0003_reconcile_heartbeat_observability.sql node:node_projection_registration node:node_projection_registration omninode_internal node:node_projection_registration:0003_reconcile_heartbeat_observability.sql 0bc9286ec9b0f7ad36e1d29e54d34bc11fe2be64cf5b874d7432384869ade69e +nodes/node_projection_registration/0004_node_service_registry_no_force_rls.sql node:node_projection_registration node:node_projection_registration omninode_internal node:node_projection_registration:0004_node_service_registry_no_force_rls.sql adb21c75ec2503b50a7272857e660ff07cfda16374698cc245af93047e3da6f2 +nodes/node_projection_routing_decision/0021_create_agent_routing_decisions.sql node:node_projection_routing_decision node:node_projection_routing_decision omninode_internal node:node_projection_routing_decision:0021_create_agent_routing_decisions.sql 46d169e595f65ad89e7f524efffedb59eb0e1a37f9527e67f1ff2e8eaaa7711e +nodes/node_projection_routing_decision/0022_agent_routing_decisions_tenant_id_and_rls.sql node:node_projection_routing_decision node:node_projection_routing_decision tenant node:node_projection_routing_decision:0022_agent_routing_decisions_tenant_id_and_rls.sql bac8be9bbe6167601ca805d35a78f469a6bce9ca9fc22e88a363c50dac0b7f83 +nodes/node_projection_sandbox_decisions/0001_create_sandbox_decisions.sql node:node_projection_sandbox_decisions node:node_projection_sandbox_decisions omninode_internal node:node_projection_sandbox_decisions:0001_create_sandbox_decisions.sql dcf92194d262bb957dd5537a2d9e16083b89d2d1c259f08f869460b76145b96c +nodes/node_projection_savings/074_create_savings_estimates.sql node:node_projection_savings node:node_projection_savings tenant node:node_projection_savings:074_create_savings_estimates.sql b78acc5ba3144f9a7c7d85fd0fd5803b02b60503765fc58f8650a6a2bde27f4e +nodes/node_projection_savings/075_add_savings_estimates_updated_at.sql node:node_projection_savings node:node_projection_savings tenant node:node_projection_savings:075_add_savings_estimates_updated_at.sql e2dc485f47b82671b06cb4cf42147d6ec410bff3803868fe375e866d407062e7 +nodes/node_projection_savings/076_create_delegation_savings_projection_view.sql node:node_projection_savings node:node_projection_savings tenant node:node_projection_savings:076_create_delegation_savings_projection_view.sql b14ba7616265359e683b5354935a7e678f9d2dc013e4ecd11f022cab6f299544 +nodes/node_projection_savings/077_create_cost_savings_overview_projection_view.sql node:node_projection_savings node:node_projection_savings tenant node:node_projection_savings:077_create_cost_savings_overview_projection_view.sql f25fdb63293786b5a1c4270aee3b28b07f66ff22dfcdbd7cd29603e9405dffd6 +nodes/node_projection_savings/078_create_delegation_savings_series_projection_view.sql node:node_projection_savings node:node_projection_savings tenant node:node_projection_savings:078_create_delegation_savings_series_projection_view.sql e424f3f6dff90182c6d72ba9601e588dfbfe1605e92cec670cd8460c634fb2c4 +nodes/node_projection_savings/079_add_savings_series_tier_mix.sql node:node_projection_savings node:node_projection_savings tenant node:node_projection_savings:079_add_savings_series_tier_mix.sql 8016a7f32ad9b317cc094f1af0e285d446646639aeba239363f84387651cf782 +nodes/node_projection_savings/080_savings_estimates_tenant_id.sql node:node_projection_savings node:node_projection_savings tenant node:node_projection_savings:080_savings_estimates_tenant_id.sql 0826ecef5bfc4764f7b6319bc9291c49f013881629fa955c3b70b52060844d2e +nodes/node_projection_savings/081_savings_estimates_rls_tenant_isolation.sql node:node_projection_savings node:node_projection_savings tenant node:node_projection_savings:081_savings_estimates_rls_tenant_isolation.sql 4eef79e5af54c1355bcefdebf1d941e081ec60525f9b19e6be21fca73d406010 +nodes/node_projection_session_outcome/0021_session_outcomes.sql node:node_projection_session_outcome node:node_projection_session_outcome omninode_internal node:node_projection_session_outcome:0021_session_outcomes.sql de421b3a1ef9bd31f29fe6bce7d1efc469a1ead9a08a1f59e9bd8fb53a1a77f0 +nodes/node_projection_session_replay/0001_create_session_replay_snapshots.sql node:node_projection_session_replay node:node_projection_session_replay omninode_internal node:node_projection_session_replay:0001_create_session_replay_snapshots.sql c27aecf66ed8feca3a95baa23d563e7d350f90fab9305ae15a6869d5369f7487 +nodes/node_projection_skill_executions/0001_create_skill_execution_snapshots.sql node:node_projection_skill_executions node:node_projection_skill_executions omninode_internal node:node_projection_skill_executions:0001_create_skill_execution_snapshots.sql 64da2834b60a19ce0291a7d505dacc97425f7c8875fa366e0f05635b6c54dfa4 +nodes/node_projection_skill_executions/0002_skill_execution_snapshots_tenant_id_and_rls.sql node:node_projection_skill_executions node:node_projection_skill_executions tenant node:node_projection_skill_executions:0002_skill_execution_snapshots_tenant_id_and_rls.sql b5949008c3361ce49554942ba8f1a2a6394d55b2e5697cfe6cd45f9ca2c2b747 +nodes/node_projection_swarm/0001_create_swarm_runs.sql node:node_projection_swarm node:node_projection_swarm omninode_internal node:node_projection_swarm:0001_create_swarm_runs.sql 371758edf852e610b60a21dda5556ae212179c5d2fa437a115bb48a9ddb26e20 +nodes/node_projection_traces/0001_create_traces.sql node:node_projection_traces node:node_projection_traces omninode_internal node:node_projection_traces:0001_create_traces.sql 3ca434ec9cad1b192179fc662467fa0bcff139810ee113dc6f7ec4330d1040d2 +nodes/node_projection_voice_sessions/0001_create_voice_sessions.sql node:node_projection_voice_sessions node:node_projection_voice_sessions omninode_internal node:node_projection_voice_sessions:0001_create_voice_sessions.sql 7929304ef3aeb0f86119e8c2e43dffce0211f859646887bc04e9e5cdaa72d946 +nodes/node_renderer_capability_projection/0001_create_renderer_capability_projection.sql node:node_renderer_capability_projection node:node_renderer_capability_projection omninode_internal node:node_renderer_capability_projection:0001_create_renderer_capability_projection.sql 3066f544ee2d6d9259af6d9b1adee6c6f435dedbb33f2de6fa4d36505a355da5 diff --git a/docker/migrations/forward/_ledger/bootstrap.sql b/docker/migrations/forward/_ledger/bootstrap.sql new file mode 100644 index 0000000000..6497b3bd4a --- /dev/null +++ b/docker/migrations/forward/_ledger/bootstrap.sql @@ -0,0 +1,826 @@ +-- SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +-- SPDX-License-Identifier: MIT +-- +-- OMN-15413: deterministic, in-place application migration ledger upgrade. +-- +-- This is a runner bootstrap, not a numbered migration: the legacy +-- filename-only schema cannot execute the numbered migration set until its +-- tracking relation has first been made readable. The runner supplies +-- a validated temporary manifest table in the same psql session. +-- +-- The selected relation is the existing checksum-capable ledger. In the +-- application database that is public.node_schema_migrations. The separate +-- service-owned infra ledger is out of scope and is never selected here. The +-- application relation is moved (same OID, rows and owner) into +-- platform_catalog and extended in place. The filename-only ledger is an +-- import source, never the selected canonical relation. +-- +-- OMN-15695 (operator ruling 2026-08-04): the four-column +-- public.schema_migrations(migration_id, applied_at, checksum, source_set) +-- relation is ambiguous by column signature alone — it is the service-owned +-- ledger in the service database and the predecessor NODE ledger in the +-- application database. It is partitioned on row content: source_set 'node' +-- rows with a node::.sql identity are adopted as an import source +-- (source preserved, never moved), source_set 'docker' rows are ignored as +-- service-owned, and anything else aborts the transaction. A relation with no +-- adoptable row is still refused outright. + +\set ON_ERROR_STOP on + +BEGIN; + +CREATE SCHEMA IF NOT EXISTS platform_catalog; + +DO $ledger_upgrade$ +DECLARE + public_ledger REGCLASS := to_regclass('public.schema_migrations'); + node_ledger REGCLASS := to_regclass('public.node_schema_migrations'); + canonical_ledger REGCLASS := to_regclass('platform_catalog.schema_migrations'); + column_count INTEGER; + origin_shape TEXT := 'canonical'; + origin_source TEXT := 'platform_catalog.schema_migrations'; + public_shape TEXT := 'absent'; + -- OMN-15695: row-content sub-classification of the migration_id shape. + -- 'none' when the shape is not migration_id, 'adopt' when the relation + -- carries the historical runner's node rows, 'service' when it carries only + -- the separate service-owned set. + migration_id_disposition TEXT := 'none'; + adoptable_row_count INTEGER := 0; + unrecognized_row_count INTEGER := 0; + primary_key_name TEXT; + primary_key_columns TEXT[]; +BEGIN + IF public_ledger IS NOT NULL THEN + SELECT count(*) INTO column_count + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations'; + + IF column_count = 2 + AND EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND column_name = 'filename' + ) + AND EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND column_name = 'applied_at' + ) + AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND ( + (column_name = 'filename' + AND (udt_name <> 'text' OR is_nullable <> 'NO')) + OR (column_name = 'applied_at' + AND (udt_name <> 'timestamptz' OR is_nullable <> 'NO')) + ) + ) THEN + public_shape := 'filename'; + ELSIF column_count = 4 + AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND column_name NOT IN ('migration_id', 'applied_at', 'checksum', 'source_set') + ) + AND EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND column_name = 'migration_id' + ) + AND EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND column_name = 'source_set' + ) + AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND ( + (column_name IN ('migration_id', 'checksum', 'source_set') + AND (udt_name <> 'text' OR is_nullable <> 'NO')) + OR (column_name = 'applied_at' + AND (udt_name <> 'timestamptz' OR is_nullable <> 'NO')) + ) + ) THEN + public_shape := 'migration_id'; + + -- OMN-15695: the column signature alone cannot tell the service-owned + -- ledger apart from the application database's predecessor NODE ledger. + -- The pre-OMN-15413 runner wrote both shapes with these four columns: + -- service rows as ('docker/', ..., 'docker') and node rows as + -- ('node::.sql', ..., 'node'). Partition on row content and + -- refuse anything that is neither. + SELECT + count(*) FILTER ( + WHERE source_set = 'node' + AND migration_id ~ + '^node:[A-Za-z0-9_][A-Za-z0-9_.-]*:[A-Za-z0-9_][A-Za-z0-9_.-]*[.]sql$' + ), + count(*) FILTER ( + WHERE NOT ( + source_set = 'node' + AND migration_id ~ + '^node:[A-Za-z0-9_][A-Za-z0-9_.-]*:[A-Za-z0-9_][A-Za-z0-9_.-]*[.]sql$' + ) + AND NOT (source_set = 'docker' AND migration_id ~ '^docker/') + ) + INTO adoptable_row_count, unrecognized_row_count + FROM public.schema_migrations; + + IF unrecognized_row_count > 0 THEN + RAISE EXCEPTION + 'unknown migration ledger shape: public.schema_migrations contains % unrecognized migration_id rows', + unrecognized_row_count; + END IF; + IF adoptable_row_count > 0 THEN + migration_id_disposition := 'adopt'; + ELSE + migration_id_disposition := 'service'; + END IF; + ELSIF column_count = 3 + AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND column_name NOT IN ('version', 'applied_at', 'checksum') + ) + AND EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND column_name = 'version' + ) + AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND ( + (column_name = 'version' + AND (udt_name <> 'text' OR is_nullable <> 'NO')) + OR (column_name = 'checksum' AND udt_name <> 'text') + OR (column_name = 'applied_at' + AND (udt_name <> 'timestamptz' OR is_nullable <> 'NO')) + ) + ) THEN + public_shape := 'version'; + ELSE + RAISE EXCEPTION + 'unknown migration ledger shape: public.schema_migrations has % columns', + column_count; + END IF; + END IF; + + IF canonical_ledger IS NOT NULL AND node_ledger IS NOT NULL THEN + RAISE EXCEPTION + 'double migration declaration: both public.node_schema_migrations and platform_catalog.schema_migrations exist'; + END IF; + -- OMN-15695: an adopted migration_id node ledger is deliberately preserved + -- beside the canonical ledger (the same non-destructive contract the + -- filename-only source has), so it is not a double declaration. Every other + -- checksum-capable public relation beside the canonical ledger still is. + IF canonical_ledger IS NOT NULL + AND (public_shape = 'version' + OR (public_shape = 'migration_id' + AND migration_id_disposition <> 'adopt')) THEN + RAISE EXCEPTION + 'double migration declaration: checksum-capable public.schema_migrations exists beside the canonical ledger'; + END IF; + + -- A migration_id relation that carries no adoptable node row is the + -- service-owned ledger. It is never selected for the application database. + IF canonical_ledger IS NULL + AND public_shape = 'migration_id' + AND migration_id_disposition <> 'adopt' THEN + RAISE EXCEPTION + 'unknown migration stream: service-owned migration_id ledger cannot be selected for the application database'; + END IF; + + IF canonical_ledger IS NULL AND node_ledger IS NOT NULL THEN + SELECT count(*) INTO column_count + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'node_schema_migrations'; + IF column_count <> 3 OR EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'node_schema_migrations' + AND ( + column_name NOT IN ('version', 'applied_at', 'checksum') + OR (column_name IN ('version', 'checksum') + AND (udt_name <> 'text' OR is_nullable <> 'NO')) + OR (column_name = 'applied_at' + AND (udt_name <> 'timestamptz' OR is_nullable <> 'NO')) + ) + ) THEN + RAISE EXCEPTION + 'unknown migration ledger shape: public.node_schema_migrations'; + END IF; + + ALTER TABLE public.node_schema_migrations SET SCHEMA platform_catalog; + ALTER TABLE platform_catalog.node_schema_migrations + RENAME TO schema_migrations; + canonical_ledger := to_regclass('platform_catalog.schema_migrations'); + origin_shape := 'node'; + origin_source := 'public.node_schema_migrations'; + ELSIF canonical_ledger IS NULL + AND public_shape = 'version' THEN + ALTER TABLE public.schema_migrations SET SCHEMA platform_catalog; + canonical_ledger := to_regclass('platform_catalog.schema_migrations'); + origin_shape := 'node'; + origin_source := 'public.schema_migrations'; + public_ledger := NULL; + public_shape := 'absent'; + ELSIF canonical_ledger IS NULL THEN + -- Fresh databases, filename-only databases, and the adopted migration_id + -- node ledger have no checksum-capable candidate that can be moved: the + -- migration_id relation cannot become the canonical shape without dropping + -- source_set. Create the selected canonical shape; that history is + -- imported below without renaming or rewriting the source relation. + CREATE TABLE platform_catalog.schema_migrations ( + migration_stream TEXT NOT NULL, + owner TEXT NOT NULL, + domain TEXT NOT NULL, + version TEXT NOT NULL, + checksum TEXT NOT NULL, + checksum_kind TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now(), + provenance TEXT NOT NULL, + PRIMARY KEY (migration_stream, domain, version) + ); + canonical_ledger := to_regclass('platform_catalog.schema_migrations'); + origin_shape := 'fresh'; + END IF; + + ALTER TABLE platform_catalog.schema_migrations + ADD COLUMN IF NOT EXISTS migration_stream TEXT, + ADD COLUMN IF NOT EXISTS owner TEXT, + ADD COLUMN IF NOT EXISTS domain TEXT, + ADD COLUMN IF NOT EXISTS checksum TEXT, + ADD COLUMN IF NOT EXISTS checksum_kind TEXT, + ADD COLUMN IF NOT EXISTS provenance TEXT; + + IF origin_shape = 'node' THEN + IF EXISTS ( + SELECT 1 + FROM platform_catalog.schema_migrations ledger + LEFT JOIN onex_application_migration_manifest manifest + ON manifest.version = ledger.version + WHERE manifest.version IS NULL + ) THEN + RAISE EXCEPTION + 'unknown migration stream/domain: historical node version has no checked-in declaration'; + END IF; + IF EXISTS ( + SELECT 1 + FROM platform_catalog.schema_migrations ledger + JOIN onex_application_migration_manifest manifest + ON manifest.version = ledger.version + WHERE ledger.checksum IS NULL + OR ledger.checksum !~ '^[0-9a-f]{64}$' + OR ledger.checksum <> manifest.checksum + ) THEN + RAISE EXCEPTION + 'conflicting migration checksum in checksum-capable node history'; + END IF; + + UPDATE platform_catalog.schema_migrations ledger + SET migration_stream = manifest.migration_stream, + owner = manifest.owner, + domain = manifest.domain, + provenance = format( + 'legacy:%s:%s:version:%s:raw-checksum=%s', + current_database(), origin_source, ledger.version, + coalesce(ledger.checksum, '') + ), + checksum_kind = 'content_sha256' + FROM onex_application_migration_manifest manifest + WHERE manifest.version = ledger.version; + END IF; + + IF EXISTS ( + SELECT 1 + FROM platform_catalog.schema_migrations + WHERE migration_stream IS NULL OR migration_stream = '' + OR owner IS NULL OR owner = '' + OR domain IS NULL OR domain = '' + OR version IS NULL OR version = '' + OR checksum IS NULL OR checksum !~ '^[0-9a-f]{64}$' + OR checksum_kind NOT IN ('content_sha256', 'legacy_attestation') + OR provenance IS NULL OR provenance = '' + ) THEN + RAISE EXCEPTION + 'canonical migration ledger contains null, empty, or malformed metadata'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM platform_catalog.schema_migrations ledger + LEFT JOIN onex_application_migration_manifest manifest + ON manifest.migration_stream = ledger.migration_stream + AND manifest.owner = ledger.owner + AND manifest.domain = ledger.domain + AND manifest.version = ledger.version + LEFT JOIN onex_legacy_node_migration_declarations legacy + ON legacy.migration_stream = ledger.migration_stream + AND legacy.owner = ledger.owner + AND legacy.domain = ledger.domain + AND legacy.version = ledger.version + WHERE manifest.version IS NULL AND legacy.version IS NULL + AND NOT ( + ledger.migration_stream = 'legacy:filename-only' + AND ledger.owner = 'legacy:filename-only' + AND ledger.domain = 'legacy_unclassified' + ) + AND NOT ( + ledger.migration_stream = 'omninode-cloud' + AND ledger.owner = 'service:onex_api' + AND ledger.domain = 'legacy_unclassified' + ) + ) THEN + RAISE EXCEPTION 'unknown migration stream/domain declaration'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM platform_catalog.schema_migrations ledger + JOIN onex_application_migration_manifest manifest + ON manifest.migration_stream = ledger.migration_stream + AND manifest.owner = ledger.owner + AND manifest.domain = ledger.domain + AND manifest.version = ledger.version + WHERE ledger.checksum_kind = 'content_sha256' + AND ledger.checksum <> manifest.checksum + ) THEN + RAISE EXCEPTION 'conflicting migration checksum in canonical node history'; + END IF; + IF EXISTS ( + SELECT 1 + FROM platform_catalog.schema_migrations ledger + JOIN onex_application_migration_manifest manifest + ON manifest.migration_stream = ledger.migration_stream + AND manifest.owner = ledger.owner + AND manifest.domain = ledger.domain + AND manifest.version = ledger.version + WHERE ledger.checksum_kind <> 'content_sha256' + ) THEN + RAISE EXCEPTION + 'checksum-capable node history cannot be downgraded to legacy attestation'; + END IF; + + SELECT c.conname, + array_agg(a.attname ORDER BY key_position.ordinality) + INTO primary_key_name, primary_key_columns + FROM pg_constraint c + CROSS JOIN LATERAL unnest(c.conkey) WITH ORDINALITY AS key_position(attnum, ordinality) + JOIN pg_attribute a + ON a.attrelid = c.conrelid AND a.attnum = key_position.attnum + WHERE c.conrelid = canonical_ledger AND c.contype = 'p' + GROUP BY c.conname; + + IF primary_key_name IS NOT NULL + AND primary_key_columns <> ARRAY['migration_stream', 'domain', 'version']::TEXT[] THEN + EXECUTE format( + 'ALTER TABLE platform_catalog.schema_migrations DROP CONSTRAINT %I', + primary_key_name + ); + primary_key_name := NULL; + END IF; + + IF primary_key_name IS NULL THEN + IF EXISTS ( + SELECT 1 + FROM platform_catalog.schema_migrations + GROUP BY migration_stream, domain, version + HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION 'duplicate migration version in canonical ledger'; + END IF; + ALTER TABLE platform_catalog.schema_migrations + ADD PRIMARY KEY (migration_stream, domain, version); + END IF; + + ALTER TABLE platform_catalog.schema_migrations + ALTER COLUMN migration_stream SET NOT NULL, + ALTER COLUMN owner SET NOT NULL, + ALTER COLUMN domain SET NOT NULL, + ALTER COLUMN version SET NOT NULL, + ALTER COLUMN checksum SET NOT NULL, + ALTER COLUMN checksum_kind SET NOT NULL, + ALTER COLUMN applied_at SET NOT NULL, + ALTER COLUMN provenance SET NOT NULL; + + SELECT count(*) INTO column_count + FROM information_schema.columns + WHERE table_schema = 'platform_catalog' + AND table_name = 'schema_migrations'; + IF column_count <> 8 OR EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'platform_catalog' + AND table_name = 'schema_migrations' + AND ( + column_name NOT IN ( + 'migration_stream', 'owner', 'domain', 'version', 'checksum', + 'checksum_kind', 'applied_at', 'provenance' + ) + OR (column_name <> 'applied_at' + AND (udt_name <> 'text' OR is_nullable <> 'NO')) + OR (column_name = 'applied_at' + AND (udt_name <> 'timestamptz' OR is_nullable <> 'NO')) + ) + ) THEN + RAISE EXCEPTION + 'unknown migration ledger shape: platform_catalog.schema_migrations'; + END IF; + + -- Recreate the named checks from the canonical predicates on every bootstrap. + -- Accepting a constraint merely because its name matches would let a stale or + -- unrelated expression disable the fail-closed boundary. + ALTER TABLE platform_catalog.schema_migrations + DROP CONSTRAINT IF EXISTS schema_migrations_stream_domain_check, + DROP CONSTRAINT IF EXISTS schema_migrations_checksum_check, + DROP CONSTRAINT IF EXISTS schema_migrations_checksum_kind_check; + ALTER TABLE platform_catalog.schema_migrations + ADD CONSTRAINT schema_migrations_stream_domain_check CHECK ( + (migration_stream ~ '^node:[A-Za-z0-9_][A-Za-z0-9_.-]*$' + AND owner = migration_stream + AND domain IN ('omninode_internal', 'tenant')) + OR (migration_stream = 'legacy:filename-only' + AND owner = 'legacy:filename-only' + AND domain = 'legacy_unclassified') + OR (migration_stream = 'omninode-cloud' + AND owner = 'service:onex_api' + AND domain = 'legacy_unclassified') + ), + ADD CONSTRAINT schema_migrations_checksum_check + CHECK (checksum ~ '^[0-9a-f]{64}$'), + ADD CONSTRAINT schema_migrations_checksum_kind_check + CHECK (checksum_kind IN ('content_sha256', 'legacy_attestation')); +END +$ledger_upgrade$; + +COMMENT ON TABLE platform_catalog.schema_migrations IS + 'Canonical application migration ledger selected by OMN-15413; evolved in place from the existing checksum-capable ledger.'; +COMMENT ON COLUMN platform_catalog.schema_migrations.migration_stream IS + 'Contract-declared producer stream; legacy:filename-only is quarantined evidence.'; +COMMENT ON COLUMN platform_catalog.schema_migrations.owner IS + 'Contract-declared migration producer owner; never inferred from the database role.'; +COMMENT ON COLUMN platform_catalog.schema_migrations.domain IS + 'Declared target schema domain; legacy_unclassified is non-executable quarantine evidence.'; +COMMENT ON COLUMN platform_catalog.schema_migrations.version IS + 'Migration identity within migration_stream; historical identity is preserved.'; +COMMENT ON COLUMN platform_catalog.schema_migrations.checksum IS + 'Lowercase SHA-256: file bytes for content_sha256, or deterministic source-record attestation for legacy_attestation.'; +COMMENT ON COLUMN platform_catalog.schema_migrations.provenance IS + 'Deterministic source record or checked-in file that produced this ledger row.'; + +-- Import filename-only omnidash history without renaming, updating, or +-- deleting the source rows. The SHA-256 is explicitly a source-record +-- attestation because this ledger never captured migration file bytes. +DO $filename_import$ +DECLARE + source_row RECORD; + existing_row RECORD; + imported_checksum TEXT; + imported_provenance TEXT; + source_column_count INTEGER; +BEGIN + IF to_regclass('public.schema_migrations') IS NULL THEN + RETURN; + END IF; + + SELECT count(*) INTO source_column_count + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations'; + + -- OMN-15695: the migration_id-shaped predecessor node ledger is adopted by + -- $migration_id_import$ below. It is a deliberately preserved adoption + -- source, not a filename-only source and not a double declaration. + IF source_column_count = 4 AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND column_name NOT IN + ('migration_id', 'applied_at', 'checksum', 'source_set') + ) THEN + RETURN; + END IF; + + IF source_column_count <> 2 OR EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND column_name NOT IN ('filename', 'applied_at') + ) THEN + RAISE EXCEPTION + 'double migration declaration: checksum-capable public.schema_migrations remains beside the canonical ledger'; + END IF; + + FOR source_row IN + SELECT filename, applied_at + FROM public.schema_migrations + ORDER BY filename + LOOP + IF source_row.filename IS NULL OR source_row.filename = '' THEN + RAISE EXCEPTION 'duplicate migration version: empty filename'; + END IF; + imported_checksum := encode(sha256(convert_to( + 'legacy:filename-only|legacy_unclassified|' || source_row.filename || '|filename-only', + 'UTF8' + )), 'hex'); + imported_provenance := format( + 'legacy:%s:public.schema_migrations:filename:%s', + current_database(), source_row.filename + ); + + SELECT * INTO existing_row + FROM platform_catalog.schema_migrations + WHERE migration_stream = 'legacy:filename-only' + AND domain = 'legacy_unclassified' + AND version = source_row.filename; + IF FOUND THEN + IF existing_row.checksum <> imported_checksum THEN + RAISE EXCEPTION 'conflicting migration checksum for version %', source_row.filename; + ELSIF existing_row.owner <> 'legacy:filename-only' + OR existing_row.domain <> 'legacy_unclassified' + OR existing_row.checksum_kind <> 'legacy_attestation' + OR existing_row.applied_at IS DISTINCT FROM source_row.applied_at + OR existing_row.provenance <> imported_provenance THEN + RAISE EXCEPTION 'double migration declaration for version %', source_row.filename; + END IF; + ELSE + INSERT INTO platform_catalog.schema_migrations ( + migration_stream, owner, domain, version, checksum, checksum_kind, + applied_at, provenance + ) VALUES ( + 'legacy:filename-only', 'legacy:filename-only', + 'legacy_unclassified', source_row.filename, + imported_checksum, 'legacy_attestation', source_row.applied_at, + imported_provenance + ); + END IF; + END LOOP; +END +$filename_import$; + +-- OMN-15695: adopt the application database's predecessor node ledger. +-- +-- The pre-OMN-15413 runner created +-- public.schema_migrations(migration_id, applied_at, checksum, source_set) +-- in the NODE database and recorded each applied node migration as +-- ('node::.sql', now(), 'applied-by-runner', 'node'). That +-- relation is the legitimate applied-history of this database, so the +-- migration_id arm's blanket refusal was a false negative for it. +-- +-- Operator ruling 2026-08-04 (ADOPT/CONVERT): preserve the applied history, +-- never re-apply an already-applied migration, never delete or rewrite the +-- source rows. The source relation is left byte-for-byte intact, exactly as +-- the filename-only import leaves its source. +-- +-- Evidence class, stated plainly: 'applied-by-runner' is a sentinel, not a +-- hash — this ledger never captured file bytes. Adoption writes the checked-in +-- manifest checksum under checksum_kind 'content_sha256', which ASSERTS that +-- the bytes applied historically equal today's checked-in bytes. That +-- assertion is the operator ruling made mechanical; it is not derivable from +-- the database. It is kept auditable and non-forgeable three ways: only the +-- exact 'applied-by-runner' literal is adoptable, a 64-hex source checksum that +-- disagrees with the manifest is still fatal, and provenance permanently +-- records the raw source checksum under an 'adopted:' prefix so an adopted row +-- can never be mistaken for a runner-verified 'file:nodes/...' row. +-- +-- Service-owned rows (source_set 'docker') belong to the separate service +-- ledger and are ignored here. Any row that is neither has already aborted the +-- transaction in the selection block above. +DO $migration_id_import$ +DECLARE + source_row RECORD; + manifest_row RECORD; + legacy_row RECORD; + existing_row RECORD; + resolved_stream TEXT; + resolved_owner TEXT; + resolved_domain TEXT; + imported_checksum TEXT; + imported_checksum_kind TEXT; + imported_provenance TEXT; + source_column_count INTEGER; +BEGIN + IF to_regclass('public.schema_migrations') IS NULL THEN + RETURN; + END IF; + + SELECT count(*) INTO source_column_count + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations'; + IF source_column_count <> 4 OR EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND column_name NOT IN + ('migration_id', 'applied_at', 'checksum', 'source_set') + ) THEN + RETURN; + END IF; + + FOR source_row IN + SELECT migration_id, applied_at, checksum + FROM public.schema_migrations + WHERE source_set = 'node' + AND migration_id ~ + '^node:[A-Za-z0-9_][A-Za-z0-9_.-]*:[A-Za-z0-9_][A-Za-z0-9_.-]*[.]sql$' + ORDER BY migration_id + LOOP + SELECT * INTO manifest_row + FROM onex_application_migration_manifest + WHERE version = source_row.migration_id; + IF NOT FOUND THEN + SELECT * INTO legacy_row + FROM onex_legacy_node_migration_declarations + WHERE version = source_row.migration_id; + IF NOT FOUND THEN + RAISE EXCEPTION + 'unknown migration stream/domain: adopted node version % has no checked-in declaration', + source_row.migration_id; + END IF; + IF source_row.checksum <> legacy_row.source_checksum THEN + RAISE EXCEPTION + 'conflicting migration checksum for version %', source_row.migration_id; + END IF; + resolved_stream := legacy_row.migration_stream; + resolved_owner := legacy_row.owner; + resolved_domain := legacy_row.domain; + imported_checksum := encode(sha256(convert_to( + 'legacy-node-attestation|' || source_row.migration_id || '|' || + source_row.checksum || '|' || legacy_row.ticket, + 'UTF8' + )), 'hex'); + imported_checksum_kind := 'legacy_attestation'; + imported_provenance := format( + 'legacy-adopted:%s:public.schema_migrations:migration_id:%s:raw-checksum=%s:ticket=%s', + current_database(), source_row.migration_id, source_row.checksum, + legacy_row.ticket + ); + ELSE + resolved_stream := manifest_row.migration_stream; + resolved_owner := manifest_row.owner; + resolved_domain := manifest_row.domain; + IF source_row.checksum ~ '^[0-9a-f]{64}$' THEN + IF source_row.checksum <> manifest_row.checksum THEN + RAISE EXCEPTION + 'conflicting migration checksum for version %', source_row.migration_id; + END IF; + imported_checksum := source_row.checksum; + ELSIF source_row.checksum = 'applied-by-runner' THEN + imported_checksum := manifest_row.checksum; + ELSE + RAISE EXCEPTION + 'conflicting migration checksum for version %', source_row.migration_id; + END IF; + imported_checksum_kind := 'content_sha256'; + imported_provenance := format( + 'adopted:%s:public.schema_migrations:migration_id:%s:raw-checksum=%s', + current_database(), source_row.migration_id, source_row.checksum + ); + END IF; + + SELECT * INTO existing_row + FROM platform_catalog.schema_migrations + WHERE migration_stream = resolved_stream + AND domain = resolved_domain + AND version = source_row.migration_id; + IF FOUND THEN + IF existing_row.checksum <> imported_checksum THEN + RAISE EXCEPTION + 'conflicting migration checksum for version %', source_row.migration_id; + ELSIF existing_row.owner <> resolved_owner + OR existing_row.domain <> resolved_domain + OR existing_row.checksum_kind <> imported_checksum_kind + OR existing_row.applied_at IS DISTINCT FROM source_row.applied_at + OR existing_row.provenance <> imported_provenance THEN + RAISE EXCEPTION + 'double migration declaration for version %', source_row.migration_id; + END IF; + ELSE + INSERT INTO platform_catalog.schema_migrations ( + migration_stream, owner, domain, version, checksum, checksum_kind, + applied_at, provenance + ) VALUES ( + resolved_stream, resolved_owner, + resolved_domain, source_row.migration_id, + imported_checksum, imported_checksum_kind, source_row.applied_at, + imported_provenance + ); + END IF; + END LOOP; +END +$migration_id_import$; + +-- Import omnimarket's specialized projection ledger when present. Its +-- (node_name, filename) identity normalizes to the already-deployed +-- node:: version grammar. Two source rows normalizing to one +-- version are a double declaration and abort the transaction. +DO $omnimarket_import$ +DECLARE + source_row RECORD; + manifest_row RECORD; + existing_row RECORD; + canonical_version TEXT; + imported_checksum TEXT; + imported_kind TEXT; + imported_provenance TEXT; + source_column_count INTEGER; +BEGIN + IF to_regclass('public.omnimarket_schema_migrations') IS NULL THEN + RETURN; + END IF; + + SELECT count(*) INTO source_column_count + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'omnimarket_schema_migrations'; + IF source_column_count <> 6 OR EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'omnimarket_schema_migrations' + AND ( + column_name NOT IN ('id', 'node_name', 'version', 'filename', 'checksum', 'applied_at') + OR (column_name = 'id' AND (udt_name <> 'int4' OR is_nullable <> 'NO')) + OR (column_name IN ('node_name', 'version', 'filename', 'checksum') + AND (udt_name <> 'text' OR is_nullable <> 'NO')) + OR (column_name = 'applied_at' + AND (udt_name <> 'timestamptz' OR is_nullable <> 'NO')) + ) + ) THEN + RAISE EXCEPTION 'unknown migration ledger shape: public.omnimarket_schema_migrations'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM public.omnimarket_schema_migrations + GROUP BY node_name, filename + HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION 'duplicate migration version in public.omnimarket_schema_migrations'; + END IF; + + FOR source_row IN + SELECT node_name, version, filename, checksum, applied_at + FROM public.omnimarket_schema_migrations + ORDER BY node_name, version + LOOP + IF source_row.node_name !~ '^[A-Za-z0-9_][A-Za-z0-9_.-]*$' + OR source_row.filename !~ '^[A-Za-z0-9_][A-Za-z0-9_.-]*[.]sql$' THEN + RAISE EXCEPTION 'unknown migration stream identity in omnimarket source'; + END IF; + IF source_row.version <> source_row.filename THEN + RAISE EXCEPTION + 'unknown migration stream identity: omnimarket version % does not equal filename %', + source_row.version, source_row.filename; + END IF; + canonical_version := format( + 'node:%s:%s', source_row.node_name, source_row.filename + ); + SELECT * INTO manifest_row + FROM onex_application_migration_manifest + WHERE artifact_path = format( + 'nodes/%s/%s', source_row.node_name, source_row.filename + ); + IF NOT FOUND OR manifest_row.version <> canonical_version THEN + RAISE EXCEPTION + 'unknown migration stream/domain for omnimarket version %', canonical_version; + END IF; + IF source_row.checksum IS NULL + OR source_row.checksum !~ '^[0-9a-f]{64}$' + OR source_row.checksum <> manifest_row.checksum THEN + RAISE EXCEPTION 'conflicting migration checksum for version %', canonical_version; + END IF; + imported_kind := 'content_sha256'; + imported_checksum := source_row.checksum; + imported_provenance := format( + 'legacy:%s:public.omnimarket_schema_migrations:%s:%s:%s:raw-checksum=%s', + current_database(), source_row.node_name, source_row.version, + source_row.filename, coalesce(source_row.checksum, '') + ); + + SELECT * INTO existing_row + FROM platform_catalog.schema_migrations + WHERE migration_stream = manifest_row.migration_stream + AND domain = manifest_row.domain + AND version = canonical_version; + IF FOUND THEN + IF existing_row.checksum <> imported_checksum THEN + RAISE EXCEPTION 'conflicting migration checksum for version %', canonical_version; + ELSIF existing_row.owner <> manifest_row.owner + OR existing_row.domain <> manifest_row.domain + OR existing_row.checksum_kind <> imported_kind + OR existing_row.applied_at IS DISTINCT FROM source_row.applied_at + OR existing_row.provenance <> imported_provenance THEN + RAISE EXCEPTION 'double migration declaration for version %', canonical_version; + END IF; + ELSE + INSERT INTO platform_catalog.schema_migrations ( + migration_stream, owner, domain, version, checksum, checksum_kind, + applied_at, provenance + ) VALUES ( + manifest_row.migration_stream, manifest_row.owner, + manifest_row.domain, canonical_version, + imported_checksum, imported_kind, source_row.applied_at, + imported_provenance + ); + END IF; + END LOOP; +END +$omnimarket_import$; + +COMMIT; diff --git a/docker/migrations/forward/_ledger/cloud-migration-aliases.tsv b/docker/migrations/forward/_ledger/cloud-migration-aliases.tsv new file mode 100644 index 0000000000..d332a62ec1 --- /dev/null +++ b/docker/migrations/forward/_ledger/cloud-migration-aliases.tsv @@ -0,0 +1,30 @@ +20251207_tenants_uuid_pk 20251207_tenants_uuid_pk.sql +20251209_m4_usage_schema 20251209_m4_usage_schema.sql +20251209b_m4_usage_privileges 20251209b_m4_usage_privileges.sql +20251210_m4_usage_privileges_seed 20251210_m4_usage_privileges_seed.sql +20251211_m4_api_keys 20251211_m4_api_keys.sql +20251212_m4_plan_metrics 20251212_m4_plan_metrics.sql +20251215_m4_stripe_billing 20251215_m4_stripe_billing.sql +20251219_m4_api_keys_key_hash_unique 20251219_m4_api_keys_key_hash_unique.sql +20251219_m4_performance_indexes 20251219_m4_performance_indexes.sql +20251222_m4_usage_events_idempotency 20251222_m4_usage_events_idempotency.sql +20260130_create_app_users 20260130_create_app_users.sql +20260226_s2_alpha_provisioning_schema 20260226_s2_alpha_provisioning_schema.sql +20260226_s2_bootstrap_tokens 20260226_s2_bootstrap_tokens.sql +20260226_s4_tenant_hardening 20260226_s4_tenant_hardening.sql +20260228_owner_email_tenants 20260228_owner_email_tenants.sql +20260427_backfill_tenant_billing_plan_code 20260427_backfill_tenant_billing_plan_code.sql +20260427_bootstrap_tokens 20260427_bootstrap_tokens.sql +20260427_provisioning_state_check_extend 20260427_provisioning_state_check_extend.sql +20260427_tenant_billing_plan_columns 20260427_tenant_billing_plan_columns.sql +20260427_tenant_plans_lifecycle_columns 20260427_tenant_plans_lifecycle_columns.sql +20260429_plan_entitlements 20260429_plan_entitlements.sql +20260512_migrations_log_direction_cleanup 20260512_migrations_log_direction_cleanup.sql +20260703_tenant_credentials 20260703_tenant_credentials.sql +20260721_gateway_workflows 20260721_gateway_workflows.sql +20260722_tenants_stripe_customer_id 20260722_tenants_stripe_customer_id.sql +20260725_backfill_tenants_provisioning_state 20260725_backfill_tenants_provisioning_state.sql +20260725_gateway_workflows_terminal 20260725_gateway_workflows_terminal.sql +20260725_tenant_suspended_plan 20260725_tenant_suspended_plan.sql +20260726_compute_minutes_metering 20260726_compute_minutes_metering.sql +20260727_storage_bytes_metering 20260727_storage_bytes_metering.sql diff --git a/docker/migrations/forward/_ledger/legacy-node-migrations.tsv b/docker/migrations/forward/_ledger/legacy-node-migrations.tsv new file mode 100644 index 0000000000..5739925761 --- /dev/null +++ b/docker/migrations/forward/_ledger/legacy-node-migrations.tsv @@ -0,0 +1,2 @@ +node:node_projection_delegation node:node_projection_delegation omninode_internal node:node_projection_delegation:0014_create_live_event_projection_view.sql hotfix-applied-by-codex OMN-15717 +node:node_projection_delegation node:node_projection_delegation omninode_internal node:node_projection_delegation:0015_create_generation_dashboard_views.sql hotfix-applied-by-codex OMN-15717 diff --git a/docker/migrations/forward/cross-database-flat-migrations.yaml b/docker/migrations/forward/cross-database-flat-migrations.yaml new file mode 100644 index 0000000000..114c0ea487 --- /dev/null +++ b/docker/migrations/forward/cross-database-flat-migrations.yaml @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# Single-sourced ledger of top-level (flat, `docker/migrations/forward/*.sql`, +# -maxdepth 1) migrations whose FIRST `\connect ` directive names a +# database OTHER than `omnibase_infra` -- the one database the k8s Job that +# applies this corpus (`omninode_infra` repo, +# `k8s/migrations/omnibase-infra-migrate.yaml`, `DB_NAME="omnibase_infra"`) +# ever connects to for the flat-loop `psql -f` apply. +# +# WHY THIS MATTERS (OMN-15819) +# That Job's flat loop is gated `directive_db == "$DB_NAME"` before it ever +# issues the `psql -f` apply -- a cross-DB file's apply step is +# unreachable, in that loop or any other in the runner. Before OMN-15819 +# the skip branch UPSERTed a version/checksum row into `omnibase_infra`'s +# OWN `schema_migrations` anyway, so every ledger read said "already +# applied" for `098_create_omninode_internal_schema.sql` and +# `099_create_omninode_internal_live_events.sql` while +# `omnidash_analytics` (their real `\connect` target) never saw either +# file. The runner fix (companion PR, `omninode_infra`) stopped writing +# that false ledger row and now FATALs the Job for any cross-DB flat file +# NOT on its own tombstone allowlist, seeded with exactly those two +# entries. +# +# WHAT THIS FILE IS +# The STATIC counterpart, enforced pre-merge in THIS repo (where the flat +# SQL corpus actually lives and where a new file would be added): +# `tests/ci/test_flat_migration_no_foreign_connect_gate.py` / +# `scripts/ci/check_flat_migration_foreign_connect.py` reject any flat +# migration whose \connect targets a foreign database UNLESS it is listed +# here, by filename, with a citation. A file is added here in exactly two +# circumstances: +# * `disposition: undeliverable` -- OMN-15819-confirmed: no code path in +# the k8s Job delivers this file to its declared target. Must name the +# replacement (a node-owned migration under +# `docker/migrations/forward/nodes//`, which connects directly to +# the target database as its own role -- OMN-15819 step 2) if one +# exists. +# * `disposition: grandfathered` -- pre-existing at the moment this gate +# was authored (this PR). Its k8s-Job delivery status was NOT +# re-audited by this gate -- do not read a `grandfathered` entry as a +# claim that the file is actually delivered anywhere. It is here so +# the gate's introduction does not retroactively fail CI on migrations +# this PR did not investigate. +# +# A NEW cross-DB flat file is a hard reject. Do not add it here to make +# the gate pass -- author a node-owned replacement instead (it is the only +# path that actually connects to the target database on the managed/RDS +# lane). Adding an entry here without a real citation defeats the point of +# the gate. +# +# Both directions are enforced: a live cross-DB flat file missing from this +# manifest fails closed (AC2 -- cannot dodge review by omission), and a +# manifest entry whose file no longer exists, or no longer targets a foreign +# database, is a stale-entry failure (remove it). +entries: + - file: 098_create_omninode_internal_schema.sql + connect_target: omnidash_analytics + disposition: undeliverable + citation: >- + OMN-15819 -- confirmed undeliverable via the k8s Job. Superseded by docker/migrations/forward/nodes/node_projection_live_events/ (asserts, does not create, the operator-provisioned omninode_internal schema as a guarded precondition of the table it owns). + - file: 099_create_omninode_internal_live_events.sql + connect_target: omnidash_analytics + disposition: undeliverable + citation: >- + OMN-15819 -- confirmed undeliverable via the k8s Job. Superseded by the node-owned replacement migration under docker/migrations/forward/nodes/node_projection_live_events/. + - file: 083_create_log_entries.sql + connect_target: omnidash_analytics + disposition: undeliverable + citation: >- + OMN-15846 -- re-audited (was grandfathered). Confirmed undeliverable via the k8s Job: to_regclass('public.log_entries') is NULL in both omnibase_infra and omnidash_analytics on onex-dev RDS (2026-08-10), and omnibase_infra's own schema_migrations carried a false "applied" row masking this. Superseded by docker/migrations/forward/nodes/node_log_persistence_effect/ (node-owned replacement, same content, delivered via the node-owned loop's role_omnidash connection). + - file: 096_grant_role_omnidash_omnidash_analytics.sql + connect_target: omnidash_analytics + disposition: undeliverable + citation: >- + OMN-15846 -- re-audited (was grandfathered). Confirmed undeliverable via the k8s Job: pg_default_acl for omnidash_analytics/public is empty on onex-dev RDS (2026-08-10), so this file's unconditional ALTER DEFAULT PRIVILEGES step never executed; the false "applied" ledger row masked this. Also unneeded on RDS: role_omnidash already owns 89/90 tables in omnidash_analytics.public there (OMN-15335 two-owner-split migration-principal model, live-verified), which strictly subsumes this file's named grants -- an owner is exempt from RLS and needs no such grant. No node-owned replacement is authored; OMN-15355 (P1, In Review) is the tracked systematic successor (generated ACL/default-privilege matrix covering role_omnidash-owned domains). File remains applicable, unchanged, on the .201 lab lane it was authored for (a different runner, unaffected by this k8s-Job-scoped classification). + - file: 097_grant_app_dashboard_connect_omnidash_analytics.sql + connect_target: omnidash_analytics + disposition: undeliverable + citation: >- + OMN-15846 -- re-audited (was grandfathered). Confirmed undeliverable via the k8s Job: app_dashboard's USAGE on schema public is granted by pg_database_owner/omninodeadmin, not role_omnibase_infra, on onex-dev RDS (2026-08-10); the false "applied" ledger row masked this. The CONNECT defect this file repairs is latent there today (PUBLIC's CONNECT has not been revoked -- pg_database.datacl still carries PUBLIC=Tc), per this file's own header framing ("the declared target state" pending OMN-15355). No node-owned replacement is authored; OMN-15355 (P1, In Review, explicit app_dashboard acceptance criterion) is the tracked systematic successor. File remains applicable, unchanged, on any lane whose own runner (scripts/run-forward-migrations.sh) can reach omnidash_analytics. diff --git a/docker/migrations/forward/fenced-node-migrations.yaml b/docker/migrations/forward/fenced-node-migrations.yaml new file mode 100644 index 0000000000..38284f391d --- /dev/null +++ b/docker/migrations/forward/fenced-node-migrations.yaml @@ -0,0 +1,139 @@ +# fenced-node-migrations.yaml — Single source of truth for the node-migration +# operator fence (OMN-15349, Option 1) +# +# Ticket: OMN-15349 (single-source the fence across omnibase_infra + +# omninode_infra — this file is that single source) +# Predecessor: OMN-15336 (parity port that first gave the compose runner a +# fence at all; left the list duplicated in two repos, which this file +# removes) +# +# PURPOSE +# This is the BASELINE operator fence over the shared node-migration id space +# (node::, minted identically by both runners over the same +# vendored docker/migrations/forward/nodes/ tree). It is consumed READ-ONLY by: +# +# - omnibase_infra/scripts/run-forward-migrations.sh (every compose lane — +# dev, stability-test, judge, prod). Reads this file directly via the +# ${MIGRATIONS_DIR} bind mount (docker/docker-compose.infra.yml mounts +# ../docker/migrations/forward:/migrations/forward:ro). +# - omninode_infra/k8s/migrations/omnibase-infra-migrate.yaml (the k8s +# Job). The Job's copy-migrations initContainer already copies this +# entire directory (docker/Dockerfile.migrate: COPY docker/migrations/ +# forward/ /migrations/forward/, recursive) out of the +# omnibase-infra-migrate image it pulls — no new image dependency, the +# coupling already existed for the SQL tree this file sits next to. +# +# BASELINE, NOT PER-LANE EFFECTIVE FENCE +# This list is the maximal fence — every id both runners fence by default. +# Each runner separately layers its OWN, independently operator-ruled, +# lane-scoped RELEASE on top of this baseline (a strict subset carve-out, +# never a widening): +# +# - compose: ONEX_MIGRATION_LANE=dev releases the registration trio on the +# lab lane only (operator ruling 15, OMN-15379). Every other lane stays +# fully fenced. See the "LANE-SCOPED FENCE RELEASE" block in +# run-forward-migrations.sh. +# - k8s: releases the registration trio unconditionally, because that Job +# serves exactly one environment (staging/onex-dev) and operator ruling +# 21 (OMN-15332 comment 1a067542, 2026-07-31T14:05Z GO) authorized a +# DURABLE release there, not an env-gated one. See the "k8s lane +# release" block in omnibase-infra-migrate.yaml. +# +# The release policies are deliberately NOT expressed in this file: they are +# per-environment operator decisions, not shared fence DATA, and folding them +# in would turn one shared baseline back into two hand-reconciled effective +# lists. What this file guarantees is that both runners start from the same +# baseline before either applies its own release — the seam OMN-15349 exists +# to close. +# +# FORMAT +# A flat list under `fenced_node_migrations`, each entry an `id:` (mandatory, +# machine-parsed) plus a `ticket:` (advisory, for humans). Order is +# significant — both runners preserve it. Parsed with a portable +# sed one-liner (no yq/python dependency in either runner's shell): +# +# sed -n 's/^[[:space:]]*-[[:space:]]*id:[[:space:]]*"\([^"]*\)".*/\1/p' \ +# fenced-node-migrations.yaml +# +# Mirrors the parsing style already used for docker/migrations/skip-manifest.yaml. +# +# CHANGING THIS LIST +# Editing this file changes the baseline for BOTH runners in one commit — +# that is the point of single-sourcing it. It does NOT by itself change +# either runner's effective (post-release) fence; a release policy change +# (e.g. widening ruling 21's scope, or adding a new ruling-15-style lane +# carve-out) is a separate, explicit edit to that runner's release block. +# +# WHY THESE NINE IDS (verbatim rationale, unchanged from the pre-OMN-15349 +# hand-maintained lists for the first seven; the eighth (node_pr_review_bot +# 001) was added under OMN-15717/OMN-15376 and the ninth +# (delegation_inference_response 0003) under OMN-15336 item 4 / OMN-15656 — +# two independent PRs landing concurrently, both additive, not conflicting): +# delegation 0023-0026 — OMN-14974 / OMN-15313. Tenant-isolation RLS on +# live, actively-written delegation tables; un-gate only in a change that +# also proves the writer sets app.tenant_id per connection. No lane may +# release these (see the "delegation ids are not releasable" checks in +# both runners). +# registration 0000/0001/0002 — OMN-15335 / OMN-15088 / OMN-15343. All +# three ALTER node_service_registry; 0002 applies FORCE ROW LEVEL +# SECURITY. 0000 (CREATE) and 0002 (the dependent ALTER) must move +# together — a fence that gates one but not the other is worse than no +# fence (OMN-15379 postmortem: a cold compose lane with 0000 fenced and +# 0002 not died with "relation node_service_registry does not exist"). +# node_pr_review_bot 001 — OMN-15717 / OMN-15376. The CREATE TABLE IF NOT +# EXISTS for review_bot_bypass_log declares 7 columns with no ALTER TABLE +# ADD COLUMN IF NOT EXISTS reconciliation, tripping the OMN-15376 static +# shape-reconciliation gate (tests/ci/test_node_migration_shape_reconciliation.py). +# Editing the SQL to add the missing ALTER statements is NOT the fix here: +# it would change the file's content sha256 (currently +# 63e2646a7f8767fad9ec969b224982e00b83aacb665107ab5b103964d5616e00), and +# that checksum is bound both by the application-migrations.tsv +# declaration and by dev's already-applied +# omnidash_analytics.platform_catalog.schema_migrations row +# (content_sha256, applied_at 2026-08-06) — a changed checksum makes +# scripts/run-forward-migrations.sh's migration_is_applied() check FATAL +# ("conflicting migration checksum") on that lane, since the runner has +# no path to reconcile a stored checksum against a changed file. Fencing +# is safe today, not merely convenient: read-only .201 probes (2026-08-10) +# confirm this migration is ALREADY applied on all four live lanes — +# dev via the content_sha256 row above, and stability-test/judge/prod via +# the legacy 'applied-by-runner' public.schema_migrations rows dated +# 2026-06-10/06-11 — and review_bot_bypass_log's live columns already +# match the 7 declared columns exactly on every lane, so there is no +# shape drift to reconcile anywhere that exists today. Residual, accepted +# risk: a genuinely new future lane (never bootstrapped from any of the +# four) would skip creating this table until an operator adds a lane +# release, same accepted-tradeoff shape as the delegation ids above. +# delegation_inference_response 0003 — OMN-15336 item 4 / OMN-15656. FORCE +# ROW LEVEL SECURITY on projection_delegation_inference_response_text. +# Contract-declared tenant domain (node_projection_delegation_inference_ +# response/contract.yaml db_io.schema), so the RLS posture itself is +# correctly classified — unlike node_service_registry below, this is not +# an internal-domain misclassification. It is held for the same reason +# the delegation quartet above is held: OMN-15301 found the projection +# writer never sets app.tenant_id per connection, so an un-gated FORCE +# apply here reproduces the identical false-clean (zero rows, no error) +# write-lockout hazard on a live, actively-written table. Was NEVER in +# this list on any runner before this entry — it applied unattended on +# the .201 dev lane (2026-07-31T06:44:32Z) purely because nothing here +# named it. Un-gate only in a change that also proves the writer sets +# app.tenant_id per connection, matching the delegation quartet's bar. +fenced_node_migrations: + - id: "node:node_projection_delegation:0023_delegation_rls_tenant_isolation.sql" + ticket: "OMN-14974" + - id: "node:node_projection_delegation:0024_drop_unwired_routing_columns.sql" + ticket: "OMN-14974" + - id: "node:node_projection_delegation:0025_delegation_judge_verdict_events_tenant_id.sql" + ticket: "OMN-15313" + - id: "node:node_projection_delegation:0026_delegation_judge_verdict_events_rls_tenant_isolation.sql" + ticket: "OMN-15313" + - id: "node:node_projection_registration:0000_create_node_service_registry.sql" + ticket: "OMN-15335" + - id: "node:node_projection_registration:0001_add_heartbeat_columns.sql" + ticket: "OMN-15088" + - id: "node:node_projection_registration:0002_node_service_registry_tenant_rls.sql" + ticket: "OMN-15343" + - id: "node:node_pr_review_bot:001_create_review_bot_bypass_log.sql" + ticket: "OMN-15717" + - id: "node:node_projection_delegation_inference_response:0003_inference_response_text_rls_tenant_isolation.sql" + ticket: "OMN-15336" diff --git a/docker/migrations/forward/flat-node-shape-parity.yaml b/docker/migrations/forward/flat-node-shape-parity.yaml new file mode 100644 index 0000000000..6aa4dee851 --- /dev/null +++ b/docker/migrations/forward/flat-node-shape-parity.yaml @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# OMN-15384 flat-vs-node dual-producer shape-parity ledger. +# +# OMN-15376 closed shape drift WITHIN the node-migration corpus (a table's own +# fresh-create path vs its own drifted-pre-existing path). It said nothing +# about a table declared by BOTH a flat migration (docker/migrations/forward/ +# *.sql, applied against the omnibase_infra DB) and a node migration +# (docker/migrations/forward/nodes//*.sql, applied against that node's +# own DB) drifting from EACH OTHER. This file is the record for that second +# axis: every table name this repo's migrations declare on both sides, and +# whether the two declared shapes currently agree. +# +# Read and enforced by tests/ci/test_flat_node_migration_shape_parity.py: +# * every table this test discovers as dual-produced (present as a guarded +# `CREATE TABLE IF NOT EXISTS` in at least one flat file AND at least one +# node file) MUST have an entry here -- an undeclared dual-producer table +# fails the gate outright, so a NEW overlap can't appear silently. +# * status: identical -- the two sides' column sets + normalized types must +# match EXACTLY. If they ever diverge, the gate goes red until this file +# is updated (either by converging the SQL, or by flipping the entry to +# accepted_divergence with a reason). +# * status: accepted_divergence -- the two sides are recorded as +# DIFFERENT ON PURPOSE, with `reason` stating why. The gate asserts the +# entry is still divergent (a stale accepted_divergence entry whose +# shapes now agree is also a gate failure -- it means the record no +# longer describes reality and must be corrected to `identical`). +# +# Do not delete an entry to make the gate pass. The gate diffs LIVE discovery +# against this file's key set and fails on any mismatch either direction. +# +# Ticket: OMN-15384 +tables: + agent_routing_decisions: + status: identical + capability_scores: + status: identical + evidence_correlation_trace_projection: + status: identical + evidence_dashboard_projection: + status: identical + evidence_readiness_aggregate_projection: + status: identical + live_events: + status: identical + llm_call_metrics: + status: identical + llm_cost_aggregates: + status: identical + llm_routing_decisions: + status: identical + log_entries: + status: identical + swarm_runs: + status: identical + baselines_comparisons: + status: accepted_divergence + reason: > + Flat producer (050_create_baselines_tables.sql) declares the original, narrower shape against the omnibase_infra DB. Node producer (node_projection_baselines/0001_create_baselines_tables.sql + 0002_realign_child_tables_to_producer_schema.sql) is the OMN-15302 shape that superseded it for that node's own DB: 14 additional comparison-detail columns (pattern_id, pattern_name, snapshot_id, window_start/end, rationale, recommendation, the *_delta columns, confidence) plus id UUID->TEXT and every REAL metric widened to DOUBLE PRECISION. The two are independently-evolving declarations against different physical databases, not an accidental drift of one shared shape. Convergence (retiring the flat side, or backporting the node shape into it) is a product decision OMN-15384 deliberately did not make -- see its "current live risk: LOW" framing. This entry exists so future drift on TOP of the current gap is caught, not to claim the gap itself is resolved. + + baselines_breakdown: + status: accepted_divergence + reason: > + Same root cause and same two producers as baselines_comparisons above: node's 0001/0002 pair adds action, avg_confidence, count, snapshot_id, projected_at, and widens id/pattern_id from UUID to TEXT and the *_rate / roi_pct metrics from REAL to DOUBLE PRECISION. Tracked, not resolved, for the same reason. + + baselines_trend: + status: accepted_divergence + reason: > + Same root cause and same two producers as baselines_comparisons above: node's 0001/0002 pair adds avg_cost_savings, avg_outcome_improvement, comparisons_evaluated, date, projected_at, snapshot_id, and widens id from UUID to TEXT and the avg_*/roi_pct/success_rate metrics from REAL to DOUBLE PRECISION. Tracked, not resolved, for the same reason. + + savings_estimates: + status: accepted_divergence + reason: > + Flat producer (074_create_savings_estimates.sql) uses bounded VARCHAR(n) / NUMERIC(14,6) columns for the omnibase_infra DB. Node producer (node_projection_savings/074_create_savings_estimates.sql) widens machine_id/model_cloud_baseline/model_local/repo_name/ session_id from VARCHAR to TEXT, widens the three cost/savings columns from NUMERIC(14,6) to NUMERIC(18,6), and adds an updated_at column the flat side does not have. Different DBs, independently evolved; convergence not attempted in this ticket. + + session_outcomes: + status: accepted_divergence + reason: > + Flat producer (063_create_session_outcomes.sql) declares correlation_id, which the node producer (node_projection_session_outcome/0021_session_outcomes.sql) does not carry; the node side instead declares created_at and updated_at, which the flat side does not carry. This is a genuine column-set divergence in both directions between the two DBs, not a formatting artifact. Convergence not attempted in this ticket. + + +# end of ledger diff --git a/docker/migrations/forward/grandfathered-force-rls-migrations.yaml b/docker/migrations/forward/grandfathered-force-rls-migrations.yaml new file mode 100644 index 0000000000..d1df02d2aa --- /dev/null +++ b/docker/migrations/forward/grandfathered-force-rls-migrations.yaml @@ -0,0 +1,94 @@ +# grandfathered-force-rls-migrations.yaml — baseline snapshot for the +# unclassified FORCE ROW LEVEL SECURITY guard (OMN-15336 item 4 repair, +# operator ruling D1 2026-08-05) +# +# WHAT THIS IS (and is NOT) +# This is a GRANDFATHER RECORD, not an operator fence. The fence +# (fenced-node-migrations.yaml, read by is_fenced_node_migration) is an +# operator GATE: an id on that list is actively held back from applying +# pending a ruling. This file is the opposite kind of fact: a frozen +# enumeration of every FORCE-ROW-LEVEL-SECURITY-enabling node migration that +# was ALREADY VENDORED, and had ALREADY BEEN APPLYING UNGATED on every warm +# lane, at the moment scripts/run-forward-migrations.sh gained the unclassified- +# FORCE-RLS guard (commit 3bc7fcaf2e, 2026-08-05). It exists solely so the +# guard can distinguish "a migration nobody has reviewed yet" (must FATAL) +# from "a migration that predates the guard and is part of the established +# baseline" (must keep applying — that is what "established baseline" means). +# +# THE DEFECT THIS FIXES +# The guard as first shipped fired for ANY FORCE-enabling migration absent +# from the operator fence, with no notion of "already part of the tree." +# The vendored tree carries 13 FORCE-enabling node migrations; the operator +# fence classifies only 4 of them (the ones an operator affirmatively chose +# to hold back). The other 9 were never held back by anyone — they are +# ordinary, already-shipped, already-applying-everywhere migrations — but +# the guard could not tell that apart from "brand new, unreviewed." Result: +# on ANY fresh/cold database (a virgin CI Postgres, a new lane bring-up), the +# guard FATALed on the first of the 9 it reached +# (node_canary_score_reducer/0002), aborting migration application entirely +# with a FATAL naming a migration nobody had asked to gate. Reproduced live: +# shipped runner against a virgin PG16 -> exit 1, 1 node migration applied, +# 87 withheld. Guard-stripped copy of the same tree -> exit 0, 87 applied. +# +# ENTRY CRITERIA (both required) +# 1. Enables FORCE ROW LEVEL SECURITY (the exact condition +# migration_declares_unclassified_force_rls tests for). +# 2. Existed in docker/migrations/forward/nodes/ at commit 3bc7fcaf2e~1 — +# i.e. BEFORE the guard first shipped. Verified per-id via +# `git show 3bc7fcaf2e~1:docker/migrations/forward/nodes/`. +# +# OMN-15831 NOTE: the commit cited above was repointed from the original +# 90cd78a580 to 3bc7fcaf2e (the PR's SQUASH MERGE SHA, permanent on `dev`) +# because 90cd78a580 was a PR-branch commit that squash-merge + branch +# deletion later made unreachable from any ref — `git show 90cd78a580~1:...` +# fails on a fresh checkout. 3bc7fcaf2e~1 resolves the identical pre-guard +# tree state. See tests/scripts/test_node_migration_fence_parity.py +# GUARD_INTRODUCTION_COMMIT for the same fix and a self-diagnosing guard +# against this recurring. +# +# THIS LIST DOES NOT GROW. +# It is a snapshot of one commit's tree, not a rolling allowlist. A NEW +# FORCE-enabling node migration written after 2026-08-05 must NEVER be added +# here to silence the guard — it must go through the operator fence +# (fenced-node-migrations.yaml) like every migration written since OMN-15336 +# shipped, or ship as a deliberately unfenced release with an operator ruling +# on record. tests/scripts/test_node_migration_fence_parity.py pins this +# file's exact, in-order content (test_grandfather_pins_the_snapshot_baseline) +# — an addition here without an accompanying test-pin update fails CI closed. +# A subtraction (an id being un-grandfathered because its FORCE posture was +# reconsidered) is the normal path off this list: move it to the operator +# fence instead, or strip FORCE entirely (see +# node_projection_registration/0004's NO FORCE precedent) and drop the id +# from both files in the same PR. +# +# FORMAT +# Same as fenced-node-migrations.yaml: a flat list under +# `grandfathered_force_rls_migrations`, each entry an `id:` (mandatory, +# machine-parsed) plus a `ticket:` (advisory). Parsed with the same portable +# sed one-liner both runners already use for the fence manifest: +# +# sed -n 's/^[[:space:]]*-[[:space:]]*id:[[:space:]]*"\([^"]*\)".*/\1/p' \ +# grandfathered-force-rls-migrations.yaml +# +# Order is significant only in that it is pinned verbatim by the test above; +# it carries no runtime semantics (membership is a set, checked by exact +# whole-line match). +grandfathered_force_rls_migrations: + - id: "node:node_canary_score_reducer:0002_capability_scores_tenant_id_and_rls.sql" + ticket: "OMN-15336" + - id: "node:node_projection_context_roi:003_context_roi_scores_tenant_id_and_rls.sql" + ticket: "OMN-15336" + - id: "node:node_projection_cost_summary:0002_llm_cost_aggregates_tenant_id_and_rls.sql" + ticket: "OMN-15336" + - id: "node:node_projection_dep_health:002_dep_health_findings_tenant_id_and_rls.sql" + ticket: "OMN-15336" + - id: "node:node_projection_instruction_eval:0002_instruction_eval_aggregate_snapshots_tenant_id_and_rls.sql" + ticket: "OMN-15336" + - id: "node:node_projection_pattern_learning:0001_pattern_learning_artifacts_tenant_id_and_rls.sql" + ticket: "OMN-15336" + - id: "node:node_projection_routing_decision:0022_agent_routing_decisions_tenant_id_and_rls.sql" + ticket: "OMN-15336" + - id: "node:node_projection_savings:081_savings_estimates_rls_tenant_isolation.sql" + ticket: "OMN-15336" + - id: "node:node_projection_skill_executions:0002_skill_execution_snapshots_tenant_id_and_rls.sql" + ticket: "OMN-15336" diff --git a/docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql b/docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql index f02a406dc8..d56892482e 100644 --- a/docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql +++ b/docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql @@ -43,6 +43,93 @@ CREATE TABLE IF NOT EXISTS public.capability_scores ( UNIQUE (model_key, task_type) ); +-- ---- BEGIN OMN-15376 shape reconciliation: capability_scores ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS id BIGSERIAL; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS model_key TEXT; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS task_type TEXT; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS success_count INT DEFAULT 0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS failure_count INT DEFAULT 0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS total_count INT DEFAULT 0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS success_rate DOUBLE PRECISION DEFAULT 0.0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS avg_latency_ms INT DEFAULT 0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS avg_tokens_per_sec DOUBLE PRECISION DEFAULT 0.0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS total_cost DOUBLE PRECISION DEFAULT 0.0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS graduated BOOLEAN DEFAULT FALSE; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS last_updated TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'model_key', 'task_type', 'success_count', 'failure_count', 'total_count', 'success_rate', 'avg_latency_ms', 'avg_tokens_per_sec', 'total_cost', 'graduated', 'last_updated'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'public.capability_scores'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'public.capability_scores'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge public.capability_scores.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'public.capability_scores'::regclass AND contype = 'p' + ) THEN + ALTER TABLE public.capability_scores ADD CONSTRAINT capability_scores_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'public.capability_scores'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['model_key', 'task_type']::text[] + ) THEN + ALTER TABLE public.capability_scores ADD CONSTRAINT capability_scores_model_key_task_type_key UNIQUE (model_key, task_type); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: capability_scores ---- + + -- Lookup by model for router input CREATE INDEX IF NOT EXISTS idx_capability_scores_model ON capability_scores (model_key); diff --git a/docker/migrations/forward/nodes/node_canary_score_reducer/0002_capability_scores_tenant_id_and_rls.sql b/docker/migrations/forward/nodes/node_canary_score_reducer/0002_capability_scores_tenant_id_and_rls.sql new file mode 100644 index 0000000000..2ca270412b --- /dev/null +++ b/docker/migrations/forward/nodes/node_canary_score_reducer/0002_capability_scores_tenant_id_and_rls.sql @@ -0,0 +1,212 @@ +-- OMN-15655 / operator ruling 2026-08-02 (house tenant): give capability_scores a +-- tenant identity and row-level tenant isolation. +-- +-- canary capability scores are per-workload model/task quality measurements THIS RELATION IS TENANT DATA +-- WHY, so the row is workload-attributable and belongs in the tenant +-- domain per the ruling's rule. Rows produced by OmniNode's own platform +-- workloads are the omninode TENANT's rows -- OmniNode is a first-class tenant, +-- not an absence of one. Only attribution-meaningless infrastructure state +-- (migration bookkeeping, registry, orchestration state, deployment +-- evidence) stays in omninode_internal. +-- +-- This migration lands alongside the contract flip +-- `db_io.db_tables[].schema: public -> tenant`. Before it, the relation +-- declared a schema the typed topology does not declare, which ADR-0027 +-- refuses by design -- `ValueError: Unknown schema 'public' for +-- database_ref 'application'` out of `ModelDeploymentTopology.schema_domain`. +-- +-- IDENTITY TYPE -- TEXT, NOT UUID, AND THAT IS DELIBERATE +-- Every landed tenant_id column on this surface is `TEXT` holding the slug +-- 'omninode' (delegation 0022/0025, savings 080, registration 0002, +-- inference-response 0002, omnidash 0001_tenant_rls), and the RLS policies +-- that already shipped (0023, 0026) compare TEXT with no `::uuid` cast. +-- OMN-15356 converts that whole set to the canonical UUID in ONE pass. +-- Adding a UUID column here alone would fork tenant identity inside a single +-- database -- the exact "one canonical model per shape" violation. The +-- canonical UUID this slug resolves to is pinned in +-- `omnimarket.projection.tenant_isolation.omninode_TENANT_UUID`. +-- +-- BACKFILL -- NO BATCHING NEEDED, AND BATCHING WOULD BE WORSE +-- `ADD COLUMN ... NOT NULL DEFAULT` with a non-volatile default does not +-- rewrite the table on PostgreSQL 11+; the default is stored in +-- `pg_attribute.attmissingval` and every pre-existing row reads back as +-- the house tenant immediately. A batched `UPDATE` would do strictly more +-- work (a real rewrite, dead tuples, a longer lock) for the same end state. +-- +-- PHYSICAL SCHEMA IS NOT MOVED HERE -- SEE OMN-15359 +-- No `ALTER TABLE ... SET SCHEMA tenant`. The `tenant` schema is created +-- by no applied migration in this repo or in omnibase_infra +-- (`docker/migrations/forward/_ledger/bootstrap.sql` creates only +-- `platform_catalog`); it exists solely in proof fixtures. Every relation +-- already classified TENANT -- delegation_events, delegation_budget_state, +-- savings_estimates -- is still physically in `public` +-- (`current_schema: ["public"]` in the OMN-15423 inventory, against +-- `target_schema: tenant`). Relocating these eight alone would create the +-- split the inventory's target/current split exists to prevent, and would +-- fail outright against a database with no `tenant` schema. The physical +-- move is OMN-15359 ("Build classified schemas and migrate internal, +-- control-plane, catalog, and tenant targets"), which moves the whole set, +-- with its trigger functions and sequences, in one governed cutover. +-- +-- FAIL-CLOSED RATCHET +-- The column DEFAULT is what supplies the house tenant today, exactly as it +-- does for savings_estimates (OMN-14058, operator-accepted): a writer that +-- resolves no tenant OMITS the key and Postgres fills it -- the key is never +-- written as NULL. That default-allowed state is PINNED, with its flip +-- condition named, by +-- `tests/unit/projection/test_house_tenant_default_ratchet.py`. When +-- customer ingress exists the writer boundary stops defaulting and refuses +-- instead; that test fails the moment the precondition changes. +-- +-- BLAST RADIUS -- FORCE constrains the table OWNER too +-- Same caveat as migration 0023: a writer connected as the `postgres` +-- SUPERUSER (the compose lanes) bypasses RLS regardless of FORCE. The real +-- isolation boundary is this policy PLUS a non-superuser, NOBYPASSRLS writer +-- role (OMN-14899 / OMN-15425). +-- +-- Fail-closed: `current_setting('app.tenant_id', true)` is NULL when the GUC +-- is unset, the predicate is NULL, and zero rows are visible. The policy has no +-- default-tenant fallback, by design. +-- +-- Idempotent: ADD COLUMN / CREATE INDEX are IF NOT EXISTS, ENABLE/FORCE are +-- idempotent, the policy is DROP + CREATE, GRANTs are idempotent. + +-- Legacy upgrade guard: this tenant/RLS migration may be the first +-- capability_scores migration a legacy application DB applies. Keep the base +-- table shape convergent here instead of assuming the preceding node-owned +-- create migration has already run in every historical ledger shape. +CREATE TABLE IF NOT EXISTS public.capability_scores ( + id BIGSERIAL PRIMARY KEY, + model_key TEXT NOT NULL, + task_type TEXT NOT NULL, + success_count INT NOT NULL DEFAULT 0, + failure_count INT NOT NULL DEFAULT 0, + total_count INT NOT NULL DEFAULT 0, + success_rate DOUBLE PRECISION NOT NULL DEFAULT 0.0, + avg_latency_ms INT NOT NULL DEFAULT 0, + avg_tokens_per_sec DOUBLE PRECISION NOT NULL DEFAULT 0.0, + total_cost DOUBLE PRECISION NOT NULL DEFAULT 0.0, + graduated BOOLEAN NOT NULL DEFAULT FALSE, + last_updated TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (model_key, task_type) +); + +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS id BIGSERIAL; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS model_key TEXT; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS task_type TEXT; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS success_count INT DEFAULT 0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS failure_count INT DEFAULT 0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS total_count INT DEFAULT 0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS success_rate DOUBLE PRECISION DEFAULT 0.0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS avg_latency_ms INT DEFAULT 0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS avg_tokens_per_sec DOUBLE PRECISION DEFAULT 0.0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS total_cost DOUBLE PRECISION DEFAULT 0.0; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS graduated BOOLEAN DEFAULT FALSE; +ALTER TABLE public.capability_scores ADD COLUMN IF NOT EXISTS last_updated TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; + v_pk_columns TEXT[]; +BEGIN + FOREACH v_col IN ARRAY ARRAY[ + 'id', 'model_key', 'task_type', 'success_count', 'failure_count', + 'total_count', 'success_rate', 'avg_latency_ms', + 'avg_tokens_per_sec', 'total_cost', 'graduated', 'last_updated' + ] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', + 'public.capability_scores'::regclass, + v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', + 'public.capability_scores'::regclass, + v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15655: cannot converge public.capability_scores.% to NOT NULL -- % pre-existing row(s) hold NULL; operator data mapping required.', + v_col, v_nulls; + END IF; + END LOOP; + + SELECT array_agg(a.attname::text ORDER BY k.ordinality) INTO v_pk_columns + FROM pg_constraint c + CROSS JOIN LATERAL unnest(c.conkey) WITH ORDINALITY AS k(attnum, ordinality) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + WHERE c.conrelid = 'public.capability_scores'::regclass + AND c.contype = 'p'; + + IF v_pk_columns IS NULL THEN + ALTER TABLE public.capability_scores + ADD CONSTRAINT capability_scores_pkey PRIMARY KEY (id); + ELSIF v_pk_columns <> ARRAY['id']::text[] THEN + RAISE EXCEPTION + 'OMN-15655: public.capability_scores primary key covers %, expected {id}; operator schema ruling required.', + v_pk_columns; + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'public.capability_scores'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY k.ordinality) + FROM unnest(c.conkey) WITH ORDINALITY AS k(attnum, ordinality) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['model_key', 'task_type']::text[] + ) THEN + ALTER TABLE public.capability_scores + ADD CONSTRAINT capability_scores_model_key_task_type_key + UNIQUE (model_key, task_type); + END IF; +END$$; + +ALTER TABLE public.capability_scores + ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'omninode'; + +CREATE INDEX IF NOT EXISTS idx_capability_scores_tenant_id + ON public.capability_scores (tenant_id); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'app_dashboard' + AND NOT rolsuper + AND NOT rolbypassrls + ) THEN + RAISE EXCEPTION + 'app_dashboard role missing or RLS-bypassing - apply omnibase_infra forward migration ' + '094_create_app_dashboard_role.sql (OMN-14899) before this RLS ' + 'migration. RLS grants without the constrained read role are the ' + 'exact bypass this work exists to prevent.'; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA public TO app_dashboard; + +ALTER TABLE public.capability_scores ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.capability_scores FORCE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS tenant_isolation ON public.capability_scores; +CREATE POLICY tenant_isolation ON public.capability_scores + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)); + +-- Read grant for the live dashboard reader. RLS still filters every +-- row this role can see -- the grant is what makes the RLS-scoped read path +-- reachable at all, not a widening of it. OMN-14894: sibling grant present on +-- context_roi_scores/instruction_eval_aggregate_snapshots/skill_execution_snapshots +-- in the same PR; capability_scores omitted it in error. +GRANT SELECT ON public.capability_scores TO app_dashboard; diff --git a/docker/migrations/forward/nodes/node_canary_score_reducer/0003_capability_scores_tenant_id_to_uuid.sql b/docker/migrations/forward/nodes/node_canary_score_reducer/0003_capability_scores_tenant_id_to_uuid.sql new file mode 100644 index 0000000000..903d9fe82a --- /dev/null +++ b/docker/migrations/forward/nodes/node_canary_score_reducer/0003_capability_scores_tenant_id_to_uuid.sql @@ -0,0 +1,172 @@ +-- OMN-15356: convert capability_scores.tenant_id from the legacy TEXT slug +-- to the canonical UUID identity, for the classified-TENANT relation set. +-- +-- OMN-15732 AC2 ADJUDICATION (2026-08-08, no gate widened) +-- An earlier revision of this migration split the mapping into a +-- standalone, schema-qualified `platform_catalog.house_tenant_map_slug_to_ +-- uuid` function so it could be ownership-declared as a new application +-- object. That placement was rejected on adjudication: the migration-ledger +-- `domain` column is the *declared target schema domain* (omnibase_infra +-- docker/migrations/forward/_ledger/bootstrap.sql), and `platform_catalog` +-- (topology domain PLATFORM_CATALOG) is inadmissible for a `node:%` +-- migration stream at both the static gate +-- (scripts/validation/validate_application_migration_manifest.py) and the +-- runtime `schema_migrations_stream_domain_check` CHECK constraint, which +-- together restrict node-stream domains to exactly {tenant, +-- omninode_internal}. `tenant` and `omninode_internal` both fail +-- apply-time schema presence today (the former needs the still-pending +-- OMN-15359 cutover; the latter is created only by the cutover-only +-- initializer, never by the forward-migration stream). The function has +-- exactly one consumer -- this file's own DDL-time `USING` clause -- and no +-- runtime caller; the runtime implementation is the single Python +-- function `omnimarket.projection.tenant_isolation.resolve_tenant_uuid`. +-- A persistent catalog object read once by one ALTER buys nothing here and +-- cannot be given a truthful ownership/domain declaration on every surface +-- at once. The mapping is therefore inlined below with no CREATE FUNCTION +-- and no new schema-qualified object; OMN-15359's governed tenant/internal +-- cutover is the right place to promote this into a shared, +-- truthfully-declared `tenant.house_tenant_map_slug_to_uuid` alongside the +-- other classified-TENANT relations' conversions. +-- +-- WHY THIS TABLE FIRST +-- capability_scores is the relation the OMN-15356 identity module +-- (`omnimarket.projection.tenant_isolation.resolve_tenant_uuid`) and its +-- test (`test_the_tenant_classified_relations_carry_a_tenant_id_migration`) +-- already use as the canonical worked example. This migration lands the +-- full end-to-end pattern -- inline fail-closed mapping, conversion, +-- index/constraint preservation, RLS policy cast -- for ONE relation as the +-- reviewable shape. The remaining classified-TENANT relations (see the +-- `expected` table in `test_house_tenant_identity.py` plus +-- delegation_events/delegation_judge_verdict_events/delegation_budget_state/ +-- inference_response_text/savings_estimates/node_service_registry) convert +-- via the identical mechanical pattern in follow-up migrations under this +-- same ticket -- NOT silently dropped, deferred and named in the PR body. +-- +-- FAIL-CLOSED: NO SENTINEL SURVIVES (two independent guards) +-- (1) A pre-guard `IF EXISTS` check below RAISEs before any DDL runs if any +-- row's tenant_id is not the one closed mapping key this codebase knows +-- about. (2) Even if that guard were ever bypassed, the `ALTER COLUMN ... +-- TYPE UUID USING (CASE ... END)` below has no ELSE branch: an unmapped +-- value evaluates the CASE to NULL, which then violates the column's +-- pre-existing NOT NULL constraint (migration 0002) and aborts the +-- statement. Postgres evaluates the `USING` expression for every row as +-- part of the single `ALTER TABLE` DDL statement, and because DDL in +-- PostgreSQL is transactional, either guard failing rolls back the entire +-- migration transaction. There is no partial conversion, no invented UUID, +-- and no 'unmapped rows keep their old value' fallback: the column stays +-- TEXT until every row maps, exactly the "no sentinel/default survives" +-- acceptance criterion. +-- +-- CONSTRAINTS AND INDEXES SURVIVE THE TYPE CHANGE +-- `ALTER COLUMN ... TYPE` rewrites the column in place; the pre-existing +-- `idx_capability_scores_tenant_id` index and the +-- `capability_scores_model_key_task_type_key` UNIQUE constraint (on +-- unrelated columns, untouched here) are NOT dropped and NOT recreated by +-- this migration -- PostgreSQL preserves an index across a column type +-- change automatically when the new type has a binary-compatible or +-- USING-expressed conversion path, which this is. The Docker proof in +-- omnibase_infra (docker/tenant-uuid-conversion-proof) asserts this +-- directly rather than assuming it. +-- +-- RLS POLICY: THE GUC COMPARISON GAINS AN EXPLICIT CAST +-- `current_setting('app.tenant_id', true)` always returns TEXT (GUCs have +-- no native UUID type); the policy predicate must cast it explicitly now +-- that the column itself is UUID. `::uuid` on a non-UUID-shaped GUC value +-- raises rather than silently coercing, so an unset or malformed GUC still +-- fails closed -- proving that behavior is OMN-15416's scope (real +-- non-owner pools), not duplicated here. +-- +-- Idempotent: the column-type guard only runs the ALTER when the column is +-- not already UUID, so a second application is a no-op. + +DO $$ +DECLARE + v_current_type TEXT; +BEGIN + SELECT atttypid::regtype::text INTO v_current_type + FROM pg_attribute + WHERE attrelid = 'public.capability_scores'::regclass + AND attname = 'tenant_id' + AND NOT attisdropped; + + IF v_current_type IS NULL THEN + RAISE EXCEPTION + 'OMN-15356: public.capability_scores.tenant_id column not found -- ' + 'expected migration 0002 to have already landed it'; + ELSIF v_current_type = 'uuid' THEN + RAISE NOTICE + 'public.capability_scores.tenant_id is already uuid; skipping conversion'; + ELSIF v_current_type = 'text' THEN + -- Pre-guard: refuse before any DDL runs if any row carries a + -- tenant_id value outside the closed mapping (today: 'omninode' + -- only). This is the first of the two independent fail-closed + -- guards described in the file header above. + IF EXISTS ( + SELECT 1 FROM public.capability_scores + WHERE tenant_id IS DISTINCT FROM 'omninode' + ) THEN + RAISE EXCEPTION + 'OMN-15356: no canonical UUID mapping for tenant value % -- ' + 'refusing to invent or default one; extend the closed mapping ' + 'in this migration only after confirming this is a real, ' + 'reviewed tenant identity', + ( + SELECT tenant_id FROM public.capability_scores + WHERE tenant_id IS DISTINCT FROM 'omninode' + LIMIT 1 + ); + END IF; + + -- The pre-existing tenant_isolation POLICY (migration 0002) depends + -- on this column -- PostgreSQL refuses ALTER COLUMN ... TYPE while + -- any policy references it, so the policy must be dropped first and + -- recreated (with the ::uuid cast) after the type change, not just + -- before/after this DO block. Caught by the Docker fixture proof + -- (docker/tenant-uuid-conversion-proof) before this ever reached a + -- real database. + DROP POLICY IF EXISTS tenant_isolation ON public.capability_scores; + + -- The pre-existing TEXT DEFAULT ('omninode') is not automatically + -- castable to uuid -- Postgres tries to cast the DEFAULT expression + -- itself (not just the stored rows) when the column type changes, + -- and 'omninode'::uuid is not a valid uuid literal, so the DEFAULT + -- must be dropped before the TYPE change and a new uuid-typed + -- DEFAULT set after it. Also caught by the same fixture proof. + ALTER TABLE public.capability_scores + ALTER COLUMN tenant_id DROP DEFAULT; + -- Second, independent fail-closed guard: the CASE has no ELSE, so + -- any value that reaches this point despite the pre-guard above + -- (e.g. a row inserted between the guard and the ALTER within the + -- same transaction) evaluates to NULL and is rejected by the + -- column's existing NOT NULL constraint, aborting the statement. + ALTER TABLE public.capability_scores + ALTER COLUMN tenant_id TYPE UUID + USING ( + CASE tenant_id + WHEN 'omninode' THEN '820272f9-4aaf-5add-a2df-0af942852ab2'::uuid + END + ); + ALTER TABLE public.capability_scores + ALTER COLUMN tenant_id SET DEFAULT '820272f9-4aaf-5add-a2df-0af942852ab2'::uuid; + ELSE + RAISE EXCEPTION + 'OMN-15356: public.capability_scores.tenant_id has unexpected type %, ' + 'expected text or uuid -- operator schema ruling required', + v_current_type; + END IF; +END$$; + +-- Idempotent whether or not the DO block above dropped it (the 'uuid' +-- branch above leaves the existing (already-cast) policy in place, so this +-- DROP + CREATE additionally covers a hand-repaired or partially-applied +-- prior state without erroring). +DROP POLICY IF EXISTS tenant_isolation ON public.capability_scores; +CREATE POLICY tenant_isolation ON public.capability_scores + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)::uuid) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)::uuid); + +-- OMN-14894 ratchet: every file that (re)creates this policy must grant +-- app_dashboard SELECT in the same file. Idempotent; already granted by +-- migration 0002, restated here so this file alone satisfies the ratchet. +GRANT SELECT ON public.capability_scores TO app_dashboard; diff --git a/docker/migrations/forward/nodes/node_contract_registry/0000_create_contract_registry.sql b/docker/migrations/forward/nodes/node_contract_registry/0000_create_contract_registry.sql index 17d190d490..af3924278d 100644 --- a/docker/migrations/forward/nodes/node_contract_registry/0000_create_contract_registry.sql +++ b/docker/migrations/forward/nodes/node_contract_registry/0000_create_contract_registry.sql @@ -16,5 +16,90 @@ CREATE TABLE IF NOT EXISTS contract_registry ( UNIQUE(node_name, contract_hash) ); +-- ---- BEGIN OMN-15376 shape reconciliation: contract_registry ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE contract_registry ADD COLUMN IF NOT EXISTS id SERIAL; +ALTER TABLE contract_registry ADD COLUMN IF NOT EXISTS node_name TEXT; +ALTER TABLE contract_registry ADD COLUMN IF NOT EXISTS contract_hash TEXT; +ALTER TABLE contract_registry ADD COLUMN IF NOT EXISTS contract_yaml TEXT; +ALTER TABLE contract_registry ADD COLUMN IF NOT EXISTS node_version JSONB DEFAULT '{}'; +ALTER TABLE contract_registry ADD COLUMN IF NOT EXISTS status TEXT; +ALTER TABLE contract_registry ADD COLUMN IF NOT EXISTS correlation_id UUID; +ALTER TABLE contract_registry ADD COLUMN IF NOT EXISTS registered_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE contract_registry ADD COLUMN IF NOT EXISTS deployer_id TEXT DEFAULT ''; +ALTER TABLE contract_registry ADD COLUMN IF NOT EXISTS target_profile TEXT DEFAULT ''; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'node_name', 'contract_hash', 'contract_yaml', 'node_version', 'status', 'correlation_id', 'registered_at', 'deployer_id', 'target_profile'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'contract_registry'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'contract_registry'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge contract_registry.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'contract_registry'::regclass AND contype = 'p' + ) THEN + ALTER TABLE contract_registry ADD CONSTRAINT contract_registry_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'contract_registry'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['contract_hash', 'node_name']::text[] + ) THEN + ALTER TABLE contract_registry ADD CONSTRAINT contract_registry_node_name_contract_hash_key UNIQUE(node_name, contract_hash); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: contract_registry ---- + + CREATE INDEX IF NOT EXISTS idx_contract_registry_node_name ON contract_registry(node_name); CREATE INDEX IF NOT EXISTS idx_contract_registry_status ON contract_registry(status); diff --git a/docker/migrations/forward/nodes/node_deployment_evidence_reducer/0001_create_deployment_evidence_projection_tables.sql b/docker/migrations/forward/nodes/node_deployment_evidence_reducer/0001_create_deployment_evidence_projection_tables.sql index 04e2f8caa9..0acc468a85 100644 --- a/docker/migrations/forward/nodes/node_deployment_evidence_reducer/0001_create_deployment_evidence_projection_tables.sql +++ b/docker/migrations/forward/nodes/node_deployment_evidence_reducer/0001_create_deployment_evidence_projection_tables.sql @@ -22,6 +22,78 @@ CREATE TABLE IF NOT EXISTS deployment_evidence_projection ( updated_at TIMESTAMPTZ NOT NULL ); +-- ---- BEGIN OMN-15376 shape reconciliation: deployment_evidence_projection ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS deployment_id TEXT; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS ticket_id TEXT; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS validation_run_id TEXT; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS repository TEXT; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS evidence_lifecycle_state TEXT; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS validation_state TEXT; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS readiness_state TEXT; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS topology_affecting BOOLEAN DEFAULT FALSE; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS blocking_reason_codes TEXT DEFAULT ''; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS contract_hash TEXT; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS evidence_bundle_hash TEXT; +ALTER TABLE deployment_evidence_projection ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['deployment_id', 'correlation_id', 'ticket_id', 'validation_run_id', 'repository', 'evidence_lifecycle_state', 'validation_state', 'readiness_state', 'topology_affecting', 'blocking_reason_codes', 'contract_hash', 'evidence_bundle_hash', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'deployment_evidence_projection'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'deployment_evidence_projection'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge deployment_evidence_projection.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'deployment_evidence_projection'::regclass AND contype = 'p' + ) THEN + ALTER TABLE deployment_evidence_projection ADD CONSTRAINT deployment_evidence_projection_pkey PRIMARY KEY (deployment_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: deployment_evidence_projection ---- + + CREATE TABLE IF NOT EXISTS deployment_readiness_projection ( deployment_id TEXT PRIMARY KEY, correlation_id TEXT NOT NULL, @@ -33,6 +105,73 @@ CREATE TABLE IF NOT EXISTS deployment_readiness_projection ( updated_at TIMESTAMPTZ NOT NULL ); +-- ---- BEGIN OMN-15376 shape reconciliation: deployment_readiness_projection ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE deployment_readiness_projection ADD COLUMN IF NOT EXISTS deployment_id TEXT; +ALTER TABLE deployment_readiness_projection ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE deployment_readiness_projection ADD COLUMN IF NOT EXISTS validation_run_id TEXT; +ALTER TABLE deployment_readiness_projection ADD COLUMN IF NOT EXISTS readiness_state TEXT; +ALTER TABLE deployment_readiness_projection ADD COLUMN IF NOT EXISTS blocking_reason_codes TEXT DEFAULT ''; +ALTER TABLE deployment_readiness_projection ADD COLUMN IF NOT EXISTS gap_report_hash TEXT; +ALTER TABLE deployment_readiness_projection ADD COLUMN IF NOT EXISTS validator_version TEXT; +ALTER TABLE deployment_readiness_projection ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['deployment_id', 'correlation_id', 'validation_run_id', 'readiness_state', 'blocking_reason_codes', 'gap_report_hash', 'validator_version', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'deployment_readiness_projection'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'deployment_readiness_projection'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge deployment_readiness_projection.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'deployment_readiness_projection'::regclass AND contype = 'p' + ) THEN + ALTER TABLE deployment_readiness_projection ADD CONSTRAINT deployment_readiness_projection_pkey PRIMARY KEY (deployment_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: deployment_readiness_projection ---- + + CREATE INDEX IF NOT EXISTS idx_deployment_evidence_projection_ticket ON deployment_evidence_projection (ticket_id); CREATE INDEX IF NOT EXISTS idx_deployment_evidence_projection_updated_at diff --git a/docker/migrations/forward/nodes/node_evidence_dashboard_reducer/0001_create_evidence_dashboard_projection_tables.sql b/docker/migrations/forward/nodes/node_evidence_dashboard_reducer/0001_create_evidence_dashboard_projection_tables.sql index 7bd893b2c8..bef46f784c 100644 --- a/docker/migrations/forward/nodes/node_evidence_dashboard_reducer/0001_create_evidence_dashboard_projection_tables.sql +++ b/docker/migrations/forward/nodes/node_evidence_dashboard_reducer/0001_create_evidence_dashboard_projection_tables.sql @@ -20,6 +20,82 @@ CREATE TABLE IF NOT EXISTS evidence_dashboard_projection ( expires_at TIMESTAMPTZ NOT NULL ); +-- ---- BEGIN OMN-15376 shape reconciliation: evidence_dashboard_projection ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS projection_key TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS ticket_id TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS repo TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS pr_number INT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS validation_run_id TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS current_stage TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS status TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS severity TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS projection_cursor TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS last_event_id TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS last_ingest_sequence BIGINT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS freshness_state TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS degraded_reason TEXT; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS observed_at TIMESTAMPTZ; +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE evidence_dashboard_projection ADD COLUMN IF NOT EXISTS expires_at TIMESTAMPTZ; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['projection_key', 'correlation_id', 'current_stage', 'status', 'severity', 'projection_cursor', 'last_event_id', 'freshness_state', 'observed_at', 'updated_at', 'expires_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'evidence_dashboard_projection'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'evidence_dashboard_projection'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge evidence_dashboard_projection.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'evidence_dashboard_projection'::regclass AND contype = 'p' + ) THEN + ALTER TABLE evidence_dashboard_projection ADD CONSTRAINT evidence_dashboard_projection_pkey PRIMARY KEY (projection_key); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: evidence_dashboard_projection ---- + + CREATE TABLE IF NOT EXISTS evidence_correlation_trace_projection ( event_id TEXT PRIMARY KEY, correlation_id TEXT NOT NULL, @@ -43,6 +119,85 @@ CREATE TABLE IF NOT EXISTS evidence_correlation_trace_projection ( payload JSONB NOT NULL DEFAULT '{}'::jsonb ); +-- ---- BEGIN OMN-15376 shape reconciliation: evidence_correlation_trace_projection ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS event_id TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS ticket_id TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS repo TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS pr_number INT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS source_topic TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS source_event_type TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS normalized_stage TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS status TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS severity TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS ingest_sequence BIGINT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS projection_cursor TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS last_event_id TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS last_ingest_sequence BIGINT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS freshness_state TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS degraded_reason TEXT; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS observed_at TIMESTAMPTZ; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS expires_at TIMESTAMPTZ; +ALTER TABLE evidence_correlation_trace_projection ADD COLUMN IF NOT EXISTS payload JSONB DEFAULT '{}'::jsonb; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['event_id', 'correlation_id', 'source_topic', 'source_event_type', 'normalized_stage', 'status', 'severity', 'projection_cursor', 'last_event_id', 'freshness_state', 'observed_at', 'updated_at', 'expires_at', 'payload'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'evidence_correlation_trace_projection'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'evidence_correlation_trace_projection'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge evidence_correlation_trace_projection.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'evidence_correlation_trace_projection'::regclass AND contype = 'p' + ) THEN + ALTER TABLE evidence_correlation_trace_projection ADD CONSTRAINT evidence_correlation_trace_projection_pkey PRIMARY KEY (event_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: evidence_correlation_trace_projection ---- + + CREATE TABLE IF NOT EXISTS evidence_readiness_aggregate_projection ( aggregate_key TEXT PRIMARY KEY, correlation_id TEXT NOT NULL, @@ -63,6 +218,82 @@ CREATE TABLE IF NOT EXISTS evidence_readiness_aggregate_projection ( expires_at TIMESTAMPTZ NOT NULL ); +-- ---- BEGIN OMN-15376 shape reconciliation: evidence_readiness_aggregate_projection ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS aggregate_key TEXT; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS ticket_id TEXT; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS repo TEXT; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS pr_number INT; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS readiness_state TEXT; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS total_events INT DEFAULT 0; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS error_events INT DEFAULT 0; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS warning_events INT DEFAULT 0; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS projection_cursor TEXT; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS last_event_id TEXT; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS last_ingest_sequence BIGINT; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS freshness_state TEXT; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS degraded_reason TEXT; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS observed_at TIMESTAMPTZ; +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE evidence_readiness_aggregate_projection ADD COLUMN IF NOT EXISTS expires_at TIMESTAMPTZ; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['aggregate_key', 'correlation_id', 'readiness_state', 'total_events', 'error_events', 'warning_events', 'projection_cursor', 'last_event_id', 'freshness_state', 'observed_at', 'updated_at', 'expires_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'evidence_readiness_aggregate_projection'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'evidence_readiness_aggregate_projection'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge evidence_readiness_aggregate_projection.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'evidence_readiness_aggregate_projection'::regclass AND contype = 'p' + ) THEN + ALTER TABLE evidence_readiness_aggregate_projection ADD CONSTRAINT evidence_readiness_aggregate_projection_pkey PRIMARY KEY (aggregate_key); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: evidence_readiness_aggregate_projection ---- + + CREATE INDEX IF NOT EXISTS idx_evidence_dashboard_projection_cursor ON evidence_dashboard_projection (projection_cursor); CREATE INDEX IF NOT EXISTS idx_evidence_dashboard_observed_at diff --git a/docker/migrations/forward/nodes/node_llm_delegation_projection/0001_create_llm_delegation_daily_projection.sql b/docker/migrations/forward/nodes/node_llm_delegation_projection/0001_create_llm_delegation_daily_projection.sql index 796f640b3f..54ec6b9bde 100644 --- a/docker/migrations/forward/nodes/node_llm_delegation_projection/0001_create_llm_delegation_daily_projection.sql +++ b/docker/migrations/forward/nodes/node_llm_delegation_projection/0001_create_llm_delegation_daily_projection.sql @@ -47,6 +47,107 @@ CREATE TABLE IF NOT EXISTS llm_delegation_daily_projection ( UNIQUE (projection_date, task_type, model_id, model_tier) ); +-- ---- BEGIN OMN-15376 shape reconciliation: llm_delegation_daily_projection ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS projection_date DATE; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS task_type TEXT; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS model_id TEXT; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS model_tier TEXT; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS total_calls INT DEFAULT 0; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS successful_calls INT DEFAULT 0; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS escalated_calls INT DEFAULT 0; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS total_tokens_in BIGINT DEFAULT 0; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS total_tokens_out BIGINT DEFAULT 0; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS total_latency_ms BIGINT DEFAULT 0; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS avg_latency_ms NUMERIC(12, 2) DEFAULT 0; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS total_actual_cost_usd NUMERIC(18, 8) DEFAULT 0; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS total_opus_equivalent_usd NUMERIC(18, 8) DEFAULT 0; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS total_savings_usd NUMERIC(18, 8) DEFAULT 0; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS avg_quality_score NUMERIC(6, 4); +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS projection_cursor TEXT; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS source_event_id TEXT; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS source_topic TEXT; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS source_partition INT; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS source_offset BIGINT; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS freshness_state TEXT DEFAULT 'FRESH'; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS reducer_version TEXT DEFAULT '1.0.0'; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS idempotency_key TEXT; +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS observed_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE llm_delegation_daily_projection ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'projection_date', 'task_type', 'model_id', 'model_tier', 'total_calls', 'successful_calls', 'escalated_calls', 'total_tokens_in', 'total_tokens_out', 'total_latency_ms', 'avg_latency_ms', 'total_actual_cost_usd', 'total_opus_equivalent_usd', 'total_savings_usd', 'projection_cursor', 'source_event_id', 'source_topic', 'source_partition', 'source_offset', 'freshness_state', 'reducer_version', 'idempotency_key', 'observed_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'llm_delegation_daily_projection'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'llm_delegation_daily_projection'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge llm_delegation_daily_projection.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_delegation_daily_projection'::regclass AND contype = 'p' + ) THEN + ALTER TABLE llm_delegation_daily_projection ADD CONSTRAINT llm_delegation_daily_projection_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'llm_delegation_daily_projection'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['model_id', 'model_tier', 'projection_date', 'task_type']::text[] + ) THEN + ALTER TABLE llm_delegation_daily_projection ADD CONSTRAINT uq_llm_delegation_daily_agg UNIQUE (projection_date, task_type, model_id, model_tier); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: llm_delegation_daily_projection ---- + + CREATE UNIQUE INDEX IF NOT EXISTS uq_llm_delegation_idempotency_key ON llm_delegation_daily_projection (idempotency_key); diff --git a/docker/migrations/forward/nodes/node_log_persistence_effect/0000_create_log_entries.sql b/docker/migrations/forward/nodes/node_log_persistence_effect/0000_create_log_entries.sql new file mode 100644 index 0000000000..6d8ad1eac7 --- /dev/null +++ b/docker/migrations/forward/nodes/node_log_persistence_effect/0000_create_log_entries.sql @@ -0,0 +1,276 @@ +-- ============================================================================= +-- MIGRATION: physically deliver log_entries through the node-owned migration +-- loop (OMN-15846) +-- ============================================================================= +-- Ticket: OMN-15846 (cross-DB flat migration classification audit — 083/096/ +-- 097 census amendment; 083 is genuinely undelivered and deliverable) +-- Related: OMN-12131 (original ticket for the flat migration this file +-- replaces the DELIVERY of — +-- docker/migrations/forward/083_create_log_entries.sql in +-- omnibase_infra), OMN-15819 (the precedent this file follows: +-- node-owned replacement for a flat migration the k8s Job cannot +-- reach), OMN-15282/OMN-15313 (node-owned migration discovery loop +-- this file runs through), OMN-15359/OMN-15423 (omninode_internal +-- domain classification this file's schema qualification follows) +-- Version: 1.0.0 +-- +-- WHY THIS FILE EXISTS (OMN-15846) +-- docker/migrations/forward/083_create_log_entries.sql (omnibase_infra) +-- carries `\connect omnidash_analytics` and is a FLAT migration +-- (docker/migrations/forward/*.sql, -maxdepth 1). The k8s Job that applies +-- that corpus (omninode_infra repo, +-- k8s/migrations/omnibase-infra-migrate.yaml) owns exactly one database, +-- omnibase_infra — its flat loop's `psql -f` apply is gated on +-- `directive_db == $DB_NAME` and is UNREACHABLE for a cross-DB file, in +-- that loop or any other in the runner (same OMN-15819 defect class as +-- 098/099). 083 carried a false "applied" ledger row in omnibase_infra's +-- own schema_migrations (applied_at 2026-07-28T22:53:47Z, checksum +-- byte-identical to the live flat file) that masked this until the +-- OMN-15846 classification-ordering fix unmasked it. Live-confirmed +-- 2026-08-10 (role_omnidash / omninode_runtime, omninode-dev-postgres +-- RDS): `to_regclass('public.log_entries')` is NULL in BOTH +-- omnibase_infra and omnidash_analytics — the table has never existed on +-- either database on this lane. +-- +-- THIS file is a NODE-OWNED migration, vendored into omnibase_infra under +-- docker/migrations/forward/nodes/node_log_persistence_effect/. The +-- node-owned loop in the SAME k8s Job (OMN-15282/OMN-15313) connects +-- DIRECTLY to omnidash_analytics as role_omnidash — it is the one code +-- path in the whole runner that can actually reach this database. That is +-- the entire fix: relocate delivery, do not relocate intent. 083 itself +-- stays in place in omnibase_infra (byte-unchanged except for a header +-- tombstone comment, OMN-15846) as the ledgered historical record of the +-- original design. +-- +-- WHY SCHEMA-QUALIFIED omninode_internal.log_entries, NOT BARE log_entries +-- 083's original design created the table unqualified (implicit `public` +-- in the omnidash_analytics session). Because this table has never +-- physically existed anywhere (see above), there is no legacy `public` +-- row set to reconcile against — this is a first physical creation, not a +-- cutover. `scripts/ci/check_application_database_sql.py` +-- (OMN-15361/OMN-15423 domain enforcement) requires every NEW deployable +-- SQL target to be schema-qualified against a declared topology domain; +-- `omninode_internal` matches this node's own `db_io.db_tables[].schema` +-- declaration in contract.yaml and the same domain node_service_registry, +-- node_projection_live_events, and every other OMNINODE_INTERNAL-domain +-- node migration in this corpus already uses. role_omnidash — the +-- identity the node-owned loop connects as — was live-confirmed +-- 2026-08-10 to already hold USAGE and CREATE on schema +-- `omninode_internal` in omnidash_analytics (the OMN-15819 step-3 +-- operator grant has since landed), so this file can create directly +-- under it with no additional grant required. +-- +-- WHY role_omnidash NEEDS NO EXPLICIT GRANT HERE +-- role_omnidash owns the `omninode_internal` schema's CREATE privilege +-- (see above) and, as the connecting/creating identity, becomes the table +-- owner automatically with full implicit privileges. No GRANT statement +-- for role_omnidash itself is required or issued. +-- +-- WHY THIS FILE ALSO GRANTS omninode_runtime (separate principal, below) +-- The shipped topology (src/omnibase_infra/topology/instances/*.yaml, +-- derived from this node's own db_io.db_tables declaration via +-- scripts/generate_application_database_table_grants.py) declares +-- INSERT/SELECT/UPDATE for omninode_runtime on omninode_internal.log_entries +-- -- the same shared runtime write-path identity 099 already grants for +-- omninode_internal.live_events. See the dedicated comment block near the +-- end of this file for the full rationale (mirrors 099's own). +-- +-- CONSUMER (currently dormant on onex-dev, tracked separately) +-- omnimarket.nodes.node_log_persistence_effect INSERTs into this table on +-- `onex.evt.platform.log-entry.v1`. On onex-dev today the handler's +-- `ONEX_PG_DSN` is unset (no wiring found in +-- omninode_infra/k8s/onex-dev/runtime/*.yaml) — the handler's own +-- documented graceful-degradation path ("no DB pool available") applies, +-- so no write is attempted either way. This migration closes the schema +-- gap; the DSN-wiring gap that keeps the feature end-to-end dormant is a +-- separate, already-filed concern (see OMN-14153 for the sibling .201 +-- dev-lane symptom of the same feature family) and is not this file's +-- scope. The handler's own `INSERT INTO log_entries` (unqualified) will +-- need a matching schema-qualification follow-up when that DSN-wiring +-- ticket is picked up — out of scope here, noted for that ticket. +-- +-- IDEMPOTENCY +-- CREATE TABLE IF NOT EXISTS and CREATE INDEX IF NOT EXISTS are safe to +-- re-run. No role or grant DDL appears in this file. +-- +-- ROLLBACK +-- DROP TABLE omninode_internal.log_entries. Safe: the table is created +-- empty by this file on every path that executes it, and nothing else in +-- the corpus references it. +-- ============================================================================= + +CREATE TABLE IF NOT EXISTS omninode_internal.log_entries ( + entry_id UUID PRIMARY KEY, + timestamp TIMESTAMPTZ NOT NULL, + node_name TEXT NOT NULL, + function_name TEXT NOT NULL DEFAULT '', + level TEXT NOT NULL DEFAULT 'info', + message TEXT NOT NULL, + correlation_id TEXT, + duration_ms DOUBLE PRECISION, + metadata JSONB NOT NULL DEFAULT '{}', + ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- ---- BEGIN OMN-15376 shape reconciliation: omninode_internal.log_entries ---- +-- CREATE TABLE IF NOT EXISTS silently no-ops against a drifted pre-existing +-- table; the guarded adds below converge such a table onto the shape +-- declared above (no-ops on the fresh-create path, since every column +-- already exists there). No DROP, no recreate, no TRUNCATE. +ALTER TABLE omninode_internal.log_entries ADD COLUMN IF NOT EXISTS entry_id UUID; +ALTER TABLE omninode_internal.log_entries ADD COLUMN IF NOT EXISTS timestamp TIMESTAMPTZ; +ALTER TABLE omninode_internal.log_entries ADD COLUMN IF NOT EXISTS node_name TEXT; +ALTER TABLE omninode_internal.log_entries ADD COLUMN IF NOT EXISTS function_name TEXT DEFAULT ''; +ALTER TABLE omninode_internal.log_entries ADD COLUMN IF NOT EXISTS level TEXT DEFAULT 'info'; +ALTER TABLE omninode_internal.log_entries ADD COLUMN IF NOT EXISTS message TEXT; +ALTER TABLE omninode_internal.log_entries ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE omninode_internal.log_entries ADD COLUMN IF NOT EXISTS duration_ms DOUBLE PRECISION; +ALTER TABLE omninode_internal.log_entries ADD COLUMN IF NOT EXISTS metadata JSONB DEFAULT '{}'; +ALTER TABLE omninode_internal.log_entries ADD COLUMN IF NOT EXISTS ingested_at TIMESTAMPTZ DEFAULT NOW(); + +-- Defaults: ADD COLUMN IF NOT EXISTS ... DEFAULT is a no-op on a column that +-- already existed without one -- restore the declared defaults explicitly so +-- a drifted pre-existing column converges too, not only a brand-new one. +ALTER TABLE omninode_internal.log_entries ALTER COLUMN function_name SET DEFAULT ''; +ALTER TABLE omninode_internal.log_entries ALTER COLUMN level SET DEFAULT 'info'; +ALTER TABLE omninode_internal.log_entries ALTER COLUMN metadata SET DEFAULT '{}'; +ALTER TABLE omninode_internal.log_entries ALTER COLUMN ingested_at SET DEFAULT NOW(); + +-- NOT NULL convergence: only promoted when every existing row already +-- satisfies it (never guessed/backfilled) -- a pre-existing row holding NULL +-- fails loud and names the exact conflict instead of silently reshaping data. +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY[ + 'entry_id', 'timestamp', 'node_name', 'function_name', 'level', + 'message', 'metadata', 'ingested_at' + ] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', + 'omninode_internal.log_entries'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', + 'omninode_internal.log_entries'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15846: cannot converge omninode_internal.log_entries.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +-- Primary key: guarded so a pre-existing table without one gets it, and a +-- table that already has it (fresh-create path, or a prior partial apply) +-- is left untouched. entry_id is NOT NULL by the block above at this point, +-- so the constraint can always be added once reached. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'omninode_internal.log_entries'::regclass AND contype = 'p' + ) THEN + ALTER TABLE omninode_internal.log_entries + ADD CONSTRAINT log_entries_pkey PRIMARY KEY (entry_id); + END IF; +END$$; +-- ---- END OMN-15376 shape reconciliation: omninode_internal.log_entries ---- + +COMMENT ON TABLE omninode_internal.log_entries IS + 'Structured log events from ONEX nodes (OMN-12131). ' + '30-day retention window anchored on ingested_at. ' + 'Consumed by node_log_persistence_effect via onex.evt.*.log-emitted.v1.'; + +COMMENT ON COLUMN omninode_internal.log_entries.entry_id IS + 'Client-supplied UUID — idempotency key for exactly-once ingestion.'; + +COMMENT ON COLUMN omninode_internal.log_entries.ingested_at IS + '30-day retention anchor. Rows with ingested_at < NOW() - INTERVAL ''30 days'' ' + 'are eligible for pruning by the nightly retention job.'; + +-- Partial index: correlation lookups (skip NULL rows to reduce index size) +CREATE INDEX IF NOT EXISTS idx_log_entries_correlation + ON omninode_internal.log_entries (correlation_id) + WHERE correlation_id IS NOT NULL; + +-- Composite: node-scoped time-range queries (dashboard primary query path) +CREATE INDEX IF NOT EXISTS idx_log_entries_node_ts + ON omninode_internal.log_entries (node_name, timestamp DESC); + +-- Composite: level-filtered time-range queries (error/warn filtering) +CREATE INDEX IF NOT EXISTS idx_log_entries_level_ts + ON omninode_internal.log_entries (level, timestamp DESC); + +-- Standalone timestamp: full time-range scans and retention sweeps +CREATE INDEX IF NOT EXISTS idx_log_entries_ts + ON omninode_internal.log_entries (timestamp DESC); + +-- ----------------------------------------------------------------------------- +-- omninode_runtime grant (topology-derived, OMN-15846) +-- ----------------------------------------------------------------------------- +-- src/omnibase_infra/topology/instances/*.yaml declares +-- principals.omninode_runtime.grants[schema: omninode_internal] for this +-- table (regenerated in the paired omnibase_infra PR via +-- scripts/generate_application_database_table_grants.py --write) -- +-- INSERT/SELECT/UPDATE only, matching the projection-writer invariant 096 +-- and 099 both already state (no DELETE: a projection writer upserts, it +-- does not reshape the table). The guarded CREATE mirrors 099's own +-- precedent exactly: omninode_runtime has never been created by any flat +-- migration in this corpus (the standalone "Migration Integration Test" CI +-- gate applies only docker/migrations/forward/*.sql via +-- scripts/run-migrations.py against a bare postgres:16-alpine service and +-- never runs 000_create_multiple_databases.sh), so an unguarded GRANT would +-- fail closed with `role "omninode_runtime" does not exist` in that CI +-- scope. NOLOGIN at create time -- LOGIN + password attach stays +-- deployment-owned, never re-asserted here even though the topology +-- declares login: true. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'omninode_runtime') THEN + BEGIN + CREATE ROLE omninode_runtime WITH + NOLOGIN + NOSUPERUSER + NOBYPASSRLS + NOCREATEDB + NOCREATEROLE + NOREPLICATION; + EXCEPTION + WHEN duplicate_object OR unique_violation THEN + NULL; -- created concurrently by another migration path + END; + END IF; +END; +$$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'omninode_runtime') THEN + RAISE EXCEPTION + 'omninode_runtime role does not exist and could not be created -- the ' + 'executing role lacks CREATEROLE. On a managed instance the role is ' + 'provisioned at the provisioning seam; this migration refuses to record ' + 'itself against a role that is not there.'; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA omninode_internal TO omninode_runtime; +GRANT SELECT, INSERT, UPDATE ON omninode_internal.log_entries TO omninode_runtime; + +-- Post-condition on the grant-gap repair itself: the write-path role must +-- actually carry INSERT on the physical table this migration just created. +-- Statically provable (no DO/RAISE), matching the OMN-15361 application +-- database gate's requirement for deployable SQL (same idiom 098/099 use). +SELECT 1 / count(*) AS omninode_runtime_log_entries_insert_grant_assertion + FROM information_schema.role_table_grants + WHERE table_schema = 'omninode_internal' + AND table_name = 'log_entries' + AND grantee = 'omninode_runtime' + AND privilege_type = 'INSERT'; diff --git a/docker/migrations/forward/nodes/node_merge_state_projection/0001_create_merge_state_transitions.sql b/docker/migrations/forward/nodes/node_merge_state_projection/0001_create_merge_state_transitions.sql index 4a2f1bd4e2..65cd584b8c 100644 --- a/docker/migrations/forward/nodes/node_merge_state_projection/0001_create_merge_state_transitions.sql +++ b/docker/migrations/forward/nodes/node_merge_state_projection/0001_create_merge_state_transitions.sql @@ -33,6 +33,97 @@ CREATE TABLE IF NOT EXISTS merge_state_transitions ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: merge_state_transitions ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS projection_cursor BIGSERIAL; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS event_id TEXT; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS repo TEXT; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS pr_number INTEGER; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS head_sha TEXT; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS branch TEXT DEFAULT ''; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS from_state TEXT; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS to_state TEXT; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS occurred_at TIMESTAMPTZ; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS reason_code TEXT; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS is_occ_evidence BOOLEAN DEFAULT FALSE; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS product_pr_number INTEGER; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS queue_wait_seconds DOUBLE PRECISION; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS product_failure_found BOOLEAN DEFAULT FALSE; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS evidence_present BOOLEAN DEFAULT FALSE; +ALTER TABLE merge_state_transitions ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['projection_cursor', 'event_id', 'repo', 'pr_number', 'head_sha', 'branch', 'from_state', 'to_state', 'occurred_at', 'is_occ_evidence', 'product_failure_found', 'evidence_present', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'merge_state_transitions'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'merge_state_transitions'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge merge_state_transitions.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'merge_state_transitions'::regclass AND contype = 'p' + ) THEN + ALTER TABLE merge_state_transitions ADD CONSTRAINT merge_state_transitions_pkey PRIMARY KEY (projection_cursor); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'merge_state_transitions'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['event_id']::text[] + ) THEN + ALTER TABLE merge_state_transitions ADD CONSTRAINT merge_state_transitions_event_id_key UNIQUE (event_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: merge_state_transitions ---- + + CREATE INDEX IF NOT EXISTS idx_merge_state_transitions_cursor ON merge_state_transitions (projection_cursor); diff --git a/docker/migrations/forward/nodes/node_nightly_loop_controller/001_create_nightly_loop_tables.sql b/docker/migrations/forward/nodes/node_nightly_loop_controller/001_create_nightly_loop_tables.sql index a470afe24b..2a4df3665a 100644 --- a/docker/migrations/forward/nodes/node_nightly_loop_controller/001_create_nightly_loop_tables.sql +++ b/docker/migrations/forward/nodes/node_nightly_loop_controller/001_create_nightly_loop_tables.sql @@ -18,6 +18,93 @@ CREATE TABLE IF NOT EXISTS nightly_loop_decisions ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: nightly_loop_decisions ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS decision_id TEXT; +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS iteration_id TEXT; +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS timestamp TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS action TEXT; +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS target TEXT; +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS outcome TEXT; +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS model_used TEXT DEFAULT ''; +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS cost_usd NUMERIC(10, 6) DEFAULT 0; +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS details TEXT DEFAULT ''; +ALTER TABLE nightly_loop_decisions ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'decision_id', 'iteration_id', 'correlation_id', 'timestamp', 'action', 'target', 'outcome', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'nightly_loop_decisions'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'nightly_loop_decisions'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge nightly_loop_decisions.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'nightly_loop_decisions'::regclass AND contype = 'p' + ) THEN + ALTER TABLE nightly_loop_decisions ADD CONSTRAINT nightly_loop_decisions_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'nightly_loop_decisions'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['decision_id']::text[] + ) THEN + ALTER TABLE nightly_loop_decisions ADD CONSTRAINT nightly_loop_decisions_decision_id_key UNIQUE (decision_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: nightly_loop_decisions ---- + + CREATE INDEX IF NOT EXISTS idx_nld_correlation_id ON nightly_loop_decisions (correlation_id); CREATE INDEX IF NOT EXISTS idx_nld_iteration_id ON nightly_loop_decisions (iteration_id); CREATE INDEX IF NOT EXISTS idx_nld_timestamp ON nightly_loop_decisions (timestamp); @@ -40,5 +127,93 @@ CREATE TABLE IF NOT EXISTS nightly_loop_iterations ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: nightly_loop_iterations ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS iteration_id TEXT; +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS iteration_number INT; +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ; +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS gaps_checked INT DEFAULT 0; +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS gaps_closed INT DEFAULT 0; +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS decisions_made INT DEFAULT 0; +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS tickets_dispatched INT DEFAULT 0; +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS total_cost_usd NUMERIC(10, 6) DEFAULT 0; +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS error TEXT; +ALTER TABLE nightly_loop_iterations ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'iteration_id', 'correlation_id', 'iteration_number', 'started_at', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'nightly_loop_iterations'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'nightly_loop_iterations'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge nightly_loop_iterations.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'nightly_loop_iterations'::regclass AND contype = 'p' + ) THEN + ALTER TABLE nightly_loop_iterations ADD CONSTRAINT nightly_loop_iterations_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'nightly_loop_iterations'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['iteration_id']::text[] + ) THEN + ALTER TABLE nightly_loop_iterations ADD CONSTRAINT nightly_loop_iterations_iteration_id_key UNIQUE (iteration_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: nightly_loop_iterations ---- + + CREATE INDEX IF NOT EXISTS idx_nli_correlation_id ON nightly_loop_iterations (correlation_id); CREATE INDEX IF NOT EXISTS idx_nli_started_at ON nightly_loop_iterations (started_at); diff --git a/docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql b/docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql index 4009439cc6..aa81ea9c48 100644 --- a/docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql +++ b/docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql @@ -25,6 +25,81 @@ CREATE TABLE IF NOT EXISTS gate_activity ( observed_at TIMESTAMPTZ NOT NULL ); +-- ---- BEGIN OMN-15376 shape reconciliation: gate_activity ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS id BIGSERIAL; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS repository_id TEXT; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS project_name TEXT DEFAULT ''; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS branch TEXT DEFAULT ''; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS base_sha TEXT DEFAULT ''; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS head_sha TEXT DEFAULT ''; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS diff_hash TEXT; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS config_hash TEXT; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS status TEXT; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS action TEXT; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS reason TEXT DEFAULT ''; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS total_checks INTEGER DEFAULT 0; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS failed_checks INTEGER DEFAULT 0; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS advisory_checks INTEGER DEFAULT 0; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS pending_checks INTEGER DEFAULT 0; +ALTER TABLE gate_activity ADD COLUMN IF NOT EXISTS observed_at TIMESTAMPTZ; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'repository_id', 'project_name', 'branch', 'base_sha', 'head_sha', 'status', 'reason', 'total_checks', 'failed_checks', 'advisory_checks', 'pending_checks', 'observed_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'gate_activity'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'gate_activity'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge gate_activity.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'gate_activity'::regclass AND contype = 'p' + ) THEN + ALTER TABLE gate_activity ADD CONSTRAINT gate_activity_pkey PRIMARY KEY (id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: gate_activity ---- + + CREATE INDEX IF NOT EXISTS gate_activity_observed_at_idx ON gate_activity (observed_at DESC); @@ -46,6 +121,72 @@ CREATE TABLE IF NOT EXISTS gate_metrics ( updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: gate_metrics ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE gate_metrics ADD COLUMN IF NOT EXISTS id INTEGER DEFAULT 1; +ALTER TABLE gate_metrics ADD COLUMN IF NOT EXISTS total_events INTEGER DEFAULT 0; +ALTER TABLE gate_metrics ADD COLUMN IF NOT EXISTS passed INTEGER DEFAULT 0; +ALTER TABLE gate_metrics ADD COLUMN IF NOT EXISTS failed INTEGER DEFAULT 0; +ALTER TABLE gate_metrics ADD COLUMN IF NOT EXISTS advisory INTEGER DEFAULT 0; +ALTER TABLE gate_metrics ADD COLUMN IF NOT EXISTS pending INTEGER DEFAULT 0; +ALTER TABLE gate_metrics ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'total_events', 'passed', 'failed', 'advisory', 'pending', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'gate_metrics'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'gate_metrics'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge gate_metrics.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'gate_metrics'::regclass AND contype = 'p' + ) THEN + ALTER TABLE gate_metrics ADD CONSTRAINT gate_metrics_pkey PRIMARY KEY (id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: gate_metrics ---- + + -- Seed the singleton metrics row so the projection API returns 1 row immediately. INSERT INTO gate_metrics (id, total_events, passed, failed, advisory, pending, updated_at) VALUES (1, 0, 0, 0, 0, 0, NOW()) diff --git a/docker/migrations/forward/nodes/node_pr_lifecycle_state_reducer/0001_create_pr_lifecycle_ledger_entries.sql b/docker/migrations/forward/nodes/node_pr_lifecycle_state_reducer/0001_create_pr_lifecycle_ledger_entries.sql index 6482b255ea..adb72e794c 100644 --- a/docker/migrations/forward/nodes/node_pr_lifecycle_state_reducer/0001_create_pr_lifecycle_ledger_entries.sql +++ b/docker/migrations/forward/nodes/node_pr_lifecycle_state_reducer/0001_create_pr_lifecycle_ledger_entries.sql @@ -49,6 +49,93 @@ CREATE TABLE IF NOT EXISTS public.pr_lifecycle_ledger_entries ( UNIQUE (sweep_id, repo, pr_number, iteration) ); +-- ---- BEGIN OMN-15376 shape reconciliation: pr_lifecycle_ledger_entries ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS id BIGSERIAL; +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS sweep_id TEXT; +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS iteration INTEGER; +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS found_at TIMESTAMPTZ; +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS repo TEXT; +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS pr_number INTEGER; +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS initial_state TEXT; +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS action_taken TEXT; +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS evidence TEXT DEFAULT ''; +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS final_state TEXT; +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS next_check_at TIMESTAMPTZ; +ALTER TABLE public.pr_lifecycle_ledger_entries ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'sweep_id', 'iteration', 'found_at', 'repo', 'pr_number', 'initial_state', 'action_taken', 'evidence', 'final_state', 'next_check_at', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'public.pr_lifecycle_ledger_entries'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'public.pr_lifecycle_ledger_entries'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge public.pr_lifecycle_ledger_entries.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'public.pr_lifecycle_ledger_entries'::regclass AND contype = 'p' + ) THEN + ALTER TABLE public.pr_lifecycle_ledger_entries ADD CONSTRAINT pr_lifecycle_ledger_entries_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'public.pr_lifecycle_ledger_entries'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['iteration', 'pr_number', 'repo', 'sweep_id']::text[] + ) THEN + ALTER TABLE public.pr_lifecycle_ledger_entries ADD CONSTRAINT pr_lifecycle_ledger_entries_sweep_id_repo_pr_number_iterati_key UNIQUE (sweep_id, repo, pr_number, iteration); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: pr_lifecycle_ledger_entries ---- + + -- Primary DoD query: count rows for a sweep. CREATE INDEX IF NOT EXISTS idx_pr_lifecycle_ledger_sweep ON public.pr_lifecycle_ledger_entries (sweep_id); diff --git a/docker/migrations/forward/nodes/node_pr_merged_projection/0001_create_pr_merged_events.sql b/docker/migrations/forward/nodes/node_pr_merged_projection/0001_create_pr_merged_events.sql index 39cf28290a..6920240846 100644 --- a/docker/migrations/forward/nodes/node_pr_merged_projection/0001_create_pr_merged_events.sql +++ b/docker/migrations/forward/nodes/node_pr_merged_projection/0001_create_pr_merged_events.sql @@ -22,6 +22,89 @@ CREATE TABLE IF NOT EXISTS pr_merged_events ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: pr_merged_events ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE pr_merged_events ADD COLUMN IF NOT EXISTS projection_cursor BIGSERIAL; +ALTER TABLE pr_merged_events ADD COLUMN IF NOT EXISTS event_id TEXT; +ALTER TABLE pr_merged_events ADD COLUMN IF NOT EXISTS repo TEXT; +ALTER TABLE pr_merged_events ADD COLUMN IF NOT EXISTS branch TEXT; +ALTER TABLE pr_merged_events ADD COLUMN IF NOT EXISTS pr_number INTEGER; +ALTER TABLE pr_merged_events ADD COLUMN IF NOT EXISTS ticket TEXT DEFAULT ''; +ALTER TABLE pr_merged_events ADD COLUMN IF NOT EXISTS merged_at TIMESTAMPTZ; +ALTER TABLE pr_merged_events ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['projection_cursor', 'event_id', 'repo', 'branch', 'pr_number', 'ticket', 'merged_at', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'pr_merged_events'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'pr_merged_events'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge pr_merged_events.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'pr_merged_events'::regclass AND contype = 'p' + ) THEN + ALTER TABLE pr_merged_events ADD CONSTRAINT pr_merged_events_pkey PRIMARY KEY (projection_cursor); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'pr_merged_events'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['event_id']::text[] + ) THEN + ALTER TABLE pr_merged_events ADD CONSTRAINT pr_merged_events_event_id_key UNIQUE (event_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: pr_merged_events ---- + + CREATE INDEX IF NOT EXISTS idx_pr_merged_events_cursor ON pr_merged_events (projection_cursor); diff --git a/docker/migrations/forward/nodes/node_pr_review_bot/001_create_review_bot_bypass_log.sql b/docker/migrations/forward/nodes/node_pr_review_bot/001_create_review_bot_bypass_log.sql new file mode 100644 index 0000000000..31294af894 --- /dev/null +++ b/docker/migrations/forward/nodes/node_pr_review_bot/001_create_review_bot_bypass_log.sql @@ -0,0 +1,17 @@ +-- Migration: Create review_bot_bypass_log table — OMN-8497 +-- Target DB: omnidash_analytics (omnibase_infra postgres on .201:5436) +-- Node: node_pr_review_bot / HandlerEmergencyBypassParser + +CREATE TABLE IF NOT EXISTS review_bot_bypass_log ( + audit_id UUID PRIMARY KEY, + pr_url TEXT NOT NULL, + actor TEXT NOT NULL, + reason TEXT NOT NULL, + bypass_timestamp TIMESTAMPTZ NOT NULL, + kafka_event_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_rbl_actor ON review_bot_bypass_log (actor); +CREATE INDEX IF NOT EXISTS idx_rbl_pr_url ON review_bot_bypass_log (pr_url); +CREATE INDEX IF NOT EXISTS idx_rbl_bypass_timestamp ON review_bot_bypass_log (bypass_timestamp); diff --git a/docker/migrations/forward/nodes/node_projection_baselines/0001_create_baselines_tables.sql b/docker/migrations/forward/nodes/node_projection_baselines/0001_create_baselines_tables.sql index 59227fcdc0..b5eec2456a 100644 --- a/docker/migrations/forward/nodes/node_projection_baselines/0001_create_baselines_tables.sql +++ b/docker/migrations/forward/nodes/node_projection_baselines/0001_create_baselines_tables.sql @@ -46,6 +46,71 @@ CREATE TABLE IF NOT EXISTS baselines_snapshots ( projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: baselines_snapshots ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE baselines_snapshots ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE baselines_snapshots ADD COLUMN IF NOT EXISTS contract_version INTEGER DEFAULT 1; +ALTER TABLE baselines_snapshots ADD COLUMN IF NOT EXISTS computed_at_utc TIMESTAMPTZ; +ALTER TABLE baselines_snapshots ADD COLUMN IF NOT EXISTS window_start_utc TIMESTAMPTZ; +ALTER TABLE baselines_snapshots ADD COLUMN IF NOT EXISTS window_end_utc TIMESTAMPTZ; +ALTER TABLE baselines_snapshots ADD COLUMN IF NOT EXISTS projected_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['snapshot_id', 'contract_version', 'projected_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'baselines_snapshots'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'baselines_snapshots'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge baselines_snapshots.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_snapshots'::regclass AND contype = 'p' + ) THEN + ALTER TABLE baselines_snapshots ADD CONSTRAINT baselines_snapshots_pkey PRIMARY KEY (snapshot_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: baselines_snapshots ---- + + CREATE INDEX IF NOT EXISTS idx_baselines_snapshots_computed_at ON baselines_snapshots (computed_at_utc DESC); @@ -72,6 +137,80 @@ CREATE TABLE IF NOT EXISTS baselines_comparisons ( rationale TEXT NOT NULL DEFAULT '' ); +-- ---- BEGIN OMN-15376 shape reconciliation: baselines_comparisons ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS id BIGSERIAL; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS pattern_id TEXT; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS pattern_name TEXT DEFAULT ''; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS sample_size BIGINT DEFAULT 0; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS window_start TEXT DEFAULT ''; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS window_end TEXT DEFAULT ''; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS token_delta JSONB DEFAULT '{}'::jsonb; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS time_delta JSONB DEFAULT '{}'::jsonb; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS retry_delta JSONB DEFAULT '{}'::jsonb; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS test_pass_rate_delta JSONB DEFAULT '{}'::jsonb; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS review_iteration_delta JSONB DEFAULT '{}'::jsonb; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS recommendation TEXT DEFAULT 'shadow'; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS confidence TEXT DEFAULT 'low'; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS rationale TEXT DEFAULT ''; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'snapshot_id', 'pattern_id', 'pattern_name', 'sample_size', 'window_start', 'window_end', 'token_delta', 'time_delta', 'retry_delta', 'test_pass_rate_delta', 'review_iteration_delta', 'recommendation', 'confidence', 'rationale'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'baselines_comparisons'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'baselines_comparisons'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge baselines_comparisons.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_comparisons'::regclass AND contype = 'p' + ) THEN + ALTER TABLE baselines_comparisons ADD CONSTRAINT baselines_comparisons_pkey PRIMARY KEY (id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: baselines_comparisons ---- + + CREATE INDEX IF NOT EXISTS idx_baselines_comparisons_snapshot ON baselines_comparisons (snapshot_id); @@ -90,6 +229,87 @@ CREATE TABLE IF NOT EXISTS baselines_trend ( CONSTRAINT uk_baselines_trend_snapshot_date UNIQUE (snapshot_id, date) ); +-- ---- BEGIN OMN-15376 shape reconciliation: baselines_trend ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS id BIGSERIAL; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS date TEXT; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS avg_cost_savings TEXT DEFAULT '0'; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS avg_outcome_improvement TEXT DEFAULT '0'; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS comparisons_evaluated BIGINT DEFAULT 0; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'snapshot_id', 'date', 'avg_cost_savings', 'avg_outcome_improvement', 'comparisons_evaluated'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'baselines_trend'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'baselines_trend'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge baselines_trend.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_trend'::regclass AND contype = 'p' + ) THEN + ALTER TABLE baselines_trend ADD CONSTRAINT baselines_trend_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'baselines_trend'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['date', 'snapshot_id']::text[] + ) THEN + ALTER TABLE baselines_trend ADD CONSTRAINT uk_baselines_trend_snapshot_date UNIQUE (snapshot_id, date); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: baselines_trend ---- + + CREATE INDEX IF NOT EXISTS idx_baselines_trend_snapshot ON baselines_trend (snapshot_id); @@ -107,5 +327,85 @@ CREATE TABLE IF NOT EXISTS baselines_breakdown ( CONSTRAINT uk_baselines_breakdown_snapshot_action UNIQUE (snapshot_id, action) ); +-- ---- BEGIN OMN-15376 shape reconciliation: baselines_breakdown ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS id BIGSERIAL; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS action TEXT; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS count BIGINT DEFAULT 0; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS avg_confidence TEXT DEFAULT '0'; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'snapshot_id', 'action', 'count', 'avg_confidence'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'baselines_breakdown'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'baselines_breakdown'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge baselines_breakdown.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_breakdown'::regclass AND contype = 'p' + ) THEN + ALTER TABLE baselines_breakdown ADD CONSTRAINT baselines_breakdown_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'baselines_breakdown'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['action', 'snapshot_id']::text[] + ) THEN + ALTER TABLE baselines_breakdown ADD CONSTRAINT uk_baselines_breakdown_snapshot_action UNIQUE (snapshot_id, action); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: baselines_breakdown ---- + + CREATE INDEX IF NOT EXISTS idx_baselines_breakdown_snapshot ON baselines_breakdown (snapshot_id); diff --git a/docker/migrations/forward/nodes/node_projection_baselines/0002_realign_child_tables_to_producer_schema.sql b/docker/migrations/forward/nodes/node_projection_baselines/0002_realign_child_tables_to_producer_schema.sql index f421576eaa..3569d3de13 100644 --- a/docker/migrations/forward/nodes/node_projection_baselines/0002_realign_child_tables_to_producer_schema.sql +++ b/docker/migrations/forward/nodes/node_projection_baselines/0002_realign_child_tables_to_producer_schema.sql @@ -71,6 +71,87 @@ CREATE TABLE IF NOT EXISTS baselines_comparisons ( projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: baselines_comparisons ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS id TEXT; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS comparison_date DATE; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS period_label TEXT; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS treatment_sessions BIGINT DEFAULT 0; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS treatment_success_rate DOUBLE PRECISION; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS treatment_avg_latency_ms DOUBLE PRECISION; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS treatment_avg_cost_tokens DOUBLE PRECISION; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS treatment_total_tokens BIGINT DEFAULT 0; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS control_sessions BIGINT DEFAULT 0; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS control_success_rate DOUBLE PRECISION; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS control_avg_latency_ms DOUBLE PRECISION; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS control_avg_cost_tokens DOUBLE PRECISION; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS control_total_tokens BIGINT DEFAULT 0; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS roi_pct DOUBLE PRECISION; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS latency_improvement_pct DOUBLE PRECISION; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS cost_improvement_pct DOUBLE PRECISION; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS sample_size BIGINT DEFAULT 0; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS computed_at TIMESTAMPTZ; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ; +ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS projected_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'snapshot_id', 'comparison_date', 'treatment_sessions', 'treatment_total_tokens', 'control_sessions', 'control_total_tokens', 'sample_size', 'computed_at', 'created_at', 'projected_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'baselines_comparisons'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'baselines_comparisons'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge baselines_comparisons.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_comparisons'::regclass AND contype = 'p' + ) THEN + ALTER TABLE baselines_comparisons ADD CONSTRAINT baselines_comparisons_pkey PRIMARY KEY (id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: baselines_comparisons ---- + + CREATE INDEX IF NOT EXISTS idx_baselines_comparisons_snapshot ON baselines_comparisons (snapshot_id); @@ -95,6 +176,77 @@ CREATE TABLE IF NOT EXISTS baselines_trend ( projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: baselines_trend ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS id TEXT; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS trend_date DATE; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS cohort TEXT; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS session_count BIGINT DEFAULT 0; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS success_rate DOUBLE PRECISION; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS avg_latency_ms DOUBLE PRECISION; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS avg_cost_tokens DOUBLE PRECISION; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS roi_pct DOUBLE PRECISION; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS computed_at TIMESTAMPTZ; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ; +ALTER TABLE baselines_trend ADD COLUMN IF NOT EXISTS projected_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'snapshot_id', 'trend_date', 'cohort', 'session_count', 'computed_at', 'created_at', 'projected_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'baselines_trend'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'baselines_trend'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge baselines_trend.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_trend'::regclass AND contype = 'p' + ) THEN + ALTER TABLE baselines_trend ADD CONSTRAINT baselines_trend_pkey PRIMARY KEY (id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: baselines_trend ---- + + CREATE INDEX IF NOT EXISTS idx_baselines_trend_snapshot ON baselines_trend (snapshot_id); @@ -123,5 +275,79 @@ CREATE TABLE IF NOT EXISTS baselines_breakdown ( projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: baselines_breakdown ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS id TEXT; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS pattern_id TEXT; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS pattern_label TEXT; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS treatment_success_rate DOUBLE PRECISION; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS control_success_rate DOUBLE PRECISION; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS roi_pct DOUBLE PRECISION; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS sample_count BIGINT DEFAULT 0; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS treatment_count BIGINT DEFAULT 0; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS control_count BIGINT DEFAULT 0; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS confidence DOUBLE PRECISION; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS computed_at TIMESTAMPTZ; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ; +ALTER TABLE baselines_breakdown ADD COLUMN IF NOT EXISTS projected_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'snapshot_id', 'pattern_id', 'sample_count', 'treatment_count', 'control_count', 'computed_at', 'created_at', 'projected_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'baselines_breakdown'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'baselines_breakdown'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge baselines_breakdown.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_breakdown'::regclass AND contype = 'p' + ) THEN + ALTER TABLE baselines_breakdown ADD CONSTRAINT baselines_breakdown_pkey PRIMARY KEY (id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: baselines_breakdown ---- + + CREATE INDEX IF NOT EXISTS idx_baselines_breakdown_snapshot ON baselines_breakdown (snapshot_id); diff --git a/docker/migrations/forward/nodes/node_projection_baselines_quality/001_create_baselines_quality_snapshots.sql b/docker/migrations/forward/nodes/node_projection_baselines_quality/001_create_baselines_quality_snapshots.sql index 21f09e5204..59d04d443b 100644 --- a/docker/migrations/forward/nodes/node_projection_baselines_quality/001_create_baselines_quality_snapshots.sql +++ b/docker/migrations/forward/nodes/node_projection_baselines_quality/001_create_baselines_quality_snapshots.sql @@ -41,6 +41,146 @@ CREATE TABLE IF NOT EXISTS baselines_quality_snapshots ( projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: baselines_quality_snapshots ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS captured_at TEXT; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS patterns_compared INTEGER DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS patterns_recommended INTEGER DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS high_confidence_count INTEGER DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS medium_confidence_count INTEGER DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS low_confidence_count INTEGER DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS quality_score NUMERIC(8, 6) DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS recommend_rate NUMERIC(8, 6) DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS projected_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'snapshot_id', 'captured_at', 'patterns_compared', 'patterns_recommended', 'high_confidence_count', 'medium_confidence_count', 'low_confidence_count', 'quality_score', 'recommend_rate', 'projected_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'baselines_quality_snapshots'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'baselines_quality_snapshots'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge baselines_quality_snapshots.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND contype = 'p' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_patterns_compared_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_patterns_compared_check CHECK (patterns_compared >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_patterns_recommended_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_patterns_recommended_check CHECK (patterns_recommended >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_high_confidence_count_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_high_confidence_count_check CHECK (high_confidence_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_medium_confidence_count_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_medium_confidence_count_check CHECK (medium_confidence_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_low_confidence_count_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_low_confidence_count_check CHECK (low_confidence_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_quality_score_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_quality_score_check CHECK (quality_score >= 0 AND quality_score <= 1); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_recommend_rate_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_recommend_rate_check CHECK (recommend_rate >= 0 AND recommend_rate <= 1); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: baselines_quality_snapshots ---- + + CREATE UNIQUE INDEX IF NOT EXISTS ux_baselines_quality_snapshots_snapshot_id ON baselines_quality_snapshots (snapshot_id); diff --git a/docker/migrations/forward/nodes/node_projection_baselines_quality/002_realign_quality_snapshots_to_producer_schema.sql b/docker/migrations/forward/nodes/node_projection_baselines_quality/002_realign_quality_snapshots_to_producer_schema.sql index b85a7fd5d3..dfe8de2be6 100644 --- a/docker/migrations/forward/nodes/node_projection_baselines_quality/002_realign_quality_snapshots_to_producer_schema.sql +++ b/docker/migrations/forward/nodes/node_projection_baselines_quality/002_realign_quality_snapshots_to_producer_schema.sql @@ -70,6 +70,146 @@ CREATE TABLE IF NOT EXISTS baselines_quality_snapshots ( projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: baselines_quality_snapshots ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS captured_at TEXT; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS patterns_compared INTEGER DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS patterns_significant INTEGER DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS high_confidence_count INTEGER DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS medium_confidence_count INTEGER DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS low_confidence_count INTEGER DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS quality_score NUMERIC(8, 6) DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS significant_rate NUMERIC(8, 6) DEFAULT 0; +ALTER TABLE baselines_quality_snapshots ADD COLUMN IF NOT EXISTS projected_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'snapshot_id', 'captured_at', 'patterns_compared', 'patterns_significant', 'high_confidence_count', 'medium_confidence_count', 'low_confidence_count', 'quality_score', 'significant_rate', 'projected_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'baselines_quality_snapshots'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'baselines_quality_snapshots'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge baselines_quality_snapshots.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND contype = 'p' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_patterns_compared_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_patterns_compared_check CHECK (patterns_compared >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_patterns_significant_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_patterns_significant_check CHECK (patterns_significant >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_high_confidence_count_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_high_confidence_count_check CHECK (high_confidence_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_medium_confidence_count_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_medium_confidence_count_check CHECK (medium_confidence_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_low_confidence_count_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_low_confidence_count_check CHECK (low_confidence_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_quality_score_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_quality_score_check CHECK (quality_score >= 0 AND quality_score <= 1); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_quality_snapshots'::regclass AND conname = 'baselines_quality_snapshots_significant_rate_check' + ) THEN + ALTER TABLE baselines_quality_snapshots ADD CONSTRAINT baselines_quality_snapshots_significant_rate_check CHECK (significant_rate >= 0 AND significant_rate <= 1); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: baselines_quality_snapshots ---- + + CREATE UNIQUE INDEX IF NOT EXISTS ux_baselines_quality_snapshots_snapshot_id ON baselines_quality_snapshots (snapshot_id); diff --git a/docker/migrations/forward/nodes/node_projection_baselines_roi/001_create_baselines_roi_snapshots.sql b/docker/migrations/forward/nodes/node_projection_baselines_roi/001_create_baselines_roi_snapshots.sql index 0b2d656c84..f22419a536 100644 --- a/docker/migrations/forward/nodes/node_projection_baselines_roi/001_create_baselines_roi_snapshots.sql +++ b/docker/migrations/forward/nodes/node_projection_baselines_roi/001_create_baselines_roi_snapshots.sql @@ -38,6 +38,94 @@ CREATE TABLE IF NOT EXISTS baselines_roi_snapshots ( projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: baselines_roi_snapshots ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS captured_at TEXT; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS token_delta BIGINT DEFAULT 0; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS time_delta_ms NUMERIC(18, 3) DEFAULT 0; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS retry_delta INTEGER DEFAULT 0; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS recommendations JSONB DEFAULT '{"promote": 0, "shadow": 0, "suppress": 0, "fork": 0}'; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS confidence NUMERIC(8, 6) DEFAULT 0; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS projected_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'snapshot_id', 'captured_at', 'token_delta', 'time_delta_ms', 'retry_delta', 'recommendations', 'confidence', 'projected_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'baselines_roi_snapshots'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'baselines_roi_snapshots'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge baselines_roi_snapshots.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_roi_snapshots'::regclass AND contype = 'p' + ) THEN + ALTER TABLE baselines_roi_snapshots ADD CONSTRAINT baselines_roi_snapshots_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_roi_snapshots'::regclass AND conname = 'baselines_roi_snapshots_retry_delta_check' + ) THEN + ALTER TABLE baselines_roi_snapshots ADD CONSTRAINT baselines_roi_snapshots_retry_delta_check CHECK (retry_delta >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_roi_snapshots'::regclass AND conname = 'baselines_roi_snapshots_confidence_check' + ) THEN + ALTER TABLE baselines_roi_snapshots ADD CONSTRAINT baselines_roi_snapshots_confidence_check CHECK (confidence >= 0 AND confidence <= 1); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: baselines_roi_snapshots ---- + + CREATE UNIQUE INDEX IF NOT EXISTS ux_baselines_roi_snapshots_snapshot_id ON baselines_roi_snapshots (snapshot_id); diff --git a/docker/migrations/forward/nodes/node_projection_baselines_roi/002_realign_roi_snapshots_to_producer_schema.sql b/docker/migrations/forward/nodes/node_projection_baselines_roi/002_realign_roi_snapshots_to_producer_schema.sql index 0a79e8b190..e13976bb99 100644 --- a/docker/migrations/forward/nodes/node_projection_baselines_roi/002_realign_roi_snapshots_to_producer_schema.sql +++ b/docker/migrations/forward/nodes/node_projection_baselines_roi/002_realign_roi_snapshots_to_producer_schema.sql @@ -61,6 +61,84 @@ CREATE TABLE IF NOT EXISTS baselines_roi_snapshots ( projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: baselines_roi_snapshots ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS captured_at TEXT; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS token_delta BIGINT DEFAULT 0; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS roi_pct_avg DOUBLE PRECISION DEFAULT 0; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS latency_improvement_pct_avg DOUBLE PRECISION DEFAULT 0; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS cost_improvement_pct_avg DOUBLE PRECISION DEFAULT 0; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS sample_size BIGINT DEFAULT 0; +ALTER TABLE baselines_roi_snapshots ADD COLUMN IF NOT EXISTS projected_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'snapshot_id', 'captured_at', 'token_delta', 'roi_pct_avg', 'latency_improvement_pct_avg', 'cost_improvement_pct_avg', 'sample_size', 'projected_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'baselines_roi_snapshots'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'baselines_roi_snapshots'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge baselines_roi_snapshots.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_roi_snapshots'::regclass AND contype = 'p' + ) THEN + ALTER TABLE baselines_roi_snapshots ADD CONSTRAINT baselines_roi_snapshots_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'baselines_roi_snapshots'::regclass AND conname = 'baselines_roi_snapshots_sample_size_check' + ) THEN + ALTER TABLE baselines_roi_snapshots ADD CONSTRAINT baselines_roi_snapshots_sample_size_check CHECK (sample_size >= 0); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: baselines_roi_snapshots ---- + + CREATE UNIQUE INDEX IF NOT EXISTS ux_baselines_roi_snapshots_snapshot_id ON baselines_roi_snapshots (snapshot_id); diff --git a/docker/migrations/forward/nodes/node_projection_capsule_store/078_create_capsule_store.sql b/docker/migrations/forward/nodes/node_projection_capsule_store/078_create_capsule_store.sql index 0a51b9eea8..f89ae2f71d 100644 --- a/docker/migrations/forward/nodes/node_projection_capsule_store/078_create_capsule_store.sql +++ b/docker/migrations/forward/nodes/node_projection_capsule_store/078_create_capsule_store.sql @@ -40,6 +40,129 @@ CREATE TABLE IF NOT EXISTS capsule_store ( ) ); +-- ---- BEGIN OMN-15376 shape reconciliation: capsule_store ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS capsule_id UUID; +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS capsule_hash TEXT; +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS factor TEXT; +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS source_commit TEXT; +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS source_artifact TEXT; +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS schema_version TEXT; +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS validity_scope TEXT; +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS success_rate NUMERIC(6, 5); +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS first_pass_rate NUMERIC(6, 5); +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS cost_per_success NUMERIC(18, 6); +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS hit_count BIGINT; +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS last_scored TIMESTAMPTZ; +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE capsule_store ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['capsule_id', 'capsule_hash', 'factor', 'source_commit', 'source_artifact', 'schema_version', 'validity_scope', 'success_rate', 'first_pass_rate', 'cost_per_success', 'hit_count', 'last_scored', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'capsule_store'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'capsule_store'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge capsule_store.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'capsule_store'::regclass AND contype = 'p' + ) THEN + ALTER TABLE capsule_store ADD CONSTRAINT capsule_store_pkey PRIMARY KEY (capsule_id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'capsule_store'::regclass AND conname = 'capsule_store_success_rate_check' + ) THEN + ALTER TABLE capsule_store ADD CONSTRAINT capsule_store_success_rate_check CHECK (success_rate >= 0 AND success_rate <= 1); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'capsule_store'::regclass AND conname = 'capsule_store_first_pass_rate_check' + ) THEN + ALTER TABLE capsule_store ADD CONSTRAINT capsule_store_first_pass_rate_check CHECK (first_pass_rate >= 0 AND first_pass_rate <= 1); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'capsule_store'::regclass AND conname = 'capsule_store_cost_per_success_check' + ) THEN + ALTER TABLE capsule_store ADD CONSTRAINT capsule_store_cost_per_success_check CHECK (cost_per_success >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'capsule_store'::regclass AND conname = 'capsule_store_hit_count_check' + ) THEN + ALTER TABLE capsule_store ADD CONSTRAINT capsule_store_hit_count_check CHECK (hit_count >= 1); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'capsule_store'::regclass AND conname = 'capsule_store_scored_effectiveness_non_null' + ) THEN + ALTER TABLE capsule_store ADD CONSTRAINT capsule_store_scored_effectiveness_non_null CHECK ( hit_count >= 1 AND success_rate IS NOT NULL AND first_pass_rate IS NOT NULL AND cost_per_success IS NOT NULL AND last_scored IS NOT NULL ); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: capsule_store ---- + + CREATE UNIQUE INDEX IF NOT EXISTS ux_capsule_identity ON capsule_store (capsule_hash); diff --git a/docker/migrations/forward/nodes/node_projection_context_roi/001_create_context_roi_scores.sql b/docker/migrations/forward/nodes/node_projection_context_roi/001_create_context_roi_scores.sql index 44ebf3dc0c..ba76c228e1 100644 --- a/docker/migrations/forward/nodes/node_projection_context_roi/001_create_context_roi_scores.sql +++ b/docker/migrations/forward/nodes/node_projection_context_roi/001_create_context_roi_scores.sql @@ -42,6 +42,146 @@ CREATE TABLE IF NOT EXISTS context_roi_scores ( updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: context_roi_scores ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS run_id TEXT; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS task_id TEXT; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS run_order INTEGER DEFAULT 0; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS context_factor_subset TEXT DEFAULT 'off'; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS context_pack_hash TEXT DEFAULT ''; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS attempt_count INTEGER DEFAULT 0; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS first_pass_success BOOLEAN DEFAULT FALSE; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS final_success BOOLEAN DEFAULT FALSE; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS failure_stage TEXT DEFAULT 'none'; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS prompt_tokens INTEGER DEFAULT 0; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS completion_tokens INTEGER DEFAULT 0; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS tokens_used INTEGER DEFAULT 0; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS estimated_cost NUMERIC(18, 6) DEFAULT 0; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS model_id TEXT DEFAULT ''; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS provider TEXT DEFAULT ''; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS endpoint_ref TEXT DEFAULT ''; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS proof_class TEXT DEFAULT 'runtime-observed-only'; +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE context_roi_scores ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'run_id', 'correlation_id', 'task_id', 'run_order', 'context_factor_subset', 'context_pack_hash', 'attempt_count', 'first_pass_success', 'final_success', 'failure_stage', 'prompt_tokens', 'completion_tokens', 'tokens_used', 'estimated_cost', 'model_id', 'provider', 'endpoint_ref', 'proof_class', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'context_roi_scores'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'context_roi_scores'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge context_roi_scores.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'context_roi_scores'::regclass AND contype = 'p' + ) THEN + ALTER TABLE context_roi_scores ADD CONSTRAINT context_roi_scores_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'context_roi_scores'::regclass AND conname = 'context_roi_scores_run_order_check' + ) THEN + ALTER TABLE context_roi_scores ADD CONSTRAINT context_roi_scores_run_order_check CHECK (run_order >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'context_roi_scores'::regclass AND conname = 'context_roi_scores_attempt_count_check' + ) THEN + ALTER TABLE context_roi_scores ADD CONSTRAINT context_roi_scores_attempt_count_check CHECK (attempt_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'context_roi_scores'::regclass AND conname = 'context_roi_scores_prompt_tokens_check' + ) THEN + ALTER TABLE context_roi_scores ADD CONSTRAINT context_roi_scores_prompt_tokens_check CHECK (prompt_tokens >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'context_roi_scores'::regclass AND conname = 'context_roi_scores_completion_tokens_check' + ) THEN + ALTER TABLE context_roi_scores ADD CONSTRAINT context_roi_scores_completion_tokens_check CHECK (completion_tokens >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'context_roi_scores'::regclass AND conname = 'context_roi_scores_tokens_used_check' + ) THEN + ALTER TABLE context_roi_scores ADD CONSTRAINT context_roi_scores_tokens_used_check CHECK (tokens_used >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'context_roi_scores'::regclass AND conname = 'context_roi_scores_estimated_cost_check' + ) THEN + ALTER TABLE context_roi_scores ADD CONSTRAINT context_roi_scores_estimated_cost_check CHECK (estimated_cost >= 0); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: context_roi_scores ---- + + CREATE UNIQUE INDEX IF NOT EXISTS ux_context_roi_scores_identity ON context_roi_scores (correlation_id); diff --git a/docker/migrations/forward/nodes/node_projection_context_roi/003_context_roi_scores_tenant_id_and_rls.sql b/docker/migrations/forward/nodes/node_projection_context_roi/003_context_roi_scores_tenant_id_and_rls.sql new file mode 100644 index 0000000000..65f828bc54 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_context_roi/003_context_roi_scores_tenant_id_and_rls.sql @@ -0,0 +1,113 @@ +-- OMN-15655 / operator ruling 2026-08-02 (house tenant): give context_roi_scores a +-- tenant identity and row-level tenant isolation. +-- +-- context-ROI scores measure a tenant's own runs THIS RELATION IS TENANT DATA +-- WHY, so the row is workload-attributable and belongs in the tenant +-- domain per the ruling's rule. Rows produced by OmniNode's own platform +-- workloads are the omninode TENANT's rows -- OmniNode is a first-class tenant, +-- not an absence of one. Only attribution-meaningless infrastructure state +-- (migration bookkeeping, registry, orchestration state, deployment +-- evidence) stays in omninode_internal. +-- +-- This migration lands alongside the contract flip +-- `db_io.db_tables[].schema: public -> tenant`. Before it, the relation +-- declared a schema the typed topology does not declare, which ADR-0027 +-- refuses by design -- `ValueError: Unknown schema 'public' for +-- database_ref 'application'` out of `ModelDeploymentTopology.schema_domain`. +-- +-- IDENTITY TYPE -- TEXT, NOT UUID, AND THAT IS DELIBERATE +-- Every landed tenant_id column on this surface is `TEXT` holding the slug +-- 'omninode' (delegation 0022/0025, savings 080, registration 0002, +-- inference-response 0002, omnidash 0001_tenant_rls), and the RLS policies +-- that already shipped (0023, 0026) compare TEXT with no `::uuid` cast. +-- OMN-15356 converts that whole set to the canonical UUID in ONE pass. +-- Adding a UUID column here alone would fork tenant identity inside a single +-- database -- the exact "one canonical model per shape" violation. The +-- canonical UUID this slug resolves to is pinned in +-- `omnimarket.projection.tenant_isolation.omninode_TENANT_UUID`. +-- +-- BACKFILL -- NO BATCHING NEEDED, AND BATCHING WOULD BE WORSE +-- `ADD COLUMN ... NOT NULL DEFAULT` with a non-volatile default does not +-- rewrite the table on PostgreSQL 11+; the default is stored in +-- `pg_attribute.attmissingval` and every pre-existing row reads back as +-- the house tenant immediately. A batched `UPDATE` would do strictly more +-- work (a real rewrite, dead tuples, a longer lock) for the same end state. +-- +-- PHYSICAL SCHEMA IS NOT MOVED HERE -- SEE OMN-15359 +-- No `ALTER TABLE ... SET SCHEMA tenant`. The `tenant` schema is created +-- by no applied migration in this repo or in omnibase_infra +-- (`docker/migrations/forward/_ledger/bootstrap.sql` creates only +-- `platform_catalog`); it exists solely in proof fixtures. Every relation +-- already classified TENANT -- delegation_events, delegation_budget_state, +-- savings_estimates -- is still physically in `public` +-- (`current_schema: ["public"]` in the OMN-15423 inventory, against +-- `target_schema: tenant`). Relocating these eight alone would create the +-- split the inventory's target/current split exists to prevent, and would +-- fail outright against a database with no `tenant` schema. The physical +-- move is OMN-15359 ("Build classified schemas and migrate internal, +-- control-plane, catalog, and tenant targets"), which moves the whole set, +-- with its trigger functions and sequences, in one governed cutover. +-- +-- FAIL-CLOSED RATCHET +-- The column DEFAULT is what supplies the house tenant today, exactly as it +-- does for savings_estimates (OMN-14058, operator-accepted): a writer that +-- resolves no tenant OMITS the key and Postgres fills it -- the key is never +-- written as NULL. That default-allowed state is PINNED, with its flip +-- condition named, by +-- `tests/unit/projection/test_house_tenant_default_ratchet.py`. When +-- customer ingress exists the writer boundary stops defaulting and refuses +-- instead; that test fails the moment the precondition changes. +-- +-- BLAST RADIUS -- FORCE constrains the table OWNER too +-- Same caveat as migration 0023: a writer connected as the `postgres` +-- SUPERUSER (the compose lanes) bypasses RLS regardless of FORCE. The real +-- isolation boundary is this policy PLUS a non-superuser, NOBYPASSRLS writer +-- role (OMN-14899 / OMN-15425). +-- +-- Fail-closed: `current_setting('app.tenant_id', true)` is NULL when the GUC +-- is unset, the predicate is NULL, and zero rows are visible. The policy has no +-- default-tenant fallback, by design. +-- +-- Idempotent: ADD COLUMN / CREATE INDEX are IF NOT EXISTS, ENABLE/FORCE are +-- idempotent, the policy is DROP + CREATE, GRANTs are idempotent. + +ALTER TABLE public.context_roi_scores + ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'omninode'; + +CREATE INDEX IF NOT EXISTS idx_context_roi_scores_tenant_id + ON public.context_roi_scores (tenant_id); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'app_dashboard' + AND NOT rolsuper + AND NOT rolbypassrls + ) THEN + RAISE EXCEPTION + 'app_dashboard role missing or RLS-bypassing - apply omnibase_infra forward migration ' + '094_create_app_dashboard_role.sql (OMN-14899) before this RLS ' + 'migration. RLS grants without the constrained read role are the ' + 'exact bypass this work exists to prevent.'; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA public TO app_dashboard; + +ALTER TABLE public.context_roi_scores ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.context_roi_scores FORCE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS tenant_isolation ON public.context_roi_scores; +CREATE POLICY tenant_isolation ON public.context_roi_scores + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)); + +-- Read grant for the live dashboard reader (omnidash ExperimentsPage reads this relation). RLS still filters every +-- row this role can see -- the grant is what makes the RLS-scoped read path +-- reachable at all, not a widening of it. Granted only where a reader is known +-- to exist; the relations with no known reader get no SELECT. +GRANT SELECT ON public.context_roi_scores TO app_dashboard; diff --git a/docker/migrations/forward/nodes/node_projection_cost_by_repo/0001_create_cost_by_repo_snapshots.sql b/docker/migrations/forward/nodes/node_projection_cost_by_repo/0001_create_cost_by_repo_snapshots.sql index b21689f953..cb61df827f 100644 --- a/docker/migrations/forward/nodes/node_projection_cost_by_repo/0001_create_cost_by_repo_snapshots.sql +++ b/docker/migrations/forward/nodes/node_projection_cost_by_repo/0001_create_cost_by_repo_snapshots.sql @@ -39,6 +39,109 @@ CREATE TABLE IF NOT EXISTS cost_by_repo_snapshots ( CONSTRAINT non_negative_cost_by_repo_total_tokens CHECK (total_tokens >= 0) ); +-- ---- BEGIN OMN-15376 shape reconciliation: cost_by_repo_snapshots ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE cost_by_repo_snapshots ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE cost_by_repo_snapshots ADD COLUMN IF NOT EXISTS repo_name VARCHAR(512); +ALTER TABLE cost_by_repo_snapshots ADD COLUMN IF NOT EXISTS "window" VARCHAR(32) DEFAULT 'latest'; +ALTER TABLE cost_by_repo_snapshots ADD COLUMN IF NOT EXISTS snapshot_timestamp_minute TIMESTAMPTZ; +ALTER TABLE cost_by_repo_snapshots ADD COLUMN IF NOT EXISTS total_cost_usd NUMERIC(14, 6) DEFAULT 0; +ALTER TABLE cost_by_repo_snapshots ADD COLUMN IF NOT EXISTS total_tokens BIGINT DEFAULT 0; +ALTER TABLE cost_by_repo_snapshots ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE cost_by_repo_snapshots ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'repo_name', 'window', 'snapshot_timestamp_minute', 'total_cost_usd', 'total_tokens', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'cost_by_repo_snapshots'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'cost_by_repo_snapshots'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge cost_by_repo_snapshots.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'cost_by_repo_snapshots'::regclass AND contype = 'p' + ) THEN + ALTER TABLE cost_by_repo_snapshots ADD CONSTRAINT cost_by_repo_snapshots_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'cost_by_repo_snapshots'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['repo_name', 'snapshot_timestamp_minute', 'window']::text[] + ) THEN + ALTER TABLE cost_by_repo_snapshots ADD CONSTRAINT uq_cost_by_repo_repo_window_minute UNIQUE (repo_name, "window", snapshot_timestamp_minute); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'cost_by_repo_snapshots'::regclass AND conname = 'non_negative_cost_by_repo_total_cost_usd' + ) THEN + ALTER TABLE cost_by_repo_snapshots ADD CONSTRAINT non_negative_cost_by_repo_total_cost_usd CHECK (total_cost_usd >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'cost_by_repo_snapshots'::regclass AND conname = 'non_negative_cost_by_repo_total_tokens' + ) THEN + ALTER TABLE cost_by_repo_snapshots ADD CONSTRAINT non_negative_cost_by_repo_total_tokens CHECK (total_tokens >= 0); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: cost_by_repo_snapshots ---- + + CREATE INDEX IF NOT EXISTS idx_cost_by_repo_snapshots_total_cost_usd ON cost_by_repo_snapshots (total_cost_usd DESC); diff --git a/docker/migrations/forward/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql b/docker/migrations/forward/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql index f0a40b98a3..dc75146ed9 100644 --- a/docker/migrations/forward/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql +++ b/docker/migrations/forward/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql @@ -60,6 +60,130 @@ CREATE TABLE IF NOT EXISTS llm_cost_aggregates ( ) ); +-- ---- BEGIN OMN-15376 shape reconciliation: llm_cost_aggregates ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS aggregation_key VARCHAR(512); +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS "window" cost_aggregation_window; +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS total_cost_usd NUMERIC(14, 6) DEFAULT 0; +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS total_tokens BIGINT DEFAULT 0; +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS call_count INTEGER DEFAULT 0; +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS estimated_coverage_pct NUMERIC(5, 2); +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'aggregation_key', 'window', 'total_cost_usd', 'total_tokens', 'call_count', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'llm_cost_aggregates'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'llm_cost_aggregates'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge llm_cost_aggregates.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_cost_aggregates'::regclass AND contype = 'p' + ) THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT llm_cost_aggregates_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'llm_cost_aggregates'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['aggregation_key', 'window']::text[] + ) THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT unique_aggregation_key_window UNIQUE (aggregation_key, "window"); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_cost_aggregates'::regclass AND conname = 'non_negative_total_cost_usd' + ) THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT non_negative_total_cost_usd CHECK (total_cost_usd >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_cost_aggregates'::regclass AND conname = 'non_negative_agg_total_tokens' + ) THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT non_negative_agg_total_tokens CHECK (total_tokens >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_cost_aggregates'::regclass AND conname = 'non_negative_call_count' + ) THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT non_negative_call_count CHECK (call_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_cost_aggregates'::regclass AND conname = 'valid_estimated_coverage_pct' + ) THEN + ALTER TABLE llm_cost_aggregates ADD CONSTRAINT valid_estimated_coverage_pct CHECK ( estimated_coverage_pct IS NULL OR (estimated_coverage_pct >= 0.00 AND estimated_coverage_pct <= 100.00) ); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: llm_cost_aggregates ---- + + CREATE INDEX IF NOT EXISTS idx_llm_cost_aggregates_aggregation_key ON llm_cost_aggregates (aggregation_key); diff --git a/docker/migrations/forward/nodes/node_projection_cost_summary/0002_llm_cost_aggregates_tenant_id_and_rls.sql b/docker/migrations/forward/nodes/node_projection_cost_summary/0002_llm_cost_aggregates_tenant_id_and_rls.sql new file mode 100644 index 0000000000..8104c6a99d --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_cost_summary/0002_llm_cost_aggregates_tenant_id_and_rls.sql @@ -0,0 +1,114 @@ +-- OMN-15655 / operator ruling 2026-08-02 (house tenant): give llm_cost_aggregates a +-- tenant identity and row-level tenant isolation. +-- +-- LLM cost aggregates are spend attributable to whoever ran the workload THIS RELATION IS TENANT DATA +-- WHY, so the row is workload-attributable and belongs in the tenant +-- domain per the ruling's rule. Rows produced by OmniNode's own platform +-- workloads are the omninode TENANT's rows -- OmniNode is a first-class tenant, +-- not an absence of one. Only attribution-meaningless infrastructure state +-- (migration bookkeeping, registry, orchestration state, deployment +-- evidence) stays in omninode_internal. +-- +-- This migration lands alongside the contract flip +-- `db_io.db_tables[].schema: public -> tenant`. Before it, the relation +-- declared a schema the typed topology does not declare, which ADR-0027 +-- refuses by design -- `ValueError: Unknown schema 'public' for +-- database_ref 'application'` out of `ModelDeploymentTopology.schema_domain`. +-- +-- IDENTITY TYPE -- TEXT, NOT UUID, AND THAT IS DELIBERATE +-- Every landed tenant_id column on this surface is `TEXT` holding the slug +-- 'omninode' (delegation 0022/0025, savings 080, registration 0002, +-- inference-response 0002, omnidash 0001_tenant_rls), and the RLS policies +-- that already shipped (0023, 0026) compare TEXT with no `::uuid` cast. +-- OMN-15356 converts that whole set to the canonical UUID in ONE pass. +-- Adding a UUID column here alone would fork tenant identity inside a single +-- database -- the exact "one canonical model per shape" violation. The +-- canonical UUID this slug resolves to is pinned in +-- `omnimarket.projection.tenant_isolation.omninode_TENANT_UUID`. +-- +-- BACKFILL -- NO BATCHING NEEDED, AND BATCHING WOULD BE WORSE +-- `ADD COLUMN ... NOT NULL DEFAULT` with a non-volatile default does not +-- rewrite the table on PostgreSQL 11+; the default is stored in +-- `pg_attribute.attmissingval` and every pre-existing row reads back as +-- the house tenant immediately. A batched `UPDATE` would do strictly more +-- work (a real rewrite, dead tuples, a longer lock) for the same end state. +-- +-- PHYSICAL SCHEMA IS NOT MOVED HERE -- SEE OMN-15359 +-- No `ALTER TABLE ... SET SCHEMA tenant`. The `tenant` schema is created +-- by no applied migration in this repo or in omnibase_infra +-- (`docker/migrations/forward/_ledger/bootstrap.sql` creates only +-- `platform_catalog`); it exists solely in proof fixtures. Every relation +-- already classified TENANT -- delegation_events, delegation_budget_state, +-- savings_estimates -- is still physically in `public` +-- (`current_schema: ["public"]` in the OMN-15423 inventory, against +-- `target_schema: tenant`). Relocating these eight alone would create the +-- split the inventory's target/current split exists to prevent, and would +-- fail outright against a database with no `tenant` schema. The physical +-- move is OMN-15359 ("Build classified schemas and migrate internal, +-- control-plane, catalog, and tenant targets"), which moves the whole set, +-- with its trigger functions and sequences, in one governed cutover. +-- +-- FAIL-CLOSED RATCHET +-- The column DEFAULT is what supplies the house tenant today, exactly as it +-- does for savings_estimates (OMN-14058, operator-accepted): a writer that +-- resolves no tenant OMITS the key and Postgres fills it -- the key is never +-- written as NULL. That default-allowed state is PINNED, with its flip +-- condition named, by +-- `tests/unit/projection/test_house_tenant_default_ratchet.py`. When +-- customer ingress exists the writer boundary stops defaulting and refuses +-- instead; that test fails the moment the precondition changes. +-- +-- BLAST RADIUS -- FORCE constrains the table OWNER too +-- Same caveat as migration 0023: a writer connected as the `postgres` +-- SUPERUSER (the compose lanes) bypasses RLS regardless of FORCE. The real +-- isolation boundary is this policy PLUS a non-superuser, NOBYPASSRLS writer +-- role (OMN-14899 / OMN-15425). +-- +-- Fail-closed: `current_setting('app.tenant_id', true)` is NULL when the GUC +-- is unset, the predicate is NULL, and zero rows are visible. The policy has no +-- default-tenant fallback, by design. +-- +-- Idempotent: ADD COLUMN / CREATE INDEX are IF NOT EXISTS, ENABLE/FORCE are +-- idempotent, and the policy is DROP + CREATE. + +ALTER TABLE public.llm_cost_aggregates + ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'omninode'; + +CREATE INDEX IF NOT EXISTS idx_llm_cost_aggregates_tenant_id + ON public.llm_cost_aggregates (tenant_id); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'app_dashboard' + AND NOT rolsuper + AND NOT rolbypassrls + ) THEN + RAISE EXCEPTION + 'app_dashboard role missing or RLS-bypassing - apply omnibase_infra forward migration ' + '094_create_app_dashboard_role.sql (OMN-14899) before this RLS ' + 'migration. RLS grants without the constrained read role are the ' + 'exact bypass this work exists to prevent.'; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA public TO app_dashboard; + +ALTER TABLE public.llm_cost_aggregates ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.llm_cost_aggregates FORCE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS tenant_isolation ON public.llm_cost_aggregates; +CREATE POLICY tenant_isolation ON public.llm_cost_aggregates + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)); + +-- Read grant for the live dashboard reader. RLS still filters every +-- row this role can see -- the grant is what makes the RLS-scoped read path +-- reachable at all, not a widening of it. OMN-14894: sibling grant present on +-- context_roi_scores/instruction_eval_aggregate_snapshots/skill_execution_snapshots +-- in the same PR; llm_cost_aggregates omitted it in error. +GRANT SELECT ON public.llm_cost_aggregates TO app_dashboard; diff --git a/docker/migrations/forward/nodes/node_projection_delegation/0007_delegation_events.sql b/docker/migrations/forward/nodes/node_projection_delegation/0007_delegation_events.sql index f481651924..7c865dacea 100644 --- a/docker/migrations/forward/nodes/node_projection_delegation/0007_delegation_events.sql +++ b/docker/migrations/forward/nodes/node_projection_delegation/0007_delegation_events.sql @@ -38,6 +38,110 @@ CREATE TABLE IF NOT EXISTS delegation_events ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: delegation_events ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS session_id TEXT; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS timestamp TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS task_type TEXT DEFAULT ''; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS delegated_to TEXT DEFAULT ''; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS model_name TEXT DEFAULT ''; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS delegated_by TEXT; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS quality_gate_passed BOOLEAN DEFAULT FALSE; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS quality_gates_checked INT DEFAULT 0; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS quality_gates_failed INT DEFAULT 0; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS quality_gates_checked_jsonb JSONB; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS quality_gates_failed_jsonb JSONB; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS quality_gate_detail TEXT; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS cost_usd NUMERIC DEFAULT 0; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS cost_savings_usd NUMERIC DEFAULT 0; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS delegation_latency_ms INT; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS latency_ms INT; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS repo TEXT; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS is_shadow BOOLEAN DEFAULT FALSE; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS llm_call_id TEXT; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS prompt_text TEXT; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS response_text TEXT; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS tokens_input INT DEFAULT 0; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS tokens_output INT DEFAULT 0; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS tokens_to_compliance INT DEFAULT 0; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS compliance_attempts INT DEFAULT 1; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS pricing_manifest_version INT DEFAULT 0; +ALTER TABLE delegation_events ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'correlation_id', 'timestamp', 'task_type', 'delegated_to', 'model_name', 'quality_gate_passed', 'quality_gates_checked', 'quality_gates_failed', 'cost_usd', 'cost_savings_usd', 'is_shadow', 'tokens_input', 'tokens_output', 'tokens_to_compliance', 'compliance_attempts', 'pricing_manifest_version', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'delegation_events'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'delegation_events'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge delegation_events.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'delegation_events'::regclass AND contype = 'p' + ) THEN + ALTER TABLE delegation_events ADD CONSTRAINT delegation_events_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'delegation_events'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['correlation_id']::text[] + ) THEN + ALTER TABLE delegation_events ADD CONSTRAINT delegation_events_correlation_id_key UNIQUE (correlation_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: delegation_events ---- + + -- Warm dev/stability volumes may already contain delegation_events from an -- older projection schema. CREATE TABLE IF NOT EXISTS does not reconcile -- missing columns, so keep this base migration idempotent before indexes and diff --git a/docker/migrations/forward/nodes/node_projection_delegation/0008_generation_events.sql b/docker/migrations/forward/nodes/node_projection_delegation/0008_generation_events.sql index 4837845796..c7a5fbe927 100644 --- a/docker/migrations/forward/nodes/node_projection_delegation/0008_generation_events.sql +++ b/docker/migrations/forward/nodes/node_projection_delegation/0008_generation_events.sql @@ -17,6 +17,93 @@ CREATE TABLE IF NOT EXISTS generation_events ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: generation_events ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS task_description TEXT DEFAULT ''; +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS provider TEXT DEFAULT ''; +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS model_id TEXT DEFAULT ''; +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS endpoint_class TEXT DEFAULT ''; +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS attempt_count INT DEFAULT 0; +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS total_latency_e2e_ms INT DEFAULT 0; +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS contract_passed BOOLEAN DEFAULT FALSE; +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS cost_inference_usd NUMERIC(18, 6) DEFAULT 0; +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS timestamp TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE generation_events ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'correlation_id', 'task_description', 'provider', 'model_id', 'endpoint_class', 'attempt_count', 'total_latency_e2e_ms', 'contract_passed', 'cost_inference_usd', 'timestamp', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'generation_events'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'generation_events'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge generation_events.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'generation_events'::regclass AND contype = 'p' + ) THEN + ALTER TABLE generation_events ADD CONSTRAINT generation_events_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'generation_events'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['correlation_id']::text[] + ) THEN + ALTER TABLE generation_events ADD CONSTRAINT generation_events_correlation_id_key UNIQUE (correlation_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: generation_events ---- + + CREATE INDEX IF NOT EXISTS idx_generation_events_contract_passed ON generation_events (contract_passed); diff --git a/docker/migrations/forward/nodes/node_projection_delegation/0009a_delegation_events_legacy_schema_reconcile.sql b/docker/migrations/forward/nodes/node_projection_delegation/0009a_delegation_events_legacy_schema_reconcile.sql new file mode 100644 index 0000000000..c666140154 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_delegation/0009a_delegation_events_legacy_schema_reconcile.sql @@ -0,0 +1,184 @@ +-- OMN-14974: reconcile warm delegation_events tables created before the +-- contract-driven projection schema. This migration intentionally sorts after +-- the base/metrics migrations and before views that require model_name. + +-- This is an explicitly bounded maintenance operation. PostgreSQL must acquire +-- the table lock before making any schema change; a busy writer makes the +-- migration fail within five seconds so the runner can retry without leaving a +-- partial conversion. The staging table was measured at 6 rows / 128 kB before +-- rollout, so the in-place JSONB-to-integer rewrite is deliberately preferred +-- over a permanent dual-column compatibility path. +-- The runner (scripts/run-forward-migrations.sh) applies node migrations with +-- `psql -f` in autocommit, one statement per implicit transaction. The SET LOCAL +-- scoping and the LOCK TABLE below are only legal inside an explicit transaction +-- block, and the partial-conversion guarantee described above requires one, so +-- this migration opens its own (OMN-15312). +BEGIN; + +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '2min'; +LOCK TABLE delegation_events IN ACCESS EXCLUSIVE MODE; + +ALTER TABLE delegation_events + ADD COLUMN IF NOT EXISTS model_name TEXT NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS llm_call_id TEXT, + ADD COLUMN IF NOT EXISTS prompt_text TEXT, + ADD COLUMN IF NOT EXISTS response_text TEXT, + ADD COLUMN IF NOT EXISTS tokens_to_compliance INT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS compliance_attempts INT NOT NULL DEFAULT 1, + ADD COLUMN IF NOT EXISTS quality_gates_checked_jsonb JSONB, + ADD COLUMN IF NOT EXISTS quality_gates_failed_jsonb JSONB; + +-- Old tables required callers to provide id explicitly, while every current +-- projection writer relies on the canonical UUID default. +ALTER TABLE delegation_events + ALTER COLUMN id SET DEFAULT gen_random_uuid(); + +-- Legacy installations stored gate labels directly in the +-- quality_gates_checked/failed JSONB columns. Current handlers store integer +-- counts there and preserve the labels in the *_jsonb evidence columns. +DO $reconcile_checked$ +DECLARE + current_type TEXT; + unsupported_shape_count BIGINT; +BEGIN + SELECT data_type + INTO current_type + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'delegation_events' + AND column_name = 'quality_gates_checked'; + + IF current_type IS NULL THEN + ALTER TABLE delegation_events + ADD COLUMN quality_gates_checked INTEGER NOT NULL DEFAULT 0; + ELSIF current_type = 'jsonb' THEN + UPDATE delegation_events + SET quality_gates_checked_jsonb = + COALESCE(quality_gates_checked_jsonb, quality_gates_checked) + WHERE quality_gates_checked IS NOT NULL; + + SELECT COUNT(*) + INTO unsupported_shape_count + FROM delegation_events + WHERE quality_gates_checked IS NOT NULL + AND NOT ( + jsonb_typeof(quality_gates_checked) = 'array' + OR ( + jsonb_typeof(quality_gates_checked) = 'number' + AND (quality_gates_checked #>> '{}') ~ '^[0-9]+$' + AND (quality_gates_checked #>> '{}')::NUMERIC <= 2147483647 + ) + ); + IF unsupported_shape_count > 0 THEN + RAISE WARNING + 'delegation_events.quality_gates_checked has % row(s) with an unsupported JSONB shape; original values are preserved in quality_gates_checked_jsonb and counts are coerced to zero', + unsupported_shape_count; + END IF; + + ALTER TABLE delegation_events + ALTER COLUMN quality_gates_checked DROP DEFAULT; + EXECUTE $sql$ + ALTER TABLE delegation_events + ALTER COLUMN quality_gates_checked TYPE INTEGER + USING ( + CASE + WHEN quality_gates_checked IS NULL THEN 0 + WHEN jsonb_typeof(quality_gates_checked) = 'array' + THEN jsonb_array_length(quality_gates_checked) + WHEN jsonb_typeof(quality_gates_checked) = 'number' + AND (quality_gates_checked #>> '{}') ~ '^[0-9]+$' + AND (quality_gates_checked #>> '{}')::NUMERIC <= 2147483647 + THEN (quality_gates_checked #>> '{}')::INTEGER + ELSE 0 + END + ) + $sql$; + ELSIF current_type <> 'integer' THEN + RAISE EXCEPTION + 'delegation_events.quality_gates_checked has unsupported type %', + current_type; + END IF; + + UPDATE delegation_events + SET quality_gates_checked = 0 + WHERE quality_gates_checked IS NULL; + ALTER TABLE delegation_events + ALTER COLUMN quality_gates_checked SET DEFAULT 0, + ALTER COLUMN quality_gates_checked SET NOT NULL; +END +$reconcile_checked$; + +DO $reconcile_failed$ +DECLARE + current_type TEXT; + unsupported_shape_count BIGINT; +BEGIN + SELECT data_type + INTO current_type + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'delegation_events' + AND column_name = 'quality_gates_failed'; + + IF current_type IS NULL THEN + ALTER TABLE delegation_events + ADD COLUMN quality_gates_failed INTEGER NOT NULL DEFAULT 0; + ELSIF current_type = 'jsonb' THEN + UPDATE delegation_events + SET quality_gates_failed_jsonb = + COALESCE(quality_gates_failed_jsonb, quality_gates_failed) + WHERE quality_gates_failed IS NOT NULL; + + SELECT COUNT(*) + INTO unsupported_shape_count + FROM delegation_events + WHERE quality_gates_failed IS NOT NULL + AND NOT ( + jsonb_typeof(quality_gates_failed) = 'array' + OR ( + jsonb_typeof(quality_gates_failed) = 'number' + AND (quality_gates_failed #>> '{}') ~ '^[0-9]+$' + AND (quality_gates_failed #>> '{}')::NUMERIC <= 2147483647 + ) + ); + IF unsupported_shape_count > 0 THEN + RAISE WARNING + 'delegation_events.quality_gates_failed has % row(s) with an unsupported JSONB shape; original values are preserved in quality_gates_failed_jsonb and counts are coerced to zero', + unsupported_shape_count; + END IF; + + ALTER TABLE delegation_events + ALTER COLUMN quality_gates_failed DROP DEFAULT; + EXECUTE $sql$ + ALTER TABLE delegation_events + ALTER COLUMN quality_gates_failed TYPE INTEGER + USING ( + CASE + WHEN quality_gates_failed IS NULL THEN 0 + WHEN jsonb_typeof(quality_gates_failed) = 'array' + THEN jsonb_array_length(quality_gates_failed) + WHEN jsonb_typeof(quality_gates_failed) = 'number' + AND (quality_gates_failed #>> '{}') ~ '^[0-9]+$' + AND (quality_gates_failed #>> '{}')::NUMERIC <= 2147483647 + THEN (quality_gates_failed #>> '{}')::INTEGER + ELSE 0 + END + ) + $sql$; + ELSIF current_type <> 'integer' THEN + RAISE EXCEPTION + 'delegation_events.quality_gates_failed has unsupported type %', + current_type; + END IF; + + UPDATE delegation_events + SET quality_gates_failed = 0 + WHERE quality_gates_failed IS NULL; + ALTER TABLE delegation_events + ALTER COLUMN quality_gates_failed SET DEFAULT 0, + ALTER COLUMN quality_gates_failed SET NOT NULL; +END +$reconcile_failed$; + +COMMIT; diff --git a/docker/migrations/forward/nodes/node_projection_delegation/0016_delegation_judge_verdict_events.sql b/docker/migrations/forward/nodes/node_projection_delegation/0016_delegation_judge_verdict_events.sql index 8fb17ffd8e..102b6216eb 100644 --- a/docker/migrations/forward/nodes/node_projection_delegation/0016_delegation_judge_verdict_events.sql +++ b/docker/migrations/forward/nodes/node_projection_delegation/0016_delegation_judge_verdict_events.sql @@ -35,6 +35,111 @@ CREATE TABLE IF NOT EXISTS delegation_judge_verdict_events ( ) ); +-- ---- BEGIN OMN-15376 shape reconciliation: delegation_judge_verdict_events ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS event_hash TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS task_type TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS score_source TEXT DEFAULT 'reproducible_judge'; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS judge_model TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS judge_model_version TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS judge_provider TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS rubric_id TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS rubric_hash TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS prompt_hash TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS input_hash TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS temperature NUMERIC(5, 3); +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS judge_node_version TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS reasoning_hash TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS verdict TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS actual_score NUMERIC(5, 3); +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS failure_kind TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS failure_message TEXT; +ALTER TABLE delegation_judge_verdict_events ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'event_hash', 'correlation_id', 'task_type', 'score_source', 'judge_model', 'judge_model_version', 'judge_provider', 'rubric_id', 'rubric_hash', 'prompt_hash', 'input_hash', 'temperature', 'judge_node_version', 'reasoning_hash', 'verdict', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'delegation_judge_verdict_events'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'delegation_judge_verdict_events'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge delegation_judge_verdict_events.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'delegation_judge_verdict_events'::regclass AND contype = 'p' + ) THEN + ALTER TABLE delegation_judge_verdict_events ADD CONSTRAINT delegation_judge_verdict_events_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'delegation_judge_verdict_events'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['event_hash']::text[] + ) THEN + ALTER TABLE delegation_judge_verdict_events ADD CONSTRAINT delegation_judge_verdict_events_event_hash_key UNIQUE (event_hash); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'delegation_judge_verdict_events'::regclass AND conname = 'delegation_judge_verdict_events_check' + ) THEN + ALTER TABLE delegation_judge_verdict_events ADD CONSTRAINT delegation_judge_verdict_events_check CHECK ( (verdict = 'judge_failed' AND actual_score IS NULL AND failure_kind IS NOT NULL AND failure_message IS NOT NULL) OR (verdict <> 'judge_failed' AND actual_score IS NOT NULL AND failure_kind IS NULL AND failure_message IS NULL) ); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: delegation_judge_verdict_events ---- + + CREATE INDEX IF NOT EXISTS idx_delegation_judge_verdict_events_correlation_id ON delegation_judge_verdict_events (correlation_id); diff --git a/docker/migrations/forward/nodes/node_projection_delegation/0019_delegation_budget_state.sql b/docker/migrations/forward/nodes/node_projection_delegation/0019_delegation_budget_state.sql index 8552080e42..446af304b0 100644 --- a/docker/migrations/forward/nodes/node_projection_delegation/0019_delegation_budget_state.sql +++ b/docker/migrations/forward/nodes/node_projection_delegation/0019_delegation_budget_state.sql @@ -35,6 +35,129 @@ CREATE TABLE IF NOT EXISTS delegation_budget_state ( updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: delegation_budget_state ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS cost_tier_name TEXT; +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS budget_period TEXT; +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS monthly_cap_usd NUMERIC(18, 6); +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS consumed_usd NUMERIC(18, 6) DEFAULT 0; +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS overage_usd NUMERIC(18, 6) DEFAULT 0; +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS headroom_remaining_usd NUMERIC(18, 6) DEFAULT 0; +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS delegation_count INTEGER DEFAULT 0; +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS last_correlation_id TEXT; +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS first_event_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS last_event_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE delegation_budget_state ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'tenant_id', 'cost_tier_name', 'budget_period', 'monthly_cap_usd', 'consumed_usd', 'overage_usd', 'headroom_remaining_usd', 'delegation_count', 'first_event_at', 'last_event_at', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'delegation_budget_state'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'delegation_budget_state'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge delegation_budget_state.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'delegation_budget_state'::regclass AND contype = 'p' + ) THEN + ALTER TABLE delegation_budget_state ADD CONSTRAINT delegation_budget_state_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'delegation_budget_state'::regclass AND conname = 'delegation_budget_state_monthly_cap_usd_check' + ) THEN + ALTER TABLE delegation_budget_state ADD CONSTRAINT delegation_budget_state_monthly_cap_usd_check CHECK (monthly_cap_usd >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'delegation_budget_state'::regclass AND conname = 'delegation_budget_state_consumed_usd_check' + ) THEN + ALTER TABLE delegation_budget_state ADD CONSTRAINT delegation_budget_state_consumed_usd_check CHECK (consumed_usd >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'delegation_budget_state'::regclass AND conname = 'delegation_budget_state_overage_usd_check' + ) THEN + ALTER TABLE delegation_budget_state ADD CONSTRAINT delegation_budget_state_overage_usd_check CHECK (overage_usd >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'delegation_budget_state'::regclass AND conname = 'delegation_budget_state_headroom_remaining_usd_check' + ) THEN + ALTER TABLE delegation_budget_state ADD CONSTRAINT delegation_budget_state_headroom_remaining_usd_check CHECK (headroom_remaining_usd >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'delegation_budget_state'::regclass AND conname = 'delegation_budget_state_delegation_count_check' + ) THEN + ALTER TABLE delegation_budget_state ADD CONSTRAINT delegation_budget_state_delegation_count_check CHECK (delegation_count >= 0); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: delegation_budget_state ---- + + CREATE UNIQUE INDEX IF NOT EXISTS ux_delegation_budget_state_identity ON delegation_budget_state (tenant_id, cost_tier_name, budget_period); diff --git a/docker/migrations/forward/nodes/node_projection_delegation/0027_generation_events_tenant_rls.sql b/docker/migrations/forward/nodes/node_projection_delegation/0027_generation_events_tenant_rls.sql new file mode 100644 index 0000000000..86cd528f34 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_delegation/0027_generation_events_tenant_rls.sql @@ -0,0 +1,87 @@ +-- OMN-14974: tenant-scoped writer/read access for generation_events. +-- +-- The cloud migration runner creates node-owned tables as the bounded RDS +-- migration principal, while the runtime and dashboard connect as the +-- non-owner role_omnidash role. generation_events was created by migration +-- 0008 without a tenant column, RLS policy, or grants, so the real generation +-- terminal reached the projection handler and failed with 42501 before a row +-- could become dashboard-visible. +-- +-- This migration deliberately uses ENABLE without FORCE. The live writer is +-- already a non-owner/NOSUPERUSER/NOBYPASSRLS role and is therefore subject to +-- RLS. FORCE decisions for the migration owner remain fenced under the +-- OMN-15088 rollout; this migration must not silently cross that boundary. +-- +-- OMN-15351 -- role_omnidash is ENVIRONMENT-provisioned, not migration-provisioned: +-- role_omnidash exists on cloud RDS (provisioned out-of-band) and on a compose +-- cluster only when ROLE_OMNIDASH_PASSWORD was configured at FIRST-STARTUP init +-- (omnibase_infra docker/migrations/forward/000_create_multiple_databases.sh). +-- NO forward migration anywhere creates it. The .201 dev lane cluster carries +-- pg_roles = {app_dashboard, postgres, role_omniweb} -- no role_omnidash -- so +-- the original fail-closed RAISE EXCEPTION on its absence made EVERY dev-lane +-- deploy fatal at this file (OMN-15348 AC4 redeploy, workflow wf_55998f90). +-- +-- The guard is therefore a WARNING that ENUMERATES the two grants it skips, and +-- those two grants execute only when the role exists. Where the role exists the +-- applied grants are identical to the pre-OMN-15351 behaviour (same statements, +-- same order). Where it does not, the database ends up with RLS enabled and NO +-- role_omnidash writer grant -- the honest state of a lane without that role, +-- logged by name, not silently skipped. A lane in that state is not exercising +-- the RDS grant path and cannot prove it. +-- +-- Creating role_omnidash locally is deliberately NOT done here: who owns role +-- creation (environment vs migration) is an architecture decision tracked +-- separately (OMN-15351 direction (b), needs OMN-14894 context). +-- +-- app_dashboard stays FATAL on purpose: omnibase_infra forward migration 094 +-- (OMN-14899) DOES create it in-repo, so its absence is a genuine migration +-- ordering bug rather than an environment difference. That is the posture every +-- sibling RLS migration takes (node_projection_delegation 0023/0026, +-- node_projection_registration 0002, node_projection_savings 081). + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'role_omnidash') THEN + RAISE WARNING + 'role_omnidash role missing — SKIPPING 2 grants on this database: ' + '(1) GRANT USAGE ON SCHEMA public TO role_omnidash; ' + '(2) GRANT SELECT, INSERT, UPDATE ON generation_events TO role_omnidash. ' + 'generation_events writer access is NOT granted here. role_omnidash is ' + 'environment-provisioned (RDS out-of-band, or ROLE_OMNIDASH_PASSWORD at ' + 'cluster init) and is never created by a migration — see OMN-15351.'; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app_dashboard') THEN + RAISE EXCEPTION + 'app_dashboard role missing — apply forward migration 094 before generation_events RLS'; + END IF; +END; +$$; + +ALTER TABLE generation_events + ADD COLUMN IF NOT EXISTS tenant_id text NOT NULL DEFAULT 'omninode'; + +CREATE INDEX IF NOT EXISTS idx_generation_events_tenant_id + ON generation_events (tenant_id); + +ALTER TABLE generation_events ENABLE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS tenant_isolation ON generation_events; +CREATE POLICY tenant_isolation ON generation_events + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)); + +-- role_omnidash grants, conditional on the environment-provisioned role existing +-- (OMN-15351 note above). The statement text and order are unchanged from the +-- pre-OMN-15351 top-level statements; only the existence guard is new. +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'role_omnidash') THEN + EXECUTE 'GRANT USAGE ON SCHEMA public TO role_omnidash'; + EXECUTE 'GRANT SELECT, INSERT, UPDATE ON generation_events TO role_omnidash'; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA public TO app_dashboard; +GRANT SELECT ON generation_events TO app_dashboard; diff --git a/docker/migrations/forward/nodes/node_projection_delegation/0028_reconcile_delegation_observability_views.sql b/docker/migrations/forward/nodes/node_projection_delegation/0028_reconcile_delegation_observability_views.sql new file mode 100644 index 0000000000..3246a2b6b4 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_delegation/0028_reconcile_delegation_observability_views.sql @@ -0,0 +1,395 @@ +-- OMN-14974: reconcile a database whose node migration ledger records the +-- historical delegation dashboard migrations while their four materialized +-- read views are absent. This net-new migration folds the final definitions +-- from 0010, 0016, and 0021 so it is forward-only and idempotent. + +CREATE OR REPLACE VIEW projection_delegation_summary AS +WITH summary AS ( + SELECT + COUNT(*)::int AS total_events, + COALESCE(SUM(CASE WHEN quality_gate_passed THEN 1 ELSE 0 END), 0)::int + AS quality_passed_count, + COALESCE(SUM(CASE WHEN NOT quality_gate_passed THEN 1 ELSE 0 END), 0)::int + AS quality_failed_count, + COALESCE(AVG(COALESCE(latency_ms, delegation_latency_ms)), 0)::float + AS avg_latency_ms, + COALESCE(MAX(EXTRACT(EPOCH FROM created_at)), 0)::float AS latest_event_at, + COALESCE(SUM(cost_savings_usd), 0)::float AS total_savings_usd, + MAX(created_at) AS latest_projection_updated_at + FROM delegation_events +), +by_task_type AS ( + SELECT COALESCE( + jsonb_agg(jsonb_build_object('taskType', task_type, 'count', count) + ORDER BY count DESC), + '[]'::jsonb + ) AS rows + FROM ( + SELECT task_type, COUNT(*)::int AS count + FROM delegation_events + GROUP BY task_type + ) grouped +), +by_model AS ( + SELECT COALESCE( + jsonb_agg(jsonb_build_object('model', delegated_to, 'count', count) + ORDER BY count DESC), + '[]'::jsonb + ) AS rows + FROM ( + SELECT delegated_to, COUNT(*)::int AS count + FROM delegation_events + GROUP BY delegated_to + ) grouped +) +SELECT + summary.total_events AS "totalDelegations", + CASE WHEN summary.total_events > 0 + THEN summary.quality_passed_count::float / summary.total_events + ELSE 0 + END AS "qualityGatePassRate", + summary.quality_passed_count AS "qualityGatePassed", + summary.total_events AS "qualityGateTotal", + summary.total_savings_usd AS "totalSavingsUsd", + summary.avg_latency_ms AS "avgLatencyMs", + summary.latest_event_at AS "latestEventAt", + summary.total_events, + summary.quality_passed_count, + summary.quality_failed_count, + summary.avg_latency_ms, + summary.latest_event_at, + by_task_type.rows AS "byTaskType", + by_model.rows AS "byModel", + summary.latest_projection_updated_at +FROM summary +CROSS JOIN by_task_type +CROSS JOIN by_model; + +CREATE OR REPLACE VIEW projection_delegation_model_routing AS +WITH grouped AS ( + SELECT + delegated_to AS model_alias, + COALESCE(NULLIF(model_name, ''), delegated_to) AS model_name, + task_type, + COUNT(*)::int AS event_count, + COALESCE(SUM(CASE WHEN quality_gate_passed THEN 1 ELSE 0 END), 0)::int + AS quality_passed, + COALESCE(AVG(COALESCE(latency_ms, delegation_latency_ms)), 0)::float + AS avg_latency_ms + FROM delegation_events + GROUP BY delegated_to, COALESCE(NULLIF(model_name, ''), delegated_to), task_type +), +totals AS ( + SELECT COUNT(*)::int AS total_delegations, + MAX(created_at) AS latest_projection_updated_at + FROM delegation_events +), +by_model AS ( + SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'model_name', model_alias, + 'total_count', total_count, + 'pct_of_total', CASE WHEN totals.total_delegations > 0 + THEN total_count::float / totals.total_delegations ELSE 0 END, + 'top_task_type', top_task_type, + 'avg_latency_ms', avg_latency_ms, + 'qg_pass_rate', CASE WHEN total_count > 0 + THEN quality_passed::float / total_count ELSE 0 END, + 'task_types', task_types + ) ORDER BY total_count DESC + ), + '[]'::jsonb + ) AS rows + FROM ( + SELECT + model_alias, + SUM(event_count)::int AS total_count, + SUM(quality_passed)::int AS quality_passed, + CASE WHEN SUM(event_count) > 0 + THEN SUM(avg_latency_ms * event_count)::float / SUM(event_count) + ELSE 0 + END AS avg_latency_ms, + (array_agg(task_type ORDER BY event_count DESC))[1] AS top_task_type, + to_jsonb(array_agg(task_type ORDER BY task_type)) AS task_types + FROM grouped + GROUP BY model_alias + ) model_totals + CROSS JOIN totals +), +routing_rows AS ( + SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'model_name', grouped.model_alias, + 'task_type', grouped.task_type, + 'count', grouped.event_count, + 'pct_of_model', CASE WHEN model_totals.total_count > 0 + THEN grouped.event_count::float / model_totals.total_count ELSE 0 END, + 'pct_of_total', CASE WHEN totals.total_delegations > 0 + THEN grouped.event_count::float / totals.total_delegations ELSE 0 END + ) ORDER BY grouped.event_count DESC + ), + '[]'::jsonb + ) AS rows + FROM grouped + JOIN ( + SELECT model_alias, SUM(event_count)::int AS total_count + FROM grouped + GROUP BY model_alias + ) model_totals USING (model_alias) + CROSS JOIN totals +), +decision_traces AS ( + SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', id, + 'correlation_id', correlation_id, + 'task_type', task_type, + 'model_name', COALESCE(NULLIF(model_name, ''), delegated_to), + 'delegated_to', delegated_to, + 'routing_rule', NULL, + 'routing_confidence', NULL, + 'routing_candidates', NULL, + 'latency_ms', COALESCE(latency_ms, delegation_latency_ms), + 'quality_gate_passed', quality_gate_passed, + 'created_at', EXTRACT(EPOCH FROM created_at) + ) ORDER BY created_at DESC + ), + '[]'::jsonb + ) AS rows + FROM ( + SELECT + row_number() OVER (ORDER BY created_at DESC)::int AS id, + correlation_id, + task_type, + model_name, + delegated_to, + latency_ms, + delegation_latency_ms, + quality_gate_passed, + created_at + FROM delegation_events + ORDER BY created_at DESC + LIMIT 20 + ) traces +), +tier_totals AS ( + SELECT + COUNT(*)::int AS total_tasks, + COALESCE(SUM(CASE WHEN cost_tier_name <> '' THEN 1 ELSE 0 END), 0)::int + AS tier_routed_total, + COALESCE(SUM(CASE WHEN cost_tier_name = '' THEN 1 ELSE 0 END), 0)::int + AS not_tier_routed_count + FROM delegation_events +), +tier_rows AS ( + SELECT + CASE WHEN cost_tier_name = '' THEN 'not_tier_routed' ELSE cost_tier_name END + AS cost_tier_name, + (cost_tier_name <> '') AS tier_routed, + COUNT(*)::int AS count + FROM delegation_events + GROUP BY 1, 2 +), +by_tier AS ( + SELECT jsonb_build_object( + 'total_tasks', tier_totals.total_tasks, + 'tier_routed_total', tier_totals.tier_routed_total, + 'not_tier_routed_count', tier_totals.not_tier_routed_count, + 'tiers', COALESCE( + ( + SELECT jsonb_agg( + jsonb_build_object( + 'cost_tier_name', tier_rows.cost_tier_name, + 'count', tier_rows.count, + 'tier_routed', tier_rows.tier_routed, + 'pct_of_tier_routed', CASE + WHEN tier_rows.tier_routed AND tier_totals.tier_routed_total > 0 + THEN tier_rows.count::float / tier_totals.tier_routed_total + ELSE 0 + END + ) ORDER BY tier_rows.tier_routed DESC, + tier_rows.count DESC, + tier_rows.cost_tier_name + ) + FROM tier_rows + ), + '[]'::jsonb + ) + ) AS summary + FROM tier_totals +) +SELECT + totals.total_delegations, + routing_rows.rows, + by_model.rows AS by_model, + decision_traces.rows AS decision_traces, + COALESCE(totals.latest_projection_updated_at, NOW()) AS captured_at, + TRUE AS provisioned, + totals.latest_projection_updated_at, + by_tier.summary AS by_tier +FROM totals +CROSS JOIN routing_rows +CROSS JOIN by_model +CROSS JOIN decision_traces +CROSS JOIN by_tier; + +CREATE OR REPLACE VIEW projection_delegation_quality_gate AS +WITH totals AS ( + SELECT + COUNT(*) FILTER (WHERE quality_gate_passed IS NOT NULL)::int AS total_checks, + COUNT(*) FILTER (WHERE quality_gate_passed IS TRUE)::int AS total_passed, + COUNT(*) FILTER (WHERE quality_gate_passed IS FALSE)::int AS total_failed, + COALESCE(SUM(escalation_count), 0)::int AS total_escalations, + COALESCE(AVG(NULLIF(tokens_to_compliance, 0)), 0)::float + AS avg_tokens_to_compliance, + percentile_cont(0.5) WITHIN GROUP ( + ORDER BY NULLIF(tokens_to_compliance, 0) + ) AS median_tokens_to_compliance, + COALESCE(AVG(NULLIF(compliance_attempts, 0)), 1)::float + AS avg_compliance_attempts, + COALESCE(AVG(actual_score), 0)::float AS avg_actual_score, + COALESCE(AVG(required_bar), 0)::float AS avg_required_bar, + MAX(created_at) AS latest_projection_updated_at + FROM delegation_events +), +failure_categories AS ( + SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'category', quality_gate_detail, + 'count', failed_count, + 'pct_of_failures', CASE WHEN totals.total_failed > 0 + THEN failed_count::float / totals.total_failed ELSE 0 END + ) ORDER BY failed_count DESC + ), + '[]'::jsonb + ) AS rows + FROM ( + SELECT quality_gate_detail, COUNT(*)::int AS failed_count + FROM delegation_events + WHERE quality_gate_detail IS NOT NULL + AND NOT quality_gate_passed + GROUP BY quality_gate_detail + ) failures + CROSS JOIN totals +), +tokens_by_model AS ( + SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'model_name', model_name, + 'avg_tokens', avg_tokens, + 'avg_attempts', avg_attempts, + 'sample_count', sample_count + ) ORDER BY avg_tokens DESC + ), + '[]'::jsonb + ) AS rows + FROM ( + SELECT + COALESCE(NULLIF(model_name, ''), delegated_to) AS model_name, + COALESCE(AVG(NULLIF(tokens_to_compliance, 0)), 0)::float AS avg_tokens, + COALESCE(AVG(NULLIF(compliance_attempts, 0)), 1)::float AS avg_attempts, + COUNT(*)::int AS sample_count + FROM delegation_events + GROUP BY COALESCE(NULLIF(model_name, ''), delegated_to) + ) by_model +) +SELECT + CASE WHEN totals.total_checks > 0 + THEN totals.total_passed::float / totals.total_checks ELSE 0 END + AS overall_pass_rate, + totals.total_passed, + totals.total_failed, + totals.total_checks, + totals.total_escalations AS escalation_count, + CASE WHEN totals.total_checks > 0 + THEN totals.total_escalations::float / totals.total_checks ELSE 0 END + AS escalation_rate, + jsonb_build_array(jsonb_build_object( + 'check_type', 'score_vs_required_bar', + 'passed', totals.total_passed, + 'failed', totals.total_failed, + 'total', totals.total_checks, + 'pass_rate', CASE WHEN totals.total_checks > 0 + THEN totals.total_passed::float / totals.total_checks ELSE 0 END + )) AS by_check_type, + failure_categories.rows AS failure_categories, + totals.avg_tokens_to_compliance, + totals.median_tokens_to_compliance, + totals.avg_compliance_attempts, + tokens_by_model.rows AS tokens_to_compliance_by_model, + COALESCE(totals.latest_projection_updated_at, NOW()) AS captured_at, + TRUE AS provisioned, + totals.latest_projection_updated_at, + totals.avg_actual_score, + totals.avg_required_bar +FROM totals +CROSS JOIN failure_categories +CROSS JOIN tokens_by_model; + +CREATE OR REPLACE VIEW projection_delegation_token_usage AS +WITH model_usage AS ( + SELECT + COALESCE(NULLIF(delegated_to, ''), NULLIF(model_name, ''), 'delegated-runtime') + AS model_id, + COALESCE(NULLIF(model_name, ''), NULLIF(delegated_to, ''), 'delegated-runtime') + AS model_name, + COALESCE(SUM(tokens_input), 0)::int AS prompt_tokens, + COALESCE(SUM(tokens_output), 0)::int AS completion_tokens, + COALESCE(SUM(tokens_input + tokens_output), 0)::int AS total_tokens, + COALESCE(SUM(cost_usd), 0)::float AS estimated_cost_usd, + CASE + WHEN COALESCE(SUM(tokens_input + tokens_output), 0) > 0 THEN 'measured' + WHEN COALESCE(SUM(cost_usd), 0) > 0 THEN 'estimated' + ELSE 'unknown' + END AS usage_source, + CASE + WHEN COALESCE(SUM(tokens_input + tokens_output), 0) > 0 THEN 'measured' + WHEN COALESCE(SUM(cost_usd), 0) > 0 THEN 'estimated' + ELSE 'unknown' + END AS token_provenance + FROM delegation_events + GROUP BY COALESCE(NULLIF(delegated_to, ''), NULLIF(model_name, ''), 'delegated-runtime'), + COALESCE(NULLIF(model_name, ''), NULLIF(delegated_to, ''), 'delegated-runtime') +), +totals AS ( + SELECT + COALESCE(SUM(tokens_input), 0)::int AS total_prompt_tokens, + COALESCE(SUM(tokens_output), 0)::int AS total_completion_tokens, + COALESCE(SUM(tokens_input + tokens_output), 0)::int AS total_tokens, + COALESCE(SUM(cost_usd), 0)::float AS total_estimated_cost_usd, + MAX(created_at) AS latest_projection_updated_at + FROM delegation_events +), +by_model AS ( + SELECT COALESCE( + jsonb_agg(to_jsonb(model_usage) ORDER BY total_tokens DESC), + '[]'::jsonb + ) AS rows + FROM model_usage +), +provenance AS ( + SELECT jsonb_build_object( + 'measured', COALESCE(SUM(CASE WHEN token_provenance = 'measured' THEN 1 ELSE 0 END), 0), + 'estimated', COALESCE(SUM(CASE WHEN token_provenance = 'estimated' THEN 1 ELSE 0 END), 0), + 'unknown', COALESCE(SUM(CASE WHEN token_provenance = 'unknown' THEN 1 ELSE 0 END), 0) + ) AS summary + FROM model_usage +) +SELECT + totals.total_prompt_tokens, + totals.total_completion_tokens, + totals.total_tokens, + totals.total_estimated_cost_usd, + provenance.summary AS provenance_summary, + by_model.rows AS by_model, + COALESCE(totals.latest_projection_updated_at, NOW()) AS captured_at, + TRUE AS provisioned, + totals.latest_projection_updated_at +FROM totals +CROSS JOIN provenance +CROSS JOIN by_model; diff --git a/docker/migrations/forward/nodes/node_projection_delegation/0029_delegation_terminal_failure_cause.sql b/docker/migrations/forward/nodes/node_projection_delegation/0029_delegation_terminal_failure_cause.sql new file mode 100644 index 0000000000..bca76e373d --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_delegation/0029_delegation_terminal_failure_cause.sql @@ -0,0 +1,46 @@ +-- OMN-15503: durable typed terminal outcome on delegation_events. +-- +-- The 2026-07-29 delegation matrix lost 5 of 13 classes to provider quota +-- exhaustion, and the durable projection could not say so. The terminal event +-- carries a typed attempt ladder and the producer-side wire DTO +-- (omnibase_core 0.46.8 ModelDelegationResult.terminal_failure_cause, typed by +-- EnumDelegationTerminalFailureCause) already speaks a machine-readable cause, +-- but the consumer side dropped both: a forced-429 command projected as +-- quality_gate_passed=true with quality_gate_detail='completed', because the +-- outer delegate-skill-completed terminal arrived LAST and won the +-- correlation_id UPSERT. +-- +-- Three columns close that seam: +-- +-- terminal_ok -- authoritative outer outcome reduced from the +-- attempt ladder, NOT the declared status. False +-- whenever no attempt produced an accepted answer. +-- terminal_failure_cause -- EnumDelegationTerminalFailureCause value +-- ('provider_quota_exhausted' today). NULL means +-- "no typed cause resolved"; it is never a claim +-- of success -- read terminal_ok for that. +-- attempt_history -- the typed per-tier ladder, so "refused after N +-- escalations" is provable from the durable row +-- rather than from a capture log. +-- +-- All three are nullable with no default: existing rows predate the reduction +-- and must stay honestly unclassified rather than be backfilled into a claim +-- the event stream never made. PostgresSyncProjectionAdapter.upsert() builds +-- its INSERT column list from row.keys(), so these must exist before the +-- OMN-15503 handler change writes them; the in-memory dict adapter used by +-- unit tests has no schema to violate and would mask the gap. + +ALTER TABLE delegation_events + ADD COLUMN IF NOT EXISTS terminal_ok BOOLEAN; + +ALTER TABLE delegation_events + ADD COLUMN IF NOT EXISTS terminal_failure_cause TEXT; + +ALTER TABLE delegation_events + ADD COLUMN IF NOT EXISTS attempt_history JSONB; + +-- Partial index: quota-exhaustion forensics scan only the failed tail, which +-- is a small minority of rows. A full index would be mostly NULLs. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_delegation_events_terminal_failure_cause + ON delegation_events (terminal_failure_cause) + WHERE terminal_failure_cause IS NOT NULL; diff --git a/docker/migrations/forward/nodes/node_projection_delegation/0030_delegation_budget_state_house_tenant_rekey.sql b/docker/migrations/forward/nodes/node_projection_delegation/0030_delegation_budget_state_house_tenant_rekey.sql new file mode 100644 index 0000000000..68a0829835 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_delegation/0030_delegation_budget_state_house_tenant_rekey.sql @@ -0,0 +1,92 @@ +-- OMN-15655 / operator ruling 2026-08-02 (house tenant): re-key +-- delegation_budget_state rows written under the orphan tenant 'default'. +-- +-- THE DEFECT +-- `handler_budget_state.py` carried `DEFAULT_TENANT = "default"` while every +-- other writer on this surface resolves to `"omninode"` and every landed +-- column DEFAULT is `'omninode'` (delegation 0022/0025, savings 080, +-- registration 0002, inference-response 0002, omnidash 0001_tenant_rls). +-- `delegation_budget_state.tenant_id` is `TEXT NOT NULL` with NO column +-- default (migration 0019), so that handler constant was the ONLY thing +-- deciding where an unattributed budget row landed -- and it landed them +-- under a tenant id that appears in no registry, no RLS policy and no other +-- relation. The ruling settles it: there is exactly ONE house tenant. +-- +-- The handler constant is fixed in the same change. This migration moves the +-- rows it already wrote, so the pre-existing budget state is not orphaned +-- from the tenant the writer will use from now on. +-- +-- COLLISION HANDLING -- NOT A BLIND UPDATE +-- `delegation_budget_state` is unique on (tenant_id, cost_tier_name, +-- budget_period) (0019). A blind `UPDATE ... SET tenant_id = 'omninode'` +-- raises a unique violation whenever BOTH a 'default' row and an 'omninode' +-- row exist for the same (tier, period). Those pairs are two divergent +-- accumulated states for one budget window; silently picking one, or summing +-- them, would invent a number nobody measured. So: +-- * non-colliding rows are moved, +-- * colliding rows are LEFT IN PLACE under 'default' and reported via +-- RAISE NOTICE with their keys, so the residual is visible and +-- hand-resolvable rather than silently destroyed or silently merged. +-- Expected count in every known lane is zero: the 'default' constant was +-- only ever reachable for a budgeted-tier event that carried no tenant. +-- +-- Idempotent: re-running moves nothing on the second pass (no 'default' rows +-- remain except the reported collisions, and those are re-reported, not +-- re-attempted). + +DO $$ +DECLARE + v_moved BIGINT := 0; + v_blocked BIGINT := 0; + v_row RECORD; + v_state_relation REGCLASS; +BEGIN + v_state_relation := to_regclass('delegation_budget_state'); + IF v_state_relation IS NULL THEN + RAISE NOTICE 'delegation_budget_state absent; nothing to re-key'; + RETURN; + END IF; + IF NOT EXISTS ( + SELECT 1 + FROM pg_class + WHERE oid = v_state_relation + AND relkind IN ('r', 'p') + ) THEN + RAISE EXCEPTION + 'OMN-15655: delegation_budget_state resolves to %, not a table-like relation', + v_state_relation; + END IF; + + WITH moved AS ( + UPDATE delegation_budget_state AS src + SET tenant_id = 'omninode' + WHERE src.tenant_id = 'default' + AND NOT EXISTS ( + SELECT 1 + FROM delegation_budget_state AS dst + WHERE dst.tenant_id = 'omninode' + AND dst.cost_tier_name = src.cost_tier_name + AND dst.budget_period = src.budget_period + ) + RETURNING 1 + ) + SELECT count(*) INTO v_moved FROM moved; + + FOR v_row IN + SELECT cost_tier_name, budget_period + FROM delegation_budget_state + WHERE tenant_id = 'default' + LOOP + v_blocked := v_blocked + 1; + RAISE NOTICE + 'delegation_budget_state: tenant ''default'' row left in place; an ' + '''omninode'' row already exists for (cost_tier_name=%, budget_period=%) ' + '-- two divergent states for one budget window, resolve by hand', + v_row.cost_tier_name, v_row.budget_period; + END LOOP; + + RAISE NOTICE + 'delegation_budget_state house-tenant re-key: % moved, % left as collisions', + v_moved, v_blocked; +END; +$$; diff --git a/docker/migrations/forward/nodes/node_projection_delegation_inference_response/0001_create_projection_delegation_inference_response_text.sql b/docker/migrations/forward/nodes/node_projection_delegation_inference_response/0001_create_projection_delegation_inference_response_text.sql index 0cb1c03060..a9cce9bbcb 100644 --- a/docker/migrations/forward/nodes/node_projection_delegation_inference_response/0001_create_projection_delegation_inference_response_text.sql +++ b/docker/migrations/forward/nodes/node_projection_delegation_inference_response/0001_create_projection_delegation_inference_response_text.sql @@ -41,6 +41,127 @@ CREATE TABLE IF NOT EXISTS projection_delegation_inference_response_text ( CHECK (singleton_key = 'global') ); +-- ---- BEGIN OMN-15376 shape reconciliation: projection_delegation_inference_response_text ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS singleton_key TEXT; +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS latest_correlation_id TEXT DEFAULT ''; +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS latest_model_name TEXT DEFAULT ''; +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS latest_task_type TEXT DEFAULT ''; +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS latest_generated_text TEXT DEFAULT ''; +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS latest_prompt_tokens INT DEFAULT 0; +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS latest_completion_tokens INT DEFAULT 0; +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS latest_latency_ms INT DEFAULT 0; +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS source_topic TEXT DEFAULT 'onex.evt.omnibase-infra.inference-response.v1'; +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS recent_responses JSONB DEFAULT '[]'::jsonb; +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS captured_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE projection_delegation_inference_response_text ADD COLUMN IF NOT EXISTS provisioned BOOLEAN DEFAULT TRUE; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['singleton_key', 'latest_correlation_id', 'latest_model_name', 'latest_task_type', 'latest_generated_text', 'latest_prompt_tokens', 'latest_completion_tokens', 'latest_latency_ms', 'source_topic', 'recent_responses', 'captured_at', 'provisioned'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'projection_delegation_inference_response_text'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'projection_delegation_inference_response_text'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge projection_delegation_inference_response_text.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'projection_delegation_inference_response_text'::regclass AND contype = 'p' + ) THEN + ALTER TABLE projection_delegation_inference_response_text ADD CONSTRAINT projection_delegation_inference_response_text_pkey PRIMARY KEY (singleton_key); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'projection_delegation_inference_response_text'::regclass AND conname = 'projection_delegation_inference_resp_latest_prompt_tokens_check' + ) THEN + ALTER TABLE projection_delegation_inference_response_text ADD CONSTRAINT projection_delegation_inference_resp_latest_prompt_tokens_check CHECK (latest_prompt_tokens >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'projection_delegation_inference_response_text'::regclass AND conname = 'projection_delegation_inference__latest_completion_tokens_check' + ) THEN + ALTER TABLE projection_delegation_inference_response_text ADD CONSTRAINT projection_delegation_inference__latest_completion_tokens_check CHECK (latest_completion_tokens >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'projection_delegation_inference_response_text'::regclass AND conname = 'projection_delegation_inference_respons_latest_latency_ms_check' + ) THEN + ALTER TABLE projection_delegation_inference_response_text ADD CONSTRAINT projection_delegation_inference_respons_latest_latency_ms_check CHECK (latest_latency_ms >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'projection_delegation_inference_response_text'::regclass AND conname = 'projection_delegation_inference_response_recent_responses_check' + ) THEN + ALTER TABLE projection_delegation_inference_response_text ADD CONSTRAINT projection_delegation_inference_response_recent_responses_check CHECK (jsonb_typeof(recent_responses) = 'array'); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'projection_delegation_inference_response_text'::regclass AND conname = 'projection_delegation_inference_response_te_singleton_key_check' + ) THEN + ALTER TABLE projection_delegation_inference_response_text ADD CONSTRAINT projection_delegation_inference_response_te_singleton_key_check CHECK (singleton_key = 'global'); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: projection_delegation_inference_response_text ---- + + -- Seed the singleton row so the projection-API always returns a row -- (avoids 404 before the first inference-response event arrives). INSERT INTO projection_delegation_inference_response_text diff --git a/docker/migrations/forward/nodes/node_projection_dep_health/001_create_dep_health_findings.sql b/docker/migrations/forward/nodes/node_projection_dep_health/001_create_dep_health_findings.sql index 7efc681b01..03ecadae9b 100644 --- a/docker/migrations/forward/nodes/node_projection_dep_health/001_create_dep_health_findings.sql +++ b/docker/migrations/forward/nodes/node_projection_dep_health/001_create_dep_health_findings.sql @@ -15,6 +15,92 @@ CREATE TABLE IF NOT EXISTS dep_health_findings ( UNIQUE (run_id, finding_type, file_path, symbol) ); +-- ---- BEGIN OMN-15376 shape reconciliation: dep_health_findings ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE dep_health_findings ADD COLUMN IF NOT EXISTS id BIGSERIAL; +ALTER TABLE dep_health_findings ADD COLUMN IF NOT EXISTS run_id VARCHAR; +ALTER TABLE dep_health_findings ADD COLUMN IF NOT EXISTS finding_type VARCHAR; +ALTER TABLE dep_health_findings ADD COLUMN IF NOT EXISTS severity VARCHAR; +ALTER TABLE dep_health_findings ADD COLUMN IF NOT EXISTS repo VARCHAR; +ALTER TABLE dep_health_findings ADD COLUMN IF NOT EXISTS file_path VARCHAR DEFAULT ''; +ALTER TABLE dep_health_findings ADD COLUMN IF NOT EXISTS symbol VARCHAR DEFAULT ''; +ALTER TABLE dep_health_findings ADD COLUMN IF NOT EXISTS detail TEXT DEFAULT ''; +ALTER TABLE dep_health_findings ADD COLUMN IF NOT EXISTS rule_id VARCHAR; +ALTER TABLE dep_health_findings ADD COLUMN IF NOT EXISTS rule_version VARCHAR; +ALTER TABLE dep_health_findings ADD COLUMN IF NOT EXISTS captured_at TIMESTAMPTZ; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'run_id', 'finding_type', 'severity', 'repo', 'file_path', 'symbol', 'detail', 'rule_id', 'rule_version', 'captured_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'dep_health_findings'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'dep_health_findings'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge dep_health_findings.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'dep_health_findings'::regclass AND contype = 'p' + ) THEN + ALTER TABLE dep_health_findings ADD CONSTRAINT dep_health_findings_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'dep_health_findings'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['file_path', 'finding_type', 'run_id', 'symbol']::text[] + ) THEN + ALTER TABLE dep_health_findings ADD CONSTRAINT dep_health_findings_run_id_finding_type_file_path_symbol_key UNIQUE (run_id, finding_type, file_path, symbol); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: dep_health_findings ---- + + CREATE INDEX IF NOT EXISTS idx_dep_health_findings_run_id ON dep_health_findings (run_id); diff --git a/docker/migrations/forward/nodes/node_projection_dep_health/002_dep_health_findings_tenant_id_and_rls.sql b/docker/migrations/forward/nodes/node_projection_dep_health/002_dep_health_findings_tenant_id_and_rls.sql new file mode 100644 index 0000000000..c4d5a7f1a6 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_dep_health/002_dep_health_findings_tenant_id_and_rls.sql @@ -0,0 +1,114 @@ +-- OMN-15655 / operator ruling 2026-08-02 (house tenant): give dep_health_findings a +-- tenant identity and row-level tenant isolation. +-- +-- dependency-health findings are scoped to the repo/run that produced them THIS RELATION IS TENANT DATA +-- WHY, so the row is workload-attributable and belongs in the tenant +-- domain per the ruling's rule. Rows produced by OmniNode's own platform +-- workloads are the omninode TENANT's rows -- OmniNode is a first-class tenant, +-- not an absence of one. Only attribution-meaningless infrastructure state +-- (migration bookkeeping, registry, orchestration state, deployment +-- evidence) stays in omninode_internal. +-- +-- This migration lands alongside the contract flip +-- `db_io.db_tables[].schema: public -> tenant`. Before it, the relation +-- declared a schema the typed topology does not declare, which ADR-0027 +-- refuses by design -- `ValueError: Unknown schema 'public' for +-- database_ref 'application'` out of `ModelDeploymentTopology.schema_domain`. +-- +-- IDENTITY TYPE -- TEXT, NOT UUID, AND THAT IS DELIBERATE +-- Every landed tenant_id column on this surface is `TEXT` holding the slug +-- 'omninode' (delegation 0022/0025, savings 080, registration 0002, +-- inference-response 0002, omnidash 0001_tenant_rls), and the RLS policies +-- that already shipped (0023, 0026) compare TEXT with no `::uuid` cast. +-- OMN-15356 converts that whole set to the canonical UUID in ONE pass. +-- Adding a UUID column here alone would fork tenant identity inside a single +-- database -- the exact "one canonical model per shape" violation. The +-- canonical UUID this slug resolves to is pinned in +-- `omnimarket.projection.tenant_isolation.omninode_TENANT_UUID`. +-- +-- BACKFILL -- NO BATCHING NEEDED, AND BATCHING WOULD BE WORSE +-- `ADD COLUMN ... NOT NULL DEFAULT` with a non-volatile default does not +-- rewrite the table on PostgreSQL 11+; the default is stored in +-- `pg_attribute.attmissingval` and every pre-existing row reads back as +-- the house tenant immediately. A batched `UPDATE` would do strictly more +-- work (a real rewrite, dead tuples, a longer lock) for the same end state. +-- +-- PHYSICAL SCHEMA IS NOT MOVED HERE -- SEE OMN-15359 +-- No `ALTER TABLE ... SET SCHEMA tenant`. The `tenant` schema is created +-- by no applied migration in this repo or in omnibase_infra +-- (`docker/migrations/forward/_ledger/bootstrap.sql` creates only +-- `platform_catalog`); it exists solely in proof fixtures. Every relation +-- already classified TENANT -- delegation_events, delegation_budget_state, +-- savings_estimates -- is still physically in `public` +-- (`current_schema: ["public"]` in the OMN-15423 inventory, against +-- `target_schema: tenant`). Relocating these eight alone would create the +-- split the inventory's target/current split exists to prevent, and would +-- fail outright against a database with no `tenant` schema. The physical +-- move is OMN-15359 ("Build classified schemas and migrate internal, +-- control-plane, catalog, and tenant targets"), which moves the whole set, +-- with its trigger functions and sequences, in one governed cutover. +-- +-- FAIL-CLOSED RATCHET +-- The column DEFAULT is what supplies the house tenant today, exactly as it +-- does for savings_estimates (OMN-14058, operator-accepted): a writer that +-- resolves no tenant OMITS the key and Postgres fills it -- the key is never +-- written as NULL. That default-allowed state is PINNED, with its flip +-- condition named, by +-- `tests/unit/projection/test_house_tenant_default_ratchet.py`. When +-- customer ingress exists the writer boundary stops defaulting and refuses +-- instead; that test fails the moment the precondition changes. +-- +-- BLAST RADIUS -- FORCE constrains the table OWNER too +-- Same caveat as migration 0023: a writer connected as the `postgres` +-- SUPERUSER (the compose lanes) bypasses RLS regardless of FORCE. The real +-- isolation boundary is this policy PLUS a non-superuser, NOBYPASSRLS writer +-- role (OMN-14899 / OMN-15425). +-- +-- Fail-closed: `current_setting('app.tenant_id', true)` is NULL when the GUC +-- is unset, the predicate is NULL, and zero rows are visible. The policy has no +-- default-tenant fallback, by design. +-- +-- Idempotent: ADD COLUMN / CREATE INDEX are IF NOT EXISTS, ENABLE/FORCE are +-- idempotent, and the policy is DROP + CREATE. + +ALTER TABLE public.dep_health_findings + ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'omninode'; + +CREATE INDEX IF NOT EXISTS idx_dep_health_findings_tenant_id + ON public.dep_health_findings (tenant_id); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'app_dashboard' + AND NOT rolsuper + AND NOT rolbypassrls + ) THEN + RAISE EXCEPTION + 'app_dashboard role missing or RLS-bypassing - apply omnibase_infra forward migration ' + '094_create_app_dashboard_role.sql (OMN-14899) before this RLS ' + 'migration. RLS grants without the constrained read role are the ' + 'exact bypass this work exists to prevent.'; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA public TO app_dashboard; + +ALTER TABLE public.dep_health_findings ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.dep_health_findings FORCE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS tenant_isolation ON public.dep_health_findings; +CREATE POLICY tenant_isolation ON public.dep_health_findings + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)); + +-- Read grant for the live dashboard reader. RLS still filters every +-- row this role can see -- the grant is what makes the RLS-scoped read path +-- reachable at all, not a widening of it. OMN-14894: sibling grant present on +-- context_roi_scores/instruction_eval_aggregate_snapshots/skill_execution_snapshots +-- in the same PR; dep_health_findings omitted it in error. +GRANT SELECT ON public.dep_health_findings TO app_dashboard; diff --git a/docker/migrations/forward/nodes/node_projection_event_chain/0001_create_event_chain.sql b/docker/migrations/forward/nodes/node_projection_event_chain/0001_create_event_chain.sql index 68b50badae..7da03f26ff 100644 --- a/docker/migrations/forward/nodes/node_projection_event_chain/0001_create_event_chain.sql +++ b/docker/migrations/forward/nodes/node_projection_event_chain/0001_create_event_chain.sql @@ -39,6 +39,84 @@ CREATE TABLE IF NOT EXISTS event_chain ( CONSTRAINT non_negative_event_chain_sequence CHECK (sequence >= 0) ); +-- ---- BEGIN OMN-15376 shape reconciliation: event_chain ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE event_chain ADD COLUMN IF NOT EXISTS correlation_id VARCHAR(256); +ALTER TABLE event_chain ADD COLUMN IF NOT EXISTS envelope_id VARCHAR(256); +ALTER TABLE event_chain ADD COLUMN IF NOT EXISTS sequence INTEGER DEFAULT 0; +ALTER TABLE event_chain ADD COLUMN IF NOT EXISTS topic TEXT DEFAULT ''; +ALTER TABLE event_chain ADD COLUMN IF NOT EXISTS source_node TEXT DEFAULT 'unknown'; +ALTER TABLE event_chain ADD COLUMN IF NOT EXISTS causation_id VARCHAR(256) DEFAULT ''; +ALTER TABLE event_chain ADD COLUMN IF NOT EXISTS captured_at TIMESTAMPTZ; +ALTER TABLE event_chain ADD COLUMN IF NOT EXISTS payload JSONB DEFAULT '{}'::JSONB; +ALTER TABLE event_chain ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['correlation_id', 'envelope_id', 'sequence', 'topic', 'source_node', 'causation_id', 'captured_at', 'payload', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'event_chain'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'event_chain'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge event_chain.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'event_chain'::regclass AND contype = 'p' + ) THEN + ALTER TABLE event_chain ADD CONSTRAINT pk_event_chain PRIMARY KEY (correlation_id, envelope_id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'event_chain'::regclass AND conname = 'non_negative_event_chain_sequence' + ) THEN + ALTER TABLE event_chain ADD CONSTRAINT non_negative_event_chain_sequence CHECK (sequence >= 0); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: event_chain ---- + + -- Ordered chain reconstruction: filter by correlation_id, order by sequence. CREATE INDEX IF NOT EXISTS idx_event_chain_correlation_sequence ON event_chain (correlation_id, sequence); diff --git a/docker/migrations/forward/nodes/node_projection_instruction_eval/0001_create_instruction_eval_aggregate_snapshots.sql b/docker/migrations/forward/nodes/node_projection_instruction_eval/0001_create_instruction_eval_aggregate_snapshots.sql index da50740753..71da5fdfb4 100644 --- a/docker/migrations/forward/nodes/node_projection_instruction_eval/0001_create_instruction_eval_aggregate_snapshots.sql +++ b/docker/migrations/forward/nodes/node_projection_instruction_eval/0001_create_instruction_eval_aggregate_snapshots.sql @@ -48,6 +48,120 @@ CREATE TABLE IF NOT EXISTS instruction_eval_aggregate_snapshots ( CHECK (runs >= 0) ); +-- ---- BEGIN OMN-15376 shape reconciliation: instruction_eval_aggregate_snapshots ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE instruction_eval_aggregate_snapshots ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE instruction_eval_aggregate_snapshots ADD COLUMN IF NOT EXISTS model VARCHAR(256); +ALTER TABLE instruction_eval_aggregate_snapshots ADD COLUMN IF NOT EXISTS task VARCHAR(256); +ALTER TABLE instruction_eval_aggregate_snapshots ADD COLUMN IF NOT EXISTS context_mode VARCHAR(64); +ALTER TABLE instruction_eval_aggregate_snapshots ADD COLUMN IF NOT EXISTS pass_rate NUMERIC(6, 4); +ALTER TABLE instruction_eval_aggregate_snapshots ADD COLUMN IF NOT EXISTS output_tokens INTEGER DEFAULT 0; +ALTER TABLE instruction_eval_aggregate_snapshots ADD COLUMN IF NOT EXISTS runs INTEGER DEFAULT 0; +ALTER TABLE instruction_eval_aggregate_snapshots ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE instruction_eval_aggregate_snapshots ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'model', 'task', 'context_mode', 'output_tokens', 'runs', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'instruction_eval_aggregate_snapshots'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'instruction_eval_aggregate_snapshots'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge instruction_eval_aggregate_snapshots.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'instruction_eval_aggregate_snapshots'::regclass AND contype = 'p' + ) THEN + ALTER TABLE instruction_eval_aggregate_snapshots ADD CONSTRAINT instruction_eval_aggregate_snapshots_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'instruction_eval_aggregate_snapshots'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['context_mode', 'model', 'task']::text[] + ) THEN + ALTER TABLE instruction_eval_aggregate_snapshots ADD CONSTRAINT uq_instruction_eval_aggregate_model_task_mode UNIQUE (model, task, context_mode); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'instruction_eval_aggregate_snapshots'::regclass AND conname = 'chk_instruction_eval_aggregate_pass_rate' + ) THEN + ALTER TABLE instruction_eval_aggregate_snapshots ADD CONSTRAINT chk_instruction_eval_aggregate_pass_rate CHECK (pass_rate IS NULL OR (pass_rate >= 0 AND pass_rate <= 1)); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'instruction_eval_aggregate_snapshots'::regclass AND conname = 'chk_instruction_eval_aggregate_output_tokens' + ) THEN + ALTER TABLE instruction_eval_aggregate_snapshots ADD CONSTRAINT chk_instruction_eval_aggregate_output_tokens CHECK (output_tokens >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'instruction_eval_aggregate_snapshots'::regclass AND conname = 'chk_instruction_eval_aggregate_runs' + ) THEN + ALTER TABLE instruction_eval_aggregate_snapshots ADD CONSTRAINT chk_instruction_eval_aggregate_runs CHECK (runs >= 0); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: instruction_eval_aggregate_snapshots ---- + + CREATE INDEX IF NOT EXISTS idx_instruction_eval_aggregate_snapshots_model ON instruction_eval_aggregate_snapshots (model); diff --git a/docker/migrations/forward/nodes/node_projection_instruction_eval/0002_instruction_eval_aggregate_snapshots_tenant_id_and_rls.sql b/docker/migrations/forward/nodes/node_projection_instruction_eval/0002_instruction_eval_aggregate_snapshots_tenant_id_and_rls.sql new file mode 100644 index 0000000000..81bdaacf83 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_instruction_eval/0002_instruction_eval_aggregate_snapshots_tenant_id_and_rls.sql @@ -0,0 +1,113 @@ +-- OMN-15655 / operator ruling 2026-08-02 (house tenant): give instruction_eval_aggregate_snapshots a +-- tenant identity and row-level tenant isolation. +-- +-- instruction-eval aggregates measure a tenant's own model/task runs THIS RELATION IS TENANT DATA +-- WHY, so the row is workload-attributable and belongs in the tenant +-- domain per the ruling's rule. Rows produced by OmniNode's own platform +-- workloads are the omninode TENANT's rows -- OmniNode is a first-class tenant, +-- not an absence of one. Only attribution-meaningless infrastructure state +-- (migration bookkeeping, registry, orchestration state, deployment +-- evidence) stays in omninode_internal. +-- +-- This migration lands alongside the contract flip +-- `db_io.db_tables[].schema: public -> tenant`. Before it, the relation +-- declared a schema the typed topology does not declare, which ADR-0027 +-- refuses by design -- `ValueError: Unknown schema 'public' for +-- database_ref 'application'` out of `ModelDeploymentTopology.schema_domain`. +-- +-- IDENTITY TYPE -- TEXT, NOT UUID, AND THAT IS DELIBERATE +-- Every landed tenant_id column on this surface is `TEXT` holding the slug +-- 'omninode' (delegation 0022/0025, savings 080, registration 0002, +-- inference-response 0002, omnidash 0001_tenant_rls), and the RLS policies +-- that already shipped (0023, 0026) compare TEXT with no `::uuid` cast. +-- OMN-15356 converts that whole set to the canonical UUID in ONE pass. +-- Adding a UUID column here alone would fork tenant identity inside a single +-- database -- the exact "one canonical model per shape" violation. The +-- canonical UUID this slug resolves to is pinned in +-- `omnimarket.projection.tenant_isolation.omninode_TENANT_UUID`. +-- +-- BACKFILL -- NO BATCHING NEEDED, AND BATCHING WOULD BE WORSE +-- `ADD COLUMN ... NOT NULL DEFAULT` with a non-volatile default does not +-- rewrite the table on PostgreSQL 11+; the default is stored in +-- `pg_attribute.attmissingval` and every pre-existing row reads back as +-- the house tenant immediately. A batched `UPDATE` would do strictly more +-- work (a real rewrite, dead tuples, a longer lock) for the same end state. +-- +-- PHYSICAL SCHEMA IS NOT MOVED HERE -- SEE OMN-15359 +-- No `ALTER TABLE ... SET SCHEMA tenant`. The `tenant` schema is created +-- by no applied migration in this repo or in omnibase_infra +-- (`docker/migrations/forward/_ledger/bootstrap.sql` creates only +-- `platform_catalog`); it exists solely in proof fixtures. Every relation +-- already classified TENANT -- delegation_events, delegation_budget_state, +-- savings_estimates -- is still physically in `public` +-- (`current_schema: ["public"]` in the OMN-15423 inventory, against +-- `target_schema: tenant`). Relocating these eight alone would create the +-- split the inventory's target/current split exists to prevent, and would +-- fail outright against a database with no `tenant` schema. The physical +-- move is OMN-15359 ("Build classified schemas and migrate internal, +-- control-plane, catalog, and tenant targets"), which moves the whole set, +-- with its trigger functions and sequences, in one governed cutover. +-- +-- FAIL-CLOSED RATCHET +-- The column DEFAULT is what supplies the house tenant today, exactly as it +-- does for savings_estimates (OMN-14058, operator-accepted): a writer that +-- resolves no tenant OMITS the key and Postgres fills it -- the key is never +-- written as NULL. That default-allowed state is PINNED, with its flip +-- condition named, by +-- `tests/unit/projection/test_house_tenant_default_ratchet.py`. When +-- customer ingress exists the writer boundary stops defaulting and refuses +-- instead; that test fails the moment the precondition changes. +-- +-- BLAST RADIUS -- FORCE constrains the table OWNER too +-- Same caveat as migration 0023: a writer connected as the `postgres` +-- SUPERUSER (the compose lanes) bypasses RLS regardless of FORCE. The real +-- isolation boundary is this policy PLUS a non-superuser, NOBYPASSRLS writer +-- role (OMN-14899 / OMN-15425). +-- +-- Fail-closed: `current_setting('app.tenant_id', true)` is NULL when the GUC +-- is unset, the predicate is NULL, and zero rows are visible. The policy has no +-- default-tenant fallback, by design. +-- +-- Idempotent: ADD COLUMN / CREATE INDEX are IF NOT EXISTS, ENABLE/FORCE are +-- idempotent, the policy is DROP + CREATE, GRANTs are idempotent. + +ALTER TABLE public.instruction_eval_aggregate_snapshots + ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'omninode'; + +CREATE INDEX IF NOT EXISTS idx_instruction_eval_aggregate_snapshots_tenant_id + ON public.instruction_eval_aggregate_snapshots (tenant_id); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'app_dashboard' + AND NOT rolsuper + AND NOT rolbypassrls + ) THEN + RAISE EXCEPTION + 'app_dashboard role missing or RLS-bypassing - apply omnibase_infra forward migration ' + '094_create_app_dashboard_role.sql (OMN-14899) before this RLS ' + 'migration. RLS grants without the constrained read role are the ' + 'exact bypass this work exists to prevent.'; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA public TO app_dashboard; + +ALTER TABLE public.instruction_eval_aggregate_snapshots ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.instruction_eval_aggregate_snapshots FORCE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS tenant_isolation ON public.instruction_eval_aggregate_snapshots; +CREATE POLICY tenant_isolation ON public.instruction_eval_aggregate_snapshots + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)); + +-- Read grant for the live dashboard reader (a live dashboard reader consumes instruction-eval aggregates). RLS still filters every +-- row this role can see -- the grant is what makes the RLS-scoped read path +-- reachable at all, not a widening of it. Granted only where a reader is known +-- to exist; the relations with no known reader get no SELECT. +GRANT SELECT ON public.instruction_eval_aggregate_snapshots TO app_dashboard; diff --git a/docker/migrations/forward/nodes/node_projection_intent_classification/0000_create_intent_classification_events.sql b/docker/migrations/forward/nodes/node_projection_intent_classification/0000_create_intent_classification_events.sql index dc23062b0f..24779b8e79 100644 --- a/docker/migrations/forward/nodes/node_projection_intent_classification/0000_create_intent_classification_events.sql +++ b/docker/migrations/forward/nodes/node_projection_intent_classification/0000_create_intent_classification_events.sql @@ -17,6 +17,91 @@ CREATE TABLE IF NOT EXISTS intent_classification_events ( updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: intent_classification_events ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE intent_classification_events ADD COLUMN IF NOT EXISTS id BIGSERIAL; +ALTER TABLE intent_classification_events ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE intent_classification_events ADD COLUMN IF NOT EXISTS session_id TEXT; +ALTER TABLE intent_classification_events ADD COLUMN IF NOT EXISTS intent_class TEXT; +ALTER TABLE intent_classification_events ADD COLUMN IF NOT EXISTS confidence FLOAT DEFAULT 0.0; +ALTER TABLE intent_classification_events ADD COLUMN IF NOT EXISTS keywords TEXT[] DEFAULT '{}'; +ALTER TABLE intent_classification_events ADD COLUMN IF NOT EXISTS emitted_at TIMESTAMPTZ; +ALTER TABLE intent_classification_events ADD COLUMN IF NOT EXISTS ingested_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE intent_classification_events ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE intent_classification_events ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'correlation_id', 'session_id', 'intent_class', 'confidence', 'keywords', 'emitted_at', 'ingested_at', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'intent_classification_events'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'intent_classification_events'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge intent_classification_events.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'intent_classification_events'::regclass AND contype = 'p' + ) THEN + ALTER TABLE intent_classification_events ADD CONSTRAINT intent_classification_events_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'intent_classification_events'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['correlation_id']::text[] + ) THEN + ALTER TABLE intent_classification_events ADD CONSTRAINT intent_classification_events_correlation_id_key UNIQUE (correlation_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: intent_classification_events ---- + + CREATE INDEX IF NOT EXISTS idx_intent_classification_events_session_id ON intent_classification_events (session_id); diff --git a/docker/migrations/forward/nodes/node_projection_live_events/0000_create_live_events.sql b/docker/migrations/forward/nodes/node_projection_live_events/0000_create_live_events.sql index e1e81bcbfe..04158a5882 100644 --- a/docker/migrations/forward/nodes/node_projection_live_events/0000_create_live_events.sql +++ b/docker/migrations/forward/nodes/node_projection_live_events/0000_create_live_events.sql @@ -22,6 +22,91 @@ CREATE TABLE IF NOT EXISTS live_events ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: live_events ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE live_events ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE live_events ADD COLUMN IF NOT EXISTS event_id TEXT; +ALTER TABLE live_events ADD COLUMN IF NOT EXISTS type TEXT DEFAULT 'ACTION'; +ALTER TABLE live_events ADD COLUMN IF NOT EXISTS timestamp TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE live_events ADD COLUMN IF NOT EXISTS source TEXT DEFAULT 'platform'; +ALTER TABLE live_events ADD COLUMN IF NOT EXISTS topic TEXT DEFAULT ''; +ALTER TABLE live_events ADD COLUMN IF NOT EXISTS summary TEXT DEFAULT ''; +ALTER TABLE live_events ADD COLUMN IF NOT EXISTS payload TEXT DEFAULT '{}'; +ALTER TABLE live_events ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE live_events ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'event_id', 'type', 'timestamp', 'source', 'topic', 'summary', 'payload', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'live_events'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'live_events'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge live_events.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'live_events'::regclass AND contype = 'p' + ) THEN + ALTER TABLE live_events ADD CONSTRAINT live_events_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'live_events'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['event_id']::text[] + ) THEN + ALTER TABLE live_events ADD CONSTRAINT live_events_event_id_key UNIQUE (event_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: live_events ---- + + CREATE INDEX IF NOT EXISTS idx_live_events_created_at ON live_events (created_at DESC); diff --git a/docker/migrations/forward/nodes/node_projection_live_events/0001_reclassify_event_lifecycle_types.sql b/docker/migrations/forward/nodes/node_projection_live_events/0001_reclassify_event_lifecycle_types.sql new file mode 100644 index 0000000000..bcdec58512 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_live_events/0001_reclassify_event_lifecycle_types.sql @@ -0,0 +1,38 @@ +-- OMN-14974: Repair persisted live-event lifecycle classifications. +-- +-- The original reducer collapsed every topic containing "delegation" into +-- ROUTING and allowed payload ``type`` / ``event_type`` fields to override the +-- contract-declared topic. The reducer now derives lifecycle type from the +-- topic; this migration applies the same ordered taxonomy to durable rows that +-- were projected before that fix. +-- +-- This update is idempotent. It touches only rows whose stored type differs +-- from the type implied by their authoritative topic. + +WITH classified AS ( + SELECT + id, + CASE + WHEN lower(topic) LIKE '%failed%' OR lower(topic) LIKE '%error%' + THEN 'ERROR' + WHEN lower(topic) LIKE 'onex.cmd.%' + THEN 'COMMAND' + WHEN lower(topic) ~ '\.routing-decision\.v[0-9]+$' + THEN 'ROUTING' + WHEN lower(topic) ~ '\.inference-response\.v[0-9]+$' + THEN 'INFERENCE' + WHEN lower(topic) ~ '\.(quality-gate-result|delegation-judge-verdict)\.v[0-9]+$' + THEN 'EVALUATION' + WHEN lower(topic) ~ '\.(delegation[^.]*|delegate-skill-[^.]+|task-delegated)\.v[0-9]+$' + THEN 'DELEGATION' + WHEN lower(topic) LIKE '%state-change%' OR lower(topic) LIKE '%transformation%' + THEN 'TRANSFORMATION' + ELSE 'ACTION' + END AS canonical_type + FROM live_events +) +UPDATE live_events AS event +SET type = classified.canonical_type +FROM classified +WHERE event.id = classified.id + AND event.type IS DISTINCT FROM classified.canonical_type; diff --git a/docker/migrations/forward/nodes/node_projection_live_events/0002_create_omninode_internal_live_events.sql b/docker/migrations/forward/nodes/node_projection_live_events/0002_create_omninode_internal_live_events.sql new file mode 100644 index 0000000000..79b378235a --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_live_events/0002_create_omninode_internal_live_events.sql @@ -0,0 +1,392 @@ +-- ============================================================================= +-- MIGRATION: physically deliver omninode_internal.live_events through the +-- node-owned migration loop (OMN-15819) +-- ============================================================================= +-- Ticket: OMN-15819 (migration runner skip-ledgers cross-DB \connect files +-- without executing them -- 098/099 recorded "applied" in the wrong +-- database while omninode_internal.live_events was never created; +-- no code path can deliver them) +-- Related: OMN-15359 (099_create_omninode_internal_live_events.sql -- the +-- flat migration this file replaces the DELIVERY of), +-- OMN-15282 (node-owned migration discovery loop this file runs +-- through), OMN-13079 (0000_create_live_events.sql -- the DDL owner +-- of public.live_events, the table this schema mirrors) +-- Version: 1.0.0 +-- +-- WHY THIS FILE EXISTS (OMN-15819 root cause) +-- docker/migrations/forward/099_create_omninode_internal_live_events.sql +-- carries `\connect omnidash_analytics` and is a FLAT migration +-- (docker/migrations/forward/*.sql, -maxdepth 1). The k8s Job that applies +-- that corpus (omninode_infra repo, +-- k8s/migrations/omnibase-infra-migrate.yaml) owns exactly one database, +-- omnibase_infra -- its flat loop's `psql -f` apply is gated on +-- `directive_db == $DB_NAME` and is UNREACHABLE for a cross-DB file, in +-- that loop or any other in the runner. 099 has therefore never executed +-- anywhere: omnidash_analytics never saw it, and +-- `omninode_internal.live_events` was never created, live-confirmed via a +-- fresh read-only probe on the exact database this file targets +-- (2026-08-10, role_omnidash, omninode-dev-postgres RDS): `to_regclass` +-- fails closed with `permission denied for schema omninode_internal` +-- (role_omnidash has neither USAGE nor CREATE there today), and the +-- runtime write path (handler_wiring._resolve_projection_database_target, +-- which has issued `INSERT INTO omninode_internal.live_events` since +-- before 099 merged) fails every write with UndefinedTable, at ~6/min +-- paired with a DLQ event on the -effects pod. +-- +-- THIS file is a NODE-OWNED migration, vendored under +-- docker/migrations/forward/nodes/node_projection_live_events/. The +-- node-owned loop in the SAME k8s Job (OMN-15282/OMN-15313) connects +-- DIRECTLY to omnidash_analytics as role_omnidash -- it is the one code +-- path in the whole runner that can actually reach this database. That is +-- the entire fix: relocate delivery, do not relocate intent. 099 itself +-- stays in place (byte-unchanged except for a header tombstone comment, +-- OMN-15819 step 1/2) as the ledgered historical record of the original +-- design; its own transform-copy/reconciliation logic for +-- PRE-EXISTING public.live_events rows is intentionally NOT duplicated +-- here -- this file's scope is closing the UndefinedTable write-path gap, +-- not backfilling history (a distinct, separately-scoped follow-up if the +-- dashboard needs continuous history in the internal-schema copy). +-- +-- THE SCHEMA TRAP THIS FILE ASSERTS, NOT WORKS AROUND +-- omninode_internal EXISTS in omnidash_analytics today (created +-- out-of-band, owner `omninodeadmin` master; live-confirmed present via +-- pg_namespace, which needs no schema-level privilege to read) but +-- role_omnidash -- the role THIS loop connects as -- has neither USAGE +-- nor CREATE on it (both live-confirmed `false` via has_schema_privilege, +-- 2026-08-10). Only the schema OWNER (or a role with GRANT OPTION) can +-- grant those, so role_omnidash cannot self-grant its way in. The +-- one-time repair is an OPERATOR action (OMN-15819 step 3, queued, +-- out-of-band, ~30s as omninodeadmin master): +-- GRANT USAGE, CREATE ON SCHEMA omninode_internal TO role_omnidash +-- WITH GRANT OPTION; +-- WITH GRANT OPTION corrects the ticket's originally-stated recipe (plain +-- GRANT, no grant option) -- live-proven while authoring this file that +-- plain USAGE lets role_omnidash create the table but leaves its OWN +-- onward `GRANT USAGE ... TO omninode_runtime` (step 5 below) a silent +-- no-op (`WARNING: no privileges were granted`), which would leave the +-- runtime write path broken while this migration reports success. See +-- precondition 2 below for the live citation. +-- This migration does NOT attempt that grant itself -- it has no rights to +-- issue it and mis-designing around that (e.g. self-escalating privilege) +-- is explicitly out of scope. Instead it ASSERTS the precondition +-- fail-fast, by design: a migration that silently no-ops or half-applies +-- against a schema it cannot use is a worse failure mode than a named, +-- loud refusal. +-- +-- WHY EVERY PRECONDITION IS A BARE SELECT, NOT A DO/RAISE BLOCK +-- The application-database SQL gate +-- (src/omnibase_infra/validation/application_database_domain_enforcement.py +-- `_requires_dynamic_sql_rejection`) rejects ANY new `DO $$ ... $$` block +-- in a migration touching an application-topology schema, unconditionally +-- -- procedural execution cannot be proven to target only statically-known +-- relations, so it is refused outright rather than inspected case-by-case. +-- 098/099 (both DO-block-heavy) predate this gate's diff scope and were +-- never re-scanned against it. This file has no such grandfathering, so +-- every precondition below uses the same statically-provable +-- `SELECT 1 / count(*)` division-by-zero idiom 098/099 already use for +-- their own POST-conditions (OMN-15361) -- division by zero is Postgres's +-- own fail-closed primitive when the guarded condition is false, and it +-- needs no procedural block to express. The tradeoff is a generic +-- `division by zero` error instead of a hand-authored message naming the +-- exact OMN-15819 remedy; that remedy is documented here and on the +-- ticket instead of in the error text itself. +-- +-- The same constraint retired the omninode_runtime guard-created-if-absent +-- DO block 099 uses (CREATE ROLE has no IF NOT EXISTS form in Postgres, so +-- idempotent creation is a DO-block-only idiom). Provisioning +-- omninode_runtime is not this migration's concern any more than +-- provisioning omninode_internal itself is (see the schema trap above) -- +-- the role already exists on the live target (pg_roles, 2026-08-10) and is +-- now asserted, not created, exactly like the schema and privilege +-- preconditions above it. +-- +-- IDEMPOTENCY +-- CREATE TABLE IF NOT EXISTS / CREATE INDEX IF NOT EXISTS are safe to +-- re-run. Every precondition is a read-only catalog probe -- re-run is a +-- no-op once the operator grant has landed. No ALTER of any kind (other +-- than the guarded ADD COLUMN IF NOT EXISTS reconciliation block) appears +-- in this file. +-- +-- CANONICAL SHAPE (do not hand-retype, OMN-15384 shape-parity gate) +-- The CREATE TABLE body below is copied VERBATIM from +-- docker/migrations/forward/099_create_omninode_internal_live_events.sql +-- (the flat migration this file replaces the delivery of), which is +-- itself already ledgered `status: identical` in +-- docker/migrations/forward/flat-node-shape-parity.yaml against this +-- node's own bare `live_events` +-- (nodes/node_projection_live_events/0000_create_live_events.sql). Keeping +-- the two declarations byte-identical (modulo the `omninode_internal.` +-- qualifier) means this file introduces no NEW shape for that gate to +-- track -- the live overlap set and ledger are unchanged by this PR +-- (verified: tests/ci/test_flat_node_migration_shape_parity.py passes +-- unmodified). +-- +-- WHY THE omninode_runtime GRANT (runtime-owns-DB doctrine) +-- handler_wiring._resolve_projection_database_target issues +-- `INSERT INTO omninode_internal.live_events` under the omninode_runtime +-- binding -- the same write-path grant 099 itself carries (SELECT, +-- INSERT, UPDATE; no DELETE -- a projection writer upserts, it does not +-- reshape the table, matching 096's role_omnidash invariant and 099's own +-- omninode_runtime invariant). +-- ============================================================================= + +-- ----------------------------------------------------------------------------- +-- 1. Precondition: the schema must exist at all. Statically provable (no +-- DO/RAISE -- see the file-header rationale above): division by zero +-- when the schema is absent. pg_namespace needs no schema-level +-- privilege to read, so this probe is safe under any connecting role. +-- has_schema_privilege() itself RAISES (not a false/NULL result) for a +-- schema name that does not exist, which is why this check runs FIRST, +-- strictly before precondition 2 below ever calls it. +-- ----------------------------------------------------------------------------- +SELECT 1 / count(*) AS omninode_internal_schema_exists_precondition + FROM pg_catalog.pg_namespace + WHERE nspname = 'omninode_internal'; + +-- ----------------------------------------------------------------------------- +-- 2. Precondition: CURRENT_USER (whichever role is actually executing this +-- file) must hold USAGE, CREATE, and USAGE WITH GRANT OPTION on +-- omninode_internal. Checked against current_user, not a hardcoded role +-- name, because the connecting identity differs by lane: role_omnidash +-- on the k8s Job / managed RDS lane (OMN-15313 NODE_DB_USER), but the +-- `postgres` superuser on the compose lane (POSTGRES_USER default, no +-- per-role split there -- see 096's own docstring for the live readback +-- proving this). A superuser always reads back `true` for every +-- has_schema_privilege() check regardless of ACL rows, so this precondition +-- is a no-op on compose and a real, live-confirmed-false gate on RDS +-- today (role_omnidash has neither USAGE nor CREATE, 2026-08-10). +-- +-- USAGE WITH GRANT OPTION is a SEPARATE requirement from plain USAGE: +-- this migration re-grants schema USAGE onward to omninode_runtime +-- (step 5 below), and Postgres requires the granting role to hold grant +-- option for that specific privilege, not merely the privilege itself, +-- or the onward GRANT silently no-ops with `WARNING: no privileges were +-- granted for "omninode_internal"` -- proven live in an ephemeral +-- sandbox while authoring this file: a plain (non-grant-option) USAGE +-- grant to role_omnidash let this migration create the table and grant +-- TABLE-level privileges (ownership of a self-created object carries its +-- own grant rights), but the SCHEMA-level forward to omninode_runtime +-- silently granted nothing, which would have left the runtime write path +-- just as broken as before this migration (permission denied for +-- schema, not UndefinedTable) while the migration itself reported +-- success. This corrects the ticket's originally-stated operator recipe +-- (OMN-15819 step 3), which did not specify WITH GRANT OPTION -- +-- flagged on the PR and on the ticket. +-- ----------------------------------------------------------------------------- +SELECT 1 / count(*) AS omninode_internal_schema_privilege_precondition + FROM ( + SELECT 1 + WHERE has_schema_privilege(current_user, 'omninode_internal', 'USAGE') + AND has_schema_privilege(current_user, 'omninode_internal', 'CREATE') + AND has_schema_privilege( + current_user, 'omninode_internal', 'USAGE WITH GRANT OPTION' + ) + ) AS assertion; + +-- ----------------------------------------------------------------------------- +-- 3. Precondition: omninode_runtime must already exist. Provisioning it is +-- not this migration's concern (see the file-header rationale above) -- +-- the role already exists on the live target (pg_roles, 2026-08-10) and +-- is asserted, not created. +-- ----------------------------------------------------------------------------- +SELECT 1 / count(*) AS omninode_runtime_role_exists_precondition + FROM pg_catalog.pg_roles + WHERE rolname = 'omninode_runtime'; + +-- ----------------------------------------------------------------------------- +-- 4. Canonical 10-column shape, copied verbatim from +-- docker/migrations/forward/099_create_omninode_internal_live_events.sql +-- -- see the file-header rationale above. Do not hand-retype; if the +-- shape ever needs to change, change 099 first and copy forward again. +-- +-- No CREATE EXTENSION pgcrypto here (unlike 099): this node's own +-- 0000_create_live_events.sql already issues +-- `CREATE EXTENSION IF NOT EXISTS pgcrypto` (unqualified) and runs +-- strictly before this file in every scope that applies node migrations +-- at all (same node directory, sorted lexical order, same database) -- +-- the extension is a database-level object that persists across the +-- separate psql connection each file gets, so it is already installed +-- and reachable via the default search_path by the time this file runs. +-- A second, redundant CREATE EXTENSION here would also need an explicit +-- `WITH SCHEMA ` target (application-database SQL +-- gate, tests/unit/validation/test_application_database_domain_enforcement.py) +-- and a matching schema-qualified gen_random_uuid() call below -- +-- strictly worse than relying on the guarantee 0000 already provides. +-- ----------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS omninode_internal.live_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + event_id TEXT UNIQUE NOT NULL, + type TEXT NOT NULL DEFAULT 'ACTION', + timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), + source TEXT NOT NULL DEFAULT 'platform', + topic TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + payload TEXT NOT NULL DEFAULT '{}', + correlation_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- ---- BEGIN OMN-15376 shape reconciliation: omninode_internal.live_events ---- +-- CREATE TABLE IF NOT EXISTS above silently no-ops if this table already +-- exists with a different shape (schema-parity gate, +-- tests/ci/test_node_migration_shape_reconciliation.py). The schema was +-- live-confirmed EMPTY of tables at the time this file was authored +-- (2026-08-10, out-of-band creation carried only the schema itself), so this +-- is a mechanical compliance guard, not a response to observed drift -- every +-- add is a no-op on the fresh-create path this migration actually exercises. +-- +-- WHAT THIS BLOCK DOES NOT DO (OMN-15819 CodeRabbit thread r3749990744): a +-- column-level `ADD COLUMN IF NOT EXISTS` can only ever add a NULLABLE +-- column -- it cannot retroactively apply the PRIMARY KEY, the `event_id` +-- UNIQUE constraint, or any NOT NULL from the CREATE TABLE above onto a +-- pre-existing, non-canonical table, and a DO block that attempted that +-- reconciliation is exactly the procedural-execution shape the +-- application-database SQL gate rejects (see the file-header rationale). +-- Rather than silently leaving a shape-degraded table able to accept +-- duplicate/incomplete events while this migration reports success, section +-- 7 below adds statically-provable post-conditions asserting the PRIMARY +-- KEY, the `event_id` UNIQUE constraint, and every required-column NOT NULL +-- are actually present after this file runs -- on a pre-existing table +-- missing any of them, the migration now fails loudly instead of no-opping. +ALTER TABLE omninode_internal.live_events + ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE omninode_internal.live_events + ADD COLUMN IF NOT EXISTS event_id TEXT; +ALTER TABLE omninode_internal.live_events + ADD COLUMN IF NOT EXISTS type TEXT DEFAULT 'ACTION'; +ALTER TABLE omninode_internal.live_events + ADD COLUMN IF NOT EXISTS timestamp TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE omninode_internal.live_events + ADD COLUMN IF NOT EXISTS source TEXT DEFAULT 'platform'; +ALTER TABLE omninode_internal.live_events + ADD COLUMN IF NOT EXISTS topic TEXT DEFAULT ''; +ALTER TABLE omninode_internal.live_events + ADD COLUMN IF NOT EXISTS summary TEXT DEFAULT ''; +ALTER TABLE omninode_internal.live_events + ADD COLUMN IF NOT EXISTS payload TEXT DEFAULT '{}'; +ALTER TABLE omninode_internal.live_events + ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE omninode_internal.live_events + ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +-- ---- END OMN-15376 shape reconciliation: omninode_internal.live_events ---- + +-- ----------------------------------------------------------------------------- +-- 5. Indexes matching public.live_events's shape +-- (nodes/node_projection_live_events/0000_create_live_events.sql) and +-- 099's own declaration for the omninode_internal copy. +-- ----------------------------------------------------------------------------- +CREATE INDEX IF NOT EXISTS idx_omninode_internal_live_events_created_at + ON omninode_internal.live_events (created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_omninode_internal_live_events_topic + ON omninode_internal.live_events (topic); + +CREATE INDEX IF NOT EXISTS idx_omninode_internal_live_events_source + ON omninode_internal.live_events (source); + +CREATE INDEX IF NOT EXISTS idx_omninode_internal_live_events_correlation_id + ON omninode_internal.live_events (correlation_id) + WHERE correlation_id IS NOT NULL; + +-- ----------------------------------------------------------------------------- +-- 6. Shape post-conditions -- BEFORE any GRANT below, deliberately (OMN-15819 +-- CodeRabbit thread r3751589783). run-forward-migrations.sh invokes +-- `psql -v ON_ERROR_STOP=1 -f` with no --single-transaction and no +-- explicit BEGIN wrapping this file, so every statement here autocommits +-- independently as psql executes it. If these assertions ran AFTER the +-- GRANT (as in an earlier revision of this file), a failed assertion +-- would still leave the runtime write grant committed on a table this +-- migration just rejected. Ordering the grant last means ON_ERROR_STOP +-- aborts the script -- and grants nothing -- before any GRANT statement +-- is reached. +-- +-- Statically provable (no DO/RAISE), matching the OMN-15361 +-- application-database gate's requirement for deployable SQL and +-- 098/099's own convention. The constraint-shape checks close the +-- reconciliation gap the ADD COLUMN IF NOT EXISTS block above cannot: +-- it can only add NULLABLE columns, never retroactively apply a PRIMARY +-- KEY, a UNIQUE constraint, or a NOT NULL onto a pre-existing, +-- non-canonical table (OMN-15819 CodeRabbit thread r3749990744). Each +-- check asserts the EXACT column set, not merely "some constraint of +-- this type exists" (OMN-15819 CodeRabbit thread r3751589796) -- a +-- pre-existing table with e.g. PRIMARY KEY on the wrong column, or a +-- composite UNIQUE(event_id, source) instead of UNIQUE(event_id), would +-- silently pass a looser check while still allowing duplicate event_id +-- values or rejecting the canonical row shape. +-- ----------------------------------------------------------------------------- +SELECT 1 / count(*) AS omninode_internal_live_events_exists_assertion + FROM information_schema.tables + WHERE table_schema = 'omninode_internal' AND table_name = 'live_events'; + +SELECT 1 / count(*) AS omninode_internal_live_events_primary_key_assertion + FROM ( + SELECT tc.constraint_name + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON kcu.constraint_name = tc.constraint_name + AND kcu.table_schema = tc.table_schema + WHERE tc.table_schema = 'omninode_internal' + AND tc.table_name = 'live_events' + AND tc.constraint_type = 'PRIMARY KEY' + GROUP BY tc.constraint_name + HAVING count(*) = 1 AND bool_and(kcu.column_name = 'id') + ) AS assertion; + +SELECT 1 / count(*) AS omninode_internal_live_events_event_id_unique_assertion + FROM ( + SELECT tc.constraint_name + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON kcu.constraint_name = tc.constraint_name + AND kcu.table_schema = tc.table_schema + WHERE tc.table_schema = 'omninode_internal' + AND tc.table_name = 'live_events' + AND tc.constraint_type = 'UNIQUE' + GROUP BY tc.constraint_name + HAVING count(*) = 1 AND bool_and(kcu.column_name = 'event_id') + ) AS assertion; + +SELECT 1 / count(*) AS omninode_internal_live_events_id_type_and_default_assertion + FROM information_schema.columns + WHERE table_schema = 'omninode_internal' + AND table_name = 'live_events' + AND column_name = 'id' + AND data_type = 'uuid' + AND column_default IS NOT NULL; + +SELECT 1 / count(*) AS omninode_internal_live_events_not_null_columns_assertion + FROM ( + SELECT 1 + WHERE ( + SELECT count(*) + FROM information_schema.columns + WHERE table_schema = 'omninode_internal' + AND table_name = 'live_events' + AND column_name IN ( + 'event_id', 'type', 'timestamp', 'source', 'topic', + 'summary', 'payload', 'created_at' + ) + AND is_nullable = 'NO' + ) = 8 + ) AS assertion; + +-- ----------------------------------------------------------------------------- +-- 7. Runtime-owns-DB doctrine: omninode_runtime read/write grant, identical +-- scope to 099's own (SELECT, INSERT, UPDATE -- no DELETE). Deliberately +-- last -- see section 6's header for why. +-- ----------------------------------------------------------------------------- +GRANT USAGE ON SCHEMA omninode_internal TO omninode_runtime; +GRANT SELECT, INSERT, UPDATE ON omninode_internal.live_events TO omninode_runtime; + +COMMENT ON TABLE omninode_internal.live_events IS + 'Platform-wide bus event projection (internal-schema copy) -- delivered by ' + 'the node-owned migration loop per OMN-15819; feeds the omnidash ' + 'live-event-stream widget via the contract-declared omninode_internal ' + 'write path.'; + +SELECT 1 / count(*) AS omninode_runtime_live_events_insert_grant_assertion + FROM information_schema.role_table_grants + WHERE table_schema = 'omninode_internal' + AND table_name = 'live_events' + AND grantee = 'omninode_runtime' + AND privilege_type = 'INSERT'; diff --git a/docker/migrations/forward/nodes/node_projection_llm_cost/0001_create_llm_call_metrics.sql b/docker/migrations/forward/nodes/node_projection_llm_cost/0001_create_llm_call_metrics.sql index 8a5671f4c0..769d3a2ab0 100644 --- a/docker/migrations/forward/nodes/node_projection_llm_cost/0001_create_llm_call_metrics.sql +++ b/docker/migrations/forward/nodes/node_projection_llm_cost/0001_create_llm_call_metrics.sql @@ -111,6 +111,143 @@ CREATE TABLE IF NOT EXISTS llm_call_metrics ( ) ); +-- ---- BEGIN OMN-15376 shape reconciliation: llm_call_metrics ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS correlation_id UUID; +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS session_id VARCHAR(255); +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS run_id VARCHAR(255); +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS model_id VARCHAR(255); +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS prompt_tokens INTEGER; +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS completion_tokens INTEGER; +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS total_tokens INTEGER; +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS estimated_cost_usd NUMERIC(12, 6); +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS latency_ms NUMERIC(10, 2); +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS usage_source usage_source_type DEFAULT 'MISSING'; +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS usage_is_estimated BOOLEAN DEFAULT FALSE; +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS usage_raw JSONB; +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS input_hash VARCHAR(71); +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS code_version VARCHAR(64); +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS contract_version VARCHAR(64); +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS source VARCHAR(255); +ALTER TABLE llm_call_metrics ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'model_id', 'usage_source', 'usage_is_estimated', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'llm_call_metrics'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'llm_call_metrics'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge llm_call_metrics.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_call_metrics'::regclass AND contype = 'p' + ) THEN + ALTER TABLE llm_call_metrics ADD CONSTRAINT llm_call_metrics_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_call_metrics'::regclass AND conname = 'non_negative_prompt_tokens' + ) THEN + ALTER TABLE llm_call_metrics ADD CONSTRAINT non_negative_prompt_tokens CHECK ( prompt_tokens IS NULL OR prompt_tokens >= 0 ); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_call_metrics'::regclass AND conname = 'non_negative_completion_tokens' + ) THEN + ALTER TABLE llm_call_metrics ADD CONSTRAINT non_negative_completion_tokens CHECK ( completion_tokens IS NULL OR completion_tokens >= 0 ); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_call_metrics'::regclass AND conname = 'non_negative_total_tokens' + ) THEN + ALTER TABLE llm_call_metrics ADD CONSTRAINT non_negative_total_tokens CHECK ( total_tokens IS NULL OR total_tokens >= 0 ); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_call_metrics'::regclass AND conname = 'non_negative_estimated_cost_usd' + ) THEN + ALTER TABLE llm_call_metrics ADD CONSTRAINT non_negative_estimated_cost_usd CHECK ( estimated_cost_usd IS NULL OR estimated_cost_usd >= 0 ); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_call_metrics'::regclass AND conname = 'non_negative_latency_ms' + ) THEN + ALTER TABLE llm_call_metrics ADD CONSTRAINT non_negative_latency_ms CHECK (latency_ms IS NULL OR latency_ms >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_call_metrics'::regclass AND conname = 'usage_raw_size_limit' + ) THEN + ALTER TABLE llm_call_metrics ADD CONSTRAINT usage_raw_size_limit CHECK ( usage_raw IS NULL OR octet_length(usage_raw::text) <= 65536 ); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: llm_call_metrics ---- + + -- ============================================================================ -- IDEMPOTENCY: input_hash unique (mirrors omnibase_infra migration 071) -- ============================================================================ diff --git a/docker/migrations/forward/nodes/node_projection_llm_routing/0000_create_llm_routing_decisions.sql b/docker/migrations/forward/nodes/node_projection_llm_routing/0000_create_llm_routing_decisions.sql index bb593e04b2..0f972f9670 100644 --- a/docker/migrations/forward/nodes/node_projection_llm_routing/0000_create_llm_routing_decisions.sql +++ b/docker/migrations/forward/nodes/node_projection_llm_routing/0000_create_llm_routing_decisions.sql @@ -51,6 +51,128 @@ CREATE TABLE IF NOT EXISTS llm_routing_decisions ( CONSTRAINT uq_llm_routing_decisions_correlation UNIQUE (correlation_id) ); +-- ---- BEGIN OMN-15376 shape reconciliation: llm_routing_decisions ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS correlation_id UUID; +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS session_id TEXT; +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS llm_agent TEXT; +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS fuzzy_agent TEXT; +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS agreement BOOLEAN DEFAULT FALSE; +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS llm_confidence NUMERIC(5, 4); +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS fuzzy_confidence NUMERIC(5, 4); +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS llm_latency_ms INTEGER DEFAULT 0; +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS fuzzy_latency_ms INTEGER DEFAULT 0; +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS used_fallback BOOLEAN DEFAULT FALSE; +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS routing_prompt_version TEXT DEFAULT 'unknown'; +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS intent TEXT; +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS model TEXT; +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS cost_usd NUMERIC(12, 8); +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE llm_routing_decisions ADD COLUMN IF NOT EXISTS projected_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'correlation_id', 'llm_agent', 'agreement', 'llm_latency_ms', 'fuzzy_latency_ms', 'used_fallback', 'routing_prompt_version', 'created_at', 'projected_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'llm_routing_decisions'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'llm_routing_decisions'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge llm_routing_decisions.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_routing_decisions'::regclass AND conname = 'llm_routing_decisions_llm_confidence_check' + ) THEN + ALTER TABLE llm_routing_decisions ADD CONSTRAINT llm_routing_decisions_llm_confidence_check CHECK (llm_confidence IS NULL OR (llm_confidence >= 0 AND llm_confidence <= 1)); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_routing_decisions'::regclass AND conname = 'llm_routing_decisions_fuzzy_confidence_check' + ) THEN + ALTER TABLE llm_routing_decisions ADD CONSTRAINT llm_routing_decisions_fuzzy_confidence_check CHECK (fuzzy_confidence IS NULL OR (fuzzy_confidence >= 0 AND fuzzy_confidence <= 1)); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_routing_decisions'::regclass AND conname = 'llm_routing_decisions_cost_usd_check' + ) THEN + ALTER TABLE llm_routing_decisions ADD CONSTRAINT llm_routing_decisions_cost_usd_check CHECK (cost_usd IS NULL OR cost_usd >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'llm_routing_decisions'::regclass AND contype = 'p' + ) THEN + ALTER TABLE llm_routing_decisions ADD CONSTRAINT pk_llm_routing_decisions PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'llm_routing_decisions'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['correlation_id']::text[] + ) THEN + ALTER TABLE llm_routing_decisions ADD CONSTRAINT uq_llm_routing_decisions_correlation UNIQUE (correlation_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: llm_routing_decisions ---- + + CREATE INDEX IF NOT EXISTS idx_lrd_created_at ON llm_routing_decisions (created_at DESC); diff --git a/docker/migrations/forward/nodes/node_projection_mcp_tools/0001_create_mcp_tools.sql b/docker/migrations/forward/nodes/node_projection_mcp_tools/0001_create_mcp_tools.sql index f1c9e82102..6fc2a7312f 100644 --- a/docker/migrations/forward/nodes/node_projection_mcp_tools/0001_create_mcp_tools.sql +++ b/docker/migrations/forward/nodes/node_projection_mcp_tools/0001_create_mcp_tools.sql @@ -18,6 +18,92 @@ CREATE TABLE IF NOT EXISTS mcp_tools ( projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: mcp_tools ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE mcp_tools ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE mcp_tools ADD COLUMN IF NOT EXISTS tool_name TEXT; +ALTER TABLE mcp_tools ADD COLUMN IF NOT EXISTS description TEXT DEFAULT ''; +ALTER TABLE mcp_tools ADD COLUMN IF NOT EXISTS model_id TEXT DEFAULT ''; +ALTER TABLE mcp_tools ADD COLUMN IF NOT EXISTS correlation_id TEXT DEFAULT ''; +ALTER TABLE mcp_tools ADD COLUMN IF NOT EXISTS status TEXT DEFAULT 'active'; +ALTER TABLE mcp_tools ADD COLUMN IF NOT EXISTS is_active BOOLEAN DEFAULT true; +ALTER TABLE mcp_tools ADD COLUMN IF NOT EXISTS mcp_tags TEXT[] DEFAULT '{}'; +ALTER TABLE mcp_tools ADD COLUMN IF NOT EXISTS metadata JSONB DEFAULT '{}'; +ALTER TABLE mcp_tools ADD COLUMN IF NOT EXISTS registered_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE mcp_tools ADD COLUMN IF NOT EXISTS projected_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'tool_name', 'description', 'model_id', 'correlation_id', 'status', 'is_active', 'mcp_tags', 'metadata', 'registered_at', 'projected_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'mcp_tools'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'mcp_tools'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge mcp_tools.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'mcp_tools'::regclass AND contype = 'p' + ) THEN + ALTER TABLE mcp_tools ADD CONSTRAINT mcp_tools_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'mcp_tools'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['tool_name']::text[] + ) THEN + ALTER TABLE mcp_tools ADD CONSTRAINT mcp_tools_tool_name_key UNIQUE (tool_name); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: mcp_tools ---- + + CREATE INDEX IF NOT EXISTS idx_mcp_tools_status ON mcp_tools (status); diff --git a/docker/migrations/forward/nodes/node_projection_overnight/0000_create_overnight_sessions_tables.sql b/docker/migrations/forward/nodes/node_projection_overnight/0000_create_overnight_sessions_tables.sql index c0d3f628f0..f2ca34663a 100644 --- a/docker/migrations/forward/nodes/node_projection_overnight/0000_create_overnight_sessions_tables.sql +++ b/docker/migrations/forward/nodes/node_projection_overnight/0000_create_overnight_sessions_tables.sql @@ -46,6 +46,88 @@ CREATE TABLE IF NOT EXISTS overnight_sessions ( ) ); +-- ---- BEGIN OMN-15376 shape reconciliation: overnight_sessions ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS session_id TEXT; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS session_start_ts TIMESTAMPTZ; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS contract_path TEXT; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS dry_run BOOLEAN DEFAULT FALSE; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS phases_run TEXT[] DEFAULT '{}'; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS phases_failed TEXT[] DEFAULT '{}'; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS phases_skipped TEXT[] DEFAULT '{}'; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS dispatch_count INT DEFAULT 0; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS halt_reason TEXT; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS session_status TEXT DEFAULT 'in_progress'; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS session_end_ts TIMESTAMPTZ; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS accumulated_cost_usd NUMERIC(10,4) DEFAULT 0; +ALTER TABLE overnight_sessions ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['session_id', 'session_start_ts', 'dry_run', 'phases_run', 'phases_failed', 'phases_skipped', 'dispatch_count', 'session_status', 'accumulated_cost_usd', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'overnight_sessions'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'overnight_sessions'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge overnight_sessions.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'overnight_sessions'::regclass AND contype = 'p' + ) THEN + ALTER TABLE overnight_sessions ADD CONSTRAINT overnight_sessions_pkey PRIMARY KEY (session_id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'overnight_sessions'::regclass AND conname = 'valid_session_status' + ) THEN + ALTER TABLE overnight_sessions ADD CONSTRAINT valid_session_status CHECK ( session_status IN ('in_progress', 'completed', 'partial', 'failed') ); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: overnight_sessions ---- + + -- Normalized phase results table — avoids JSONB overhead, enables per-phase indexing CREATE TABLE IF NOT EXISTS overnight_session_phases ( id BIGSERIAL PRIMARY KEY, @@ -63,6 +145,94 @@ CREATE TABLE IF NOT EXISTS overnight_session_phases ( ) ); +-- ---- BEGIN OMN-15376 shape reconciliation: overnight_session_phases ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE overnight_session_phases ADD COLUMN IF NOT EXISTS id BIGSERIAL; +ALTER TABLE overnight_session_phases ADD COLUMN IF NOT EXISTS session_id TEXT; +ALTER TABLE overnight_session_phases ADD COLUMN IF NOT EXISTS phase_name TEXT; +ALTER TABLE overnight_session_phases ADD COLUMN IF NOT EXISTS phase_status TEXT; +ALTER TABLE overnight_session_phases ADD COLUMN IF NOT EXISTS duration_ms INT DEFAULT 0; +ALTER TABLE overnight_session_phases ADD COLUMN IF NOT EXISTS side_effect_summary TEXT DEFAULT ''; +ALTER TABLE overnight_session_phases ADD COLUMN IF NOT EXISTS error_message TEXT; +ALTER TABLE overnight_session_phases ADD COLUMN IF NOT EXISTS sequence_number INT DEFAULT 0; +ALTER TABLE overnight_session_phases ADD COLUMN IF NOT EXISTS recorded_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'session_id', 'phase_name', 'phase_status', 'duration_ms', 'side_effect_summary', 'sequence_number', 'recorded_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'overnight_session_phases'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'overnight_session_phases'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge overnight_session_phases.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'overnight_session_phases'::regclass AND contype = 'p' + ) THEN + ALTER TABLE overnight_session_phases ADD CONSTRAINT overnight_session_phases_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'overnight_session_phases'::regclass AND conname = 'overnight_session_phases_session_id_fkey' + ) THEN + ALTER TABLE overnight_session_phases ADD CONSTRAINT overnight_session_phases_session_id_fkey FOREIGN KEY (session_id) REFERENCES overnight_sessions(session_id) ON DELETE CASCADE; + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'overnight_session_phases'::regclass AND conname = 'valid_phase_status' + ) THEN + ALTER TABLE overnight_session_phases ADD CONSTRAINT valid_phase_status CHECK ( phase_status IN ('success', 'failed', 'skipped') ); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: overnight_session_phases ---- + + CREATE UNIQUE INDEX IF NOT EXISTS ix_session_phases_unique ON overnight_session_phases(session_id, phase_name, sequence_number); diff --git a/docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql b/docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql index 976b9d0b17..c520111d39 100644 --- a/docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql +++ b/docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql @@ -34,6 +34,97 @@ CREATE TABLE IF NOT EXISTS pattern_learning_artifacts ( CONSTRAINT uq_pattern_learning_pattern_id UNIQUE (pattern_id) ); +-- ---- BEGIN OMN-15376 shape reconciliation: pattern_learning_artifacts ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS pattern_id UUID; +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS pattern_name VARCHAR(255) DEFAULT ''; +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS pattern_type VARCHAR(100) DEFAULT ''; +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS language VARCHAR(50); +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS lifecycle_state TEXT DEFAULT 'candidate'; +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS state_changed_at TIMESTAMPTZ; +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS composite_score NUMERIC(10, 6) DEFAULT 0; +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS scoring_evidence JSONB DEFAULT '{}'; +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS signature JSONB DEFAULT '{}'; +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS metrics JSONB DEFAULT '{}'; +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS metadata JSONB DEFAULT '{}'; +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE pattern_learning_artifacts ADD COLUMN IF NOT EXISTS projected_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'pattern_id', 'pattern_name', 'pattern_type', 'lifecycle_state', 'composite_score', 'scoring_evidence', 'signature', 'created_at', 'updated_at', 'projected_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'pattern_learning_artifacts'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'pattern_learning_artifacts'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge pattern_learning_artifacts.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'pattern_learning_artifacts'::regclass AND contype = 'p' + ) THEN + ALTER TABLE pattern_learning_artifacts ADD CONSTRAINT pk_pattern_learning_artifacts PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'pattern_learning_artifacts'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['pattern_id']::text[] + ) THEN + ALTER TABLE pattern_learning_artifacts ADD CONSTRAINT uq_pattern_learning_pattern_id UNIQUE (pattern_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: pattern_learning_artifacts ---- + + -- Backfill correlation_id on pre-existing deployments of this table that were -- created by the legacy omnibase_infra 064 migration (which lacked the column). ALTER TABLE pattern_learning_artifacts diff --git a/docker/migrations/forward/nodes/node_projection_pattern_learning/0001_pattern_learning_artifacts_tenant_id_and_rls.sql b/docker/migrations/forward/nodes/node_projection_pattern_learning/0001_pattern_learning_artifacts_tenant_id_and_rls.sql new file mode 100644 index 0000000000..845ba6ba99 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_pattern_learning/0001_pattern_learning_artifacts_tenant_id_and_rls.sql @@ -0,0 +1,114 @@ +-- OMN-15655 / operator ruling 2026-08-02 (house tenant): give pattern_learning_artifacts a +-- tenant identity and row-level tenant isolation. +-- +-- learned patterns are derived from a tenant's own execution history THIS RELATION IS TENANT DATA +-- WHY, so the row is workload-attributable and belongs in the tenant +-- domain per the ruling's rule. Rows produced by OmniNode's own platform +-- workloads are the omninode TENANT's rows -- OmniNode is a first-class tenant, +-- not an absence of one. Only attribution-meaningless infrastructure state +-- (migration bookkeeping, registry, orchestration state, deployment +-- evidence) stays in omninode_internal. +-- +-- This migration lands alongside the contract flip +-- `db_io.db_tables[].schema: public -> tenant`. Before it, the relation +-- declared a schema the typed topology does not declare, which ADR-0027 +-- refuses by design -- `ValueError: Unknown schema 'public' for +-- database_ref 'application'` out of `ModelDeploymentTopology.schema_domain`. +-- +-- IDENTITY TYPE -- TEXT, NOT UUID, AND THAT IS DELIBERATE +-- Every landed tenant_id column on this surface is `TEXT` holding the slug +-- 'omninode' (delegation 0022/0025, savings 080, registration 0002, +-- inference-response 0002, omnidash 0001_tenant_rls), and the RLS policies +-- that already shipped (0023, 0026) compare TEXT with no `::uuid` cast. +-- OMN-15356 converts that whole set to the canonical UUID in ONE pass. +-- Adding a UUID column here alone would fork tenant identity inside a single +-- database -- the exact "one canonical model per shape" violation. The +-- canonical UUID this slug resolves to is pinned in +-- `omnimarket.projection.tenant_isolation.omninode_TENANT_UUID`. +-- +-- BACKFILL -- NO BATCHING NEEDED, AND BATCHING WOULD BE WORSE +-- `ADD COLUMN ... NOT NULL DEFAULT` with a non-volatile default does not +-- rewrite the table on PostgreSQL 11+; the default is stored in +-- `pg_attribute.attmissingval` and every pre-existing row reads back as +-- the house tenant immediately. A batched `UPDATE` would do strictly more +-- work (a real rewrite, dead tuples, a longer lock) for the same end state. +-- +-- PHYSICAL SCHEMA IS NOT MOVED HERE -- SEE OMN-15359 +-- No `ALTER TABLE ... SET SCHEMA tenant`. The `tenant` schema is created +-- by no applied migration in this repo or in omnibase_infra +-- (`docker/migrations/forward/_ledger/bootstrap.sql` creates only +-- `platform_catalog`); it exists solely in proof fixtures. Every relation +-- already classified TENANT -- delegation_events, delegation_budget_state, +-- savings_estimates -- is still physically in `public` +-- (`current_schema: ["public"]` in the OMN-15423 inventory, against +-- `target_schema: tenant`). Relocating these eight alone would create the +-- split the inventory's target/current split exists to prevent, and would +-- fail outright against a database with no `tenant` schema. The physical +-- move is OMN-15359 ("Build classified schemas and migrate internal, +-- control-plane, catalog, and tenant targets"), which moves the whole set, +-- with its trigger functions and sequences, in one governed cutover. +-- +-- FAIL-CLOSED RATCHET +-- The column DEFAULT is what supplies the house tenant today, exactly as it +-- does for savings_estimates (OMN-14058, operator-accepted): a writer that +-- resolves no tenant OMITS the key and Postgres fills it -- the key is never +-- written as NULL. That default-allowed state is PINNED, with its flip +-- condition named, by +-- `tests/unit/projection/test_house_tenant_default_ratchet.py`. When +-- customer ingress exists the writer boundary stops defaulting and refuses +-- instead; that test fails the moment the precondition changes. +-- +-- BLAST RADIUS -- FORCE constrains the table OWNER too +-- Same caveat as migration 0023: a writer connected as the `postgres` +-- SUPERUSER (the compose lanes) bypasses RLS regardless of FORCE. The real +-- isolation boundary is this policy PLUS a non-superuser, NOBYPASSRLS writer +-- role (OMN-14899 / OMN-15425). +-- +-- Fail-closed: `current_setting('app.tenant_id', true)` is NULL when the GUC +-- is unset, the predicate is NULL, and zero rows are visible. The policy has no +-- default-tenant fallback, by design. +-- +-- Idempotent: ADD COLUMN / CREATE INDEX are IF NOT EXISTS, ENABLE/FORCE are +-- idempotent, the policy is DROP + CREATE, GRANTs are idempotent. + +ALTER TABLE public.pattern_learning_artifacts + ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'omninode'; + +CREATE INDEX IF NOT EXISTS idx_pattern_learning_artifacts_tenant_id + ON public.pattern_learning_artifacts (tenant_id); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'app_dashboard' + AND NOT rolsuper + AND NOT rolbypassrls + ) THEN + RAISE EXCEPTION + 'app_dashboard role missing or RLS-bypassing - apply omnibase_infra forward migration ' + '094_create_app_dashboard_role.sql (OMN-14899) before this RLS ' + 'migration. RLS grants without the constrained read role are the ' + 'exact bypass this work exists to prevent.'; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA public TO app_dashboard; + +ALTER TABLE public.pattern_learning_artifacts ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.pattern_learning_artifacts FORCE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS tenant_isolation ON public.pattern_learning_artifacts; +CREATE POLICY tenant_isolation ON public.pattern_learning_artifacts + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)); + +-- Read grant for the live dashboard reader. RLS still filters every +-- row this role can see -- the grant is what makes the RLS-scoped read path +-- reachable at all, not a widening of it. OMN-14894: sibling grant present on +-- context_roi_scores/instruction_eval_aggregate_snapshots/skill_execution_snapshots +-- in the same PR; pattern_learning_artifacts omitted it in error. +GRANT SELECT ON public.pattern_learning_artifacts TO app_dashboard; diff --git a/docker/migrations/forward/nodes/node_projection_receipt_gate/0000_create_receipt_gate_projection_table.sql b/docker/migrations/forward/nodes/node_projection_receipt_gate/0000_create_receipt_gate_projection_table.sql index d0cfc9ac8b..d24e556377 100644 --- a/docker/migrations/forward/nodes/node_projection_receipt_gate/0000_create_receipt_gate_projection_table.sql +++ b/docker/migrations/forward/nodes/node_projection_receipt_gate/0000_create_receipt_gate_projection_table.sql @@ -20,6 +20,76 @@ CREATE TABLE IF NOT EXISTS receipt_gate_rows ( observed_at TIMESTAMPTZ NOT NULL ); +-- ---- BEGIN OMN-15376 shape reconciliation: receipt_gate_rows ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE receipt_gate_rows ADD COLUMN IF NOT EXISTS id BIGSERIAL; +ALTER TABLE receipt_gate_rows ADD COLUMN IF NOT EXISTS name TEXT; +ALTER TABLE receipt_gate_rows ADD COLUMN IF NOT EXISTS pass BOOLEAN; +ALTER TABLE receipt_gate_rows ADD COLUMN IF NOT EXISTS detail TEXT DEFAULT ''; +ALTER TABLE receipt_gate_rows ADD COLUMN IF NOT EXISTS pr_ref TEXT; +ALTER TABLE receipt_gate_rows ADD COLUMN IF NOT EXISTS worker TEXT; +ALTER TABLE receipt_gate_rows ADD COLUMN IF NOT EXISTS verifier TEXT; +ALTER TABLE receipt_gate_rows ADD COLUMN IF NOT EXISTS evidence_count INTEGER; +ALTER TABLE receipt_gate_rows ADD COLUMN IF NOT EXISTS evidence_hash TEXT; +ALTER TABLE receipt_gate_rows ADD COLUMN IF NOT EXISTS signed_at TEXT; +ALTER TABLE receipt_gate_rows ADD COLUMN IF NOT EXISTS observed_at TIMESTAMPTZ; + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'name', 'pass', 'detail', 'observed_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'receipt_gate_rows'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'receipt_gate_rows'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge receipt_gate_rows.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'receipt_gate_rows'::regclass AND contype = 'p' + ) THEN + ALTER TABLE receipt_gate_rows ADD CONSTRAINT receipt_gate_rows_pkey PRIMARY KEY (id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: receipt_gate_rows ---- + + CREATE INDEX IF NOT EXISTS receipt_gate_rows_observed_at_idx ON receipt_gate_rows (observed_at DESC); diff --git a/docker/migrations/forward/nodes/node_projection_registration/0003_reconcile_heartbeat_observability.sql b/docker/migrations/forward/nodes/node_projection_registration/0003_reconcile_heartbeat_observability.sql new file mode 100644 index 0000000000..dcc1ec8625 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_registration/0003_reconcile_heartbeat_observability.sql @@ -0,0 +1,28 @@ +-- OMN-14974: reconcile a staging database whose node migration ledger drifted +-- ahead of the materialized node_service_registry shape. This net-new migration +-- is intentionally idempotent so the runner applies it even when historical +-- 0001 is recorded while its columns are absent. + +-- The registration owner migrations are operator-fenced while the RLS ruling is +-- unresolved. A fresh database therefore does not have the base table. Warm +-- databases do, and are the only databases this reconciliation is meant to +-- repair. Keep the migration deterministic on both paths. +DO $reconcile$ +BEGIN + IF to_regclass('public.node_service_registry') IS NOT NULL THEN + ALTER TABLE node_service_registry + ADD COLUMN IF NOT EXISTS last_heartbeat_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS uptime_seconds BIGINT NOT NULL DEFAULT 0; + + UPDATE node_service_registry + SET last_heartbeat_at = last_health_check + WHERE last_heartbeat_at IS NULL + AND last_health_check IS NOT NULL; + + CREATE INDEX IF NOT EXISTS idx_node_service_registry_last_heartbeat_at + ON node_service_registry (last_heartbeat_at); + ELSE + RAISE NOTICE 'node_service_registry is absent; heartbeat reconciliation is a no-op'; + END IF; +END +$reconcile$; diff --git a/docker/migrations/forward/nodes/node_projection_registration/0004_node_service_registry_no_force_rls.sql b/docker/migrations/forward/nodes/node_projection_registration/0004_node_service_registry_no_force_rls.sql new file mode 100644 index 0000000000..f63fc1acb3 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_registration/0004_node_service_registry_no_force_rls.sql @@ -0,0 +1,57 @@ +-- OMN-15336 item 4 (required-fix #4) / operator ruling R-q (2026-08-05, +-- rolling ledger 23:45Z): registry and orchestration state +-- (node_service_registry) is omninode_internal runtime state, not tenant +-- data. FORCE ROW LEVEL SECURITY on internal-domain runtime state is the +-- wrong posture: it forces EVERY reader, including the table owner, through +-- a tenant_id predicate that internal registry rows do not carry +-- meaningfully (0002 stamped a DEFAULT 'omninode' single-tenant value — +-- there is no real multi-tenant partition of "which node is running where"). +-- +-- This reverses ONLY the FORCE half of 0002_node_service_registry_tenant_rls +-- (leaves RLS ENABLEd and the tenant_isolation POLICY in place, so a future +-- tenant-scoped consumer is not blocked from reintroducing FORCE explicitly +-- with a considered ruling of its own) — narrower than DISABLE ROW LEVEL +-- SECURITY or DROP POLICY, matching item 4's own recommendation "(b) strip +-- FORCE-RLS" precisely rather than removing the RLS posture wholesale. +-- +-- Idempotent on both paths, matching 0003's own precedent: the registration +-- trio (0000/0001/0002) is operator-fenced by default (OMN-15379/OMN-15349) +-- and released only on the compose dev lane and the k8s onex-dev Job +-- (ruling 21) — a fresh, still-fenced database has no node_service_registry +-- table at all, and this migration must be a deterministic no-op there, not +-- an error. +-- +-- Does NOT need a fence-manifest entry: the OMN-15336 item-4 guard +-- (migration_declares_unclassified_force_rls in run-forward-migrations.sh) +-- only refuses migrations that ENABLE FORCE ROW LEVEL SECURITY without a +-- fence entry. `NO FORCE ROW LEVEL SECURITY` is the disabling form and is +-- explicitly excluded from that match — the guard exists to gate the +-- hazard this migration removes, not to block its own remedy. +-- +-- Reconciles with ruling 15 (OMN-15379, "the registration trio moves +-- together") and ruling 21 (OMN-15332 comment 1a067542, the k8s durable +-- release): neither ruling is reversed by this migration. Both rulings +-- govern whether 0000/0001/0002 apply at all (fence membership); this +-- migration only changes the FORCE posture of a table that has already been +-- released and created, and ships unfenced so it applies on every lane +-- where the trio has already landed, converging them to the corrected +-- posture without re-litigating the release decision itself. +-- +-- Ticket: OMN-15336 (item 4). Introduced-by: OMN-15343 (0002's FORCE). +-- Domain corroboration: node_projection_registration/contract.yaml +-- db_io.schema=omninode_internal; 2026-08-02 operator domain ruling +-- ("registry... stays omninode_internal"); OMN-15656's landed +-- grants-derivation correction ("node_service_registry/live_events are +-- internal not tenant"); R-q's generalized precedent ("FORCE-RLS on +-- internal-domain runtime state is wrong"). +DO $unforce_registry_rls$ +BEGIN + IF to_regclass('public.node_service_registry') IS NOT NULL THEN + ALTER TABLE public.node_service_registry NO FORCE ROW LEVEL SECURITY; + ELSE + RAISE NOTICE + 'node_service_registry is absent; FORCE-RLS reversal is a no-op ' + '(the registration trio is still fenced on this database)'; + END IF; +END +$unforce_registry_rls$; diff --git a/docker/migrations/forward/nodes/node_projection_routing_decision/0021_create_agent_routing_decisions.sql b/docker/migrations/forward/nodes/node_projection_routing_decision/0021_create_agent_routing_decisions.sql index fe97833b59..3c6168131b 100644 --- a/docker/migrations/forward/nodes/node_projection_routing_decision/0021_create_agent_routing_decisions.sql +++ b/docker/migrations/forward/nodes/node_projection_routing_decision/0021_create_agent_routing_decisions.sql @@ -51,6 +51,78 @@ CREATE TABLE IF NOT EXISTS agent_routing_decisions ( claude_session_id VARCHAR(255) ); +-- ---- BEGIN OMN-15376 shape reconciliation: agent_routing_decisions ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS id UUID; +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS correlation_id UUID; +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS selected_agent VARCHAR(255); +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS confidence_score DECIMAL(5, 4); +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS request_type VARCHAR(100); +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS alternatives JSONB; +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS routing_reason TEXT; +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS domain VARCHAR(255); +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS metadata JSONB; +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS project_path TEXT; +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS project_name VARCHAR(255); +ALTER TABLE agent_routing_decisions ADD COLUMN IF NOT EXISTS claude_session_id VARCHAR(255); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'agent_routing_decisions'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'agent_routing_decisions'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge agent_routing_decisions.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'agent_routing_decisions'::regclass AND contype = 'p' + ) THEN + ALTER TABLE agent_routing_decisions ADD CONSTRAINT agent_routing_decisions_pkey PRIMARY KEY (id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: agent_routing_decisions ---- + + -- Minimal indexing for write-heavy workload - only TTL cleanup index. CREATE INDEX IF NOT EXISTS idx_agent_routing_decisions_created_at ON agent_routing_decisions (created_at); diff --git a/docker/migrations/forward/nodes/node_projection_routing_decision/0022_agent_routing_decisions_tenant_id_and_rls.sql b/docker/migrations/forward/nodes/node_projection_routing_decision/0022_agent_routing_decisions_tenant_id_and_rls.sql new file mode 100644 index 0000000000..676745688d --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_routing_decision/0022_agent_routing_decisions_tenant_id_and_rls.sql @@ -0,0 +1,114 @@ +-- OMN-15655 / operator ruling 2026-08-02 (house tenant): give agent_routing_decisions a +-- tenant identity and row-level tenant isolation. +-- +-- a routing decision is made for a specific workload THIS RELATION IS TENANT DATA +-- WHY, so the row is workload-attributable and belongs in the tenant +-- domain per the ruling's rule. Rows produced by OmniNode's own platform +-- workloads are the omninode TENANT's rows -- OmniNode is a first-class tenant, +-- not an absence of one. Only attribution-meaningless infrastructure state +-- (migration bookkeeping, registry, orchestration state, deployment +-- evidence) stays in omninode_internal. +-- +-- This migration lands alongside the contract flip +-- `db_io.db_tables[].schema: public -> tenant`. Before it, the relation +-- declared a schema the typed topology does not declare, which ADR-0027 +-- refuses by design -- `ValueError: Unknown schema 'public' for +-- database_ref 'application'` out of `ModelDeploymentTopology.schema_domain`. +-- +-- IDENTITY TYPE -- TEXT, NOT UUID, AND THAT IS DELIBERATE +-- Every landed tenant_id column on this surface is `TEXT` holding the slug +-- 'omninode' (delegation 0022/0025, savings 080, registration 0002, +-- inference-response 0002, omnidash 0001_tenant_rls), and the RLS policies +-- that already shipped (0023, 0026) compare TEXT with no `::uuid` cast. +-- OMN-15356 converts that whole set to the canonical UUID in ONE pass. +-- Adding a UUID column here alone would fork tenant identity inside a single +-- database -- the exact "one canonical model per shape" violation. The +-- canonical UUID this slug resolves to is pinned in +-- `omnimarket.projection.tenant_isolation.omninode_TENANT_UUID`. +-- +-- BACKFILL -- NO BATCHING NEEDED, AND BATCHING WOULD BE WORSE +-- `ADD COLUMN ... NOT NULL DEFAULT` with a non-volatile default does not +-- rewrite the table on PostgreSQL 11+; the default is stored in +-- `pg_attribute.attmissingval` and every pre-existing row reads back as +-- the house tenant immediately. A batched `UPDATE` would do strictly more +-- work (a real rewrite, dead tuples, a longer lock) for the same end state. +-- +-- PHYSICAL SCHEMA IS NOT MOVED HERE -- SEE OMN-15359 +-- No `ALTER TABLE ... SET SCHEMA tenant`. The `tenant` schema is created +-- by no applied migration in this repo or in omnibase_infra +-- (`docker/migrations/forward/_ledger/bootstrap.sql` creates only +-- `platform_catalog`); it exists solely in proof fixtures. Every relation +-- already classified TENANT -- delegation_events, delegation_budget_state, +-- savings_estimates -- is still physically in `public` +-- (`current_schema: ["public"]` in the OMN-15423 inventory, against +-- `target_schema: tenant`). Relocating these eight alone would create the +-- split the inventory's target/current split exists to prevent, and would +-- fail outright against a database with no `tenant` schema. The physical +-- move is OMN-15359 ("Build classified schemas and migrate internal, +-- control-plane, catalog, and tenant targets"), which moves the whole set, +-- with its trigger functions and sequences, in one governed cutover. +-- +-- FAIL-CLOSED RATCHET +-- The column DEFAULT is what supplies the house tenant today, exactly as it +-- does for savings_estimates (OMN-14058, operator-accepted): a writer that +-- resolves no tenant OMITS the key and Postgres fills it -- the key is never +-- written as NULL. That default-allowed state is PINNED, with its flip +-- condition named, by +-- `tests/unit/projection/test_house_tenant_default_ratchet.py`. When +-- customer ingress exists the writer boundary stops defaulting and refuses +-- instead; that test fails the moment the precondition changes. +-- +-- BLAST RADIUS -- FORCE constrains the table OWNER too +-- Same caveat as migration 0023: a writer connected as the `postgres` +-- SUPERUSER (the compose lanes) bypasses RLS regardless of FORCE. The real +-- isolation boundary is this policy PLUS a non-superuser, NOBYPASSRLS writer +-- role (OMN-14899 / OMN-15425). +-- +-- Fail-closed: `current_setting('app.tenant_id', true)` is NULL when the GUC +-- is unset, the predicate is NULL, and zero rows are visible. The policy has no +-- default-tenant fallback, by design. +-- +-- Idempotent: ADD COLUMN / CREATE INDEX are IF NOT EXISTS, ENABLE/FORCE are +-- idempotent, the policy is DROP + CREATE, GRANTs are idempotent. + +ALTER TABLE public.agent_routing_decisions + ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'omninode'; + +CREATE INDEX IF NOT EXISTS idx_agent_routing_decisions_tenant_id + ON public.agent_routing_decisions (tenant_id); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'app_dashboard' + AND NOT rolsuper + AND NOT rolbypassrls + ) THEN + RAISE EXCEPTION + 'app_dashboard role missing or RLS-bypassing - apply omnibase_infra forward migration ' + '094_create_app_dashboard_role.sql (OMN-14899) before this RLS ' + 'migration. RLS grants without the constrained read role are the ' + 'exact bypass this work exists to prevent.'; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA public TO app_dashboard; + +ALTER TABLE public.agent_routing_decisions ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.agent_routing_decisions FORCE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS tenant_isolation ON public.agent_routing_decisions; +CREATE POLICY tenant_isolation ON public.agent_routing_decisions + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)); + +-- Read grant for the live dashboard reader. RLS still filters every +-- row this role can see -- the grant is what makes the RLS-scoped read path +-- reachable at all, not a widening of it. OMN-14894: sibling grant present on +-- context_roi_scores/instruction_eval_aggregate_snapshots/skill_execution_snapshots +-- in the same PR; agent_routing_decisions omitted it in error. +GRANT SELECT ON public.agent_routing_decisions TO app_dashboard; diff --git a/docker/migrations/forward/nodes/node_projection_sandbox_decisions/0001_create_sandbox_decisions.sql b/docker/migrations/forward/nodes/node_projection_sandbox_decisions/0001_create_sandbox_decisions.sql index 12431945d3..89bcb7fce7 100644 --- a/docker/migrations/forward/nodes/node_projection_sandbox_decisions/0001_create_sandbox_decisions.sql +++ b/docker/migrations/forward/nodes/node_projection_sandbox_decisions/0001_create_sandbox_decisions.sql @@ -15,6 +15,92 @@ CREATE TABLE IF NOT EXISTS sandbox_decisions ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: sandbox_decisions ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE sandbox_decisions ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE sandbox_decisions ADD COLUMN IF NOT EXISTS node_name TEXT; +ALTER TABLE sandbox_decisions ADD COLUMN IF NOT EXISTS status TEXT; +ALTER TABLE sandbox_decisions ADD COLUMN IF NOT EXISTS runtime_backend TEXT DEFAULT 'sandbox'; +ALTER TABLE sandbox_decisions ADD COLUMN IF NOT EXISTS hot_load BOOLEAN DEFAULT FALSE; +ALTER TABLE sandbox_decisions ADD COLUMN IF NOT EXISTS error TEXT; +ALTER TABLE sandbox_decisions ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['correlation_id', 'node_name', 'status', 'runtime_backend', 'hot_load', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'sandbox_decisions'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'sandbox_decisions'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge sandbox_decisions.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'sandbox_decisions'::regclass AND contype = 'p' + ) THEN + ALTER TABLE sandbox_decisions ADD CONSTRAINT sandbox_decisions_pkey PRIMARY KEY (correlation_id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'sandbox_decisions'::regclass AND conname = 'sandbox_decisions_status_check' + ) THEN + ALTER TABLE sandbox_decisions ADD CONSTRAINT sandbox_decisions_status_check CHECK (status IN ('completed', 'failed')); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'sandbox_decisions'::regclass AND conname = 'sandbox_decisions_runtime_backend_check' + ) THEN + ALTER TABLE sandbox_decisions ADD CONSTRAINT sandbox_decisions_runtime_backend_check CHECK (runtime_backend IN ('sandbox', 'runtime')); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: sandbox_decisions ---- + + CREATE INDEX IF NOT EXISTS idx_sandbox_decisions_created_at ON sandbox_decisions (created_at); diff --git a/docker/migrations/forward/nodes/node_projection_savings/074_create_savings_estimates.sql b/docker/migrations/forward/nodes/node_projection_savings/074_create_savings_estimates.sql index 2afd84540a..3750e9363f 100644 --- a/docker/migrations/forward/nodes/node_projection_savings/074_create_savings_estimates.sql +++ b/docker/migrations/forward/nodes/node_projection_savings/074_create_savings_estimates.sql @@ -19,6 +19,107 @@ CREATE TABLE IF NOT EXISTS savings_estimates ( CHECK (savings_usd = cloud_cost_usd - local_cost_usd) ); +-- ---- BEGIN OMN-15376 shape reconciliation: savings_estimates ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS event_timestamp TIMESTAMPTZ; +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS session_id TEXT; +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS model_local TEXT; +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS model_cloud_baseline TEXT; +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS local_cost_usd NUMERIC(18, 6); +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS cloud_cost_usd NUMERIC(18, 6); +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS savings_usd NUMERIC(18, 6); +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS repo_name TEXT; +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS machine_id TEXT; +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE savings_estimates ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'event_timestamp', 'session_id', 'model_local', 'model_cloud_baseline', 'local_cost_usd', 'cloud_cost_usd', 'savings_usd', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'savings_estimates'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'savings_estimates'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge savings_estimates.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'savings_estimates'::regclass AND contype = 'p' + ) THEN + ALTER TABLE savings_estimates ADD CONSTRAINT savings_estimates_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'savings_estimates'::regclass AND conname = 'savings_estimates_local_cost_usd_check' + ) THEN + ALTER TABLE savings_estimates ADD CONSTRAINT savings_estimates_local_cost_usd_check CHECK (local_cost_usd >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'savings_estimates'::regclass AND conname = 'savings_estimates_cloud_cost_usd_check' + ) THEN + ALTER TABLE savings_estimates ADD CONSTRAINT savings_estimates_cloud_cost_usd_check CHECK (cloud_cost_usd >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'savings_estimates'::regclass AND conname = 'savings_estimates_amounts_match' + ) THEN + ALTER TABLE savings_estimates ADD CONSTRAINT savings_estimates_amounts_match CHECK (savings_usd = cloud_cost_usd - local_cost_usd); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: savings_estimates ---- + + CREATE UNIQUE INDEX IF NOT EXISTS ux_savings_estimates_identity ON savings_estimates ( session_id, diff --git a/docker/migrations/forward/nodes/node_projection_session_outcome/0021_session_outcomes.sql b/docker/migrations/forward/nodes/node_projection_session_outcome/0021_session_outcomes.sql index 0c2d4b3e49..b808dd4c34 100644 --- a/docker/migrations/forward/nodes/node_projection_session_outcome/0021_session_outcomes.sql +++ b/docker/migrations/forward/nodes/node_projection_session_outcome/0021_session_outcomes.sql @@ -12,6 +12,71 @@ CREATE TABLE IF NOT EXISTS session_outcomes ( updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: session_outcomes ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE session_outcomes ADD COLUMN IF NOT EXISTS session_id TEXT; +ALTER TABLE session_outcomes ADD COLUMN IF NOT EXISTS outcome TEXT; +ALTER TABLE session_outcomes ADD COLUMN IF NOT EXISTS emitted_at TIMESTAMPTZ; +ALTER TABLE session_outcomes ADD COLUMN IF NOT EXISTS ingested_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE session_outcomes ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE session_outcomes ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['session_id', 'outcome', 'emitted_at', 'ingested_at', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'session_outcomes'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'session_outcomes'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge session_outcomes.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'session_outcomes'::regclass AND contype = 'p' + ) THEN + ALTER TABLE session_outcomes ADD CONSTRAINT session_outcomes_pkey PRIMARY KEY (session_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: session_outcomes ---- + + CREATE INDEX IF NOT EXISTS idx_session_outcomes_emitted_at ON session_outcomes (emitted_at); diff --git a/docker/migrations/forward/nodes/node_projection_session_replay/0001_create_session_replay_snapshots.sql b/docker/migrations/forward/nodes/node_projection_session_replay/0001_create_session_replay_snapshots.sql index 29cf6a5dce..34d4b8173e 100644 --- a/docker/migrations/forward/nodes/node_projection_session_replay/0001_create_session_replay_snapshots.sql +++ b/docker/migrations/forward/nodes/node_projection_session_replay/0001_create_session_replay_snapshots.sql @@ -36,6 +36,91 @@ CREATE TABLE IF NOT EXISTS public.session_replay_snapshots ( UNIQUE (session_id, sequence) ); +-- ---- BEGIN OMN-15376 shape reconciliation: session_replay_snapshots ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE public.session_replay_snapshots ADD COLUMN IF NOT EXISTS snapshot_id TEXT; +ALTER TABLE public.session_replay_snapshots ADD COLUMN IF NOT EXISTS session_id TEXT; +ALTER TABLE public.session_replay_snapshots ADD COLUMN IF NOT EXISTS sequence INT DEFAULT 0; +ALTER TABLE public.session_replay_snapshots ADD COLUMN IF NOT EXISTS timestamp TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE public.session_replay_snapshots ADD COLUMN IF NOT EXISTS event_type TEXT; +ALTER TABLE public.session_replay_snapshots ADD COLUMN IF NOT EXISTS node_name TEXT DEFAULT ''; +ALTER TABLE public.session_replay_snapshots ADD COLUMN IF NOT EXISTS state_delta JSONB DEFAULT '{}'; +ALTER TABLE public.session_replay_snapshots ADD COLUMN IF NOT EXISTS cumulative_tokens INT DEFAULT 0; +ALTER TABLE public.session_replay_snapshots ADD COLUMN IF NOT EXISTS is_checkpoint BOOLEAN DEFAULT FALSE; +ALTER TABLE public.session_replay_snapshots ADD COLUMN IF NOT EXISTS ingested_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['snapshot_id', 'session_id', 'sequence', 'timestamp', 'event_type', 'node_name', 'state_delta', 'cumulative_tokens', 'is_checkpoint', 'ingested_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'public.session_replay_snapshots'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'public.session_replay_snapshots'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge public.session_replay_snapshots.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'public.session_replay_snapshots'::regclass AND contype = 'p' + ) THEN + ALTER TABLE public.session_replay_snapshots ADD CONSTRAINT session_replay_snapshots_pkey PRIMARY KEY (snapshot_id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'public.session_replay_snapshots'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['sequence', 'session_id']::text[] + ) THEN + ALTER TABLE public.session_replay_snapshots ADD CONSTRAINT session_replay_snapshots_session_id_sequence_key UNIQUE (session_id, sequence); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: session_replay_snapshots ---- + + -- Primary lookup: all snapshots for a session in order. CREATE INDEX IF NOT EXISTS idx_session_replay_session_sequence ON public.session_replay_snapshots (session_id, sequence ASC); diff --git a/docker/migrations/forward/nodes/node_projection_skill_executions/0001_create_skill_execution_snapshots.sql b/docker/migrations/forward/nodes/node_projection_skill_executions/0001_create_skill_execution_snapshots.sql index f306f5e8dc..9825635a74 100644 --- a/docker/migrations/forward/nodes/node_projection_skill_executions/0001_create_skill_execution_snapshots.sql +++ b/docker/migrations/forward/nodes/node_projection_skill_executions/0001_create_skill_execution_snapshots.sql @@ -80,6 +80,144 @@ CREATE TABLE IF NOT EXISTS skill_execution_snapshots ( CONSTRAINT non_negative_skill_exec_partial_count CHECK (partial_count >= 0) ); +-- ---- BEGIN OMN-15376 shape reconciliation: skill_execution_snapshots ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS skill_name VARCHAR(256); +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS repo_id VARCHAR(256); +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS "window" VARCHAR(32) DEFAULT 'latest'; +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS snapshot_timestamp_minute TIMESTAMPTZ; +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS started_count BIGINT DEFAULT 0; +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS completed_count BIGINT DEFAULT 0; +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS success_count BIGINT DEFAULT 0; +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS failed_count BIGINT DEFAULT 0; +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS partial_count BIGINT DEFAULT 0; +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS receipt_coverage NUMERIC(5, 4) GENERATED ALWAYS AS ( CASE WHEN started_count > 0 THEN LEAST(1.0, completed_count::numeric / started_count) ELSE 0 END ) STORED; +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE skill_execution_snapshots ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'skill_name', 'repo_id', 'window', 'snapshot_timestamp_minute', 'started_count', 'completed_count', 'success_count', 'failed_count', 'partial_count', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'skill_execution_snapshots'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'skill_execution_snapshots'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge skill_execution_snapshots.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'skill_execution_snapshots'::regclass AND contype = 'p' + ) THEN + ALTER TABLE skill_execution_snapshots ADD CONSTRAINT skill_execution_snapshots_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'skill_execution_snapshots'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['repo_id', 'skill_name', 'snapshot_timestamp_minute', 'window']::text[] + ) THEN + ALTER TABLE skill_execution_snapshots ADD CONSTRAINT uq_skill_exec_skill_repo_window_minute UNIQUE (skill_name, repo_id, "window", snapshot_timestamp_minute); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'skill_execution_snapshots'::regclass AND conname = 'non_negative_skill_exec_started_count' + ) THEN + ALTER TABLE skill_execution_snapshots ADD CONSTRAINT non_negative_skill_exec_started_count CHECK (started_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'skill_execution_snapshots'::regclass AND conname = 'non_negative_skill_exec_completed_count' + ) THEN + ALTER TABLE skill_execution_snapshots ADD CONSTRAINT non_negative_skill_exec_completed_count CHECK (completed_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'skill_execution_snapshots'::regclass AND conname = 'non_negative_skill_exec_success_count' + ) THEN + ALTER TABLE skill_execution_snapshots ADD CONSTRAINT non_negative_skill_exec_success_count CHECK (success_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'skill_execution_snapshots'::regclass AND conname = 'non_negative_skill_exec_failed_count' + ) THEN + ALTER TABLE skill_execution_snapshots ADD CONSTRAINT non_negative_skill_exec_failed_count CHECK (failed_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'skill_execution_snapshots'::regclass AND conname = 'non_negative_skill_exec_partial_count' + ) THEN + ALTER TABLE skill_execution_snapshots ADD CONSTRAINT non_negative_skill_exec_partial_count CHECK (partial_count >= 0); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: skill_execution_snapshots ---- + + CREATE INDEX IF NOT EXISTS idx_skill_execution_snapshots_started_count ON skill_execution_snapshots (started_count DESC); diff --git a/docker/migrations/forward/nodes/node_projection_skill_executions/0002_skill_execution_snapshots_tenant_id_and_rls.sql b/docker/migrations/forward/nodes/node_projection_skill_executions/0002_skill_execution_snapshots_tenant_id_and_rls.sql new file mode 100644 index 0000000000..fe5fc0574f --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_skill_executions/0002_skill_execution_snapshots_tenant_id_and_rls.sql @@ -0,0 +1,113 @@ +-- OMN-15655 / operator ruling 2026-08-02 (house tenant): give skill_execution_snapshots a +-- tenant identity and row-level tenant isolation. +-- +-- skill-execution snapshots count a tenant's own skill runs THIS RELATION IS TENANT DATA +-- WHY, so the row is workload-attributable and belongs in the tenant +-- domain per the ruling's rule. Rows produced by OmniNode's own platform +-- workloads are the omninode TENANT's rows -- OmniNode is a first-class tenant, +-- not an absence of one. Only attribution-meaningless infrastructure state +-- (migration bookkeeping, registry, orchestration state, deployment +-- evidence) stays in omninode_internal. +-- +-- This migration lands alongside the contract flip +-- `db_io.db_tables[].schema: public -> tenant`. Before it, the relation +-- declared a schema the typed topology does not declare, which ADR-0027 +-- refuses by design -- `ValueError: Unknown schema 'public' for +-- database_ref 'application'` out of `ModelDeploymentTopology.schema_domain`. +-- +-- IDENTITY TYPE -- TEXT, NOT UUID, AND THAT IS DELIBERATE +-- Every landed tenant_id column on this surface is `TEXT` holding the slug +-- 'omninode' (delegation 0022/0025, savings 080, registration 0002, +-- inference-response 0002, omnidash 0001_tenant_rls), and the RLS policies +-- that already shipped (0023, 0026) compare TEXT with no `::uuid` cast. +-- OMN-15356 converts that whole set to the canonical UUID in ONE pass. +-- Adding a UUID column here alone would fork tenant identity inside a single +-- database -- the exact "one canonical model per shape" violation. The +-- canonical UUID this slug resolves to is pinned in +-- `omnimarket.projection.tenant_isolation.omninode_TENANT_UUID`. +-- +-- BACKFILL -- NO BATCHING NEEDED, AND BATCHING WOULD BE WORSE +-- `ADD COLUMN ... NOT NULL DEFAULT` with a non-volatile default does not +-- rewrite the table on PostgreSQL 11+; the default is stored in +-- `pg_attribute.attmissingval` and every pre-existing row reads back as +-- the house tenant immediately. A batched `UPDATE` would do strictly more +-- work (a real rewrite, dead tuples, a longer lock) for the same end state. +-- +-- PHYSICAL SCHEMA IS NOT MOVED HERE -- SEE OMN-15359 +-- No `ALTER TABLE ... SET SCHEMA tenant`. The `tenant` schema is created +-- by no applied migration in this repo or in omnibase_infra +-- (`docker/migrations/forward/_ledger/bootstrap.sql` creates only +-- `platform_catalog`); it exists solely in proof fixtures. Every relation +-- already classified TENANT -- delegation_events, delegation_budget_state, +-- savings_estimates -- is still physically in `public` +-- (`current_schema: ["public"]` in the OMN-15423 inventory, against +-- `target_schema: tenant`). Relocating these eight alone would create the +-- split the inventory's target/current split exists to prevent, and would +-- fail outright against a database with no `tenant` schema. The physical +-- move is OMN-15359 ("Build classified schemas and migrate internal, +-- control-plane, catalog, and tenant targets"), which moves the whole set, +-- with its trigger functions and sequences, in one governed cutover. +-- +-- FAIL-CLOSED RATCHET +-- The column DEFAULT is what supplies the house tenant today, exactly as it +-- does for savings_estimates (OMN-14058, operator-accepted): a writer that +-- resolves no tenant OMITS the key and Postgres fills it -- the key is never +-- written as NULL. That default-allowed state is PINNED, with its flip +-- condition named, by +-- `tests/unit/projection/test_house_tenant_default_ratchet.py`. When +-- customer ingress exists the writer boundary stops defaulting and refuses +-- instead; that test fails the moment the precondition changes. +-- +-- BLAST RADIUS -- FORCE constrains the table OWNER too +-- Same caveat as migration 0023: a writer connected as the `postgres` +-- SUPERUSER (the compose lanes) bypasses RLS regardless of FORCE. The real +-- isolation boundary is this policy PLUS a non-superuser, NOBYPASSRLS writer +-- role (OMN-14899 / OMN-15425). +-- +-- Fail-closed: `current_setting('app.tenant_id', true)` is NULL when the GUC +-- is unset, the predicate is NULL, and zero rows are visible. The policy has no +-- default-tenant fallback, by design. +-- +-- Idempotent: ADD COLUMN / CREATE INDEX are IF NOT EXISTS, ENABLE/FORCE are +-- idempotent, the policy is DROP + CREATE, GRANTs are idempotent. + +ALTER TABLE public.skill_execution_snapshots + ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'omninode'; + +CREATE INDEX IF NOT EXISTS idx_skill_execution_snapshots_tenant_id + ON public.skill_execution_snapshots (tenant_id); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'app_dashboard' + AND NOT rolsuper + AND NOT rolbypassrls + ) THEN + RAISE EXCEPTION + 'app_dashboard role missing or RLS-bypassing - apply omnibase_infra forward migration ' + '094_create_app_dashboard_role.sql (OMN-14899) before this RLS ' + 'migration. RLS grants without the constrained read role are the ' + 'exact bypass this work exists to prevent.'; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA public TO app_dashboard; + +ALTER TABLE public.skill_execution_snapshots ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.skill_execution_snapshots FORCE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS tenant_isolation ON public.skill_execution_snapshots; +CREATE POLICY tenant_isolation ON public.skill_execution_snapshots + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)); + +-- Read grant for the live dashboard reader (omnidash SkillAdoptionWidget reads this relation). RLS still filters every +-- row this role can see -- the grant is what makes the RLS-scoped read path +-- reachable at all, not a widening of it. Granted only where a reader is known +-- to exist; the relations with no known reader get no SELECT. +GRANT SELECT ON public.skill_execution_snapshots TO app_dashboard; diff --git a/docker/migrations/forward/nodes/node_projection_swarm/0001_create_swarm_runs.sql b/docker/migrations/forward/nodes/node_projection_swarm/0001_create_swarm_runs.sql index 7911f66ab8..5dd50fff3f 100644 --- a/docker/migrations/forward/nodes/node_projection_swarm/0001_create_swarm_runs.sql +++ b/docker/migrations/forward/nodes/node_projection_swarm/0001_create_swarm_runs.sql @@ -47,5 +47,93 @@ CREATE TABLE IF NOT EXISTS swarm_runs ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ---- BEGIN OMN-15376 shape reconciliation: swarm_runs ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS run_id TEXT; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS correlation_id TEXT; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS status TEXT; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS task_hash TEXT DEFAULT ''; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS subtask_count INTEGER DEFAULT 0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS succeeded_count INTEGER DEFAULT 0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS failed_count INTEGER DEFAULT 0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS skipped_count INTEGER DEFAULT 0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS models_used TEXT[] DEFAULT '{}'; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS machines_used TEXT[] DEFAULT '{}'; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS total_cost_usd DOUBLE PRECISION DEFAULT 0.0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS cloud_equivalent_cost_usd DOUBLE PRECISION DEFAULT 0.0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS savings_usd DOUBLE PRECISION DEFAULT 0.0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS parallelism_speedup_ratio DOUBLE PRECISION DEFAULT 1.0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS decomposition_latency_ms INTEGER DEFAULT 0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS dispatch_wall_latency_ms INTEGER DEFAULT 0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS aggregation_latency_ms INTEGER DEFAULT 0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS total_latency_ms INTEGER DEFAULT 0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS endpoint_registry_hash TEXT DEFAULT ''; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS registry_schema_version TEXT DEFAULT ''; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS projection_cursor TEXT DEFAULT ''; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS source_event_id TEXT DEFAULT ''; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS source_topic TEXT DEFAULT ''; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS source_partition INTEGER DEFAULT 0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS source_offset INTEGER DEFAULT 0; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS reducer_version TEXT DEFAULT '1.0.0'; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS freshness_state TEXT DEFAULT 'fresh'; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS observed_at TIMESTAMPTZ; +ALTER TABLE swarm_runs ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['run_id', 'correlation_id', 'status', 'task_hash', 'subtask_count', 'succeeded_count', 'failed_count', 'skipped_count', 'created_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'swarm_runs'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'swarm_runs'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge swarm_runs.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'swarm_runs'::regclass AND contype = 'p' + ) THEN + ALTER TABLE swarm_runs ADD CONSTRAINT swarm_runs_pkey PRIMARY KEY (run_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: swarm_runs ---- + + -- Projection API orders by created_at DESC (most recent runs first). CREATE INDEX IF NOT EXISTS idx_swarm_runs_created_at ON swarm_runs (created_at DESC); diff --git a/docker/migrations/forward/nodes/node_projection_traces/0001_create_traces.sql b/docker/migrations/forward/nodes/node_projection_traces/0001_create_traces.sql index 27d3f45e34..8b3d87ea92 100644 --- a/docker/migrations/forward/nodes/node_projection_traces/0001_create_traces.sql +++ b/docker/migrations/forward/nodes/node_projection_traces/0001_create_traces.sql @@ -41,6 +41,96 @@ CREATE TABLE IF NOT EXISTS traces ( CONSTRAINT non_negative_traces_duration_ms CHECK (duration_ms >= 0) ); +-- ---- BEGIN OMN-15376 shape reconciliation: traces ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE traces ADD COLUMN IF NOT EXISTS correlation_id VARCHAR(256); +ALTER TABLE traces ADD COLUMN IF NOT EXISTS nodes_involved TEXT[] DEFAULT ARRAY[]::TEXT[]; +ALTER TABLE traces ADD COLUMN IF NOT EXISTS event_count INTEGER DEFAULT 0; +ALTER TABLE traces ADD COLUMN IF NOT EXISTS first_event_at TIMESTAMPTZ; +ALTER TABLE traces ADD COLUMN IF NOT EXISTS last_event_at TIMESTAMPTZ; +ALTER TABLE traces ADD COLUMN IF NOT EXISTS duration_ms BIGINT DEFAULT 0; +ALTER TABLE traces ADD COLUMN IF NOT EXISTS has_error BOOLEAN DEFAULT FALSE; +ALTER TABLE traces ADD COLUMN IF NOT EXISTS is_running BOOLEAN DEFAULT TRUE; +ALTER TABLE traces ADD COLUMN IF NOT EXISTS latest_message TEXT DEFAULT ''; +ALTER TABLE traces ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE traces ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['correlation_id', 'nodes_involved', 'event_count', 'first_event_at', 'last_event_at', 'duration_ms', 'has_error', 'is_running', 'latest_message', 'created_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'traces'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'traces'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge traces.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'traces'::regclass AND contype = 'p' + ) THEN + ALTER TABLE traces ADD CONSTRAINT traces_pkey PRIMARY KEY (correlation_id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'traces'::regclass AND conname = 'non_negative_traces_event_count' + ) THEN + ALTER TABLE traces ADD CONSTRAINT non_negative_traces_event_count CHECK (event_count >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'traces'::regclass AND conname = 'non_negative_traces_duration_ms' + ) THEN + ALTER TABLE traces ADD CONSTRAINT non_negative_traces_duration_ms CHECK (duration_ms >= 0); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: traces ---- + + CREATE INDEX IF NOT EXISTS idx_traces_last_event_at ON traces (last_event_at DESC); diff --git a/docker/migrations/forward/nodes/node_projection_voice_sessions/0001_create_voice_sessions.sql b/docker/migrations/forward/nodes/node_projection_voice_sessions/0001_create_voice_sessions.sql index 5d01f9bf91..b469ab3322 100644 --- a/docker/migrations/forward/nodes/node_projection_voice_sessions/0001_create_voice_sessions.sql +++ b/docker/migrations/forward/nodes/node_projection_voice_sessions/0001_create_voice_sessions.sql @@ -20,6 +20,115 @@ CREATE TABLE IF NOT EXISTS voice_sessions ( CHECK (ended_at IS NULL OR ended_at >= started_at) ); +-- ---- BEGIN OMN-15376 shape reconciliation: voice_sessions ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE voice_sessions ADD COLUMN IF NOT EXISTS session_id TEXT; +ALTER TABLE voice_sessions ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ; +ALTER TABLE voice_sessions ADD COLUMN IF NOT EXISTS ended_at TIMESTAMPTZ; +ALTER TABLE voice_sessions ADD COLUMN IF NOT EXISTS is_active BOOLEAN DEFAULT TRUE; +ALTER TABLE voice_sessions ADD COLUMN IF NOT EXISTS total_turns INTEGER DEFAULT 0; +ALTER TABLE voice_sessions ADD COLUMN IF NOT EXISTS total_duration_ms BIGINT DEFAULT 0; +ALTER TABLE voice_sessions ADD COLUMN IF NOT EXISTS agent_name TEXT DEFAULT ''; +ALTER TABLE voice_sessions ADD COLUMN IF NOT EXISTS transcript_turns JSONB DEFAULT '[]'::jsonb; +ALTER TABLE voice_sessions ADD COLUMN IF NOT EXISTS ingested_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE voice_sessions ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['session_id', 'started_at', 'is_active', 'total_turns', 'total_duration_ms', 'agent_name', 'transcript_turns', 'ingested_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'voice_sessions'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'voice_sessions'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge voice_sessions.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'voice_sessions'::regclass AND contype = 'p' + ) THEN + ALTER TABLE voice_sessions ADD CONSTRAINT voice_sessions_pkey PRIMARY KEY (session_id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'voice_sessions'::regclass AND conname = 'voice_sessions_total_turns_check' + ) THEN + ALTER TABLE voice_sessions ADD CONSTRAINT voice_sessions_total_turns_check CHECK (total_turns >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'voice_sessions'::regclass AND conname = 'voice_sessions_total_duration_ms_check' + ) THEN + ALTER TABLE voice_sessions ADD CONSTRAINT voice_sessions_total_duration_ms_check CHECK (total_duration_ms >= 0); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'voice_sessions'::regclass AND conname = 'voice_sessions_transcript_turns_check' + ) THEN + ALTER TABLE voice_sessions ADD CONSTRAINT voice_sessions_transcript_turns_check CHECK (jsonb_typeof(transcript_turns) = 'array'); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'voice_sessions'::regclass AND conname = 'voice_sessions_check' + ) THEN + ALTER TABLE voice_sessions ADD CONSTRAINT voice_sessions_check CHECK (ended_at IS NULL OR ended_at >= started_at); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: voice_sessions ---- + + CREATE INDEX IF NOT EXISTS idx_voice_sessions_started_at ON voice_sessions (started_at DESC); diff --git a/docker/migrations/forward/nodes/node_renderer_capability_projection/0001_create_renderer_capability_projection.sql b/docker/migrations/forward/nodes/node_renderer_capability_projection/0001_create_renderer_capability_projection.sql index 3e1432262e..17a4254865 100644 --- a/docker/migrations/forward/nodes/node_renderer_capability_projection/0001_create_renderer_capability_projection.sql +++ b/docker/migrations/forward/nodes/node_renderer_capability_projection/0001_create_renderer_capability_projection.sql @@ -31,5 +31,93 @@ CREATE TABLE IF NOT EXISTS renderer_capability_projection ( CONSTRAINT uq_renderer_capability_renderer_id UNIQUE (renderer_id) ); +-- ---- BEGIN OMN-15376 shape reconciliation: renderer_capability_projection ---- +-- The CREATE TABLE IF NOT EXISTS above SILENTLY NO-OPS when a table of this +-- name already exists with a DIFFERENT shape (an out-of-band or legacy apply +-- that predates this migration). Everything below it in this file is NOT so +-- forgiving: CREATE INDEX IF NOT EXISTS guards the index NAME, not the COLUMN, +-- so the first column-dependent statement raises +-- ERROR: column "" does not exist +-- and ON_ERROR_STOP=1 kills the whole migration Job there. Because the runner +-- halts at the first failure, instances of this class surface strictly one per +-- deploy cycle -- OMN-15376 (llm_cost_aggregates.aggregation_key, run +-- 30418878385) and OMN-15302 (baselines_comparisons.snapshot_id) each cost one. +-- +-- The guarded adds below converge a drifted pre-existing table onto the shape +-- declared above. On the fresh-create path every one is a no-op (the column +-- already exists), so BOTH paths end at the same schema. No DROP, no recreate, +-- no TRUNCATE: pre-existing rows are preserved. A column that cannot be made +-- NOT NULL without inventing data fails LOUD and names the exact conflict +-- instead of guessing. +-- +-- Gated by tests/ci/test_node_migration_shape_reconciliation.py (static) and +-- tests/integration/migrations/test_node_migration_shape_drift_omn15376.py +-- (RED/GREEN + fresh-vs-drifted schema equality on real Postgres). + +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS id UUID DEFAULT gen_random_uuid(); +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS renderer_id TEXT; +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS platform TEXT; +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS supported_component_kinds TEXT[] DEFAULT '{}'; +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS interaction_model TEXT; +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS accessibility_tier TEXT; +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS contract_version TEXT; +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS declared_at TIMESTAMPTZ; +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS last_heartbeat TIMESTAMPTZ; +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS is_degraded BOOLEAN DEFAULT FALSE; +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS empty_state_reason TEXT; +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS observed_at TIMESTAMPTZ DEFAULT NOW(); +ALTER TABLE renderer_capability_projection ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +DO $$ +DECLARE + v_col TEXT; + v_nulls BIGINT; +BEGIN + FOREACH v_col IN ARRAY ARRAY['id', 'renderer_id', 'platform', 'supported_component_kinds', 'interaction_model', 'accessibility_tier', 'contract_version', 'declared_at', 'last_heartbeat', 'is_degraded', 'observed_at', 'updated_at'] + LOOP + EXECUTE format( + 'SELECT count(*) FROM %s WHERE %I IS NULL', 'renderer_capability_projection'::regclass, v_col + ) INTO v_nulls; + IF v_nulls = 0 THEN + EXECUTE format( + 'ALTER TABLE %s ALTER COLUMN %I SET NOT NULL', 'renderer_capability_projection'::regclass, v_col + ); + ELSE + RAISE EXCEPTION + 'OMN-15376: cannot converge renderer_capability_projection.% to NOT NULL -- % pre-existing row(s) hold NULL. This needs a data ruling (backfill value, or drop the NOT NULL from the contract); the migration refuses to guess.', + v_col, v_nulls; + END IF; + END LOOP; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'renderer_capability_projection'::regclass AND contype = 'p' + ) THEN + ALTER TABLE renderer_capability_projection ADD CONSTRAINT renderer_capability_projection_pkey PRIMARY KEY (id); + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + WHERE c.conrelid = 'renderer_capability_projection'::regclass + AND c.contype IN ('p', 'u') + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(c.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ) = ARRAY['renderer_id']::text[] + ) THEN + ALTER TABLE renderer_capability_projection ADD CONSTRAINT uq_renderer_capability_renderer_id UNIQUE (renderer_id); + END IF; +END$$; + +-- ---- END OMN-15376 shape reconciliation: renderer_capability_projection ---- + + CREATE INDEX IF NOT EXISTS ix_renderer_capability_last_heartbeat ON renderer_capability_projection (last_heartbeat DESC); diff --git a/docker/migrations/intelligence/026_create_code_entities.sql b/docker/migrations/intelligence/026_create_code_entities.sql new file mode 100644 index 0000000000..17b199ac41 --- /dev/null +++ b/docker/migrations/intelligence/026_create_code_entities.sql @@ -0,0 +1,129 @@ +-- Migration: 026_create_code_entities.sql +-- Canonical DDL for the AST code-intelligence store: code_entities + code_relationships. +-- +-- Ticket: OMN-15276 (supersedes the false-Done OMN-5765 "reconcile competing +-- code_entities migrations (025 vs 025_create)", closed 2026-03-21 with +-- both conflicting files still on omniintelligence dev four months later). +-- Origin: omniintelligence/deployment/database/migrations/025_code_entities.sql (OMN-5661) +-- + omniintelligence/deployment/database/migrations/026_create_code_relationships.sql (OMN-5709) -- REJECTED shape, see below +-- + omniintelligence/deployment/database/migrations/027_code_entity_enrichment_part2.sql (OMN-5676) -- folded in +-- +-- WHY THIS FILE LIVES HERE (DDL ownership, OMN-15276 scope item 2) +-- --------------------------------------------------------------- +-- The .201 docker lanes apply THIS directory, not omniintelligence's: +-- docker/docker-compose.infra.yml -> intelligence-migration service +-- MIGRATIONS_DIR: /migrations/intelligence +-- ../docker/migrations/intelligence:/migrations/intelligence:ro +-- scripts/run-intelligence-migrations.sh -> applies ${MIGRATIONS_DIR}/*.sql in sorted +-- order, tracking basenames in omniintelligence.schema_migrations +-- The same binding appears in docker/docker-compose.judge.yml and +-- docker/catalog/services/intelligence-migration.yaml. +-- +-- The two conflicting 025_* files lived in omniintelligence/deployment/database/migrations/, +-- which no .201 lane reads. That is why they were never applied: omniintelligence.schema_migrations +-- held 27 identical rows on the stability-test and prod lanes, ending at +-- 025_fix_llm_delegation_call_log_date_index (2026-06-11T09:37:18Z), and +-- code_entities/code_relationships were absent from 20/20 databases across both lanes +-- (read-only probe, 2026-07-27T23:46Z). A fix landed in the omniintelligence tree would +-- have read green in CI and changed nothing on any lane. +-- +-- WHICH SCHEMA WON, AND WHY +-- ------------------------- +-- The OMN-5661 shape (025_code_entities.sql). It is the shape both live consumers +-- actually query, column-for-column: +-- * omnimarket RepositoryCodeEntityPostgres (omnimarket#1923) projects +-- id/entity_name/entity_type/qualified_name/source_repo/source_path/docstring/ +-- signature/classification/llm_description (embedding batch) and +-- .../bases/methods/fields/decorators (enrichment batch); it predicates on +-- last_embedded_at < last_extracted_at and classification IS NULL, and writes +-- architectural_pattern/classification_confidence/enrichment_version/last_enriched_at. +-- * omniintelligence RepositoryCodeEntity (node_ast_extraction_compute) upserts on +-- ON CONFLICT (qualified_name, source_repo) and selects WHERE source_path = $1. +-- +-- The rejected OMN-5709 shape (025_create_code_entities.sql) named the same concepts +-- differently -- name/file_path/line_start/line_end, bases and decorators as JSONB +-- rather than TEXT[] -- and carried no qualified_name, no signature, no llm_description +-- and none of the last_*_at freshness stamps. Every consumer statement above would have +-- raised UndefinedColumn against it. Because both files were CREATE TABLE IF NOT EXISTS, +-- whichever sorted first would have silently won and the other's columns would never +-- have appeared: applying both did not converge, it just picked a winner quietly. +-- +-- code_relationships takes the OMN-5661 shape for the same reason: the producer's +-- INSERT writes evidence, inject_into_context, source_repo and updated_at, none of +-- which exist in the OMN-5709 026_create_code_relationships.sql variant. + +-- Latest-state entity table. One row per entity per repo. +-- Upsert key: (qualified_name, source_repo) +CREATE TABLE IF NOT EXISTS code_entities ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + entity_name TEXT NOT NULL, + entity_type TEXT NOT NULL, -- class, protocol, model, function, import, constant + qualified_name TEXT NOT NULL, -- module.ClassName.method_name + source_repo TEXT NOT NULL, + source_path TEXT NOT NULL, + line_number INT, + bases TEXT[], -- base classes + methods JSONB, -- [{name, args, return_type, decorators}] + fields JSONB, -- for models: [{name, type, default}] + decorators TEXT[], + docstring TEXT, + signature TEXT, -- function signature string + file_hash TEXT NOT NULL, -- SHA256 for change detection + -- LLM enrichment fields (NULL until enriched) + classification TEXT, + llm_description TEXT, + architectural_pattern TEXT, + classification_confidence FLOAT, + enrichment_version TEXT, + -- Deterministic classification (OMN-5674, folded from 027) -- fast, no LLM + deterministic_node_type TEXT, + deterministic_confidence FLOAT, + deterministic_alternatives JSONB, + -- Quality scoring (OMN-5675, folded from 027) -- multi-dimensional + quality_score FLOAT, + quality_dimensions JSONB, -- {"complexity": 0.7, "maintainability": 0.8, ...} + -- Config-aware idempotency metadata (folded from 027). Operational state kept + -- separate from domain data; RepositoryCodeEntity merges into it with `||`, + -- so it must default to '{}' and never be NULL. + enrichment_metadata JSONB NOT NULL DEFAULT '{}', + -- Multi-language support (OMN-5679, folded from 027) + source_language TEXT DEFAULT 'python', + -- Freshness timestamps for derived-store coordination + last_extracted_at TIMESTAMPTZ DEFAULT NOW(), + last_enriched_at TIMESTAMPTZ, + last_embedded_at TIMESTAMPTZ, + last_graph_synced_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(qualified_name, source_repo) +); + +-- Latest-state relationship table. +-- Upsert key: (source_entity_id, target_entity_id, relationship_type) +CREATE TABLE IF NOT EXISTS code_relationships ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_entity_id UUID REFERENCES code_entities(id) ON DELETE CASCADE, + target_entity_id UUID REFERENCES code_entities(id) ON DELETE CASCADE, + relationship_type TEXT NOT NULL, + trust_tier TEXT NOT NULL DEFAULT 'strong', + confidence FLOAT DEFAULT 1.0, + evidence TEXT[], + inject_into_context BOOLEAN DEFAULT true, + source_repo TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(source_entity_id, target_entity_id, relationship_type) +); + +CREATE INDEX IF NOT EXISTS idx_code_entities_repo ON code_entities(source_repo); +CREATE INDEX IF NOT EXISTS idx_code_entities_type ON code_entities(entity_type); +CREATE INDEX IF NOT EXISTS idx_code_entities_qualified ON code_entities(qualified_name); +CREATE INDEX IF NOT EXISTS idx_code_entities_classification ON code_entities(classification); +CREATE INDEX IF NOT EXISTS idx_code_entities_file_path ON code_entities(source_path); +CREATE INDEX IF NOT EXISTS idx_code_entities_det_node_type ON code_entities(deterministic_node_type); +CREATE INDEX IF NOT EXISTS idx_code_entities_quality ON code_entities(quality_score); +CREATE INDEX IF NOT EXISTS idx_code_entities_language ON code_entities(source_language); +CREATE INDEX IF NOT EXISTS idx_code_relationships_source ON code_relationships(source_entity_id); +CREATE INDEX IF NOT EXISTS idx_code_relationships_target ON code_relationships(target_entity_id); +CREATE INDEX IF NOT EXISTS idx_code_relationships_type ON code_relationships(relationship_type); +CREATE INDEX IF NOT EXISTS idx_code_relationships_injectable ON code_relationships(inject_into_context) WHERE inject_into_context = true; diff --git a/docker/migrations/intelligence/README.md b/docker/migrations/intelligence/README.md new file mode 100644 index 0000000000..b71b675a47 --- /dev/null +++ b/docker/migrations/intelligence/README.md @@ -0,0 +1,87 @@ +# `docker/migrations/intelligence/` — the applied migration set for the `omniintelligence` database + +**Ticket:** OMN-15276 (DDL-ownership decision, scope item 2). + +## This directory is what the `.201` docker lanes actually apply + +The `intelligence-migration` one-shot service mounts **this** directory and applies +every `*.sql` in it, in sorted order, recording basenames in +`omniintelligence.schema_migrations`: + +| Surface | Binding | +| --- | --- | +| `docker/docker-compose.infra.yml` (dev / stability-test / prod lanes) | `MIGRATIONS_DIR: /migrations/intelligence`, `../docker/migrations/intelligence:/migrations/intelligence:ro` | +| `docker/docker-compose.judge.yml` (judge lane) | same pair | +| `docker/catalog/services/intelligence-migration.yaml` | same pair | +| `scripts/run-intelligence-migrations.sh` | `MIGRATIONS_DIR="${MIGRATIONS_DIR:-/migrations/intelligence}"`, `for migration_file in $(ls "${MIGRATIONS_DIR}"/*.sql \| sort)` | + +`intelligence-api` depends on this service with `service_completed_successfully`, so a +migration failure here blocks the lane rather than silently degrading it. + +## There is a second, drifted tree — do not confuse them + +`omniintelligence/deployment/database/migrations/` is a **separate** tree that feeds the +`omniintelligence-migrate` ECR image (`omniintelligence/deployment/docker/Dockerfile.migrate`, +built by `build-and-push-migrate-image.yml` on pushes to `main`) for the cloud k8s +migration Job. No `.201` lane reads it. + +The two trees have **already drifted and no gate compares them** (measured 2026-07-27 +against `omniintelligence@1af0132e`): of the 26 `.sql` files here, 8 differ byte-for-byte +from the same-named file over there, 3 exist only here, and 5 exist only there. + +**This is the trap OMN-15276 exists to name:** the two conflicting `code_entities` +migrations (`025_code_entities.sql` / `025_create_code_entities.sql`) lived in the +omniintelligence tree, so neither was ever applied on any lane — +`omniintelligence.schema_migrations` held 27 identical rows on stability-test and prod, +ending at `025_fix_llm_delegation_call_log_date_index`, and `code_entities` was absent +from 20/20 databases across both lanes. A fix landed in the omniintelligence tree would +have passed CI and changed nothing on any lane. + +## Decision + +**`docker/migrations/intelligence/` owns the `code_entities` / `code_relationships` DDL.** + +- The canonical DDL is `026_create_code_entities.sql` (OMN-15276). It is the only file + repo-wide that creates either table; the omniintelligence copies were retired in the + companion PR. +- The number `026` is a **fresh slot in this directory's own namespace** (this set topped + out at `025_fix_llm_delegation_call_log_date_index`). Nothing already recorded in + `schema_migrations` was renumbered — the runner keys on basename, so renumbering an + applied migration would re-apply it under a new id. + +### Known gaps this decision does not close + +1. **Cloud parity.** The `omniintelligence-migrate` ECR image no longer carries any + `code_entities` DDL. Because the table has never existed on any probed lane, this + removes an unapplied file rather than regressing a live schema — but cloud lanes will + need this DDL (or a real sync path between the two trees) before the code-intelligence + nodes run there. Follow-up, not slice 1. +2. **No duplicate-prefix gate covers this directory.** + `scripts/validation/validate_migration_sequence.py` scans only + `docker/migrations/forward/` and `src/omnibase_infra/migrations/forward/`. That is why + the four-month-old triple-claimed `025` prefix never fired a gate — the check exists, + but not over this path. `023` is duplicated *here* today for the same reason + (`023_create_debug_intelligence_tables.sql` / `023_create_dispatch_eval_results.sql`), + and both are already applied, so it cannot be fixed by renumbering. + `tests/unit/migrations/test_code_entities_canonical_ddl.py` installs a **ratchet**: + `023` is the sole grandfathered duplicate and any *new* duplicate prefix in this + directory fails. Promoting that ratchet into the pre-commit/CI sequence validator is + the enforcement follow-up. +3. **`schema_fingerprint.sha256` in this directory is stale and unverified.** It records + `migration_file_count: 25` at `2026-04-30`; the directory held 26 files before this + change. `scripts/check_schema_fingerprint.py` defaults to `docker/migrations/forward` + and has no caller pointed here, so the artifact is decorative. Left untouched + deliberately — stamping it would manufacture a green signal for a check nothing runs. + +## Adding a migration here + +1. Take the next free `NNN` **in this directory** (`ls docker/migrations/intelligence/`). +2. Write idempotent SQL (`CREATE TABLE IF NOT EXISTS`, `ADD COLUMN IF NOT EXISTS`) — warm + lanes re-run the runner on every bring-up. +3. Never renumber an applied file. `schema_migrations` keys on the basename; a rename + re-applies the SQL under a new id. +4. Retiring a superseded migration = **delete the file** (the OMN-13124 idiom). The runner + discovers files on disk, so a removed migration is simply never applied on fresh + volumes and leaves no dangling `schema_migrations` reference. The + `docker/migrations/skip-manifest.yaml` tombstone mechanism belongs to + `run-forward-migrations.sh` and is not read by `run-intelligence-migrations.sh`. diff --git a/docker/migrations/rollback/rollback_096_grant_role_omnidash_omnidash_analytics.sql b/docker/migrations/rollback/rollback_096_grant_role_omnidash_omnidash_analytics.sql new file mode 100644 index 0000000000..109599af21 --- /dev/null +++ b/docker/migrations/rollback/rollback_096_grant_role_omnidash_omnidash_analytics.sql @@ -0,0 +1,35 @@ +-- ============================================================================= +-- ROLLBACK: 096_grant_role_omnidash_omnidash_analytics.sql +-- ============================================================================= +-- Ticket: OMN-15363 +-- +-- SCOPE +-- Revokes the grants migration 095 issued. It deliberately does NOT drop +-- role_omnidash: the role predates 095 on every lane that has one +-- (000_create_multiple_databases.sh on compose, out-of-band provisioning on +-- cloud RDS), and dropping it would take down the cloud migration principal. +-- +-- WARNING BEFORE RUNNING THIS +-- Any lane whose analytics DSN connects AS role_omnidash — the .201 lab lane +-- after docker/docker-compose.dev-lane.yml — loses its projection write path +-- the moment these revokes land. Repoint that DSN FIRST. Rolling back to +-- the `postgres` DSN also restores the superuser/BYPASSRLS connection, which +-- makes every RLS/FORCE state on this database inert again; that is the +-- condition OMN-15363 exists to remove, so this rollback is an emergency +-- surface, not a maintenance one. +-- ============================================================================= + +\connect omnidash_analytics + +ALTER DEFAULT PRIVILEGES IN SCHEMA public + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM role_omnidash; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + REVOKE USAGE, SELECT ON SEQUENCES FROM role_omnidash; + +REVOKE SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public FROM role_omnidash; +REVOKE USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public FROM role_omnidash; +REVOKE USAGE ON SCHEMA public FROM role_omnidash; + +\connect omnibase_infra + +REVOKE CONNECT ON DATABASE omnidash_analytics FROM role_omnidash; diff --git a/docker/migrations/rollback/rollback_097_grant_app_dashboard_connect_omnidash_analytics.sql b/docker/migrations/rollback/rollback_097_grant_app_dashboard_connect_omnidash_analytics.sql new file mode 100644 index 0000000000..bef77bf30b --- /dev/null +++ b/docker/migrations/rollback/rollback_097_grant_app_dashboard_connect_omnidash_analytics.sql @@ -0,0 +1,75 @@ +-- ============================================================================= +-- ROLLBACK: 097_grant_app_dashboard_connect_omnidash_analytics.sql +-- ============================================================================= +-- Ticket: OMN-15297 +-- +-- SCOPE +-- Revokes exactly what 097 granted: CONNECT on omnidash_analytics, USAGE on +-- schema public, and SELECT on the tenant-isolated tables. It deliberately +-- does NOT drop app_dashboard — the role is created by 094 and dropping it +-- here would roll back a different migration (see +-- rollback_094_create_app_dashboard_role.sql for that). +-- +-- It also does not touch PUBLIC's privileges. 097 never changed them, and +-- OMN-15355 owns that surface. +-- +-- WARNING BEFORE RUNNING THIS +-- Any application pool connected AS app_dashboard loses its read path the +-- moment the CONNECT revoke lands — existing sessions survive (Postgres +-- checks CONNECT at session establishment), new ones fail with +-- `permission denied for database`, which is the exact symptom OMN-15297 +-- exists to remove. Repoint the DSN FIRST. +-- +-- Repointing that DSN back to a superuser role also restores a connection +-- that is exempt from row-level security, which makes every RLS/FORCE state +-- on this database inert again. This rollback is an emergency surface, not a +-- maintenance one. +-- +-- IDEMPOTENCY +-- REVOKE on a privilege that was never granted is a no-op in Postgres, so +-- this file is safe to re-run and safe to run on a lane where 097 never +-- applied. The SELECT revoke is table-driven rather than a blanket +-- `ALL TABLES` so it cannot revoke a grant that some other migration owns. +-- ============================================================================= + +\connect omnidash_analytics + +DO $$ +DECLARE + covered RECORD; +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app_dashboard') THEN + RAISE NOTICE 'app_dashboard does not exist — nothing to revoke'; + RETURN; + END IF; + + FOR covered IN + SELECT c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' + AND c.relkind = 'r' + AND c.relrowsecurity + AND EXISTS ( + SELECT 1 FROM pg_policy p + WHERE p.polrelid = c.oid AND p.polname = 'tenant_isolation' + ) + ORDER BY c.relname + LOOP + EXECUTE format('REVOKE ALL ON public.%I FROM app_dashboard', covered.relname); + END LOOP; + + EXECUTE 'REVOKE USAGE ON SCHEMA public FROM app_dashboard'; +END; +$$; + +\connect omnibase_infra + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app_dashboard') + AND EXISTS (SELECT 1 FROM pg_database WHERE datname = 'omnidash_analytics') THEN + EXECUTE 'REVOKE CONNECT ON DATABASE omnidash_analytics FROM app_dashboard'; + END IF; +END; +$$; diff --git a/docker/migrations/rollback/rollback_098_create_omninode_internal_schema.sql b/docker/migrations/rollback/rollback_098_create_omninode_internal_schema.sql new file mode 100644 index 0000000000..2f3a7f66eb --- /dev/null +++ b/docker/migrations/rollback/rollback_098_create_omninode_internal_schema.sql @@ -0,0 +1,25 @@ +-- ============================================================================= +-- ROLLBACK: 098_create_omninode_internal_schema.sql +-- ============================================================================= +-- Ticket: OMN-15359 +-- +-- SCOPE +-- Drops the `omninode_internal` schema created by 098. RESTRICT, never +-- CASCADE: this rollback is safe ONLY while the schema remains empty (its +-- state as of this ticket — no table has been copied into it). If a later +-- migration has copied any relation into `omninode_internal`, RESTRICT +-- makes this statement fail closed instead of silently destroying that +-- data; do not change it to CASCADE to force the rollback through. +-- +-- IDEMPOTENCY +-- DROP SCHEMA IF EXISTS is safe to re-run and safe on a lane where 098 +-- never applied. +-- +-- MANUAL EXECUTION ONLY +-- This file lives under rollback/ and is never auto-applied by the forward +-- runner or docker-entrypoint-initdb.d. +-- ============================================================================= + +\connect omnidash_analytics + +DROP SCHEMA IF EXISTS omninode_internal RESTRICT; diff --git a/docker/migrations/rollback/rollback_099_create_omninode_internal_live_events.sql b/docker/migrations/rollback/rollback_099_create_omninode_internal_live_events.sql new file mode 100644 index 0000000000..3ba1c76f9e --- /dev/null +++ b/docker/migrations/rollback/rollback_099_create_omninode_internal_live_events.sql @@ -0,0 +1,56 @@ +-- ============================================================================= +-- ROLLBACK: 099_create_omninode_internal_live_events.sql +-- ============================================================================= +-- Ticket: OMN-15359 +-- +-- SCOPE +-- Drops omninode_internal.live_events. RESTRICT (the default DROP TABLE +-- behavior with no CASCADE): fails closed instead of silently taking a +-- dependent object down with it, if anything has come to depend on this +-- table since 099 applied. public.live_events is NEVER touched by this +-- rollback -- 099 only ever copies into omninode_internal.live_events, so +-- the source of truth survives regardless of whether this rollback runs. +-- +-- Also reverts the grant-gap repair (099 step 4/5): the ALTER DEFAULT +-- PRIVILEGES and the schema USAGE grant, so a re-run of 099 starts from a +-- clean slate. Deliberately does NOT drop the omninode_runtime role or +-- revoke its CONNECT/DATABASE grant -- same rationale rollback_096 and +-- rollback_094 already state for role_omnidash/app_dashboard: the role may +-- predate 099 (out-of-band provisioning on a managed instance) or be +-- needed by another already-cutover table, so this rollback only undoes +-- what 099 itself added. +-- +-- WARNING BEFORE RUNNING THIS +-- The live runtime write path (handler_wiring._resolve_projection_database_target) +-- issues `INSERT INTO omninode_internal.live_events` unconditionally -- +-- rolling this back re-introduces the original UndefinedTable failure this +-- migration exists to close. Emergency surface, not a maintenance one. +-- +-- IDEMPOTENCY +-- DROP TABLE IF EXISTS is safe to re-run and safe on a lane where 099 never +-- applied. The REVOKE/ALTER DEFAULT PRIVILEGES statements are guarded on +-- the role existing so this file is also safe on a lane where the role was +-- never created. +-- +-- MANUAL EXECUTION ONLY +-- This file lives under rollback/ and is never auto-applied by the forward +-- runner or docker-entrypoint-initdb.d. +-- ============================================================================= + +\connect omnidash_analytics + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'omninode_runtime') THEN + EXECUTE + 'ALTER DEFAULT PRIVILEGES IN SCHEMA omninode_internal ' + 'REVOKE SELECT, INSERT, UPDATE ON TABLES FROM omninode_runtime'; + EXECUTE + 'REVOKE SELECT, INSERT, UPDATE ON omninode_internal.live_events ' + 'FROM omninode_runtime'; + EXECUTE 'REVOKE USAGE ON SCHEMA omninode_internal FROM omninode_runtime'; + END IF; +END; +$$; + +DROP TABLE IF EXISTS omninode_internal.live_events; diff --git a/docker/migrations/schema_fingerprint.sha256 b/docker/migrations/schema_fingerprint.sha256 index baf56a5a26..816115f458 100644 --- a/docker/migrations/schema_fingerprint.sha256 +++ b/docker/migrations/schema_fingerprint.sha256 @@ -1,6 +1,6 @@ # Schema migration fingerprint for omnibase_infra (auto-generated) # Regenerate: python scripts/check_schema_fingerprint.py stamp # Verify: python scripts/check_schema_fingerprint.py verify -sha256:d332f6a1dc8bdca0dfd100471f61efa4563c4447b3586a6756bb24d8e516df19 -generated_at: 2026-07-25T04:22:56Z -migration_file_count: 79 +sha256:9a5a9e3f3ec61f88abfeba77a0108476b8848a7d24926b18ec0ce4c3602248d8 +generated_at: 2026-08-10T23:12:56Z +migration_file_count: 84 diff --git a/docker/runners/Dockerfile b/docker/runners/Dockerfile index 214370328d..1bbb8afd24 100644 --- a/docker/runners/Dockerfile +++ b/docker/runners/Dockerfile @@ -9,7 +9,7 @@ # contract). Build args MUST match docker/runners/runner-image.lock.json — CI # verifies the bound identity via scripts/ci/runner_image_identity.py. -ARG RUNNER_VERSION=2.334.0 +ARG RUNNER_VERSION=2.336.0 ARG GH_VERSION=2.67.0 ARG KUBECTL_VERSION=1.32.1 # uv pinned to the shared CI env (canary) version so the image binding is truthful. @@ -184,8 +184,8 @@ RUN groupadd --gid 1001 runner \ && usermod -aG docker runner # Download and SHA256-verify GitHub Actions runner binary -# SHA256 for actions-runner-linux-x64-2.334.0.tar.gz -ENV RUNNER_SHA256=048024cd2c848eb6f14d5646d56c13a4def2ae7ee3ad12122bee960c56f3d271 +# SHA256 for actions-runner-linux-x64-2.336.0.tar.gz +ENV RUNNER_SHA256=04cf0be1aff4c3ec3554466c39124ca250e3effd8873bb7e8d68535aa9505d5d RUN mkdir -p "${RUNNER_HOME}" \ && cd "${RUNNER_HOME}" \ diff --git a/docker/runners/entrypoint.sh b/docker/runners/entrypoint.sh index 48c5b132ec..91654a95fd 100755 --- a/docker/runners/entrypoint.sh +++ b/docker/runners/entrypoint.sh @@ -114,19 +114,78 @@ LISTENER_RESTART_MAX="${LISTENER_RESTART_MAX:-50}" # recycle NEVER fires while a Runner.Worker is executing a job. # # The KILL threshold is deliberately DECOUPLED from the healthcheck's ALERT -# threshold (RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS, 900s). Live readback +# threshold (RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS). Live readback # 2026-07-23T05:25-06:02Z: a fleet-wide broker-quiet window silenced _diag on # 53/64 listeners for 35-50 min while GitHub kept every one of them online # and docker-"unhealthy" runners were actively EXECUTING jobs (runners 4 and # 40 busy while heartbeat-stale); runner-2 and runner-45 both resumed on # their own after ~37 min blocked in the same token-refresh path that hangs -# the true zombies forever. Killing at 900s would have mass-recycled ~50 -# healthy-but-quiet listeners mid-window. 3600s clears the observed benign -# ceiling (~50 min) with margin while still recovering a true AAD-deadlock -# zombie in ~1h instead of the 6 days the 2026-07-16..23 incident took. +# the true zombies forever. Killing at the then-current 900s alert threshold +# would have mass-recycled ~50 healthy-but-quiet listeners mid-window. 3600s +# clears the observed benign ceiling (~50 min) with margin while still +# recovering a true AAD-deadlock zombie in ~1h instead of the 6 days the +# 2026-07-16..23 incident took. +# +# OMN-15233 flipped the ORDER of the two thresholds, not their independence: +# the alert threshold is now 4500s, ABOVE this 3600s kill threshold. That is +# intentional. This watchdog is the narrower signal — it additionally requires +# LISTENER_HEARTBEAT_MISSES consecutive ticks and refuses to fire while a +# Runner.Worker is executing — so it can afford to act on staleness the +# unconditional, retry-free healthcheck must not flag. Do NOT "restore +# ordering" by dropping the alert threshold back toward 900s: that is the +# arithmetic false positive OMN-15233 removed. +# +# RE-DERIVATION ATTEMPT (2026-08-10, omn15776-wedge-verify): OMN-15776 asked +# whether this threshold could be tightened toward sub-900s to close the gap +# between GitHub's ~600s broker-dispatch-wedge orphan timeout and this +# watchdog's eventual recycle (see scripts/ci/runner_broker_dispatch_wedge_rerun.sh +# for that mechanism). Measured LIVE, read-only, against 5 healthy fleet +# runners' own _diag/Runner_*.log inter-line gaps (not a hypothetical): +# +# runner log window (real) max observed gap +# ----- ------------------------ ----------------- +# 19 04:50-08:09 UTC (~19.7h) 3007s (~50.1min) +# 20 Aug8 22:14-Aug9 08:09 3009s (~50.2min) +# 29 04:04-08:09 UTC (~4.1h) 2960s (~49.3min) +# 2, 4 partial windows (fleet 319-508s (windows +# under load, no full did not span a +# idle cycle observed) full idle cycle) +# +# Max observed normal-idle gap across the sample: 3009s. This corroborates, +# not contradicts, the 2026-07-23 finding above (~50min benign ceiling) and +# the independent OMN-15233 healthcheck.sh derivation (~50min OAuth/AAD +# token-refresh cadence while idle). CONCLUSION: sub-900s detection is NOT +# supported by evidence — a threshold anywhere near 900s-3000s would +# reproduce the exact mass-recycle false-positive class OMN-15233 fixed, +# just on this watchdog's KILL path instead of the healthcheck's ALERT path. +# 3600s already sits ~590-600s (~20%) above the measured ceiling, which is +# the intended margin, not slack to cut. Threshold left UNCHANGED at 3600s. +# The broker-dispatch-wedge false-red this was evaluated against is instead +# closed by scripts/ci/runner_broker_dispatch_wedge_rerun.sh's targeted +# rerun, which does not depend on this watchdog's timing. LISTENER_HEARTBEAT_MAX_AGE_SECONDS="${LISTENER_HEARTBEAT_MAX_AGE_SECONDS:-3600}" LISTENER_HEARTBEAT_MISSES="${LISTENER_HEARTBEAT_MISSES:-3}" +# --------------------------------------------------------------------------- +# Orphan reap before respawn (OMN-15233) +# --------------------------------------------------------------------------- +# Incident (2026-07-27, runners 1/43/55/57): a Runner.Listener survived the +# death of its wrapper tree, was reparented to PPID 1, and kept holding this +# runner's GitHub broker session. This loop then spawned a REPLACEMENT listener +# on top of it, which crash-looped every ~5 min on +# TaskAgentSessionConflictException — the orphan still owned the session. Each +# crash minted a fresh Runner_*.log, so the _diag mtime heartbeat read HEALTHY +# forever and no signal surfaced the state (88-234 log files vs 3-7 normal). +# +# Spawn-without-reap is what MANUFACTURES the session conflict. Reaping is +# therefore unconditional and precedes every run.sh spawn, not just the +# watchdog-recycle path: any listener still matching this runner home's pattern +# at the top of the loop is by definition left over from a previous +# incarnation (the loop only ever gets here after the previous run.sh exited or +# was recycled), so it is killed and confirmed gone before a replacement is +# started. +LISTENER_REAP_TIMEOUT_SECONDS="${LISTENER_REAP_TIMEOUT_SECONDS:-30}" + # --------------------------------------------------------------------------- # Credential cache helpers # --------------------------------------------------------------------------- @@ -283,6 +342,18 @@ LISTENER_PGREP_PATTERN="${LISTENER_PGREP_PATTERN:-${RUNNER_HOME//./\\.}/bin/Runn # Worker pattern (OMN-14564): a Runner.Worker process means a job is executing # — the heartbeat watchdog must NEVER recycle mid-job. WORKER_PGREP_PATTERN="${WORKER_PGREP_PATTERN:-${RUNNER_HOME//./\\.}/bin/Runner\.Worker}" +# run-helper pattern (OMN-15776): match THIS runner home's run.sh->run-helper.sh +# wrapper (the direct parent of Runner.Listener — see healthcheck.sh's process +# tree comment). MUST be RUNNER_HOME-anchored like the two patterns above: an +# unanchored "run-helper" pkill matches ANY process on the host whose cmdline +# contains that substring, including the real fleet runner's own run-helper.sh +# when a nested/synthetic entrypoint.sh (e.g. the functional tests in +# tests/ci/test_runner_listener_liveness.py) runs in the same PID namespace — +# this is the confirmed root cause of self-hosted CI jobs mid-run receiving an +# out-of-band shutdown signal ("[HostContext] Runner will be shutdown for +# UserCancelled") while test_entrypoint_recycles_hung_listener exercises this +# recycle path elsewhere in the same container. +RUN_HELPER_PGREP_PATTERN="${RUN_HELPER_PGREP_PATTERN:-${RUNNER_HOME//./\\.}.*run-helper}" # OMN-14564: same find-mmin condition as healthcheck.sh layer 2 — returns 0 # (stale) when no _diag *.log was modified within @@ -308,14 +379,47 @@ _listener_heartbeat_stale() { _recycle_runner_tree() { local pid="${1}" kill -TERM "${pid}" 2>/dev/null || true - pkill -TERM -f "run-helper" 2>/dev/null || true + pkill -TERM -f "${RUN_HELPER_PGREP_PATTERN}" 2>/dev/null || true pkill -TERM -f "${LISTENER_PGREP_PATTERN}" 2>/dev/null || true sleep 10 kill -KILL "${pid}" 2>/dev/null || true - pkill -KILL -f "run-helper" 2>/dev/null || true + pkill -KILL -f "${RUN_HELPER_PGREP_PATTERN}" 2>/dev/null || true pkill -KILL -f "${LISTENER_PGREP_PATTERN}" 2>/dev/null || true } +# OMN-15233: reap any listener left over from a previous incarnation BEFORE +# spawning a replacement. Returns 0 when no listener remains (safe to spawn), +# 1 when one survived even SIGKILL (the caller must not spawn into a contested +# session). TERM first so a healthy-but-stranded listener can deregister its +# session cleanly; escalate to KILL after LISTENER_REAP_TIMEOUT_SECONDS because +# a listener deadlocked in its token-refresh call ignores TERM (OMN-14564). +_reap_orphaned_listeners() { + local pids + pids=$(pgrep -f "${LISTENER_PGREP_PATTERN}" 2>/dev/null || true) + if [[ -z "${pids}" ]]; then + return 0 + fi + echo "[entrypoint] REAP: Runner.Listener survived from a previous incarnation (pids: ${pids//$'\n'/ }) — killing before spawning a replacement (OMN-15233: spawn-without-reap manufactures TaskAgentSessionConflictException)" + pkill -TERM -f "${LISTENER_PGREP_PATTERN}" 2>/dev/null || true + local waited=0 + while pgrep -f "${LISTENER_PGREP_PATTERN}" >/dev/null 2>&1; do + if [[ ${waited} -ge ${LISTENER_REAP_TIMEOUT_SECONDS} ]]; then + echo "[entrypoint] REAP: listener ignored TERM for ${waited}s — escalating to KILL" + pkill -KILL -f "${LISTENER_PGREP_PATTERN}" 2>/dev/null || true + sleep 2 + break + fi + sleep 1 + waited=$((waited + 1)) + done + if pgrep -f "${LISTENER_PGREP_PATTERN}" >/dev/null 2>&1; then + echo "[entrypoint] REAP: FAILED — a Runner.Listener survived SIGKILL; refusing to spawn a replacement into a contested session" + return 1 + fi + echo "[entrypoint] REAP: no Runner.Listener remains — safe to spawn replacement" + return 0 +} + # Check for credentials in priority order: # 1. In-place (container restart — files already in RUNNER_HOME) # 2. Volume cache (fresh container — restore from mounted volume) @@ -340,6 +444,16 @@ fi attempt=0 listener_restarts=0 while true; do + # OMN-15233: never spawn on top of a surviving listener. A leftover listener + # still owns the GitHub broker session, so the replacement would crash-loop + # on TaskAgentSessionConflictException while keeping _diag fresh enough to + # read HEALTHY. Exiting here surfaces the state to the container restart + # policy + runner-monitor instead of hiding it in a silent loop. + if ! _reap_orphaned_listeners; then + echo "[entrypoint] REAP: unreapable listener — exiting so the container restart policy replaces the whole PID namespace" + exit 1 + fi + echo "[entrypoint] Starting runner (attempt $((attempt + 1)))" set +e _as_runner "${RUNNER_HOME}/run.sh" > >(tee "${LOG_FILE}") 2>&1 & diff --git a/docker/runners/healthcheck.sh b/docker/runners/healthcheck.sh index d8370acde9..c3be006d75 100755 --- a/docker/runners/healthcheck.sh +++ b/docker/runners/healthcheck.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Docker healthcheck for the GitHub Actions runner container. -# Tickets: OMN-12433 (egress), OMN-13915 (listener liveness + heartbeat freshness). +# Tickets: OMN-12433 (egress), OMN-13915 (listener liveness + heartbeat +# freshness), OMN-15233 (threshold recalibration + orphan/crash-loop +# detection), OMN-15311 (broker session state — the fourth state). # # History of what each layer catches: # - The original check only asserted container liveness — 37/48 runners sat @@ -9,19 +11,77 @@ # pgrep still passes when the listener is hung/zombied or when a wrapper # process keeps the tree "alive-looking" while no work flows. # - OMN-13915 adds a HEARTBEAT FRESHNESS check: a live, registered listener -# appends to ${RUNNER_HOME}/_diag continuously (long-poll cycle ~50s). If the -# newest _diag file is older than RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS the -# listener is not actually talking to GitHub, whatever the process table says. +# appends to ${RUNNER_HOME}/_diag continuously. If the newest _diag file is +# older than RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS the listener is not actually +# talking to GitHub, whatever the process table says. +# - OMN-15233 fixes the two defects the OMN-13915 layer shipped with: +# (a) FALSE POSITIVE by arithmetic — the 900s threshold sat far below the +# ~50-minute IDLE _diag write cadence, so an idle runner read unhealthy +# for ~35 of every 50 minutes with nothing wrong. See the threshold +# comment below. +# (b) INVERSION — the zombie shape this check exists to catch scored +# HEALTHY. An orphaned listener reparented to PPID 1 keeps the GitHub +# session; the watchdog's replacement crash-loops on +# TaskAgentSessionConflictException every ~5 min, and each crash mints +# a fresh Runner_*.log — which keeps _diag "fresh" forever. Layers 2 +# (process topology) and 4 (crash-loop rate) catch that shape. +# - OMN-15311 adds a BROKER SESSION STATE check (layer 3b) for the FOURTH +# state, measured live on 2026-07-27 during the OMN-15233 fan-out: runners +# 36, 38 and 56 sat GitHub-registry-OFFLINE for ~20 minutes while every +# local layer above passed — one live non-orphaned listener, _diag kept +# FRESH by the listener's own reconnect RETRY traffic, normal start rate, +# github.com reachable. The layers above assert that a listener process +# exists, is singular, is parented, is writing, and can reach github.com. +# None of them assert the thing that actually matters: that the listener +# HOLDS A LIVE BROKER SESSION and can therefore be handed a job. A +# state-4 runner is counted as capacity by Docker and is suppressed from +# auto-bounce by runner-monitor.sh's local-listener evidence rule, so it +# silently absorbs zero jobs until something else restarts it. # -# Tunables (env, defaults chosen for the 48-runner .201 fleet): -# RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS heartbeat staleness threshold (default 900) -# RUNNER_HEALTH_EGRESS_CHECK set to 0 to skip the github.com egress -# probe (used by offline CI tests only; -# production compose leaves it enabled) +# Tunables (env, defaults chosen for the 64-runner .201 fleet): +# RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS heartbeat staleness threshold (4500) +# RUNNER_HEALTH_MAX_LOG_STARTS_PER_HOUR listener starts/hour before the +# crash-loop layer fails (6). A RATE, +# normalized to the window below — not +# a raw count of files in the window. +# RUNNER_HEALTH_LOG_RATE_WINDOW_MINUTES crash-loop rate window (60) +# RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS how long the broker session may stay +# broken before the runner fails (900). +# A GRACE, not a threshold on a +# measurement — see layer 3b. +# RUNNER_HEALTH_SESSION_STATE_CHECK set to 0 to skip the broker-session +# layer entirely (fleet-wide kill +# switch by env, no file swap needed) +# RUNNER_HEALTH_EGRESS_CHECK set to 0 to skip the github.com egress +# probe (used by offline CI tests only; +# production compose leaves it enabled) set -u RUNNER_HOME="${RUNNER_HOME:-/home/runner/actions-runner}" -RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS="${RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS:-900}" +# OMN-15233 (a): 4500s (75 min), NOT the original 900s. When a runner is IDLE the +# only thing that writes _diag is the OAuth/AAD token refresh, on a ~50-minute +# cadence; the minutes-scale cadence only holds while jobs are running. 900s +# therefore flagged every idle runner for ~35 of every 50 minutes purely by +# threshold arithmetic — that is what produced the 13 -> 37 -> 59 "unhealthy +# growth" on 2026-07-27 while the GitHub registry reported 64/64 online +# throughout (59 -> 4 resolved with only 8 restarts; the untouched control group +# self-healed). 4500s clears the observed ~50-min idle write cadence with 50% +# margin. It is deliberately NOT the liveness signal of record — the GitHub +# registry (runner-fleet-canary, layer 4 of the runbook) is; this threshold only +# has to avoid manufacturing false unhealthy on an idle fleet while still +# surfacing a listener that has gone permanently silent. +RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS="${RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS:-4500}" +RUNNER_HEALTH_MAX_LOG_STARTS_PER_HOUR="${RUNNER_HEALTH_MAX_LOG_STARTS_PER_HOUR:-6}" +RUNNER_HEALTH_LOG_RATE_WINDOW_MINUTES="${RUNNER_HEALTH_LOG_RATE_WINDOW_MINUTES:-60}" +# OMN-15311: 900s (15 min). This is a GRACE on a state that is already known to +# be bad, not a threshold on a noisy measurement. A listener that loses its +# broker session normally re-establishes it in well under a minute (the +# 2026-07-27 network fault's registry-offline spike mostly self-healed within +# one poll); the cohort that did NOT recover held the broken state for ~20 min +# and only cleared on restart. 900s therefore sits above every recovery +# observed and below the shortest unrecovered case. +RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS="${RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS:-900}" +RUNNER_HEALTH_SESSION_STATE_CHECK="${RUNNER_HEALTH_SESSION_STATE_CHECK:-1}" RUNNER_HEALTH_EGRESS_CHECK="${RUNNER_HEALTH_EGRESS_CHECK:-1}" # 1. Listener process must be alive. Match THIS runner home's listener BINARY @@ -29,16 +89,46 @@ RUNNER_HEALTH_EGRESS_CHECK="${RUNNER_HEALTH_EGRESS_CHECK:-1}" # scripts, log paths, or another runner's listener must never satisfy the # liveness assertion. Dots in the path are escaped for pgrep's ERE matching. listener_pattern="${RUNNER_HOME//./\\.}/bin/Runner\.Listener" -if ! pgrep -f "${listener_pattern}" >/dev/null 2>&1; then +listener_pids=$(pgrep -f "${listener_pattern}" 2>/dev/null) +if [[ -z "${listener_pids}" ]]; then echo "unhealthy: Runner.Listener not running" exit 1 fi -# 2. Listener heartbeat must be FRESH (OMN-13915). The listener writes to -# _diag on every long-poll cycle; a listener that stopped talking to GitHub -# stops writing. Fail when no _diag file was modified within the threshold. -# A missing _diag directory with a "live" listener process is the same -# divergence — fail closed (compose start_period covers first registration). +# 2. Listener process TOPOLOGY must be sane (OMN-15233 b). Process EXISTENCE is +# not the assertion — process SINGULARITY and PARENTAGE are. +# +# - Duplicate listeners: exactly one Runner.Listener may hold this runner's +# GitHub broker session. Two means the session is contested, which is the +# TaskAgentSessionConflictException crash-loop from the inside. +# - PPID 1: a healthy listener's parent chain is +# entrypoint.sh(PID 1) -> run.sh -> run-helper.sh -> Runner.Listener, so a +# healthy listener is NEVER a direct child of PID 1. PPID 1 means its +# parent tree died and it was reparented — an orphan still holding the +# session while the entrypoint spawns replacements that cannot register. +# This is exactly the shape found on runners 1/43/55/57 (88-234 Runner_*.log +# files vs 3-7 on normal runners), which the mtime-only check scored +# HEALTHY because the crash-looping replacement kept _diag fresh. +listener_count=$(printf '%s\n' "${listener_pids}" | grep -c '^[0-9][0-9]*$') +if [[ "${listener_count}" -gt 1 ]]; then + echo "unhealthy: ${listener_count} Runner.Listener processes (pids: ${listener_pids//$'\n'/ }) — duplicate listeners contest the GitHub session (OMN-15233 orphan/conflict mode)" + exit 1 +fi +for pid in ${listener_pids}; do + listener_ppid=$(ps -o ppid= -p "${pid}" 2>/dev/null | tr -d '[:space:]') + if [[ "${listener_ppid}" == "1" ]]; then + echo "unhealthy: Runner.Listener pid ${pid} has PPID 1 — orphaned listener reparented after its wrapper tree died (OMN-15233 orphan mode)" + exit 1 + fi +done + +# 3. Listener heartbeat must be FRESH (OMN-13915, threshold recalibrated by +# OMN-15233 a). The listener writes to _diag on every long-poll cycle while +# busy and on every token refresh while idle; a listener that stopped talking +# to GitHub entirely stops writing. Fail when no _diag file was modified +# within the threshold. A missing _diag directory with a "live" listener +# process is the same divergence — fail closed (compose start_period covers +# first registration). diag_dir="${RUNNER_HOME}/_diag" max_age_minutes=$(( (RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS + 59) / 60 )) if [[ ! -d "${diag_dir}" ]]; then @@ -51,7 +141,160 @@ if [[ -z "${fresh_file}" ]]; then exit 1 fi -# 3. github.com must be reachable (OMN-12433). A connected listener with no +# 3b. Broker SESSION state must be CONNECTED (OMN-15311 — the FOURTH state). +# +# Every layer above is satisfied by a runner that GitHub considers OFFLINE. +# That is not hypothetical: on 2026-07-27 a transient host<->GitHub network +# fault left runners 36/38/56 registry-offline for ~20 min with a single +# non-orphaned live listener, a FRESH _diag (the reconnect retries are +# themselves _diag writes), a normal listener start rate, and github.com +# reachable. Exit 0 on all five layers; zero jobs accepted. +# +# The registry is the authoritative liveness surface, but the container has +# no GitHub credential to query it (and an unauthenticated poll from 64 +# containers would rate-limit itself). The listener's OWN log is the local +# projection of that same fact: it records when the broker session is +# established and when it drops. +# +# STATE, NOT PRESENCE. What matters is which marker class appears LAST in +# the newest Runner_*.log. "A connect error appears somewhere in the log" is +# true of essentially every long-lived healthy runner and would be a +# fleet-wide false positive; "the last session marker is an error, with no +# re-establish after it" is the actual broken state. +# +# PERSISTENCE, NOT INSTANT. Reconnects are normal and fast. A single check +# that fails the moment a drop is observed would flap the whole fleet on +# every blip. The broken state is therefore stamped on first observation and +# only fails once the stamp is older than the grace window; recovery deletes +# the stamp, so a later blip restarts the clock instead of inheriting an +# ancient one. Age is measured with find -mmin — the same idiom layers 3/4 +# already use — deliberately NOT by parsing the log's "[YYYY-MM-DD HH:MM:SSZ]" +# prefix, which needs GNU `date -d` and would make the layer unexercisable +# on the BSD-date gate host. +if [[ "${RUNNER_HEALTH_SESSION_STATE_CHECK}" != "0" ]]; then + if ! [[ "${RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS}" =~ ^[0-9]+$ ]] || + [[ "${RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS}" -lt 1 ]]; then + echo "unhealthy: RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS='${RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS}' is not a positive integer — refusing to guess a broker-session grace (fail closed)" + exit 1 + fi + + # Markers the runner itself writes. CONNECTED means the broker session is + # established and the runner is reachable for job assignment; BROKEN means the + # session was lost, refused, or is contested. + # + # SocketException is DELIBERATELY ABSENT from the broken set (removed + # 2026-07-28 by adversarial fleet probe, OMN-15311). BrokerServer emits + # "System.Net.Sockets.SocketException (125): Operation canceled" ~45-150x per + # listener log as ORDINARY long-poll cancellation - the very next line is + # "Get messages has been cancelled using local token source. Continue to get + # messages with new status." and the session is still up. Ordering does not + # rescue it: the connected markers only fire at session establishment / job + # assignment, so on any runner idle for >15 min that retry noise IS the last + # marker. Measured against all 64 live listeners on omninode-pc, every one of + # them Up-healthy and registry-online: WITH SocketException 64/64 classified + # broken; WITHOUT it 0/64. A permanently-red check is a disabled check - the + # same reasoning as the rate-vs-cumulative note in layer 4 below. + # tests/ci/fixtures/runner_diag_real_tail.log.gz pins this to a real log tail. + session_connected_patterns='Listening for Jobs|Runner reconnected|Job message received' + session_broken_patterns='Runner connect error|TaskAgentSessionConflictException|A session for this runner already exists|Unable to connect to the server|Failed to create session' + + # SC2012: `ls -t` is the portable newest-first ordering here; find -printf + # '%T@' is GNU-only and this script must also run under the BSD find on the + # gate host. Runner_*.log names are runner-generated and contain no spaces. + # shellcheck disable=SC2012 + newest_runner_log=$(ls -1t "${diag_dir}"/Runner_*.log 2>/dev/null | head -n 1) + if [[ -z "${newest_runner_log}" ]]; then + # Same divergence class as a missing _diag directory: a live listener always + # mints a Runner_-utc.log at start, so its absence means the + # process we matched is not a listener that ever registered. Fail closed. + echo "unhealthy: listener process present but no ${diag_dir}/Runner_*.log exists — broker session state is unreadable (OMN-15311 fail-closed)" + exit 1 + fi + + session_stamp="${diag_dir}/.session_broken_since" + last_connected_line=$(grep -nE "${session_connected_patterns}" "${newest_runner_log}" 2>/dev/null | tail -n 1 | cut -d: -f1) + last_broken_line=$(grep -nE "${session_broken_patterns}" "${newest_runner_log}" 2>/dev/null | tail -n 1 | cut -d: -f1) + + session_is_broken=0 + if [[ -n "${last_broken_line}" ]]; then + if [[ -z "${last_connected_line}" ]] || [[ "${last_broken_line}" -gt "${last_connected_line}" ]]; then + session_is_broken=1 + fi + fi + + if [[ "${session_is_broken}" -eq 1 ]]; then + if [[ ! -e "${session_stamp}" ]]; then + if ! : >"${session_stamp}" 2>/dev/null; then + # Without the stamp the grace cannot be measured, so "transient" and + # "stuck for an hour" become indistinguishable. Indeterminate is not + # health — and a non-writable _diag is itself a real fault, since the + # listener writes there continuously. + echo "unhealthy: broker session is broken and ${session_stamp} is not writable — cannot measure how long it has been broken (OMN-15311 fail-closed)" + exit 1 + fi + fi + session_grace_minutes=$(((RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS + 59) / 60)) + if [[ -z "$(find "${session_stamp}" -mmin "-${session_grace_minutes}" -print 2>/dev/null)" ]]; then + echo "unhealthy: GitHub broker session broken for more than ${RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS}s — listener alive and writing _diag, but its last session marker in $(basename "${newest_runner_log}") is an error with no re-establish after it (OMN-15311 broken-session mode; GitHub reports this runner OFFLINE)" + exit 1 + fi + session_state="reconnecting (broken, inside the ${RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS}s grace)" + else + # Recovered (or never broken): drop the stamp so the grace clock restarts + # from the NEXT drop rather than from an old one. + rm -f "${session_stamp}" 2>/dev/null || true + session_state="connected" + fi +else + session_state="unchecked" +fi + +# 4. Listener restart RATE must be sane (OMN-15233 b). The runner mints one +# Runner_-utc.log per listener process start, so listener starts +# inside a bounded window are directly countable. A replacement listener +# crash-looping against an orphan that owns the session restarts every ~5 min +# (~12/hour); a normal runner restarts a handful of times over its entire +# lifetime. +# +# This is deliberately RATE-BASED, NOT CUMULATIVE. A cumulative Runner_*.log +# count persists across restarts and grows monotonically with container +# uptime, so any long-lived healthy container would eventually cross a fixed +# total and flag forever — a permanently-red check is a disabled check. Only +# logs touched inside RUNNER_HEALTH_LOG_RATE_WINDOW_MINUTES are counted; a +# runner with 234 historical logs and one active log reads 1. +# +# -mmin (not birth time) is the portable proxy: a Runner_*.log is only +# appended to while its listener process lives, so a closed log's mtime is +# within seconds of that listener's death and an active log's mtime is now. +# -maxdepth 1 keeps the per-job page logs under _diag/pages/ out of the count. +# +# NORMALIZATION (the two tunables are in DIFFERENT units): the count is taken +# over RUNNER_HEALTH_LOG_RATE_WINDOW_MINUTES but the threshold is expressed +# PER HOUR, so comparing them directly is only correct at the 60-minute +# default. A 30m window would silently halve the effective threshold (6/hour +# enforced as 6-per-30m = 12/hour) and a 120m window would double it +# (6-per-120m = 3/hour). The per-hour budget is therefore scaled to the window +# actually measured before the comparison. Integer arithmetic only (no bc in +# the runner image): the +59 numerator rounds the allowance UP, so a +# fractional budget (6/hour over a 5m window = 0.5) never floors to 0 — a +# zero allowance would fail on the first legitimate listener start. +window_minutes="${RUNNER_HEALTH_LOG_RATE_WINDOW_MINUTES}" +if ! [[ "${window_minutes}" =~ ^[0-9]+$ ]] || [[ "${window_minutes}" -lt 1 ]]; then + echo "unhealthy: RUNNER_HEALTH_LOG_RATE_WINDOW_MINUTES='${window_minutes}' is not a positive integer — refusing to guess a crash-loop window (fail closed)" + exit 1 +fi +if ! [[ "${RUNNER_HEALTH_MAX_LOG_STARTS_PER_HOUR}" =~ ^[0-9]+$ ]]; then + echo "unhealthy: RUNNER_HEALTH_MAX_LOG_STARTS_PER_HOUR='${RUNNER_HEALTH_MAX_LOG_STARTS_PER_HOUR}' is not a non-negative integer — refusing to guess a crash-loop threshold (fail closed)" + exit 1 +fi +max_starts_in_window=$(( (RUNNER_HEALTH_MAX_LOG_STARTS_PER_HOUR * window_minutes + 59) / 60 )) +recent_starts=$(find "${diag_dir}" -maxdepth 1 -type f -name 'Runner_*.log' -mmin "-${window_minutes}" -print 2>/dev/null | wc -l | tr -d '[:space:]') +if [[ "${recent_starts}" -gt "${max_starts_in_window}" ]]; then + echo "unhealthy: ${recent_starts} listener starts in the last ${window_minutes}m (> ${max_starts_in_window} allowed — ${RUNNER_HEALTH_MAX_LOG_STARTS_PER_HOUR}/hour normalized to a ${window_minutes}m window) — listener crash-looping (OMN-15233 crash-loop rate)" + exit 1 +fi + +# 5. github.com must be reachable (OMN-12433). A connected listener with no # in-flight job is expected; an egress fault that drops the GitHub # connection is what we catch. Use a bounded HEAD request instead of the # unauthenticated API rate_limit endpoint so shared-IP API limits cannot @@ -63,5 +306,5 @@ if [[ "${RUNNER_HEALTH_EGRESS_CHECK}" != "0" ]]; then fi fi -echo "healthy: listener up, heartbeat fresh, github.com reachable" +echo "healthy: single non-orphaned listener, heartbeat fresh, broker session ${session_state}, ${recent_starts}/${max_starts_in_window} allowed start(s) in ${window_minutes}m, github.com reachable" exit 0 diff --git a/docker/runners/runner-image.lock.json b/docker/runners/runner-image.lock.json index fcb6c1550e..feafe0e9fe 100644 --- a/docker/runners/runner-image.lock.json +++ b/docker/runners/runner-image.lock.json @@ -1,12 +1,12 @@ { "base_image_digest": "sha256:3ba65aa20f86a0fad9df2b2c259c613df006b2e6d0bfcc8a146afb8c525a9751", "gh_version": "2.67.0", - "identity_digest": "b7dda174b00fcb4405dd15f4f5d1609f", - "image_version": 6, + "identity_digest": "17b97156cc173de6faa32489b88ac961", + "image_version": 7, "kubectl_version": "1.32.1", "python_version": "3.12", - "runner_version": "2.334.0", - "shared_env_digest": "d6b23d12c45638a081ec280b", + "runner_version": "2.336.0", + "shared_env_digest": "bffad7795606a13c66aefdac", "shared_env_install_args": "--frozen --all-extras --all-groups --no-install-project", "uv_version": "0.6.14" } diff --git a/docker/runtime-policy.env b/docker/runtime-policy.env index 8288ebd118..d75947cfa7 100644 --- a/docker/runtime-policy.env +++ b/docker/runtime-policy.env @@ -4,7 +4,7 @@ ARCH_GRAPH_BOLT_URI=bolt://omnibase-infra-memgraph:7687 AUXILIARY_SERVICES_OMNIMEMORY_ENABLED=false BIFROST_VERIFY_ENDPOINTS=1 BIFROST_VERTEX_GEMINI_ENDPOINT_URL=https://us-central1-aiplatform.googleapis.com/v1beta1/projects/gen-lang-client-0084338881/locations/us-central1/endpoints/openapi/chat/completions -DEV_BOUNDARY_DLQ_ENABLED=false +DEV_BOUNDARY_DLQ_ENABLED=true DEV_COMPOSE_PROJECT=omnibase-infra DEV_RUNTIME_EFFECTS_BIFROST_VERIFY_ENDPOINTS=1 DEV_RUNTIME_EFFECTS_CAPABILITIES=effects.consumer,market.skill-proof,runtime.effects diff --git a/docs/decisions/README.md b/docs/decisions/README.md index c755600d34..593a52518a 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -30,6 +30,7 @@ ADRs are immutable once accepted. Superseded decisions are marked but not delete | [ADR-007](adr-007-vault-to-infisical-migration.md) | Vault to Infisical Migration | Accepted | | [ADR-008](adr-008-realm-agnostic-topics.md) | Realm-Agnostic Topic Format | Accepted | | [ADR-009](adr-009-llm-cost-tracking-spi.md) | LLM Cost Tracking at the Infrastructure Layer | Accepted | +| [ADR-010](adr-010-authoritative-environment-topology-owner.md) | OmniBase Infra Owns Authoritative Environment Topology | Accepted | ### Topic-Based ADRs @@ -75,6 +76,7 @@ ADRs are immutable once accepted. Superseded decisions are marked but not delete - ADR-007: Security + Configuration (Vault to Infisical migration) - ADR-008: Event Bus (realm-agnostic topic format) - ADR-009: Data Layer (LLM cost tracking SPI) +- ADR-010: Architecture + Configuration (authoritative environment topology owner) - The canonical publish interface policy ADR is also Architecture. ## Writing ADRs diff --git a/docs/decisions/adr-010-authoritative-environment-topology-owner.md b/docs/decisions/adr-010-authoritative-environment-topology-owner.md new file mode 100644 index 0000000000..01f0cabb3f --- /dev/null +++ b/docs/decisions/adr-010-authoritative-environment-topology-owner.md @@ -0,0 +1,61 @@ +> **Navigation**: [Home](../index.md) > [Decisions](README.md) > ADR-010 Environment Topology Owner + +# ADR-010: OmniBase Infra Owns Authoritative Environment Topology + +## Status + +Accepted + +## Date + +2026-07-29 + +## Context + +OMN-15414 requires one checked-in, secret-free authority for application-database +topology. `omnibase_infra` already owns the typed Docker service catalog and its +generator, ships the runtime topology consumer package, and directly consumes +`omnibase_core.models.core.ModelDeploymentTopology`. `omninode_infra` owns Kubernetes +manifests, but those manifests are deployment projections and must not become a second +database-semantics registry. Host-local `~/.omnibase/topology.yaml` is mutable setup +state and cannot be platform authority. + +## Decision + +`omnibase_infra/src/omnibase_infra/topology/instances/*.yaml` is the authoritative +environment topology. Each file parses through the frozen, extra-forbid +`ModelDeploymentTopology` contract. The files may contain logical database names, +schema domains, NOLOGIN owner names, workload-principal names, explicit grant +declarations, and DSN environment-variable names. They contain no credentials or DSN +values. + +Docker database bindings under `docker/catalog/` and Kubernetes database/secret +bindings under `omninode_infra/k8s/` are generated or parity-validated projections. +The Kubernetes secret-ownership manifest remains authoritative for credential +synchronization only; it does not own database, schema, or principal semantics. + +The `omninode_runtime` service name and `omninode_runtime` PostgreSQL principal are +separate namespaces. The service consumes the `omninode_runtime_service` binding, +which resolves to the principal. Consumers load an explicitly named checked-in +environment and never fall back to a host-local file or an inferred database. + +## Consequences + +### Positive + +- Docker, Kubernetes, runtime catalogs, and DSN maps can prove parity against one typed + instance. +- Seeded database, schema, principal, or DSN-environment drift fails before deployment. +- Local setup remains useful without gaining authority over shared environments. +- Secret values remain solely in secret-management systems. + +### Negative + +- Cross-repository Kubernetes changes require an exact `omnibase_infra` source revision + or projection checksum until the topology is distributed as a released artifact. +- A topology-model change must land in `omnibase_core` before consumer PRs may land. + +### Neutral + +- This decision declares target semantics only. It performs no DDL, GRANT, secret + rotation, deploy, restart, workload cutover, or database retirement. diff --git a/docs/env-example-full.txt b/docs/env-example-full.txt index f59d48caac..4f23d1d51a 100644 --- a/docs/env-example-full.txt +++ b/docs/env-example-full.txt @@ -271,6 +271,10 @@ ONEX_LOG_LEVEL=INFO # The same rule applies to any future ONEX_EFFECTS_*_TOPIC overrides. # ONEX_EFFECTS_GROUP_ID=onex-runtime-effects +# HMAC-SHA256 key shared by runtime-effects and deploy-agent. Required for +# signed rebuild commands; generate once and store in ~/.omnibase/.env. +# DEPLOY_AGENT_HMAC_SECRET=__REPLACE_WITH_SECURE_RANDOM_SECRET__ + # Worker runtime Kafka topics (optional) # Default: worker-requests, worker-responses, onex-runtime-workers # ONEX_WORKER_INPUT_TOPIC=worker-requests @@ -658,8 +662,11 @@ ONEX_LOG_LEVEL=INFO # RUNTIME_MAIN_PORT=8085 # RUNTIME_EFFECTS_PORT=8086 -# Number of worker replicas for horizontal scaling -# WORKER_REPLICAS=2 +# Number of worker replicas for horizontal scaling. OMN-14968: the knob is the +# lane-prefixed DEV_WORKER_REPLICAS, `:?`-required in docker-compose.infra.yml +# (the render fails closed when unset instead of silently rendering 0 replicas). +# The ledgered pin lives in docker/runtime-policy.env. +# DEV_WORKER_REPLICAS=2 # ============================================================================= # Agent Actions Consumer Configuration (Observability Profile) diff --git a/docs/evidence/OMN-15123/2026-08-03-one-tenant-contract-freeze.md b/docs/evidence/OMN-15123/2026-08-03-one-tenant-contract-freeze.md new file mode 100644 index 0000000000..8c8d1ae47c --- /dev/null +++ b/docs/evidence/OMN-15123/2026-08-03-one-tenant-contract-freeze.md @@ -0,0 +1,130 @@ +# Managed-staging one-tenant contract — FREEZE PACKET (partial, dated instance) + +**Ticket:** OMN-15123 · **Plan row:** rolling plan §3 B3 (structural note below — this +citation no longer resolves against the live plan) · **Parent epic:** OMN-14724 +**Field manifest (the seam):** [`docs/runbooks/managed-staging-proof-kit/fields.yaml`](../../runbooks/managed-staging-proof-kit/fields.yaml) +**Template this was copied from:** [`docs/runbooks/managed-staging-one-tenant-contract-freeze.md`](../../runbooks/managed-staging-one-tenant-contract-freeze.md) +**Seam test:** `tests/ci/test_managed_staging_proof_kit_seam.py` +**Instance date:** 2026-08-03 (UTC) · **Author lane:** fable-beta-0803/G1 (build agent, Sonnet 5) + +> ## THIS IS NOT A FREEZE. Read this before reading the table. +> +> A freeze requires every row below to carry a real, current readback (AC #1/#2). +> **9 of 19 rows are BLOCKED and carry no value** — this host has no live AWS +> session (SSO expired, human login pending) and no DB/deploy access, per this +> session's operating constraints. **A 10th row (`plan_row_binding`) is blocked +> for a structural reason unrelated to AWS** (see §ac3 below). Only the rows +> answerable from already-committed, offline repo state are filled. +> +> **Do not treat the commit that lands this file as the OMN-15123 freeze event.** +> `freeze_signature` below records only that this partial snapshot was committed; +> the real freeze happens when a later, complete instance supersedes this one +> with every BLOCKED row cleared by a live readback. Until then, OMN-15123's +> acceptance criteria remain unmet and the ticket must not be moved to Done from +> this artifact alone. + +## AC-by-AC disposition + +| # | Acceptance criterion | Status | Why | +|---|---|---|---| +| 1 | Committed, immutable contract artifact enumerating all 19 fields | **NOT MET** | This artifact exists and is committed, but only 8 of 19 fields carry a real value; 11 are BLOCKED (see table). Not immutable in the AC's sense until complete. | +| 2 | Digest fields read back equal to the live candidate at freeze time, command output attached | **NOT MET** | `source_digest`/`image_digest` are BLOCKED. The nearest known digest (`sha256:414944a4…`, OMN-14974, 2026-07-27T02:12Z per `ROLLING_WORK_LEDGER.md:8439`) is **7 days stale** relative to this instance date and is explicitly not usable as "at freeze time" — cited for context only, not as a filled value. | +| 3 | Rolling plan §3 B3's "unverifiable by construction" tag bound to this ticket id (plan diff cited) | **STRUCTURALLY BLOCKED, not AWS-blocked** | See §ac3. | +| 4 | OMN-14736 (B11 canary) references this frozen tuple as its input | **NOT MET** | The tuple isn't frozen (AC #1 unmet), so linking OMN-14736 to it now would misrepresent an incomplete artifact as authoritative input. Deferred to the complete instance. | + +### §ac3 — the plan-row-binding field is blocked for a structural reason + +The manifest's `plan_row_binding` evidence source reads: + +``` +git -C $OMNI_HOME/omni_home log -p -- docs/plans/ROLLING_SEVEN_DAY_PLAN.md | grep -n 'OMN-15123' # cite the plan diff that replaced the 'unverifiable by construction' tag +``` + +Live grep against the current `docs/plans/ROLLING_SEVEN_DAY_PLAN.md` (2026-08-03), +run from this instance: + +- No `§3` section heading exists anywhere in the file. The plan's decisions section + is now `## 3. Decisions` with subsections `### 3a.` / `### 3b.` — a different + structure than the "§3 B3" row the ticket and this manifest cite. +- The literal string `unverifiable by construction` does not appear anywhere in the + current plan. +- `OMN-15123` **is** present in the plan (three hits, `§2` "Ranked seven-day work + queue" chain-ordering prose, e.g. line 38: `(OMN-10858, OMN-15123, OMN-15124, + OMN-15125)`), but none of those hits replace a "§3 B3 unverifiable-by-construction" + tag — that row no longer exists in that form. The plan was restructured by the + 2026-08-01 §0-AIM rewrite (`docs/plans/ROLLING_SEVEN_DAY_PLAN.md` revision log, + 2026-08-01 entry) after the ticket was filed 2026-07-25 against the pre-rewrite + section numbering. + +This is a **plan-governor reconciliation gap**, not something a build lane can +close by editing the ticket's own citation: closing AC #3 correctly requires a +plan-governor pass over the live plan to either (a) locate the successor row the +old "§3 B3" content moved to and bind `OMN-15123` there explicitly, or (b) rule +that the row was subsumed by the §2 chain-ordering prose and record that ruling. +Neither is a build-lane decision. **Flagged here, not fixed.** + +## Fields + +| Field | What it is | Value | Evidence source | Status | +|---|---|---|---|---| +| `aws_account` | AWS account | _BLOCKED_ | `aws sts get-caller-identity --query Account --output text` | **BLOCKED — AWS SSO expired on this host, human login pending.** (Committed docs, e.g. `docs/runbooks/managed-staging-canary-teardown-rollback.md` §"Plane scope", name account `272493677981` — cited for context only; that is a doc reference, not the live readback this field requires, and is not treated as satisfying the field.) | +| `aws_region` | AWS region | _BLOCKED_ | `aws configure get region` | **BLOCKED — same AWS SSO gap.** (Same runbook names `us-east-1` for context only.) | +| `k8s_namespace` | Kubernetes namespace (single canary namespace) | _BLOCKED_ | `kubectl get ns -o jsonpath=...` | **BLOCKED — no live k8s access from this host/session.** | +| `msk_cluster_arn` | MSK cluster ARN | _BLOCKED_ | `aws kafka list-clusters-v2 ...` | **BLOCKED — AWS SSO gap.** | +| `rds_instance_identifier` | RDS instance identifier | _BLOCKED_ | `aws rds describe-db-instances ...` | **BLOCKED — AWS SSO gap.** | +| `gateway_endpoint` | The one gateway | _BLOCKED_ | `kubectl -n get svc,ingress ...` | **BLOCKED — no live k8s access.** | +| `synthetic_tenant_id` | The one synthetic tenant (UUID) | _BLOCKED_ | `psql "$CANARY_DSN" ...` | **BLOCKED — no DB access; the canary DB's provisioning status is itself unconfirmed from this host.** | +| `source_digest` | Candidate source digest | _BLOCKED (stale reference only)_ | `git -C rev-parse HEAD` | **BLOCKED as "at freeze time."** Nearest known value: commit `710197a6` (omnibase_infra dev, per `ROLLING_WORK_LEDGER.md:8438`, 2026-07-27T02:05Z) — 7 days stale, not usable as current. | +| `image_digest` | Candidate image digest | _BLOCKED (stale reference only)_ | `aws ecr describe-images ...` | **BLOCKED as "at freeze time."** Nearest known value: `sha256:414944a4f1d543bd63119aa4172e0e90edfe2717e8d68dae6bdc5323fd81788a` (per `ROLLING_WORK_LEDGER.md:8439`, 2026-07-27T02:12Z) — 7 days stale, not usable as current, and not confirmed to be the same candidate tuple this freeze targets. | +| `config_digest` | Rendered runtime config digest | _BLOCKED_ | `kubectl -n get cm ... \| shasum -a 256` | **BLOCKED — no live k8s access.** | +| `topic_catalog` | Approved `onex.mstg1.` topic/group catalog | **164 topics / 56 groups — see `probes/topic_catalog.txt`** | `uv run python -c '...build_canary_catalog_from_candidate...'` | **FILLED — offline, in-repo.** Ran on this host 2026-08-03; full sorted list committed alongside this packet (`probes/topic_catalog.txt`, 228 lines incl. headers). Generator: `src/omnibase_infra/topics/managed_staging_canary_catalog.py` (OMN-14727). | +| `zero_collision_readback` | Zero-collision readback | **`is_clean: True` against an EMPTY snapshot — see caveat** | `uv run python -c '...verify_zero_collision...'` | **FILLED, but OFFLINE-ONLY — not the live check the field label implies.** The module's own docstring is explicit: "The catalog + namespace live entirely in-repo, so the readback runs offline. The live readback against the actual 1089 topics + live consumer groups needs a broker connection from inside the VPC and is therefore deferred to Phase 3 (apply-to-cluster)." This run passed `existing_topics=[]`/`existing_groups=[]` (no live snapshot available), so `is_clean: True` proves only that the freshly generated 164-topic/56-group catalog has no *internal* collisions and the check code runs cleanly — it does **not** prove disjointness from the live cluster's 1089 topics. Full output: `probes/zero_collision_readback_offline.txt`. Live Phase-3 readback remains BLOCKED (no VPC/broker access from this host). | +| `msk_epoch` | Unique MSK epoch | **`mstg1`** | `src/omnibase_infra/topics/managed_staging_canary_catalog_namespace.yaml -> epoch` | **FILLED.** Read directly from the committed namespace file (line 22 area, `epoch: "mstg1"`). | +| `group_start_reset_policy` | Signed consumer-group start/reset policy | **`earliest` (policy value only — NOT signed)** | `..._namespace.yaml -> group_start_policy` + operator signature line | **PARTIALLY FILLED.** The policy value (`group_start_policy: "earliest"`) is a committed fact, read from the same namespace file. The manifest also requires an **operator signature line** in the filled packet — that signature does not exist yet; this row is not complete until an operator signs it. Not counted as BLOCKED (the value is known and committed) but not counted as satisfying the AC's "signed" requirement either. | +| `rollback_authority` | Named rollback authority | **See ownership table citation** | `docs/runbooks/managed-staging-canary-teardown-rollback.md` §0 | **FILLED.** §0 ownership table: trigger owner Jonah (planned teardown) / staffed operator Jonah (abort, on agent's B10 breach signal) / Jonah (rollback); live-execution owner is Jonah in all three paths; the agent prepares/validates/captures evidence. Supporting identity: the A3 create/delete-capable operator IAM identity (contractor-team-owned) for topic/group deletion — never the runtime identity. | +| `zero_prod_diff` | Zero-prod-diff assertion | **PASS (no prod resource named), with one documented self-match** | `grep -nE 'omnibase-infra-prod\|:28085\|:28086' ` | **FILLED.** Run against this file after it was written (`probes/zero_prod_diff_grep.txt`): exactly **one** match, this row's own `Evidence source` cell text (the grep pattern itself, quoting the strings it checks for) — that is the check's own documentation, not a prod resource used by the tuple. Excluding that self-referential row, zero matches; no actual prod resource is named anywhere in the tuple's data rows. | +| `omnidash_exclusion` | Omnidash exclusion | _BLOCKED_ | `kubectl -n get deploy -o jsonpath=...` | **BLOCKED — no live k8s access.** | +| `freeze_signature` | Freeze signature | **See note — not a valid freeze signature yet** | `git log -1 --format='%H %aI %an' -- ` | **DELIBERATELY NOT FILLED AS A FREEZE.** This row is reserved for the commit that lands the *complete* packet. This commit is not that commit (11 of 19 rows are BLOCKED or partial). Recording the landing commit here would misstate an incomplete artifact as the freeze event, which AC #1 explicitly guards against ("immutable" implies complete). Left unfilled by design; fill it only on the instance that clears every BLOCKED row above. | +| `plan_row_binding` | Rolling plan §3 B3 bound to OMN-15123 | _BLOCKED (structural)_ | `git log -p -- docs/plans/ROLLING_SEVEN_DAY_PLAN.md \| grep -n 'OMN-15123'` | **BLOCKED — see §ac3 above.** Not an AWS gap; a plan-governor reconciliation gap. | + +## Probes committed alongside this packet + +- `probes/topic_catalog.txt` — full offline-generated topic/group catalog (164 topics, 56 groups), `build_canary_catalog_from_candidate()` run 2026-08-03 on this host. +- `probes/zero_collision_readback_offline.txt` — offline `verify_zero_collision()` run against an empty snapshot; explicitly captioned as not the live check. +- `probes/zero_prod_diff_grep.txt` — raw `grep -nE` output against this file, with the two self-referential matches called out. + +## Known drift flagged (do not inherit silently) + +`src/omnibase_infra/topics/managed_staging_canary_catalog_namespace.yaml` line +~47's comment still reads *"The 2x kafka.t3.small managed cluster is already +~2.7x over AWS partition guidance"*. **This is stale.** OMN-15253's 2026-07-27 +live readback (`ROLLING_WORK_LEDGER.md:8745`) corrected the live broker sizing to +**2x kafka.m5.large**. This packet does not carry a sizing field itself (sizing +lives in `default_partitions`/`default_replication_factor`, not in the frozen +tuple's field list), so there is nothing to correct in the table above — but any +future artifact that cites broker sizing from that yaml comment must use the +corrected `kafka.m5.large` value and note the comment itself is stale. Filing a +doc-fix for that comment is outside this ticket's scope (it lives in +`src/`, not `docs/evidence/`) and is not attempted here to keep this PR +evidence-only. + +## What would close this ticket for real + +1. A fresh AWS SSO session on a host with live access (human login, per this + session's hard constraint that SSO is dead here) to fill the 9 AWS/k8s/DB + rows with real command output. +2. A source/image/config digest pinned **at the moment of freeze**, not a + 7-day-old reference — meaning the freeze must happen at the same time as (or + immediately after) a fresh candidate build, not asynchronously from it. +3. A plan-governor pass resolving §ac3 (bind `OMN-15123` to whatever row the old + "§3 B3 unverifiable by construction" content became, or rule it subsumed). +4. Only once 1–3 land: a complete instance superseding this one, with + `freeze_signature` filled on *that* commit, followed by a comment/link on + OMN-14736 (AC #4). + +## Related + +- `docs/runbooks/managed-staging-canary-postgres-provisioning.md` — B12 landing table + §3.4 readback +- `docs/runbooks/managed-staging-canary-teardown-rollback.md` — teardown / abort / rollback ownership (B13) +- `src/omnibase_infra/topics/managed_staging_canary_catalog.py` + `..._namespace.yaml` — B7 `onex.mstg1.` catalog, epoch, zero-collision readback (OMN-14727) +- `docs/runbooks/managed-staging-proof-kit/fields.yaml` — the field manifest (seam) this packet fills diff --git a/docs/evidence/OMN-15123/probes/topic_catalog.txt b/docs/evidence/OMN-15123/probes/topic_catalog.txt new file mode 100644 index 0000000000..b6395660a9 --- /dev/null +++ b/docs/evidence/OMN-15123/probes/topic_catalog.txt @@ -0,0 +1,228 @@ +# topic_prefix: onex.mstg1. +# group_prefix: onex.mstg1. +# topic_count: 164 +# group_count: 56 + +## topics +onex.mstg1.onex.cmd.artifact.reconcile.v1 +onex.mstg1.onex.cmd.omnibase-infra.baselines-batch-compute.v1 +onex.mstg1.onex.cmd.omnibase-infra.build-loop-append.v1 +onex.mstg1.onex.cmd.omnibase-infra.chain-learn.v1 +onex.mstg1.onex.cmd.omnibase-infra.coding-agent-effect-invoke.v1 +onex.mstg1.onex.cmd.omnibase-infra.coding-agent-invoke.v1 +onex.mstg1.onex.cmd.omnibase-infra.coding-agent-workspace-validate.v1 +onex.mstg1.onex.cmd.omnibase-infra.consumer-restart.v1 +onex.mstg1.onex.cmd.omnibase-infra.llm-completion-request.v1 +onex.mstg1.onex.cmd.omnibase-infra.llm-embedding-request.v1 +onex.mstg1.onex.cmd.omnibase-infra.llm-inference-request.v1 +onex.mstg1.onex.cmd.omnibase-infra.onboarding-start.v1 +onex.mstg1.onex.cmd.omnibase-infra.pattern-b-dispatch.v1 +onex.mstg1.onex.cmd.omnibase-infra.pr-state-upsert.v1 +onex.mstg1.onex.cmd.omnibase-infra.remote-agent-invoke.v1 +onex.mstg1.onex.cmd.omnibase-infra.runner-fleet-health-evaluate.v1 +onex.mstg1.onex.cmd.omnibase-infra.runner-fleet-maintain-start.v1 +onex.mstg1.onex.cmd.omnibase-infra.runner-fleet-snapshot-gather.v1 +onex.mstg1.onex.cmd.omnibase-infra.topic-migration-execute.v1 +onex.mstg1.onex.cmd.omnibase-infra.validation-ledger-append.v1 +onex.mstg1.onex.cmd.omnibase-infra.vector-store-request.v1 +onex.mstg1.onex.cmd.omnimarket.redeploy-start.v1 +onex.mstg1.onex.cmd.platform.contract-resolve-requested.v1 +onex.mstg1.onex.cmd.platform.ledger-append.v1 +onex.mstg1.onex.cmd.platform.ledger-query.v1 +onex.mstg1.onex.cmd.platform.node-registration-acked.v1 +onex.mstg1.onex.cmd.platform.request-introspection.v1 +onex.mstg1.onex.cmd.platform.topic-catalog-query.v1 +onex.mstg1.onex.cmd.router.route-request.v1 +onex.mstg1.onex.cmd.rsd.score.v1 +onex.mstg1.onex.cmd.skill.merge-sweep.v1 +onex.mstg1.onex.cmd.skill.scope-check.v1 +onex.mstg1.onex.dlq.omnibase-infra.commands.v1 +onex.mstg1.onex.dlq.omnibase-infra.events.v1 +onex.mstg1.onex.dlq.omnibase-infra.intents.v1 +onex.mstg1.onex.dlq.omnibase-infra.omnibase-infra.v1 +onex.mstg1.onex.dlq.omnibase-infra.platform.v1 +onex.mstg1.onex.dlq.omnibase-infra.quarantine.v1 +onex.mstg1.onex.dlq.omnibase-infra.router.v1 +onex.mstg1.onex.dlq.omnibase-infra.rsd.v1 +onex.mstg1.onex.dlq.omnibase-infra.skill.v1 +onex.mstg1.onex.evt.artifact.change-detected.v1 +onex.mstg1.onex.evt.artifact.impact-analyzed.v1 +onex.mstg1.onex.evt.artifact.pr-comment-posted.v1 +onex.mstg1.onex.evt.artifact.update-plan-created.v1 +onex.mstg1.onex.evt.artifact.update-plan-emitted.v1 +onex.mstg1.onex.evt.github.pr-status.v1 +onex.mstg1.onex.evt.github.pr-webhook.v1 +onex.mstg1.onex.evt.occ.nightly-promotion.v1 +onex.mstg1.onex.evt.omnibase-infra.agent-task-lifecycle.v1 +onex.mstg1.onex.evt.omnibase-infra.baselines-computed.v1 +onex.mstg1.onex.evt.omnibase-infra.build-loop-appended.v1 +onex.mstg1.onex.evt.omnibase-infra.chain-learn-complete.v1 +onex.mstg1.onex.evt.omnibase-infra.chain-learn-failed.v1 +onex.mstg1.onex.evt.omnibase-infra.chain-replay-result.v1 +onex.mstg1.onex.evt.omnibase-infra.chain-retrieval-result.v1 +onex.mstg1.onex.evt.omnibase-infra.chain-stored.v1 +onex.mstg1.onex.evt.omnibase-infra.chain-verified.v1 +onex.mstg1.onex.evt.omnibase-infra.coding-agent-completed.v1 +onex.mstg1.onex.evt.omnibase-infra.coding-agent-failed.v1 +onex.mstg1.onex.evt.omnibase-infra.coding-agent-fsm-advance.v1 +onex.mstg1.onex.evt.omnibase-infra.coding-agent-fsm-state-updated.v1 +onex.mstg1.onex.evt.omnibase-infra.coding-agent-invoke-completed.v1 +onex.mstg1.onex.evt.omnibase-infra.coding-agent-invoke-failed.v1 +onex.mstg1.onex.evt.omnibase-infra.coding-agent-workspace-validated.v1 +onex.mstg1.onex.evt.omnibase-infra.consumer-health.v1 +onex.mstg1.onex.evt.omnibase-infra.db-error.v1 +onex.mstg1.onex.evt.omnibase-infra.event-forwarded.v1 +onex.mstg1.onex.evt.omnibase-infra.gmail-intent-received.v1 +onex.mstg1.onex.evt.omnibase-infra.inference-response.v1 +onex.mstg1.onex.evt.omnibase-infra.llm-call-completed.v1 +onex.mstg1.onex.evt.omnibase-infra.llm-completion-completed.v1 +onex.mstg1.onex.evt.omnibase-infra.network-pool-status.v1 +onex.mstg1.onex.evt.omnibase-infra.onboarding-completed.v1 +onex.mstg1.onex.evt.omnibase-infra.onboarding-step-verified.v1 +onex.mstg1.onex.evt.omnibase-infra.pattern-b-dispatch-completed.v1 +onex.mstg1.onex.evt.omnibase-infra.row-count-diagnostic.v1 +onex.mstg1.onex.evt.omnibase-infra.runner-fleet-health-verdict.v1 +onex.mstg1.onex.evt.omnibase-infra.runner-fleet-maintain-completed.v1 +onex.mstg1.onex.evt.omnibase-infra.runner-health-snapshot.v1 +onex.mstg1.onex.evt.omnibase-infra.runtime-booted.v1 +onex.mstg1.onex.evt.omnibase-infra.runtime-error.v1 +onex.mstg1.onex.evt.omnibase-infra.runtime-manifest-published.v1 +onex.mstg1.onex.evt.omnibase-infra.savings-estimated.v1 +onex.mstg1.onex.evt.omnibase-infra.service-lifecycle.v1 +onex.mstg1.onex.evt.omnibase-infra.system-alert.v1 +onex.mstg1.onex.evt.omnibase-infra.tool-update.v1 +onex.mstg1.onex.evt.omnibase-infra.topic-migration-lifecycle.v1 +onex.mstg1.onex.evt.omnibase-infra.vector-store-completed.v1 +onex.mstg1.onex.evt.omniclaude.context-audit-dlq.v1 +onex.mstg1.onex.evt.omniclaude.context-injected.v1 +onex.mstg1.onex.evt.omniclaude.notification-blocked.v1 +onex.mstg1.onex.evt.omniclaude.notification-completed.v1 +onex.mstg1.onex.evt.omniclaude.pattern-enforcement.v1 +onex.mstg1.onex.evt.omniclaude.phase-metrics.v1 +onex.mstg1.onex.evt.omniclaude.session-ended.v1 +onex.mstg1.onex.evt.omniclaude.session-outcome.v1 +onex.mstg1.onex.evt.omniclaude.validator-catch.v1 +onex.mstg1.onex.evt.omniintelligence.dispatch-outcome-evaluated.v1 +onex.mstg1.onex.evt.omniintelligence.llm-call-completed.v1 +onex.mstg1.onex.evt.omniintelligence.waste-detected.v1 +onex.mstg1.onex.evt.omnimarket.build-loop-orchestrator-completed.v1 +onex.mstg1.onex.evt.omnimarket.runtime-deployment-proof.v1 +onex.mstg1.onex.evt.omnimemory.policy-state-updated.v1 +onex.mstg1.onex.evt.omnimemory.reward-assigned.v1 +onex.mstg1.onex.evt.omninode.runner-usage-recorded.v1 +onex.mstg1.onex.evt.onex-change-control.contract-drift-detected.v1 +onex.mstg1.onex.evt.onex-change-control.cosmetic-compliance-scored.v1 +onex.mstg1.onex.evt.onex-change-control.governance-check-completed.v1 +onex.mstg1.onex.evt.platform.contract-deregistered.v1 +onex.mstg1.onex.evt.platform.contract-registered.v1 +onex.mstg1.onex.evt.platform.fsm-state-transitions.v1 +onex.mstg1.onex.evt.platform.ledger-appended.v1 +onex.mstg1.onex.evt.platform.ledger-query-result.v1 +onex.mstg1.onex.evt.platform.merge-gate-decision.v1 +onex.mstg1.onex.evt.platform.node-became-active.v1 +onex.mstg1.onex.evt.platform.node-heartbeat.v1 +onex.mstg1.onex.evt.platform.node-introspection.v1 +onex.mstg1.onex.evt.platform.node-liveness-expired.v1 +onex.mstg1.onex.evt.platform.node-registration-accepted.v1 +onex.mstg1.onex.evt.platform.node-registration-ack-received.v1 +onex.mstg1.onex.evt.platform.node-registration-ack-timed-out.v1 +onex.mstg1.onex.evt.platform.node-registration-initiated.v1 +onex.mstg1.onex.evt.platform.node-registration-rejected.v1 +onex.mstg1.onex.evt.platform.node-registration-result.v1 +onex.mstg1.onex.evt.platform.node-registration.v1 +onex.mstg1.onex.evt.platform.registration-snapshots.v1 +onex.mstg1.onex.evt.platform.registry-request-introspection.v1 +onex.mstg1.onex.evt.platform.topic-catalog-changed.v1 +onex.mstg1.onex.evt.platform.topic-catalog-response.v1 +onex.mstg1.onex.evt.platform.validation-adjudication-completed.v1 +onex.mstg1.onex.evt.platform.validation-adjudication-requested.v1 +onex.mstg1.onex.evt.platform.validation-candidate-submitted.v1 +onex.mstg1.onex.evt.platform.validation-checks-completed.v1 +onex.mstg1.onex.evt.platform.validation-execution-requested.v1 +onex.mstg1.onex.evt.platform.validation-lifecycle-updated.v1 +onex.mstg1.onex.evt.platform.validation-plan-created.v1 +onex.mstg1.onex.evt.platform.validation-result-published.v1 +onex.mstg1.onex.evt.router.health-snapshot.v1 +onex.mstg1.onex.evt.router.routing-complete.v1 +onex.mstg1.onex.evt.router.routing-failed.v1 +onex.mstg1.onex.evt.router.routing-outcome.v1 +onex.mstg1.onex.evt.router.scoring-decision.v1 +onex.mstg1.onex.evt.rsd.data-fetched.v1 +onex.mstg1.onex.evt.rsd.score-complete.v1 +onex.mstg1.onex.evt.rsd.score-failed.v1 +onex.mstg1.onex.evt.rsd.scores-calculated.v1 +onex.mstg1.onex.evt.rsd.scores-stored.v1 +onex.mstg1.onex.evt.skill.merge-sweep-auto-merged.v1 +onex.mstg1.onex.evt.skill.merge-sweep-classified.v1 +onex.mstg1.onex.evt.skill.merge-sweep-complete.v1 +onex.mstg1.onex.evt.skill.merge-sweep-failed.v1 +onex.mstg1.onex.evt.skill.merge-sweep-pr-list.v1 +onex.mstg1.onex.evt.skill.scope-check-complete.v1 +onex.mstg1.onex.evt.skill.scope-check-failed.v1 +onex.mstg1.onex.evt.skill.scope-extracted.v1 +onex.mstg1.onex.evt.skill.scope-file-read.v1 +onex.mstg1.onex.evt.skill.scope-manifest-written.v1 +onex.mstg1.onex.evt.steel-onslaught.match-terminal.v1 +onex.mstg1.onex.evt.validation.cross-repo-run-completed.v1 +onex.mstg1.onex.evt.validation.cross-repo-run-started.v1 +onex.mstg1.onex.evt.validation.cross-repo-violations-batch.v1 +onex.mstg1.onex.intent.platform.runtime-tick.v1 +onex.mstg1.onex.snapshot.projection.coding-agent.correlation-trace.v1 + +## groups +onex.mstg1.node_artifact_change_detector_effect.consume.v1 +onex.mstg1.node_artifact_reconciliation_orchestrator.consume.v1 +onex.mstg1.node_baselines_batch_compute.consume.v1 +onex.mstg1.node_build_loop_projection_compute.consume.v1 +onex.mstg1.node_build_loop_write_effect.consume.v1 +onex.mstg1.node_chain_orchestrator.consume.v1 +onex.mstg1.node_chain_retrieval_effect.consume.v1 +onex.mstg1.node_chain_store_effect.consume.v1 +onex.mstg1.node_coding_agent_fsm_reducer.consume.v1 +onex.mstg1.node_coding_agent_invoke_effect.consume.v1 +onex.mstg1.node_coding_agent_orchestrator.consume.v1 +onex.mstg1.node_coding_agent_workspace_compute.consume.v1 +onex.mstg1.node_consumer_health_triage_effect.consume.v1 +onex.mstg1.node_context_audit_dlq_effect.consume.v1 +onex.mstg1.node_contract_registry_reducer.consume.v1 +onex.mstg1.node_contract_resolver_bridge.consume.v1 +onex.mstg1.node_db_error_linear_effect.consume.v1 +onex.mstg1.node_dlq_replay_effect.consume.v1 +onex.mstg1.node_emit_daemon_runtime.consume.v1 +onex.mstg1.node_event_forward_effect.consume.v1 +onex.mstg1.node_github_pr_poller_effect.consume.v1 +onex.mstg1.node_gmail_archive_cleanup_effect.consume.v1 +onex.mstg1.node_impact_analyzer_compute.consume.v1 +onex.mstg1.node_ledger_projection_compute.consume.v1 +onex.mstg1.node_ledger_write_effect.consume.v1 +onex.mstg1.node_llm_completion_effect.consume.v1 +onex.mstg1.node_llm_embedding_effect.consume.v1 +onex.mstg1.node_llm_inference_effect.consume.v1 +onex.mstg1.node_merge_gate_effect.consume.v1 +onex.mstg1.node_merge_sweep_workflow_orchestrator.consume.v1 +onex.mstg1.node_onboarding_orchestrator.consume.v1 +onex.mstg1.node_pr_state_projection_compute.consume.v1 +onex.mstg1.node_pr_state_write_effect.consume.v1 +onex.mstg1.node_registration_orchestrator.consume.v1 +onex.mstg1.node_remote_agent_invoke_effect.consume.v1 +onex.mstg1.node_routing_orchestrator.consume.v1 +onex.mstg1.node_rsd_orchestrator.consume.v1 +onex.mstg1.node_runner_fleet_health_compute.consume.v1 +onex.mstg1.node_runner_fleet_maintain_orchestrator.consume.v1 +onex.mstg1.node_runner_health_snapshot_effect.consume.v1 +onex.mstg1.node_runner_usage_effect.consume.v1 +onex.mstg1.node_runtime_error_triage_effect.consume.v1 +onex.mstg1.node_runtime_manifest_reducer.consume.v1 +onex.mstg1.node_runtime_orchestrator.consume.v1 +onex.mstg1.node_runtime_source_attestor_effect.consume.v1 +onex.mstg1.node_savings_estimation_compute.consume.v1 +onex.mstg1.node_scope_workflow_orchestrator.consume.v1 +onex.mstg1.node_topic_migration_executor_effect.consume.v1 +onex.mstg1.node_topic_migration_projection.consume.v1 +onex.mstg1.node_update_plan_reducer.consume.v1 +onex.mstg1.node_validation_adjudicator.consume.v1 +onex.mstg1.node_validation_ledger_projection_compute.consume.v1 +onex.mstg1.node_validation_ledger_write_effect.consume.v1 +onex.mstg1.node_validation_orchestrator.consume.v1 +onex.mstg1.node_vector_store_effect.consume.v1 +onex.mstg1.node_waste_detection_compute.consume.v1 diff --git a/docs/evidence/OMN-15123/probes/zero_collision_readback_offline.txt b/docs/evidence/OMN-15123/probes/zero_collision_readback_offline.txt new file mode 100644 index 0000000000..922291bd9c --- /dev/null +++ b/docs/evidence/OMN-15123/probes/zero_collision_readback_offline.txt @@ -0,0 +1,17 @@ +# OFFLINE zero-collision readback (in-repo generated catalog vs an EMPTY existing-state snapshot). +# This is NOT the live cluster readback the field label implies; the module docstring is +# explicit that the live readback (real 1089 topics) needs a broker connection from inside +# the VPC and is deferred to Phase 3 (apply-to-cluster). This proves the catalog/report code +# path executes cleanly and the freshly generated namespace has zero self-collisions; it does +# NOT prove disjointness from the live cluster state. +is_clean: True +topic_prefix: onex.mstg1. +group_prefix: onex.mstg1. +checked_topic_count: 164 +checked_group_count: 56 +existing_topic_count (snapshot size): 0 +existing_group_count (snapshot size): 0 +colliding_topics: () +colliding_groups: () +prefix_conflicting_topics: () +prefix_conflicting_groups: () diff --git a/docs/evidence/OMN-15123/probes/zero_prod_diff_grep.txt b/docs/evidence/OMN-15123/probes/zero_prod_diff_grep.txt new file mode 100644 index 0000000000..bdacf1d0b8 --- /dev/null +++ b/docs/evidence/OMN-15123/probes/zero_prod_diff_grep.txt @@ -0,0 +1 @@ +85:| `zero_prod_diff` | Zero-prod-diff assertion | **PASS (no prod resource named), with one documented self-match** | `grep -nE 'omnibase-infra-prod\|:28085\|:28086' ` | **FILLED.** Run against this file after it was written (`probes/zero_prod_diff_grep.txt`): exactly **one** match, this row's own `Evidence source` cell text (the grep pattern itself, quoting the strings it checks for) — that is the check's own documentation, not a prod resource used by the tuple. Excluding that self-referential row, zero matches; no actual prod resource is named anywhere in the tuple's data rows. | diff --git a/docs/evidence/OMN-15124/2026-08-03-candidate-isolation-static-evidence-partial.md b/docs/evidence/OMN-15124/2026-08-03-candidate-isolation-static-evidence-partial.md new file mode 100644 index 0000000000..5973a83495 --- /dev/null +++ b/docs/evidence/OMN-15124/2026-08-03-candidate-isolation-static-evidence-partial.md @@ -0,0 +1,129 @@ +# OMN-15124 — candidate-in-isolation compatibility proof: PARTIAL static evidence + +**Ticket:** OMN-15124 · **Parent epic:** OMN-14724 · **Status:** PARTIAL, NOT a completed packet +**Field manifest (the seam):** [`docs/runbooks/managed-staging-proof-kit/fields.yaml`](../../runbooks/managed-staging-proof-kit/fields.yaml) +**Packet template (unfilled, unmodified by this doc):** [`docs/runbooks/managed-staging-candidate-isolation-compatibility-proof.md`](../../runbooks/managed-staging-candidate-isolation-compatibility-proof.md) +**Captured:** 2026-08-03T18:09:04Z, host `Stickybeatz-Studio.local` (`.200`), repo `omnibase_infra` at `origin/dev` HEAD `3860bec762`. + +## Why this is PARTIAL, not a filled packet + +The ticket's 5 acceptance criteria all require a **live isolation-environment run** +against real MSK and RDS with negative controls (AC1–AC4) plus a rolling-plan diff +citation for a plan row that does not exist in the current plan structure (AC5). AWS +SSO is dead on every host available to this session (human login pending) — every +live AWS/k8s/psql step is BLOCKED. None of the 5 ACs can be checked off by this +session. **Do not read this document as progress against any AC checkbox.** + +Of the manifest's 12 `candidate_isolation_compatibility` fields, exactly **2** are +answerable with zero live AWS/network dependency — pure static code-path checks run +on this host today. They are recorded below as labeled evidence. The remaining 10 are +enumerated in the gap statement and require a live isolation lane this session does +not have. + +## What was run (static, no AWS, no network) + +### Field: `typed_config_authority` + +> Typed config authority (config comes from the typed model, not ad-hoc env reads) + +``` +$ cd omnibase_infra && env -u PYTHONPATH uv run python -c " +from omnibase_infra.event_bus.kafka_auth import build_aiokafka_auth_kwargs_from_env +print(build_aiokafka_auth_kwargs_from_env.__module__) +" +omnibase_infra.event_bus.kafka_auth +``` + +Confirms the function the candidate must call for its Kafka auth kwargs resolves to +the typed `omnibase_infra.event_bus.kafka_auth` module, not an ad-hoc env-read call +site elsewhere. **This proves only that the typed entry point exists and is what a +correctly-wired caller would import** — it does not prove any live candidate process +actually calls it instead of a bypass, and it does not run +`scripts/check_required_env_vars.py` against a candidate's real env contract (that +script validates local `docker/docker-compose.infra.yml` + local env files, not an +isolation-lane candidate's live config — running it here would not add real signal +toward this field and was skipped as out of scope). + +### Field: `no_raw_endpoint_fallback` — STATIC HALF ONLY + +> NEGATIVE CONTROL: no raw-endpoint / plaintext fallback path was exercised + +The manifest evidence source splits this field into a live half ("unset the IAM env, +start the candidate, and show it FAILS CLOSED") and a static half. Only the static +half was run: + +``` +$ cd omnibase_infra && bash scripts/check_no_cloud_bus_wrapper.sh +(no output, exit 0) + +$ grep -rn "PLAINTEXT" src/omnibase_infra/event_bus/ +src/omnibase_infra/event_bus/kafka_auth.py:107: if config.security_protocol == "PLAINTEXT": +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:113: Default: "PLAINTEXT" +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:114: Options: "PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL" +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:119: Requires: security_protocol must be SASL_PLAINTEXT or SASL_SSL +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:382: default="PLAINTEXT", +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:385: "Valid values: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL" +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:387: pattern=r"^(PLAINTEXT|SSL|SASL_PLAINTEXT|SASL_SSL)$", +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:394: "Requires security_protocol to be SASL_PLAINTEXT or SASL_SSL. " +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:562: - If security_protocol is SASL_PLAINTEXT or SASL_SSL, sasl_mechanism must be set +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:563: - If sasl_mechanism is set, security_protocol must be SASL_PLAINTEXT or SASL_SSL +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:579: self.security_protocol in ("SASL_PLAINTEXT", "SASL_SSL") +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:590: if self.security_protocol not in ("SASL_PLAINTEXT", "SASL_SSL"): +src/omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py:593: f"'SASL_PLAINTEXT' or 'SASL_SSL', got {self.security_protocol!r}", +``` + +Reading `kafka_auth.py:105-107`: + +```python +def build_aiokafka_auth_kwargs(config: ModelKafkaEventBusConfig) -> dict[str, object]: + """Build auth/TLS kwargs for aiokafka clients from runtime Kafka config.""" + if config.security_protocol == "PLAINTEXT": + return {} +``` + +**This is not a fail-closed guard and must not be read as one.** `PLAINTEXT` is a +valid, config-declared value of `security_protocol` (used for local dev), and the +function honors it by returning empty auth kwargs — a legitimate branch, not a silent +fallback bypassing a failed IAM path. `check_no_cloud_bus_wrapper.sh` passed clean +(no disallowed direct cloud-bus construction found), and no hidden alternate +construction path exists in `src/omnibase_infra/event_bus/` outside this typed +module. **What this does NOT prove:** that a candidate configured for +`AWS_MSK_IAM` actually fails closed rather than silently degrading to `PLAINTEXT` +when its IAM env is unset — that is exactly the live half of this negative control +("unset the IAM env, start the candidate, show FAILS CLOSED"), which requires a +running candidate process in an isolation lane and was not exercised. + +## Gap statement — everything else this ticket's ACs require, and why it is blocked + +All 5 ACs remain **unchecked**. Per field: + +| Field | AC | Blocked on | +|---|---|---| +| `isolation_lane` | AC1 | No isolation host/k8s context available this session — AWS SSO dead, human login pending. | +| `candidate_image_digest` | AC1 | Requires `kubectl get pod ... imageID` against a live isolation lane. | +| `msk_iam_signer` (live half) | AC1 | Requires a running candidate process actually invoking the MSK IAM signer against a real broker. | +| `token_refresh_cycle` | AC1 | Requires a soaked live session with ≥2 observed token mints — no live process exists. | +| `auto_create_off` | AC2 | Requires `aws kafka describe-configuration-revision` — AWS API, no credentials. | +| `explicit_topic_bootstrap` | AC2 | Requires a live broker + `kafka-topics --bootstrap-server` with IAM creds. | +| `negative_control_out_of_catalog` | AC2 | Requires a live broker to observe the actual denial. | +| `broker_group_perms` | (supporting AC2) | Requires `aws iam get-role-policy` — AWS API, no credentials. | +| `rds_verify_full` | AC3 | Requires a live `psql` connection to a real RDS instance. | +| `dashboard_zero_authority` | AC4 | Requires `kubectl get cm,secret` against a live isolation/candidate namespace. | +| `plan_row_binding` | AC5 | **Not AWS-blocked — structurally absent.** `docs/plans/ROLLING_SEVEN_DAY_PLAN.md` has no `§3 B5` row: live section headings are `0-AIM`, `0-CHAIN`, `0. Operating rules`, `1. Current ground state`, `2. Ranked seven-day work queue`, `3. Decisions`, `4. Parked...`, `5. Source trail`, `6. Revision log`, and `git log -p -- docs/plans/ROLLING_SEVEN_DAY_PLAN.md \| grep -n 'OMN-15124'` returns no plan-diff hit — only the single line-38 chain-list mention (`## 0-CHAIN`) and the §3b decisions table have no B5/OMN-15124 row. Same class of gap as OMN-15123 AC #3: the manifest's `plan_row: "rolling plan §3 B5"` cites a row that does not exist in the live plan document. This is a doc-authoring/reconciliation gap, not an AWS dependency, and is out of this session's scope to resolve unilaterally (it requires a plan-governance decision on how/whether to bind the tag). | + +## Adjacent, out-of-scope context (recorded per ticket comment, not actioned here) + +Per the 2026-08-01 ticket comment (Jonah Gray): OMN-15639 tracks a separate, +group-authorization-level MSK IAM defect (`GroupAuthorizationFailedError` on +`runtime-local-*` consumer groups) adjacent to but explicitly **out of** this +ticket's connection-level scope. Even a fully-satisfied OMN-15124 does not clear +OMN-15639 — recorded here for continuity, not addressed by this PR. + +## Bottom line + +0/5 ACs satisfied. This PR adds labeled PARTIAL static evidence for 2 of 12 manifest +fields and a field-by-field gap statement for the remaining 10, so the next session +with live AWS/isolation-lane access has a starting inventory instead of an empty +template. **No AC checkbox in the live ticket is flipped by this PR** — that would be +a false-Done claim: only a live isolation run (AC1–AC4) and an operator plan-binding +decision (AC5) close this ticket. diff --git a/docs/evidence/OMN-15124/2026-08-05-candidate-isolation-round4-evidence.md b/docs/evidence/OMN-15124/2026-08-05-candidate-isolation-round4-evidence.md new file mode 100644 index 0000000000..85c78738ce --- /dev/null +++ b/docs/evidence/OMN-15124/2026-08-05-candidate-isolation-round4-evidence.md @@ -0,0 +1,225 @@ +# OMN-15124 — candidate-in-isolation compatibility proof: round-4 evidence (still PARTIAL) + +**Ticket:** OMN-15124 · **Parent epic:** OMN-14724 · **Status:** PARTIAL, NOT a completed packet +**Field manifest (the seam):** [`docs/runbooks/managed-staging-proof-kit/fields.yaml`](../../runbooks/managed-staging-proof-kit/fields.yaml) +**Packet template (unfilled, unmodified by this doc):** [`docs/runbooks/managed-staging-candidate-isolation-compatibility-proof.md`](../../runbooks/managed-staging-candidate-isolation-compatibility-proof.md) +**Captured:** 2026-08-05 (UTC), host `Stickybeatz-Studio.local` (`.200`), repo `omnibase_infra` at `origin/dev` HEAD `d1c5927cb7`. +**Supersedes:** does not edit `docs/evidence/OMN-15124/2026-08-03-candidate-isolation-static-evidence-partial.md` — this is a new dated instance adding what AWS SSO recovery unlocks. + +## Why this is still PARTIAL, not a filled packet + +AWS SSO recovered 2026-08-04T15:47:47Z and is confirmed live this session. That +unlocks **control-plane, read-only AWS API calls** this instance adds below. It does +**not** unlock the ticket's core requirement: all 4 real ACs (AC1–AC4) need a **live +candidate process actually running in a named isolation lane**, pointed at real +MSK/RDS, with negative controls exercised against it. No isolation lane exists — +there is no k8s/kubectl access from this host (direct `:6443` times out, same finding +as the OMN-15123 round-4 packet), so no candidate pod is running anywhere this +session can observe. **0/5 ACs remain unchecked. Do not read this document as +progress against any AC checkbox.** + +## What changed vs. the 2026-08-03 instance + +The 2026-08-03 instance filled 2/12 fields (both pure static code-path checks, zero +AWS dependency). This instance re-runs those 2 unchanged (values do not depend on +AWS/network state) and adds **2 new fields answerable via live, read-only AWS +control-plane API now that SSO works** — neither requires a running candidate or a +named isolation lane, so neither closes an AC, but both are real signal a future live +run will consume: + +| Field | 2026-08-03 | 2026-08-05 round-4 | Why | +|---|---|---|---| +| `typed_config_authority` | FILLED (static) | **FILLED (re-run, unchanged)** | Pure code-path check, no AWS dependency | +| `no_raw_endpoint_fallback` (static half) | FILLED (static) | **FILLED (re-run, unchanged)** | Pure code-path check, no AWS dependency | +| `auto_create_off` | BLOCKED (no AWS creds) | **FILLED — `auto.create.topics.enable=false`** | `aws kafka describe-configuration-revision`, live, read-only | +| *(new finding, not a manifest field)* broker network reachability | not probed | **MSK IAM broker port `9098` TCP-reachable from this host; RDS `5432` is NOT** | See "New finding" below — flags a possible narrower isolation-lane option, does not itself satisfy any field | +| `isolation_lane`, `candidate_image_digest`, `msk_iam_signer` (live half), `token_refresh_cycle`, `explicit_topic_bootstrap`, `negative_control_out_of_catalog`, `broker_group_perms`, `rds_verify_full`, `dashboard_zero_authority` | BLOCKED | **still BLOCKED** | No isolation lane / no k8s access / require the candidate's own scoped IAM identity, not this session's admin SSO role — see per-field reasons below | +| `plan_row_binding` | BLOCKED (structural) | **RESOLVED** | 2026-08-04 plan-governor reconcile, same fix as OMN-15123 (`ROLLING_WORK_LEDGER.md:12253`) | + +## What was run (live, read-only AWS control-plane; zero mutation, zero isolation-lane execution) + +### Field: `auto_create_off` + +> Broker auto-create is OFF + +```console +$ aws kafka describe-configuration-revision \ + --arn arn:aws:kafka:us-east-1:272493677981:configuration/omninode-dev-msk-config/930d30cc-105c-4c3e-ab2a-aa6cfb0a5b0b-14 \ + --revision 1 --region us-east-1 --query ServerProperties --output text | base64 -d +auto.create.topics.enable=false +default.replication.factor=2 +delete.topic.enable=true +log.retention.hours=168 +min.insync.replicas=1 +num.partitions=3 +``` + +Full output: `probes-round4/msk_configuration_server_properties.txt`. This is +revision **1** of the config, which is **not** proof it is the revision bound to the +live cluster — the round-4 evidence CodeRabbit review (2026-08-08) correctly flagged +that the field was FILLED off a configuration revision without confirming that +revision is the one the active cluster actually runs. + +**Cluster-binding readback, added to close that gap** (2026-08-09, same AWS SSO +session, live/read-only): + +```console +$ aws kafka describe-cluster --region us-east-1 \ + --cluster-arn arn:aws:kafka:us-east-1:272493677981:cluster/omninode-dev-msk/88ad72bc-f70c-4549-93bc-c392b965f424-14 +ClusterArn: arn:aws:kafka:us-east-1:272493677981:cluster/omninode-dev-msk/88ad72bc-f70c-4549-93bc-c392b965f424-14 +State: ACTIVE +PublicAccess.Type: DISABLED +CurrentBrokerSoftwareInfo.ConfigurationArn: arn:aws:kafka:us-east-1:272493677981:configuration/omninode-dev-msk-config/930d30cc-105c-4c3e-ab2a-aa6cfb0a5b0b-14 +CurrentBrokerSoftwareInfo.ConfigurationRevision: 2 +``` + +Full output: `probes-round4/msk_describe_cluster.json`. **The active revision is 2, +not 1** — the original probe queried a stale revision. Revision 2 was re-read +directly to confirm the field value still holds on the actually-bound config: + +```console +$ aws kafka describe-configuration-revision \ + --arn arn:aws:kafka:us-east-1:272493677981:configuration/omninode-dev-msk-config/930d30cc-105c-4c3e-ab2a-aa6cfb0a5b0b-14 \ + --revision 2 --region us-east-1 --query ServerProperties --output text | base64 -d +auto.create.topics.enable=false +default.replication.factor=2 +delete.topic.enable=true +log.retention.hours=168 +min.insync.replicas=1 +num.partitions=3 +``` + +Full output: `probes-round4/msk_configuration_revision2_server_properties.txt` — +identical values to revision 1, `auto.create.topics.enable=false` unchanged. +**FILLED — live, now bound to the cluster's active configuration (ArN + revision 2), +cluster state ACTIVE, `PublicAccess.Type: DISABLED` confirmed.** + +### New finding (not a manifest field): broker network reachability from this host + +```console +$ nc -zv -w5 b-1.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com 9098 +Connection to b-1.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com port 9098 [tcp/*] succeeded! + +$ nc -zv -w5 omninode-dev-postgres.cqjkkokeaqd2.us-east-1.rds.amazonaws.com 5432 +nc: connectx to ... port 5432 (tcp) failed: Operation timed out +``` + +Full output: `probes-round4/nc_msk_broker_9098.txt`, `probes-round4/nc_rds_5432.txt`. +**Genuinely surprising** given the cluster's `PublicAccess.Type: DISABLED` and this +host being outside the VPC — a bare TCP handshake to the MSK IAM listener (9098) +succeeds, while the RDS Postgres port (5432, also VPC-private) times out as expected. +**This is recorded as a finding, not exploited.** It does not by itself prove +anything about the candidate: (1) TCP reachability is not the same as a completed +SASL/IAM handshake — no Kafka protocol exchange was attempted; (2) even if a real +handshake succeeded, testing with this session's own admin SSO identity +(`AROAT64PDRWOQRVJSLB6P:jonah-iam-main`) would prove that identity's permissions, not +the candidate's scoped node-role permissions — a negative-control test run under an +admin identity is not a real negative control (admin can do everything an +under-scoped role cannot, so a "denied" result would be impossible to obtain and a +"succeeded" result would prove nothing about the intended restriction). No topic +create/list/describe was attempted against the live broker in this lane — that would +require either the candidate's own IAM identity or would-be canary-scoped +credentials, neither available here, and any topic mutation is out of this lane's +authorized scope (artifacts/proofs only, no canary execution). **Handoff:** whoever +stands up the isolation lane should know this host may not need an SSM tunnel for +MSK specifically (unlike RDS and k8s, both confirmed still blocked) — worth a quick +re-check before assuming full VPC access is required for the MSK-facing half of this +proof. + +### Fields carried forward unchanged (re-run, same result) + +`typed_config_authority`: +```console +$ env -u PYTHONPATH uv run python -c " +from omnibase_infra.event_bus.kafka_auth import build_aiokafka_auth_kwargs_from_env +print(build_aiokafka_auth_kwargs_from_env.__module__) +" +omnibase_infra.event_bus.kafka_auth +``` +Full output: `probes-round4/typed_config_authority.txt`. Unchanged from 2026-08-03 — +this module boundary has not moved. + +`no_raw_endpoint_fallback` (static half): +```console +$ bash scripts/check_no_cloud_bus_wrapper.sh +(no output, exit 0) +$ grep -n "PLAINTEXT" src/omnibase_infra/event_bus/kafka_auth.py +107: if config.security_protocol == "PLAINTEXT": +``` +Full output: `probes-round4/check_no_cloud_bus_wrapper.txt`, +`probes-round4/grep_plaintext_kafka_auth.txt`. Same reading as the 2026-08-03 +instance: this is a legitimate config-declared branch (local dev), not a silent +fallback — the live half (unset IAM env on a running candidate, observe fail-closed) +remains unexercised. + +## Gap statement — everything still blocking AC1–AC4, and why + +| Field | AC | Blocked on | +|---|---|---| +| `isolation_lane` | AC1 | No k8s/kubectl access from this host (`:6443` direct times out — same finding as OMN-15123 round-4). No named isolation host exists to record. | +| `candidate_image_digest` | AC1 | Requires `kubectl get pod ... imageID` against a live isolation lane — none exists. (Cross-reference only, not a fill: the round-4 target is `sha256:b829c56f...`, per the OMN-15123 round-4 packet — but this field means the digest **actually running in the isolation pod**, which this session cannot observe.) | +| `msk_iam_signer` (live half) | AC1 | Requires a running candidate process actually invoking the signer against the real broker; only the static module-resolution half is answerable without a live process. | +| `token_refresh_cycle` | AC1 | Requires a soaked live session with ≥2 observed token mints — no live process exists. | +| `explicit_topic_bootstrap` | AC2 | Requires `kafka-topics --list` with IAM auth configured — no `kafka-topics` CLI + AWS MSK IAM auth JAAS setup exists on this host, and even if it did, listing under this session's admin identity would not represent the candidate's actual bootstrap path. | +| `negative_control_out_of_catalog` | AC2 | Same identity problem as above, doubled: a negative control run under an admin identity cannot fail the way the candidate's scoped identity would — see "New finding" above. | +| `broker_group_perms` | (supporting AC2) | Requires `aws iam get-role-policy --role-name ` — the candidate's actual node/pod IAM role name is not known from this host (would need IRSA/service-account annotation lookup via `kubectl`, which is blocked). `aws iam list-roles` in this account shows only cluster/infra-management roles (`omninode-k3s-*-node-role`, ASG lifecycle, GitHub Actions) — none obviously named as the runtime pod's scoped role, and guessing would risk citing the wrong policy as evidence. | +| `rds_verify_full` | AC3 | RDS port 5432 confirmed unreachable from this host (`probes-round4/nc_rds_5432.txt`, timeout) — VPC-private, `PubliclyAccessible: false`. | +| `dashboard_zero_authority` | AC4 | Requires `kubectl get cm,secret` against a live isolation/candidate namespace — blocked, same as above. | +| `plan_row_binding` | AC5 | **RESOLVED**, not blocked — see below. | + +### `plan_row_binding` (AC5) — resolved, not blocked + +The 2026-08-03 instance found this structurally blocked (the plan's `§3 B5` row no +longer exists in that form). The 2026-08-04T03:58:31Z plan-governor reconcile +(`ROLLING_WORK_LEDGER.md:12253`, plan commit `7fb2e9853`) fixed this for both +OMN-15123 and OMN-15124 together: the plan now cites `OMN-15124` by ID directly at +two live anchors — `§0-CHAIN` link 6 and `§2` "Fastest readiness order" item 6 — both +independently re-confirmed by this instance's own grep against the live plan +(`OMN-15124` present at lines 38 and 93 of `docs/plans/ROLLING_SEVEN_DAY_PLAN.md`). +**AC5 is the one AC of the five that this instance closes cleanly.** + +## Adjacent, out-of-scope context (unchanged from 2026-08-03, still relevant) + +OMN-15639 tracks a separate, group-authorization-level MSK IAM defect +(`GroupAuthorizationFailedError` on `runtime-local-*` consumer groups) adjacent to +but explicitly out of this ticket's connection-level scope. Even a fully-satisfied +OMN-15124 does not clear OMN-15639. + +## Operator / Daniyal handoff + +1. Same SSM-tunnel gap as OMN-15123: k8s access from this host needs a tunnel, not + direct `:6443`. Once available, `isolation_lane`, `candidate_image_digest`, + `dashboard_zero_authority`, and `broker_group_perms` (via the pod's IRSA + annotation) all become reachable. +2. **Worth a quick check before assuming MSK also needs the tunnel**: the MSK IAM + listener (`:9098`) is TCP-reachable from this host right now, unlike RDS. + Standing up a real candidate process (or even a scoped test client with the + node's actual IAM identity, not admin SSO) directly from a host like this one + might close `msk_iam_signer` (live half), `token_refresh_cycle`, + `explicit_topic_bootstrap`, and `negative_control_out_of_catalog` without a full + k8s isolation lane — worth scoping as a cheaper path than standing up the whole + isolation lane, but requires the candidate's actual scoped credentials, which + this session does not have and should not substitute admin credentials for (a + negative control run under admin creds is not a real negative control). +3. `rds_verify_full` genuinely needs in-VPC or SSM-tunneled access; no shortcut + found this session. + +## Bottom line + +0/5 ACs satisfied (AC5/`plan_row_binding` is resolved, but that resolution landed via +a separate plan-governor commit, not this PR, so it is recorded here rather than +claimed as this PR's own closure). This instance adds 2 new live, read-only AWS +findings (`auto_create_off` filled; MSK broker network reachability noted as a +scoping lead) on top of the 2 static fields carried forward unchanged. The +candidate-isolation manifest (`docs/runbooks/managed-staging-proof-kit/fields.yaml`, +`isolation_lane` … `dashboard_zero_authority`) declares 12 unique field IDs backing +AC1–AC4; `plan_row_binding` (AC5) is a separate, thirteenth field not counted in that +12. Of the 12: **3 FILLED** (`auto_create_off`, `typed_config_authority`, +`no_raw_endpoint_fallback` static half) and **9 BLOCKED** — every BLOCKED field is +enumerated with its reason in the gap table above (`isolation_lane`, +`candidate_image_digest`, `msk_iam_signer` live half, `token_refresh_cycle`, +`explicit_topic_bootstrap`, `negative_control_out_of_catalog`, `broker_group_perms`, +`rds_verify_full`, `dashboard_zero_authority`). A prior revision of this document +undercounted this as "8 of 12" — corrected here after CodeRabbit flagged the +mismatch (2026-08-08); each next session should derive the count from the gap +table's row count, not restate a number by hand. diff --git a/docs/evidence/OMN-15124/probes-round4/check_no_cloud_bus_wrapper.txt b/docs/evidence/OMN-15124/probes-round4/check_no_cloud_bus_wrapper.txt new file mode 100644 index 0000000000..9200c2a7b2 --- /dev/null +++ b/docs/evidence/OMN-15124/probes-round4/check_no_cloud_bus_wrapper.txt @@ -0,0 +1 @@ +exit=0 diff --git a/docs/evidence/OMN-15124/probes-round4/grep_plaintext_kafka_auth.txt b/docs/evidence/OMN-15124/probes-round4/grep_plaintext_kafka_auth.txt new file mode 100644 index 0000000000..ddf0e73e16 --- /dev/null +++ b/docs/evidence/OMN-15124/probes-round4/grep_plaintext_kafka_auth.txt @@ -0,0 +1 @@ +107: if config.security_protocol == "PLAINTEXT": diff --git a/docs/evidence/OMN-15124/probes-round4/msk_client_authentication.json b/docs/evidence/OMN-15124/probes-round4/msk_client_authentication.json new file mode 100644 index 0000000000..727a0ea1d9 --- /dev/null +++ b/docs/evidence/OMN-15124/probes-round4/msk_client_authentication.json @@ -0,0 +1,13 @@ +{ + "Sasl": { + "Scram": { + "Enabled": false + }, + "Iam": { + "Enabled": true + } + }, + "Unauthenticated": { + "Enabled": false + } +} diff --git a/docs/evidence/OMN-15124/probes-round4/msk_configuration_revision2_server_properties.txt b/docs/evidence/OMN-15124/probes-round4/msk_configuration_revision2_server_properties.txt new file mode 100644 index 0000000000..1ccc2308f9 --- /dev/null +++ b/docs/evidence/OMN-15124/probes-round4/msk_configuration_revision2_server_properties.txt @@ -0,0 +1,6 @@ +auto.create.topics.enable=false +default.replication.factor=2 +delete.topic.enable=true +log.retention.hours=168 +min.insync.replicas=1 +num.partitions=3 diff --git a/docs/evidence/OMN-15124/probes-round4/msk_configuration_server_properties.txt b/docs/evidence/OMN-15124/probes-round4/msk_configuration_server_properties.txt new file mode 100644 index 0000000000..8055cd7083 --- /dev/null +++ b/docs/evidence/OMN-15124/probes-round4/msk_configuration_server_properties.txt @@ -0,0 +1,6 @@ +auto.create.topics.enable=false +default.replication.factor=2 +delete.topic.enable=true +log.retention.hours=168 +min.insync.replicas=1 +num.partitions=3 diff --git a/docs/evidence/OMN-15124/probes-round4/msk_describe_cluster.json b/docs/evidence/OMN-15124/probes-round4/msk_describe_cluster.json new file mode 100644 index 0000000000..70c542cfc3 --- /dev/null +++ b/docs/evidence/OMN-15124/probes-round4/msk_describe_cluster.json @@ -0,0 +1,111 @@ +{ + "ClusterInfo": { + "BrokerNodeGroupInfo": { + "BrokerAZDistribution": "DEFAULT", + "ClientSubnets": [ + "subnet-0f35663430d232833", + "subnet-06fe4fab38749131b" + ], + "InstanceType": "kafka.m5.large", + "SecurityGroups": [ + "sg-0183e3c660adf6ef1" + ], + "StorageInfo": { + "EbsStorageInfo": { + "VolumeSize": 100 + } + }, + "ConnectivityInfo": { + "PublicAccess": { + "Type": "DISABLED" + }, + "VpcConnectivity": { + "ClientAuthentication": { + "Sasl": { + "Scram": { + "Enabled": false + }, + "Iam": { + "Enabled": false + } + }, + "Tls": { + "Enabled": false + } + } + }, + "NetworkType": "IPV4" + }, + "ZoneIds": [ + "use1-az2", + "use1-az1" + ] + }, + "ClientAuthentication": { + "Sasl": { + "Scram": { + "Enabled": false + }, + "Iam": { + "Enabled": true + } + }, + "Unauthenticated": { + "Enabled": false + } + }, + "ClusterArn": "arn:aws:kafka:us-east-1:272493677981:cluster/omninode-dev-msk/88ad72bc-f70c-4549-93bc-c392b965f424-14", + "ClusterName": "omninode-dev-msk", + "CreationTime": "2026-06-19T21:55:58.133000+00:00", + "CurrentBrokerSoftwareInfo": { + "ConfigurationArn": "arn:aws:kafka:us-east-1:272493677981:configuration/omninode-dev-msk-config/930d30cc-105c-4c3e-ab2a-aa6cfb0a5b0b-14", + "ConfigurationRevision": 2, + "KafkaVersion": "3.9.x" + }, + "CurrentVersion": "K27OOP63XLO9TI", + "EncryptionInfo": { + "EncryptionAtRest": { + "DataVolumeKMSKeyId": "arn:aws:kms:us-east-1:272493677981:key/3523ae04-ec42-4104-967e-cbb3eacf0bd6" + }, + "EncryptionInTransit": { + "ClientBroker": "TLS", + "InCluster": true + } + }, + "EnhancedMonitoring": "PER_TOPIC_PER_BROKER", + "OpenMonitoring": { + "Prometheus": { + "JmxExporter": { + "EnabledInBroker": true + }, + "NodeExporter": { + "EnabledInBroker": true + } + } + }, + "LoggingInfo": { + "BrokerLogs": { + "CloudWatchLogs": { + "Enabled": true, + "LogGroup": "/aws/msk/omninode-dev" + } + } + }, + "NumberOfBrokerNodes": 6, + "State": "ACTIVE", + "Tags": { + "Project": "omninode", + "Repository": "omninode_infra", + "ManagedBy": "terraform", + "Environment": "staging", + "Option": "B", + "Component": "managed-kafka", + "Module": "cluster-dev", + "Name": "omninode-staging-msk" + }, + "ZookeeperConnectString": "z-1.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com:2181,z-3.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com:2181,z-2.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com:2181", + "ZookeeperConnectStringTls": "z-1.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com:2182,z-3.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com:2182,z-2.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com:2182", + "StorageMode": "LOCAL", + "CustomerActionStatus": "NONE" + } +} diff --git a/docs/evidence/OMN-15124/probes-round4/nc_msk_broker_9098.txt b/docs/evidence/OMN-15124/probes-round4/nc_msk_broker_9098.txt new file mode 100644 index 0000000000..87d7d91873 --- /dev/null +++ b/docs/evidence/OMN-15124/probes-round4/nc_msk_broker_9098.txt @@ -0,0 +1,2 @@ +Connection to b-1.omninodedevmsk.7ozyd3.c14.kafka.us-east-1.amazonaws.com port 9098 [tcp/*] succeeded! +exit=0 diff --git a/docs/evidence/OMN-15124/probes-round4/nc_rds_5432.txt b/docs/evidence/OMN-15124/probes-round4/nc_rds_5432.txt new file mode 100644 index 0000000000..79c8687043 --- /dev/null +++ b/docs/evidence/OMN-15124/probes-round4/nc_rds_5432.txt @@ -0,0 +1,2 @@ +nc: connectx to omninode-dev-postgres.cqjkkokeaqd2.us-east-1.rds.amazonaws.com port 5432 (tcp) failed: Operation timed out +exit=1 diff --git a/docs/evidence/OMN-15124/probes-round4/typed_config_authority.txt b/docs/evidence/OMN-15124/probes-round4/typed_config_authority.txt new file mode 100644 index 0000000000..a3b4f00ba6 --- /dev/null +++ b/docs/evidence/OMN-15124/probes-round4/typed_config_authority.txt @@ -0,0 +1,12 @@ +Using CPython 3.12.12 +Creating virtual environment at: .venv + Building omnibase-infra @ file:///Users/jonah/Code/omni_home/omni_worktrees/OMN-15124-round4/omnibase_infra + Built omnibase-infra @ file:///Users/jonah/Code/omni_home/omni_worktrees/OMN-15124-round4/omnibase_infra +Installed 195 packages in 458ms +/Users/jonah/Code/omni_home/omni_worktrees/OMN-15124-round4/omnibase_infra/.venv/lib/python3.12/site-packages/omnibase_core/models/core/model_deployment_topology_database_migration_ledger.py:13: UserWarning: Field name "schema" in "ModelDeploymentTopologyDatabaseMigrationLedger" shadows an attribute in parent "BaseModel" + class ModelDeploymentTopologyDatabaseMigrationLedger(BaseModel): +/Users/jonah/Code/omni_home/omni_worktrees/OMN-15124-round4/omnibase_infra/.venv/lib/python3.12/site-packages/omnibase_core/models/core/model_deployment_topology_database_grant.py:56: UserWarning: Field name "schema" in "ModelDeploymentTopologyDatabaseGrant" shadows an attribute in parent "BaseModel" + class ModelDeploymentTopologyDatabaseGrant(BaseModel): +/Users/jonah/Code/omni_home/omni_worktrees/OMN-15124-round4/omnibase_infra/.venv/lib/python3.12/site-packages/omnibase_core/models/contracts/subcontracts/model_db_table_declaration.py:13: UserWarning: Field name "schema" in "ModelDbTableDeclaration" shadows an attribute in parent "BaseModel" + class ModelDbTableDeclaration(BaseModel): +omnibase_infra.event_bus.kafka_auth diff --git a/docs/evidence/OMN-15125/2026-08-05-aug5-readiness-rollback-packet.md b/docs/evidence/OMN-15125/2026-08-05-aug5-readiness-rollback-packet.md new file mode 100644 index 0000000000..895d1965b2 --- /dev/null +++ b/docs/evidence/OMN-15125/2026-08-05-aug5-readiness-rollback-packet.md @@ -0,0 +1,120 @@ +# Aug-5 managed-staging readiness / rollback — GO-NO-GO PACKET (dated instance, NO-GO) + +**Ticket:** OMN-15125 · **Parent epic:** OMN-14724 +**Field manifest (the seam):** [`docs/runbooks/managed-staging-proof-kit/fields.yaml`](../../runbooks/managed-staging-proof-kit/fields.yaml) +**Template this was copied from:** [`docs/runbooks/managed-staging-aug5-readiness-rollback-packet.md`](../../runbooks/managed-staging-aug5-readiness-rollback-packet.md) +**Seam test:** `tests/ci/test_managed_staging_proof_kit_seam.py` +**Instance date:** 2026-08-05 (UTC) · **Author lane:** fable-epsilon-0805-eve (build subagent, Sonnet 5) + +> **This is a template instance, not an authorization.** Assembling this packet does +> not authorize the Aug-5 window. Per the packet's own standing rule (quoted from the +> unfilled template): *"Until `reconciled_blocker_graph`, `dated_chain_with_slack`, +> and `t20_handoff` carry real content, the correct `go_no_go_decision` is NO-GO by +> construction."* All three are filled below with real (if partial/gapped) content — +> and the honest reading of that content is **NO-GO for tonight's Aug-5 window**. See +> `go_no_go_decision` at the bottom; read it before anything else in this packet. + +## Fields + +| Field | What it is | Value | Evidence source | Status | +|---|---|---|---|---| +| `source_digest` | Source digest being promoted | **`a853500ab3c74620acdba34b418f6bd087081153`** | ECR tag on the round-4 candidate; must equal `one_tenant_contract_freeze.source_digest` — cross-checked against `docs/evidence/OMN-15123/2026-08-05-one-tenant-contract-freeze-round4.md`, **matches exactly** | **FILLED — matches the OMN-15123 round-4 packet.** | +| `previous_digest` | Previous digest (the rollback target) | **runtime `sha256:d2a0cac015bb86fb3dd35939d5b9f064f8b69deb17f68f6bdd019986132eda46` (tag `candidate-d53e3cf-20260805004651`) · migrate `sha256:268061a07e069c89c1fefd3b824a28ddf45ddd24a1dc8a8df8b5e28d114aa4da` @ commit `d53e3cfa9038fa1f066ed2269486b0df523d38a7`** | Manifest evidence source calls for `kubectl -n rollout history` — **not available, no k8s access.** Substituted with the immediately-prior pin in the build lineage (`omninode_infra#814`, OMN-15655 round 3, per `ROLLING_WORK_LEDGER.md:12496`), independently re-read from ECR (`probes/round_history_runtime.json`), not trusted from the ledger citation alone. | **FILLED, WITH A CAVEAT.** This is the round-3 build lineage predecessor, **not a live "currently-serving" readback** — nothing is confirmed actually deployed/serving anywhere this session can observe, so there is no live rollback target to read back from a running deployment. Treat this as the best-available candidate rollback target, not a proven one. | +| `amd64_manifest` | linux/amd64 manifest present for the promoted digest | **PRESENT — `architecture: amd64, os: linux`** (round-4 runtime) | `docker manifest inspect --verbose 272493677981.dkr.ecr.us-east-1.amazonaws.com/omninode-runtime@sha256:b829c56f...` — `probes/manifest_round4_runtime.json` | **FILLED — live, this instance.** Cross-check: the `previous_digest` (round-3) runtime image is also confirmed `amd64`/`linux` (`probes/manifest_round3_runtime.json`), so a rollback would not hit an architecture mismatch. | +| `config_hash` | Rendered config hash | _BLOCKED_ | `kubectl -n get cm ... | shasum -a 256` | **BLOCKED — no k8s access from this host** (same finding as OMN-15123/15124 round-4: direct `:6443` times out). Must equal `one_tenant_contract_freeze.config_digest`, which is itself BLOCKED for the same reason. | +| `policy_hash` | IAM/broker policy hash | _BLOCKED_ | `aws iam get-role-policy --role-name ...` | **BLOCKED — the candidate's actual node/pod IAM role name is not known from this host.** `aws iam list-roles` in this account surfaces only cluster/infra-management roles (`omninode-k3s-*-node-role`, ASG lifecycle, GitHub Actions) — none is unambiguously the runtime pod's scoped IRSA role; resolving it needs a `kubectl` lookup of the service-account annotation, which is blocked. | +| `vulnerability_result` | Vulnerability scan result for the promoted digest | **CRITICAL: 4 · HIGH: 12 · MEDIUM: 6 · LOW: 1** (scan status: `COMPLETE`) | `aws ecr describe-image-scan-findings --repository-name omninode-runtime --image-id imageDigest=sha256:b829c56f...` — `probes/vuln_scan_round4.json` | **FILLED — live, this instance.** **Discrepancy flagged:** `aws ecr describe-images ... --query imageDetails[0].imageScanStatus` returns `null` for this same digest — the two ECR read APIs disagree; `describe-image-scan-findings` (used here) returns an explicit `{"status":"COMPLETE",...}` block and real findings, so it is treated as authoritative. **4 CRITICAL findings is a real signal against promotion** — not evaluated further here (severity triage is outside this packet's scope), but it is a fact the go/no-go decision must weigh, not a clean bill of health. | +| `a6_thresholds_with_live_samples` | A6 numeric thresholds loaded, each with a live sample value | **Loaded: 5/5. Live-sampled: 0/5.** | `omnimarket/src/omnimarket/nodes/node_canary_monitoring_gate_compute/thresholds.yaml` (OMN-14732/OMN-14948) + Linear readback of OMN-14736 | **PARTIALLY FILLED — see sub-table below.** All 5 thresholds are loaded and wired (OMN-14735/OMN-14948 both Done); **zero have a live sampled value**, because no evidence surface reachable this session shows an actual canary soak run's observed numbers. | +| `monitoring_owner_actions` | Staffed monitoring owner + the action each breach triggers | **Owner named; staffed window NOT named.** | `docs/runbooks/managed-staging-canary-teardown-rollback.md` §0/§3 | **PARTIALLY FILLED.** §0 ownership table: abort-call owner and live-execution owner = **Jonah** (staffed operator), on the **agent's** B10-breach detection signal (agent owns detection + presents the halt, per §3). Abort sequence (§3): stop producer → halt consumers → snapshot breach evidence → run teardown from T-4. **No specific on-call staffed *window* (hours/timezone coverage) is named anywhere in the runbook** — it names the person (Jonah), not a shift. Flagged as a real gap, not filled with an invented window. | +| `b12_psql_readback` | B12 psql readback proving the landing table exists | _BLOCKED — genuine tension flagged_ | `docs/runbooks/managed-staging-canary-postgres-provisioning.md` §3.4 | **BLOCKED, AND FLAGGED.** §3.4 ("Readback — prove the landing table exists (ACCEPTANCE GATE)") is marked `HELD` in the live runbook — no filled `psql \d+` output is committed there. **Yet Linear OMN-14737 ("B12: Provision the canary Postgres landing table") shows `status: Done`, `completedAt: 2026-07-27`.** This is an unresolved discrepancy between the ticket status and the artifact its own field manifest cites as evidence — not resolved here (this session has no RDS access to independently re-verify either way; `nc -zv` to the RDS endpoint times out from this host). **Flagged for the next session/operator to reconcile: either the runbook needs its §3.4 filled with the actual readback that justified Done, or the Done flip needs re-examination.** | +| `teardown_readback` | OMN-14772 teardown readback | _BLOCKED_ | `docs/runbooks/managed-staging-canary-teardown-rollback.md` §2 T-4..T-8 | **BLOCKED, consistent with ticket state.** OMN-14772 (IMDS hop-limit revert, the T-7-adjacent residual) is live-checked this instance: `status: In Progress` (re-entered In Progress 2026-08-05T17:32:26Z after an In Review round), not Done — so a post-teardown steady-state assertion cannot exist yet. No tension here; ticket state and artifact absence agree. | +| `executable_rollback` | Executable rollback procedure, proven by dry-run (not asserted) | **Tuple NAMED, dry-run NOT run** | `docs/runbooks/managed-staging-canary-teardown-rollback.md` §4.1 + dry-run `kubectl rollout undo --dry-run=server` | **PARTIALLY FILLED.** The rollback tuple is named per §4.1's own required shape: previous image digest = round-3 (`previous_digest` row above), rollback owner = Jonah (live), rollback commands = standard `kubectl -n rollout undo deploy/ --to-revision=` against the round-3-pinned deployment once one exists, reconciliation query = the same `b12_psql_readback`/topic-catalog checks this packet already cites, abort thresholds = the A6 table above. **The dry-run itself is BLOCKED — no k8s access.** Per the ticket's own AC wording ("proven by dry-run, not asserted"), naming the tuple does not satisfy this field; only the dry-run output would. | +| `reconciled_blocker_graph` | Reconciled blocker graph (every open blocker, with its owner) | **See table below — live `list_issues(parentId=OMN-14724)` readback, 11 children, this instant** | Linear MCP `list_issues` + `get_issue`, this session | **FILLED — live, this instance.** | +| `dated_chain_with_slack` | Dated critical-path chain with explicit slack | **See table below** | Derived from the reconciled blocker graph + this session's own live findings | **FILLED, with an honest verdict: negative slack.** | +| `t20_handoff` | T20 handoff (final linux/amd64 build + digest handoff from the contractor lane) | _NOT FOUND_ | `docs/plans/` / `docs/handoff/` search for a T20/B1 contractor handoff artifact | **BLOCKED — genuinely not found, not just unreached.** `docs/plans/2026-07-17-managed-staging-verified-state-and-task-split.md:135` maps `T20 (main-lineage candidate image) → B1 (Lane B)`, but no discrete "T20 handoff" document exists under `docs/plans/` or `docs/handoff/` naming a final linux/amd64 build handoff from a contractor lane. The closest analog is the round-4 build provenance itself — GitHub Actions runs `31048750499` (runtime) / `31048074889` (migrate), both ECR-verified this session — but that is a CI build log, not the contractor handoff artifact the manifest specifically asks for. Not force-fit; recorded as a real gap. | +| `go_no_go_decision` | Go / no-go decision, decider, UTC timestamp | **NO-GO — see below** | this packet + `docs/tracking/ROLLING_WORK_LEDGER.md` | **FILLED — see "Go/no-go decision" section below.** | +| `plan_row_binding` | Rolling plan §3 B7 bound to OMN-15125 | **RESOLVED — same fix as OMN-15123/OMN-15124** | live grep, `docs/plans/ROLLING_SEVEN_DAY_PLAN.md` | **FILLED.** `OMN-15125` present at `§0-CHAIN` link 6 (line 38) and `§2` "Fastest readiness order" item 6 (line 93), both citing it by ID directly. Resolved by the 2026-08-04T03:58:31Z plan-governor reconcile (`ROLLING_WORK_LEDGER.md:12253`) alongside OMN-15123/OMN-15124; re-confirmed live here, not trusted from the ledger claim alone. | + +## `a6_thresholds_with_live_samples` — sub-table (required by the template) + +| Signal | Numeric threshold | Comparison | Live sampled value | Sample timestamp | +|---|---|---|---|---| +| auth | 2 (count) | gte | **NONE — unloaded, no live sample exists** | — | +| tls | 1 (count) | gte | **NONE — unloaded, no live sample exists** | — | +| broker | 3 (count) | gte | **NONE — unloaded, no live sample exists** | — | +| lag | 5 (messages) | gte | **NONE — unloaded, no live sample exists** | — | +| rds | 2 (count) | gte | **NONE — unloaded, no live sample exists** | — | + +Per the manifest's own field definition ("a threshold with no live sample counts as unloaded"): **all 5 signals count as unloaded**, despite all 5 being correctly configured and wired in code (OMN-14735/OMN-14948, both Done, thresholds sourced verbatim from the A6 contractor deliverable OMN-14732). This is a code-readiness fact, not a proof-readiness fact — the gate is wired, it has simply never fired against a real soak. + +## `reconciled_blocker_graph` — live children of OMN-14724, this instant + +| Ticket | Title (short) | Status | Owner | Note | +|---|---|---|---|---| +| OMN-15123 | Freeze the immutable canary tuple | In Progress | Jonah Gray | This session's own round-4 PR (#2667) advances it; not closed. | +| OMN-15124 | Candidate-isolation compatibility proof | In Progress | Jonah Gray | This session's own round-4 PR (#2668) advances it; not closed. | +| OMN-15125 | THIS ticket | In Progress | Jonah Gray | This packet is evidence toward, not closure of, this ticket. | +| OMN-14733 | B2: scale worker capacity off desired=0 | **Canceled** (2026-08-05T18:17:07Z) | Daniyal Abbas | **Canceled today**, was `axis:awaiting-operator-decision`. If worker capacity is genuinely needed to run the canary and this stays canceled, that is itself a live-blocking gap for B11-class work — not re-litigated here, flagged for the operator. | +| OMN-15253 | Typed staging-readiness contract (slice 1) | Done | Jonah Gray | Closed. | +| OMN-14734 | B3: reset Valkey credential | Done | Daniyal Abbas | Closed. | +| OMN-14737 | B12: provision canary Postgres landing table | Done (completedAt 2026-07-27) | Jonah Gray | **Tension flagged in the `b12_psql_readback` row above** — the runbook artifact this field cites as evidence shows §3.4 still `HELD`. | +| OMN-14736 | B11: run the one-tenant MSK backend canary | Done (completedAt 2026-07-27) | Daniyal Abbas | No independent evidence of a completed run was found reachable from this host this session (RDS/k8s both unreachable here); Daniyal likely has in-VPC access this host does not, so this is **not** asserted as a false-Done — recorded as unverified-from-this-vantage-point, consistent with the `a6_thresholds_with_live_samples` finding of zero live samples anywhere this session could reach. | +| OMN-14735 | B10: wire monitoring to numeric thresholds | Done | Jonah Gray | Closed, code-verified (thresholds.yaml exists, cited above). | +| OMN-14779 | B12↔B6 seam CI test | Done | Jonah Gray | Closed. | +| OMN-14642 | Dashboard reads via effect-node adapter (bridge) | Done | Jonah Gray | Closed. | +| OMN-14738 | B13: define teardown/rollback | Done | Jonah Gray | Closed — this is the runbook itself, which is why its still-`HELD` execution markers are meaningful (the *spec* is Done; the *execution* is not, by the spec's own design). | + +**Net:** of 12 non-terminal-or-just-closed children, 3 are the OMN-15123/15124/15125 triad this lane is actively advancing, 1 (B2) went Canceled today with an unresolved capacity question, and 2 "Done" tickets (B11, B12) have artifacts that either can't be independently verified from this host (B11) or show a directly contradicting `HELD` marker in their own cited evidence surface (B12). **A clean blocker graph would show zero such tensions; this one shows two, both worth an operator pass**, not silently inherited as clean. + +## `dated_chain_with_slack` + +| Item | Owner | Start | Target end | Slack (vs. tonight's Aug-5 window) | +|---|---|---|---|---| +| Round-4 candidate build (runtime+migrate) | build lane | 2026-08-05T21:22Z (migrate push) / 21:33Z (runtime push) | complete | 0 — already built | +| Pin PRs `omninode_infra#818`/`#819` merge | Codex (queue) | opened 2026-08-05T21:35Z / later | **unmerged as of this packet** | **negative** — both still OPEN, one (`#818`) has a genuine unresolved test conflict (`test_migration_image_digest_matches_the_pinned_job` asserts the round-3 digest, not round-4's — per `ROLLING_WORK_LEDGER.md:12715`), not a flake | +| OMN-15123 freeze (`freeze_signature`) | this ticket chain | blocked on the pin PRs above | **not started** | **negative** — cannot start until the pins land | +| k8s/DB access (SSM tunnel) for the 5+ BLOCKED fields across OMN-15123/15124/15125 | unowned this session | not started | **unknown** | **unknown — no owner named, no ETA found** | +| Canary Postgres §3.4 readback (b12) | unresolved (see tension above) | unknown | unknown | **unknown** | +| Aug-5 window itself (soak Aug 6) | — | — | **tonight, 2026-08-05** | **the window is now; every upstream item above still shows 0 or negative slack against it** | + +**Verdict: negative slack across the chain.** The pin PRs that gate everything downstream are open with a live, non-flake test conflict; the k8s/DB access needed to clear the remaining BLOCKED fields has no named owner or ETA in this session's reach; and two "Done" tickets in the blocker graph show artifacts that don't independently confirm from this vantage point. This is not a chain with a tight-but-real Aug-5 landing — it is a chain that has not yet resolved its own gating conflict. + +## Go/no-go decision + +**Decision: NO-GO for the 2026-08-05 window.** +**Decider:** this build lane (fable-epsilon-0805-eve), stating the mechanical conclusion the template's own rule requires — **not** an operator override. An operator can still say GO on different information; this packet's job is to make sure that decision is made with the real state in front of it, not a clean-looking but incomplete packet. +**UTC timestamp:** 2026-08-05 (packet assembly time, this instance). + +**Why NO-GO, not "packet incomplete, defer decision":** +1. The candidate tuple is not yet pinned — `omninode_infra#818`/`#819` open, `#818` has a live test-assertion conflict against the round-3 digest (not resolved by this packet). +2. `freeze_signature` (OMN-15123) is correctly unfilled — there is no frozen tuple to promote. +3. 4 CRITICAL vulnerability findings on the round-4 image are unaddressed (not triaged in this packet — flagged, not waived). +4. Two blocker-graph tensions (B11/B12 "Done" vs. their own cited evidence) are unresolved. +5. Zero live A6 threshold samples exist anywhere this session could reach. +6. `t20_handoff` genuinely does not exist as a named artifact. + +This mirrors the corresponding conclusion in `docs/tracking/2026-08-05-beta-blocking-axis-register.md` (EOD-refreshed the same day, `docs/tracking/ROLLING_WORK_LEDGER.md:12681`), which independently tracks OMN-15123/OMN-15124/OMN-15125 as still not-closed under the broader beta-blocking axis register — no contradiction found between that surface and this one. + +## Probes committed alongside this packet + +- `probes/vuln_scan_round4.json` +- `probes/manifest_round4_runtime.json` +- `probes/manifest_round3_runtime.json` +- `probes/round_history_runtime.json` + +## Operator / Daniyal handoff + +1. Resolve the `omninode_infra#818` digest-assertion conflict (round-3 vs round-4) before expecting either pin PR to merge cleanly. +2. Reconcile the B11/B12 "Done" vs. cited-evidence tension — either fill the runbook's §3.4 with the readback that justified B12's Done, or re-open and re-verify. +3. Name an owner + ETA for the SSM-tunnel k8s access this and the sibling OMN-15123/15124 packets both need to clear their remaining BLOCKED fields. +4. Decide the B2 (worker capacity) cancellation's downstream effect on B11-class execution capacity — recorded as canceled today with no visible successor ticket. +5. Locate or explicitly declare-absent the T20 contractor handoff artifact the manifest expects. + +## Related + +- `docs/evidence/OMN-15123/2026-08-05-one-tenant-contract-freeze-round4.md` — sibling round-4 freeze packet (source of `source_digest` cross-check) +- `docs/evidence/OMN-15124/2026-08-05-candidate-isolation-round4-evidence.md` — sibling round-4 isolation-proof evidence +- `docs/tracking/2026-08-05-beta-blocking-axis-register.md` — independent, same-day reconciliation of the broader beta blocking axes +- `docs/runbooks/managed-staging-canary-teardown-rollback.md` — rollback tuple §4.1, abort §3, ownership §0 +- `docs/runbooks/managed-staging-canary-postgres-provisioning.md` — B12 §3.4 (still HELD) +- `omninode_infra#818` / `#819` — the open round-4 pin PRs this packet's `source_digest`/`previous_digest` depend on diff --git a/docs/evidence/OMN-15125/probes/manifest_round3_runtime.json b/docs/evidence/OMN-15125/probes/manifest_round3_runtime.json new file mode 100644 index 0000000000..63d017d044 --- /dev/null +++ b/docs/evidence/OMN-15125/probes/manifest_round3_runtime.json @@ -0,0 +1,184 @@ +{ + "Ref": "272493677981.dkr.ecr.us-east-1.amazonaws.com/omninode-runtime@sha256:d2a0cac015bb86fb3dd35939d5b9f064f8b69deb17f68f6bdd019986132eda46", + "Descriptor": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:d2a0cac015bb86fb3dd35939d5b9f064f8b69deb17f68f6bdd019986132eda46", + "size": 7004, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + "Raw": "ewogICAic2NoZW1hVmVyc2lvbiI6IDIsCiAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5kaXN0cmlidXRpb24ubWFuaWZlc3QudjIranNvbiIsCiAgICJjb25maWciOiB7CiAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5jb250YWluZXIuaW1hZ2UudjEranNvbiIsCiAgICAgICJzaXplIjogMjcxODEsCiAgICAgICJkaWdlc3QiOiAic2hhMjU2OjA5NDAwYTk4NDIxYTcwM2M3NDY3NDFlY2JjYTljYTZkOTQ4NmUwNTNhN2MxZTZiYTI2NjliNGRjNTNhMjI5YjQiCiAgIH0sCiAgICJsYXllcnMiOiBbCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiAzMDc4MjY1MSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6Zjk2ZmRlNWM3NTc5N2M2MDZmMTZhN2M2MDAxY2NiYjJhZThkNDU4OTMzZTRlZjhhYzkyNTgxZmI0YjUzMGRhNiIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDEyOTMzMDEsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2Ojk4ZGIyNDg1YTBkMDdhODkxNDU4NmIwMjM4N2UzODEzYWE3ZTlmZWQ3OWFiMjUyODk4ZDNlOTZlMjFjNzE3ZWEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiAxMjEwODQwNSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NDgzNDdiMTVjODVmZDZkZGU5YzViMDI1OWYzNzhmYmFlZTNjZTIzMWIzMGE0MmYyZjJiY2M0ZWEwMjg1Y2JjOSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDI1MCwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6ZmQwNzk2MzJlZGMwYWI0ZTlkMTBjNzdlYzM0OGQ1MDU3YTk3NmU2ZmM1MDhlOTM4NTU1NDgwOTZkZWMyYWUxZSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDMyLAogICAgICAgICAiZGlnZXN0IjogInNoYTI1Njo0ZjRmYjcwMGVmNTQ0NjFjZmEwMjU3MWFlMGRiOWEwZGMxZTBjZGI1NTc3NDg0YTZkNzVlNjhkYzM4ZThhY2MxIgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogMzIsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OjRmNGZiNzAwZWY1NDQ2MWNmYTAyNTcxYWUwZGI5YTBkYzFlMGNkYjU1Nzc0ODRhNmQ3NWU2OGRjMzhlOGFjYzEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiA0OTgzNDcxOSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6OTM3ZDcxNGJlMzYwZGU2ODM4NmEyNTFmN2UxYWRkODUyZDJmYzQ0MDczZTNmZjYzNDFlNjRjNjAyNzFkNzI3NiIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDYyNjgxODQ4LAogICAgICAgICAiZGlnZXN0IjogInNoYTI1NjozOTAxNTNlM2FmNjMwOWM3ZDM4NTEwZGI0NDA2ZDM1OGFlMjBhNDUzZGFkODYzMzA4MDUxYTc4Yzc0NTlhNWY5IgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogMTgxODAyMDI1LAogICAgICAgICAiZGlnZXN0IjogInNoYTI1NjoyNmNiNGNhM2U1ZTdjNThkYzE3Mjg5MzZiNzM5YjhjOTMwODRlNzMwM2MzOWFiNWE5MzEzODMyMDMwNjc3MTVlIgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogNTIxNDA3NiwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6MmRkMDNmMDExYTI1MTBlY2ViZDY5YzNjZGFhNDAyOTc5MWYyYTBiYzRkNDE0ODhhYjhmNDVhZWIwYWYzZGJlOSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDM1MiwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6ZTY0MTRmMDc5N2Q5NzZlY2QzZTg0MDA3Y2M1NDA5YTM2ZmUwMmU1NmEyNWE5MmU3ODU2YjdiYzc2MWIzOTQxMiIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDE2NzY4NDczLAogICAgICAgICAiZGlnZXN0IjogInNoYTI1Njo5NmRjYzJiOTBmMGY3NTBjODJhNjgyNGJmOWEyNzI1YmZmY2E3OTZlZmFhODRjOWMyMGJlNDk3MjJiMjAyODE5IgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogMzIsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OjRmNGZiNzAwZWY1NDQ2MWNmYTAyNTcxYWUwZGI5YTBkYzFlMGNkYjU1Nzc0ODRhNmQ3NWU2OGRjMzhlOGFjYzEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiA4MzQyMzAsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OjFjMDYwZWEwZDQwNmIwMTRmZDUyMDk1NjFmYzM2Nzc5MzJlODFkN2VmNWZmMjQ5YjRmMzYxNjg1NDAyNzljMGUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiAyMjA5MCwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NDdlY2E0MDAyZGZhNWQxNzY4MDM2NWQ2ZmZiYmY1OTk2MTA0ZmY4NTAxZTg2Y2QwNzc3MDhkYjg3NGEwMDc0OSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDMzMjgsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OmNhNmIyYWEwYTg3MWQwODlkNGIxZDM5MWMyMzZmMmJlZGNjOTJhYTczZjRjYjhkNjBlMDk2MTgxNjJiZDJlY2YiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiAxNjgsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OjRmZjY2ZTA3NThiODRiZDcwZjFjYTk5M2U4NDg5MzdmOTM4NWYxMzYxMTA4N2NkZTMwMDU4MWQwNWUzMGM0YTQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiA5MywKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6ZmU4NDZlYmY3NWE5YWQ2YTIwODE4ODM4MjBhZjU0NzZjNTc2OGJlMTBiMzIzZTM3MThmMmQ4YTQzZTE3YmU3MSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDMyLAogICAgICAgICAiZGlnZXN0IjogInNoYTI1Njo0ZjRmYjcwMGVmNTQ0NjFjZmEwMjU3MWFlMGRiOWEwZGMxZTBjZGI1NTc3NDg0YTZkNzVlNjhkYzM4ZThhY2MxIgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogNDE4MjA2OTEyLAogICAgICAgICAiZGlnZXN0IjogInNoYTI1Njo4ZWVjNWQ5NGUwNzc5YWVhYmVjMmYxNjY3NWNmNzhlYTA3YzllZWNiZDg0NGM0OTVlYzVjM2E1OWNmZTBjZGE3IgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogMzkwNDgyNiwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6ZGIwMTE2Y2JjMzU3ODU4M2JlZTNhNDk5MzY1NzIzZmE1YmE4ZDcxN2UzMjJiMWI1MzQ5MDg5NmUzZDIxMDE5NSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDc5MSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6ZWIyNjI0YjI2YWQ3ZTc2YmFmY2E5MDQ2MTg0MGExZDViZTczZTM1NDQ4N2VkNDI5NTYyMThkMDQ4NWM1ZGU4MCIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDEzMTY0NCwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6ZjA4MzM3MTU2ZTg5YTFiZjUwNmM5MDFkNjhmOWIwYTI3MzUwYzAwZGZlMGFhZWUwNjE0MzVlOGQ4NzY4MTk2YiIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDkxMTAwMCwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NWVlMzFiZDdiNzg5N2I2YzkzNTYxMDk4OGZjNWFmZjQ0ZTYwYzQ1Y2FhOWMyZmFhNDE0ZWViODExNjI2MzU5YyIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDE5NTg4LAogICAgICAgICAiZGlnZXN0IjogInNoYTI1NjozNGE3Y2U4MmY3NjE2Yzc2MWEyYThmOGRiMTI4MmI2NjdjMmExOWViNWZkOWVlMGRlNjQ3NmEyYWRhMmY3ZmFkIgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogNjAzMywKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6OTlkZmFkOWMzMzM2OWJiMTRhNmQ1M2QwNGIxMDZjNjVlY2U0Y2IxZGY2Mjc4YTQ1YjE2Y2M4YjQyMmZjY2EwYiIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDEzMDIsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OmJmNTM5NjU2YzE1NDMyNzRmYWI5ZjE2YTVkMGFlMTNmM2E0ZGEwYjVlOTBlNGIyNDk1M2I4MjBkYmRkNGM3YjEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiAxNzg5LAogICAgICAgICAiZGlnZXN0IjogInNoYTI1NjozMzMxMmExY2FlZDlmNGI0YjM2MGExMTZlYTBkOWQ4MzZhNGUyZWQ3ZGJmZDA2OGViY2MyNWRmNDYzNzllYWU0IgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogMzY4OSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NmU5NjAzMGQxNzQ2ZWU1MjQxZTg5YTk1NTBjYzU5NTE3NGYxNDU4NjUyMWUzNDBkYTQ0MGUzYjM5YmI2Nzc2ZiIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDMyLAogICAgICAgICAiZGlnZXN0IjogInNoYTI1Njo0ZjRmYjcwMGVmNTQ0NjFjZmEwMjU3MWFlMGRiOWEwZGMxZTBjZGI1NTc3NDg0YTZkNzVlNjhkYzM4ZThhY2MxIgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogNTQ3NSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6YzUwYzc3YjFmMDA2ZTAyODk2MjA0YzA2YTE1OGJjZmRmMmQwMGEwMWIwYTc0ZDE5YjY1MzljN2ExZGNiZjkyNSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDE0OSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NDhhMmE0NWNkMDY4YTFhMGY1ODQwNTgzZTcyOGVmMjQzNTA4MDIxMTEzYTMwNjY3MTQ5ZDQ4YTVjNGZhNzljNyIKICAgICAgfQogICBdCn0=", + "SchemaV2Manifest": { + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 27181, + "digest": "sha256:09400a98421a703c746741ecbca9ca6d9486e053a7c1e6ba2669b4dc53a229b4" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 30782651, + "digest": "sha256:f96fde5c75797c606f16a7c6001ccbb2ae8d458933e4ef8ac92581fb4b530da6" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 1293301, + "digest": "sha256:98db2485a0d07a8914586b02387e3813aa7e9fed79ab252898d3e96e21c717ea" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 12108405, + "digest": "sha256:48347b15c85fd6dde9c5b0259f378fbaee3ce231b30a42f2f2bcc4ea0285cbc9" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 250, + "digest": "sha256:fd079632edc0ab4e9d10c77ec348d5057a976e6fc508e93855548096dec2ae1e" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32, + "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32, + "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 49834719, + "digest": "sha256:937d714be360de68386a251f7e1add852d2fc44073e3ff6341e64c60271d7276" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 62681848, + "digest": "sha256:390153e3af6309c7d38510db4406d358ae20a453dad863308051a78c7459a5f9" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 181802025, + "digest": "sha256:26cb4ca3e5e7c58dc1728936b739b8c93084e7303c39ab5a931383203067715e" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 5214076, + "digest": "sha256:2dd03f011a2510ecebd69c3cdaa4029791f2a0bc4d41488ab8f45aeb0af3dbe9" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 352, + "digest": "sha256:e6414f0797d976ecd3e84007cc5409a36fe02e56a25a92e7856b7bc761b39412" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 16768473, + "digest": "sha256:96dcc2b90f0f750c82a6824bf9a2725bffca796efaa84c9c20be49722b202819" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32, + "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 834230, + "digest": "sha256:1c060ea0d406b014fd5209561fc3677932e81d7ef5ff249b4f36168540279c0e" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 22090, + "digest": "sha256:47eca4002dfa5d17680365d6ffbbf5996104ff8501e86cd077708db874a00749" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 3328, + "digest": "sha256:ca6b2aa0a871d089d4b1d391c236f2bedcc92aa73f4cb8d60e09618162bd2ecf" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 168, + "digest": "sha256:4ff66e0758b84bd70f1ca993e848937f9385f13611087cde300581d05e30c4a4" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 93, + "digest": "sha256:fe846ebf75a9ad6a2081883820af5476c5768be10b323e3718f2d8a43e17be71" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32, + "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 418206912, + "digest": "sha256:8eec5d94e0779aeabec2f16675cf78ea07c9eecbd844c495ec5c3a59cfe0cda7" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 3904826, + "digest": "sha256:db0116cbc3578583bee3a499365723fa5ba8d717e322b1b53490896e3d210195" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 791, + "digest": "sha256:eb2624b26ad7e76bafca90461840a1d5be73e354487ed42956218d0485c5de80" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 131644, + "digest": "sha256:f08337156e89a1bf506c901d68f9b0a27350c00dfe0aaee061435e8d8768196b" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 911000, + "digest": "sha256:5ee31bd7b7897b6c935610988fc5aff44e60c45caa9c2faa414eeb811626359c" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 19588, + "digest": "sha256:34a7ce82f7616c761a2a8f8db1282b667c2a19eb5fd9ee0de6476a2ada2f7fad" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 6033, + "digest": "sha256:99dfad9c33369bb14a6d53d04b106c65ece4cb1df6278a45b16cc8b422fcca0b" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 1302, + "digest": "sha256:bf539656c1543274fab9f16a5d0ae13f3a4da0b5e90e4b24953b820dbdd4c7b1" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 1789, + "digest": "sha256:33312a1caed9f4b4b360a116ea0d9d836a4e2ed7dbfd068ebcc25df46379eae4" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 3689, + "digest": "sha256:6e96030d1746ee5241e89a9550cc595174f14586521e340da440e3b39bb6776f" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32, + "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 5475, + "digest": "sha256:c50c77b1f006e02896204c06a158bcfdf2d00a01b0a74d19b6539c7a1dcbf925" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 149, + "digest": "sha256:48a2a45cd068a1a0f5840583e728ef243508021113a30667149d48a5c4fa79c7" + } + ] + } +} diff --git a/docs/evidence/OMN-15125/probes/manifest_round4_runtime.json b/docs/evidence/OMN-15125/probes/manifest_round4_runtime.json new file mode 100644 index 0000000000..341b5d9b54 --- /dev/null +++ b/docs/evidence/OMN-15125/probes/manifest_round4_runtime.json @@ -0,0 +1,184 @@ +{ + "Ref": "272493677981.dkr.ecr.us-east-1.amazonaws.com/omninode-runtime@sha256:b829c56f547340363be347391275cc847b1f465c18fa275836dd098989f7ecb7", + "Descriptor": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:b829c56f547340363be347391275cc847b1f465c18fa275836dd098989f7ecb7", + "size": 7004, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + "Raw": "ewogICAic2NoZW1hVmVyc2lvbiI6IDIsCiAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5kaXN0cmlidXRpb24ubWFuaWZlc3QudjIranNvbiIsCiAgICJjb25maWciOiB7CiAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5jb250YWluZXIuaW1hZ2UudjEranNvbiIsCiAgICAgICJzaXplIjogMjcxOTcsCiAgICAgICJkaWdlc3QiOiAic2hhMjU2OjM0OGM5ZjBkOGI3OWU1Mjk1MTA5ODAwMjA2NGY0N2I2N2MwOTk0YmNhNjQ3MWJhZjUyZmM1NDJkMGU5NDEyZjkiCiAgIH0sCiAgICJsYXllcnMiOiBbCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiAzMDc4MjYwOSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6MDg1OTkyZTQwY2MzNmI0N2QxNjZkOWU5OTMyOTFlMGFiNzU5NDFhMDZlNjg5YzlkYjI3ZGQ0NGVlYWVhM2FiMCIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDEyOTMyNzYsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OjVhMzFkYjRjZDQ3ODk4ZTg2MmE1NjcwMjhhYTQxZjdhMTFkODE0Y2Y2OWJlNjkwNDE3MzQwMmE4YjhlYWM1Y2IiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiAxMjExMzczMywKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6Yzg1YWQwYmNhY2E4OTVhZmEwODA1MzUzOGVjMWVhNzY0MmM3ZTllMDVmNTY4NGQxOGUzNjI3YTIzMTZjYThhZSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDI0OSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6YjNjN2E5YmRiNGYyYTMwZDBhMTM1ODQzOGQxNTM5YzA2Yjg2ZGE5OTEzNWNmZDlkOTI3NTY4NjNjYzg3YWUxOCIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDMyLAogICAgICAgICAiZGlnZXN0IjogInNoYTI1Njo0ZjRmYjcwMGVmNTQ0NjFjZmEwMjU3MWFlMGRiOWEwZGMxZTBjZGI1NTc3NDg0YTZkNzVlNjhkYzM4ZThhY2MxIgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogMzIsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OjRmNGZiNzAwZWY1NDQ2MWNmYTAyNTcxYWUwZGI5YTBkYzFlMGNkYjU1Nzc0ODRhNmQ3NWU2OGRjMzhlOGFjYzEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiA0OTgzNDc3NCwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6YWRhZGMxODhhZmMzZjRjNTUwMDdjM2FmZjk4NGZlMWNjNmVjMGVlM2UyMjEyMmViMGMzNzFiMjE3YThiMGM3OSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDYyNjgyNTczLAogICAgICAgICAiZGlnZXN0IjogInNoYTI1NjozYzhhMzgzZDc1OTE3OTQ1MThkNjU0YzE1MzYwNzIzYzRiYjg5NzdlZTk1NjcwY2I0NmJlNmJlNWY2MmQ4YjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogMTgxODAyMDU1LAogICAgICAgICAiZGlnZXN0IjogInNoYTI1NjoyMTk4NzQ0YmEzOWQ0N2Y2ODAxZGU5NTNiMDAxODIwZTdjOWFmNDg3YTk1N2Y1N2Y1MjEyMmZiYmRlZjRjNTI1IgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogNTIxMzczNSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6OGM3ZWJhMGU5MDZiZWUwZDI0MDVkMTc5OGY4NWNkMjI0ZTAzMTA2MGVlNzJkODRiODg3ZDcwNmE4OGJjNTc5YyIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDM1MiwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NWZiYTdjMGJkNzJhNTNmMmUzMzI2NDY0NzZjMzNiMzIxNWEzNzRiYWZlMjYzNmJkMzJiNTQ0ZmExNmUzMGU0NSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDE2NzY4NDc2LAogICAgICAgICAiZGlnZXN0IjogInNoYTI1Njo2ZjVmODc5NDQyM2QzMmY5OWNjOTM4ZDQ3YmUxNjc5Y2Q0Yjg0YWI1N2IzYzVhMDFmYjA4NzdjOGI1OTlmODkwIgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogMzIsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OjRmNGZiNzAwZWY1NDQ2MWNmYTAyNTcxYWUwZGI5YTBkYzFlMGNkYjU1Nzc0ODRhNmQ3NWU2OGRjMzhlOGFjYzEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiA4MzQyMjIsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OjZjNDVjYTgwNWIzNDcwYTI3YzIxMWVmYzgxYmI5MjAwMjQ2MjMzYTU3ODFlY2JmYTNmMjEwNDk1MDdhNjAyOTEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiAyMjA4NywKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NTQwNDU0OTc2NTk1MjU2NDYxNzhmMzAxYWMyZmEyNTdhZGUwYTQxYTdjN2VjOWUyOTM3MDNhYzFiMTllMzlmNCIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDMzMjYsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OmRmYzQwZWQzZWIwMTY4NTYxMjdmODlhYmRmMzA4NGRmMzBhZDI5YjE4M2RlODQzZjZhMDU1YjNlNWE5OGViOTEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiAxNjksCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OmFmMDdmNTVkYWFiYWJmNmE3NjJkMjJjNjBkNWI0NmZlNTI5ODVjNDY4NTgzODJiNDZmYmRiOTA4ZmU0N2E3YWUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiA5MywKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NTM5YTQ2YmMyNjg3NzAxOWQxOTNlZjExYjI3MTRhYjZhYmMzYTQwNzA1ODMyMDc4ZWEyN2MyMzA0NjlkMTE2MyIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDMyLAogICAgICAgICAiZGlnZXN0IjogInNoYTI1Njo0ZjRmYjcwMGVmNTQ0NjFjZmEwMjU3MWFlMGRiOWEwZGMxZTBjZGI1NTc3NDg0YTZkNzVlNjhkYzM4ZThhY2MxIgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogNDE4MjA0NzM3LAogICAgICAgICAiZGlnZXN0IjogInNoYTI1Njo5NzA4MjlkYTMxNzZkZTYzYmI1MGQ2ZWU5OWU2MWZkOWIzNzVlYTRlYTJiYTgyMTU1MGU5NDM0MDRiODI2YzEyIgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogMzkwNTkzNiwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6YjQ4ZTc1YjE4YzEwZGViYmQ3N2ZmMGQyNWVjZjQzN2EwZjA3ZTRjMjBlYzA3NDNjMjg3Mzg5ZTk4MzgxMGI4NyIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDc4OSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6YTJlNzNhN2IwMzFhZDZhNDUwM2MxNTVhMTgxOTQ2Nzc4NDMwNjI2NWJhZTVkYWU0NzJhODk0YzZhM2Y3NDcxOSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDEzMTYzOCwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6YmRhNDQzYWNjY2Y1MjgwNzAzYWM4OWJiMTE1MTgwNjQ3NWEyMmVmYjM1MTBhMWVmODU1OTQ0ZjVkMjFhMWIwZSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDkxMDk1MSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6MTAzNGJkMjM2ZmFlZDU0ZTMyODdmODk2MmM3YzhiNWY5YzZmNjAzN2ZhYTUwOGYyYmY2NWE2ZjI3NmJhYWMxMSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDE5NTg1LAogICAgICAgICAiZGlnZXN0IjogInNoYTI1NjowMzhjNGQ3ZGQyMmM5N2U5MzRjYmQwM2ViMjIxZTVjNWZlNjEwOTIwZGU4NGVkYjc5OTkxMmM2MTBlNzkwNzE3IgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogNjAzMSwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6MWRiNmQxNzMzYzA3MjRkNTc3OGFhZWY5MTVkYzFjMjk5OTE0NzBhMzM0ZmU4ZjliYTBiMDlkNjM2MDQ2MmM5NSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDEzMDMsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2Ojg5MjAzZjA2ODY5MzNhMzQ2ODA0MWY4MzI5YWM3NGEzMjU2NTViYTgxYWMyNTYxYjdkYjc0NGU4ODRkNGMxNzciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAgIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwKICAgICAgICAgInNpemUiOiAxNzg2LAogICAgICAgICAiZGlnZXN0IjogInNoYTI1NjpmODIzOGQ0YWRhMzRhY2YyNGZkNjgyMWZhNTVjYmNiODIxNjA1NDRmMTU1ZDgyMmVkMjI4M2RkMTJiZDcwZTBiIgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogMzY4NCwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NzExZDNhZDExODMwMzI3ZmZkMWI0YTI2NWRlNjA2MGRiODFhZWUyOTIwM2QxYTAxYmZkY2FhMmVmYmM3OWVhZSIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDMyLAogICAgICAgICAiZGlnZXN0IjogInNoYTI1Njo0ZjRmYjcwMGVmNTQ0NjFjZmEwMjU3MWFlMGRiOWEwZGMxZTBjZGI1NTc3NDg0YTZkNzVlNjhkYzM4ZThhY2MxIgogICAgICB9LAogICAgICB7CiAgICAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsCiAgICAgICAgICJzaXplIjogNTQ3MywKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NDI1ZjY5ZGEwNTVlZjg1YWY3YmNkZWI3MGU3NzQyZTUwOWY1NDQ2YjQxMzBkY2Y3OThhMjk5ZjY2ZTU2MDE3MCIKICAgICAgfSwKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDE1MCwKICAgICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6ZWZkYjE3OTg4ZDY1NjJlNzQ5Y2NjNWYxZTY4ZDdiYzVjOTJlZGFjZTVkY2ZmN2UxNjk1MjA1NmUzZTFkMDJkNyIKICAgICAgfQogICBdCn0=", + "SchemaV2Manifest": { + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 27197, + "digest": "sha256:348c9f0d8b79e52951098002064f47b67c0994bca6471baf52fc542d0e9412f9" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 30782609, + "digest": "sha256:085992e40cc36b47d166d9e993291e0ab75941a06e689c9db27dd44eeaea3ab0" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 1293276, + "digest": "sha256:5a31db4cd47898e862a567028aa41f7a11d814cf69be6904173402a8b8eac5cb" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 12113733, + "digest": "sha256:c85ad0bcaca895afa08053538ec1ea7642c7e9e05f5684d18e3627a2316ca8ae" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 249, + "digest": "sha256:b3c7a9bdb4f2a30d0a1358438d1539c06b86da99135cfd9d92756863cc87ae18" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32, + "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32, + "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 49834774, + "digest": "sha256:adadc188afc3f4c55007c3aff984fe1cc6ec0ee3e22122eb0c371b217a8b0c79" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 62682573, + "digest": "sha256:3c8a383d7591794518d654c15360723c4bb8977ee95670cb46be6be5f62d8b56" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 181802055, + "digest": "sha256:2198744ba39d47f6801de953b001820e7c9af487a957f57f52122fbbdef4c525" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 5213735, + "digest": "sha256:8c7eba0e906bee0d2405d1798f85cd224e031060ee72d84b887d706a88bc579c" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 352, + "digest": "sha256:5fba7c0bd72a53f2e332646476c33b3215a374bafe2636bd32b544fa16e30e45" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 16768476, + "digest": "sha256:6f5f8794423d32f99cc938d47be1679cd4b84ab57b3c5a01fb0877c8b599f890" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32, + "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 834222, + "digest": "sha256:6c45ca805b3470a27c211efc81bb9200246233a5781ecbfa3f21049507a60291" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 22087, + "digest": "sha256:54045497659525646178f301ac2fa257ade0a41a7c7ec9e293703ac1b19e39f4" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 3326, + "digest": "sha256:dfc40ed3eb016856127f89abdf3084df30ad29b183de843f6a055b3e5a98eb91" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 169, + "digest": "sha256:af07f55daababf6a762d22c60d5b46fe52985c46858382b46fbdb908fe47a7ae" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 93, + "digest": "sha256:539a46bc26877019d193ef11b2714ab6abc3a40705832078ea27c230469d1163" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32, + "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 418204737, + "digest": "sha256:970829da3176de63bb50d6ee99e61fd9b375ea4ea2ba821550e943404b826c12" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 3905936, + "digest": "sha256:b48e75b18c10debbd77ff0d25ecf437a0f07e4c20ec0743c287389e983810b87" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 789, + "digest": "sha256:a2e73a7b031ad6a4503c155a1819467784306265bae5dae472a894c6a3f74719" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 131638, + "digest": "sha256:bda443acccf5280703ac89bb1151806475a22efb3510a1ef855944f5d21a1b0e" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 910951, + "digest": "sha256:1034bd236faed54e3287f8962c7c8b5f9c6f6037faa508f2bf65a6f276baac11" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 19585, + "digest": "sha256:038c4d7dd22c97e934cbd03eb221e5c5fe610920de84edb799912c610e790717" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 6031, + "digest": "sha256:1db6d1733c0724d5778aaef915dc1c29991470a334fe8f9ba0b09d6360462c95" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 1303, + "digest": "sha256:89203f0686933a3468041f8329ac74a325655ba81ac2561b7db744e884d4c177" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 1786, + "digest": "sha256:f8238d4ada34acf24fd6821fa55cbcb82160544f155d822ed2283dd12bd70e0b" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 3684, + "digest": "sha256:711d3ad11830327ffd1b4a265de6060db81aee29203d1a01bfdcaa2efbc79eae" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32, + "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 5473, + "digest": "sha256:425f69da055ef85af7bcdeb70e7742e509f5446b4130dcf798a299f66e560170" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 150, + "digest": "sha256:efdb17988d6562e749ccc5f1e68d7bc5c92edace5dcff7e16952056e3e1d02d7" + } + ] + } +} diff --git a/docs/evidence/OMN-15125/probes/round_history_runtime.json b/docs/evidence/OMN-15125/probes/round_history_runtime.json new file mode 100644 index 0000000000..bedb803641 --- /dev/null +++ b/docs/evidence/OMN-15125/probes/round_history_runtime.json @@ -0,0 +1,42 @@ +[ + { + "Tag": "candidate-d42cbe7-20260731022315", + "Pushed": "2026-07-30T22:27:14.600000-04:00", + "Digest": "sha256:40a744dfea44ced2b1b11cef0352e92d99323358cf06c721fbd8a6ae22bf62f1" + }, + { + "Tag": "candidate-6644803-20260801191103", + "Pushed": "2026-08-01T15:15:29.002000-04:00", + "Digest": "sha256:5cca91c6d92b50485491e516f46ef7edb603edacc46d80f6232b50cbf6d8bb1b" + }, + { + "Tag": "candidate-72d27ef-20260801212041", + "Pushed": "2026-08-01T17:25:16.890000-04:00", + "Digest": "sha256:a45fab64f77c84978244767a5127b624927faa4df072ec1afa472db6eccfecf9" + }, + { + "Tag": "candidate-bf070a9-20260802180459", + "Pushed": "2026-08-02T14:09:39.785000-04:00", + "Digest": "sha256:70d7880a8096dcf597c9c3e925baaaf848c49f02d9a0062cddcb08b165dd42f9" + }, + { + "Tag": "candidate-2e5ef7d-20260804155301", + "Pushed": "2026-08-04T11:57:31.384000-04:00", + "Digest": "sha256:5507667152e2dc420fc4dd6bd337f8dbe2ecf592939572dc8f353efb10a05dd6" + }, + { + "Tag": "candidate-234f7bd-20260804192314", + "Pushed": "2026-08-04T15:27:47.315000-04:00", + "Digest": "sha256:8535742ee43968809e3efe7ed87a4cf7a9ff8621b22fe46f10e0860290dc1c7d" + }, + { + "Tag": "candidate-d53e3cf-20260805004651", + "Pushed": "2026-08-04T20:51:27.557000-04:00", + "Digest": "sha256:d2a0cac015bb86fb3dd35939d5b9f064f8b69deb17f68f6bdd019986132eda46" + }, + { + "Tag": "candidate-a853500-20260805212844", + "Pushed": "2026-08-05T17:33:12.278000-04:00", + "Digest": "sha256:b829c56f547340363be347391275cc847b1f465c18fa275836dd098989f7ecb7" + } +] diff --git a/docs/evidence/OMN-15125/probes/vuln_scan_round4.json b/docs/evidence/OMN-15125/probes/vuln_scan_round4.json new file mode 100644 index 0000000000..1f1cc51be7 --- /dev/null +++ b/docs/evidence/OMN-15125/probes/vuln_scan_round4.json @@ -0,0 +1,575 @@ +{ + "imageScanFindings": { + "findings": [ + { + "name": "CVE-2026-12087", + "description": "Socket versions before 2.041 for Perl have an out-of-bounds heap read. In Socket.xs, pack_ip_mreq_source() checks the length of its source argument before the argument is read, so the check tests the byte length carried over from the preceding multiaddr argument instead. Both addresses occupy a 4-byte field, so a valid multiaddr lets a source of any length pass the check, and the source is then copied into the 4-byte imr_sourceaddr field with a fixed-size copy. A source shorter than 4 bytes is not rejected, and the copy reads up to 3 bytes past the end of its buffer. Calling pack_ip_mreq_source() with a source value shorter than 4 bytes copies adjacent heap memory into the returned packed structure.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-12087", + "severity": "CRITICAL", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "9.1" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:H" + }, + { + "key": "package_version", + "value": "5.40.1-6" + }, + { + "key": "package_name", + "value": "perl" + } + ] + }, + { + "name": "CVE-2026-11822", + "description": "SQLite before 3.53.2 contains memory corruption vulnerabilities in the FTS5 full-text search extension that allow attackers to cause process crashes, memory exhaustion, or arbitrary code execution by supplying a crafted database with malformed FTS5 page data. Attackers can trigger an out-of-bounds read in fts5LeafSeek() via an attacker-controlled loop bound and a heap buffer overflow write in fts5ChunkIterate() through a crafted continuation page causing an integer underflow, exploitable when an FTS5 MATCH query is executed against the malicious database.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-11822", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS4_SCORE", + "value": "8.5" + }, + { + "key": "CVSS4_VECTOR", + "value": "CVSS:4.0/AV:L/AC:L/AT:N/PR:N/UI:P/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N/E:X/CR:X/IR:X/AR:X/MAV:X/MAC:X/MAT:X/MPR:X/MUI:X/MVC:X/MVI:X/MVA:X/MSC:X/MSI:X/MSA:X/S:X/AU:X/R:X/V:X/RE:X/U:X" + }, + { + "key": "package_version", + "value": "3.46.1-7+deb13u1" + }, + { + "key": "package_name", + "value": "sqlite3" + } + ] + }, + { + "name": "CVE-2026-11824", + "description": "SQLite before 3.53.2 contains a heap-based buffer overflow vulnerability in the FTS5 full-text search extension that allows attackers to cause a crash or execute arbitrary code by supplying a crafted database with malicious continuation page metadata specifying a szLeaf value smaller than 4. Attackers can trigger an integer underflow in fts5ChunkIterate() causing an inflated remaining byte count during FTS5 MATCH query processing, leading to a heap buffer overflow of attacker-controlled data in applications compiled with SQLITE_ENABLE_FTS5.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-11824", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS4_SCORE", + "value": "8.5" + }, + { + "key": "CVSS4_VECTOR", + "value": "CVSS:4.0/AV:L/AC:L/AT:N/PR:N/UI:P/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N/E:X/CR:X/IR:X/AR:X/MAV:X/MAC:X/MAT:X/MPR:X/MUI:X/MVC:X/MVI:X/MVA:X/MSC:X/MSI:X/MSA:X/S:X/AU:X/R:X/V:X/RE:X/U:X" + }, + { + "key": "package_version", + "value": "3.46.1-7+deb13u1" + }, + { + "key": "package_name", + "value": "sqlite3" + } + ] + }, + { + "name": "CVE-2026-5928", + "description": "Calling the ungetwc function on a FILE stream with wide characters encoded in a character set that has overlaps between its single byte and multi-byte character encodings, in the GNU C Library version 2.43 or earlier, may result in an attempt to read bytes before an allocated buffer, potentially resulting in unintentional disclosure of neighboring data in the heap, or a program crash. A bug in the wide character pushback implementation (_IO_wdefault_pbackfail in libio/wgenops.c) causes ungetwc() to operate on the regular character buffer (fp->_IO_read_ptr) instead of the actual wide-stream read pointer (fp->_wide_data->_IO_read_ptr). The program crash may happen in cases where fp->_IO_read_ptr is not initialized and hence points to NULL. The buffer under-read requires a special situation where the input character encoding is such that there are overlaps between single byte representations and multibyte representations in that encoding, resulting in spurious matches. The spurious match case is not possible in the standard Unicode character sets.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-5928", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "7.5" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "key": "package_version", + "value": "2.41-12+deb13u3" + }, + { + "key": "package_name", + "value": "glibc" + } + ] + }, + { + "name": "CVE-2026-57062", + "description": "CMS (Cryptographic Message Syntax) parsing in gpgsm in GnuPG through 2.5.20 mishandles the CMS format for AES-GCM because aes-ICVlen is supposed to be 12 bytes but 4 bytes is accepted. NOTE: this is related to CVE-2026-34182.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-57062", + "severity": "LOW", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "2.9" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:L/AC:H/PR:N/UI:N/S:U/C:N/I:L/A:N" + }, + { + "key": "package_version", + "value": "2.4.7-21+deb13u1" + }, + { + "key": "package_name", + "value": "gnupg2" + } + ] + }, + { + "name": "CVE-2026-18508", + "description": "A flaw was found in GNU tar. When extracting an archive with the --one-top-level option, hardlink targets are not confined to the designated top-level directory and may resolve relative to the extraction working directory. A crafted archive can create hardlinks that escape the intended boundary and, when combined with a preexisting symbolic link under the working directory, may allow writing outside that boundary during a single extraction.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-18508", + "severity": "MEDIUM", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "4.4" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:L/AC:L/PR:N/UI:R/S:U/C:L/I:L/A:N" + }, + { + "key": "package_version", + "value": "1.35+dfsg-3.1" + }, + { + "key": "package_name", + "value": "tar" + } + ] + }, + { + "name": "CVE-2026-48959", + "description": "IO::Uncompress::Unzip versions before 2.220 for Perl allow CPU exhaustion via per-byte read loop in fastForward. fastForward() compares length $offset (the digit count of the offset, 1 to 19) against the chunk size $c instead of $offset itself, so $c shrinks from 16 KiB to 1-19 bytes per iteration. Extracting a named entry from an attacker supplied zip via IO::Uncompress::Unzip->new($zip, Name => $target) drives a per-byte read loop scaling with the entry's compressed size, up to the non-Zip64 4 GiB cap.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-48959", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "7.5" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "key": "package_version", + "value": "5.40.1-6" + }, + { + "key": "package_name", + "value": "perl" + } + ] + }, + { + "name": "CVE-2026-66035", + "description": "libssh2 through 1.11.1, fixed in commit 42e33d8, contains a pre-authentication heap buffer overflow vulnerability that allows a malicious SSH server to corrupt heap metadata in any connecting client by sending a packet with a packet_length smaller than the cipher's block size during Encrypt-then-MAC cipher negotiation. In the fullpacket() function in src/transport.c, the ETM path allocates a buffer of packet_length bytes but copies blocksize minus one bytes via memcpy, causing an overflow that on 32-bit glibc writes attacker-controlled bytes into an adjacent chunk's SIZE field, enabling tcache bin confusion, overlapping live objects, and function pointer overwrite during the session handshake before authentication.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-66035", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS4_SCORE", + "value": "7.7" + }, + { + "key": "CVSS4_VECTOR", + "value": "CVSS:4.0/AV:N/AC:H/AT:N/PR:N/UI:P/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N/E:X/CR:X/IR:X/AR:X/MAV:X/MAC:X/MAT:X/MPR:X/MUI:X/MVC:X/MVI:X/MVA:X/MSC:X/MSI:X/MSA:X/S:X/AU:X/R:X/V:X/RE:X/U:X" + }, + { + "key": "package_version", + "value": "1.11.1-1+deb13u1" + }, + { + "key": "package_name", + "value": "libssh2" + } + ] + }, + { + "name": "CVE-2026-66033", + "description": "libssh2 through 1.11.1, fixed in commit a2ed82d, contains a pre-authentication integer underflow vulnerability in the ssh2_cipher_crypt() function in src/openssl.c that allows a malicious SSH server to crash any connecting client by negotiating AES-GCM ciphers during handshake. Attackers can exploit the underflow in the expression computing blocksize minus aadlen minus authentication tag length to trigger an out-of-bounds read and a memcpy call with a near-SIZE_MAX length argument, causing immediate process crash before any authentication occurs.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-66033", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS4_SCORE", + "value": "8.7" + }, + { + "key": "CVSS4_VECTOR", + "value": "CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:N/VI:N/VA:H/SC:N/SI:N/SA:N/E:X/CR:X/IR:X/AR:X/MAV:X/MAC:X/MAT:X/MPR:X/MUI:X/MVC:X/MVI:X/MVA:X/MSC:X/MSI:X/MSA:X/S:X/AU:X/R:X/V:X/RE:X/U:X" + }, + { + "key": "package_version", + "value": "1.11.1-1+deb13u1" + }, + { + "key": "package_name", + "value": "libssh2" + } + ] + }, + { + "name": "CVE-2026-48961", + "description": "IO::Compress versions from 2.207 before 2.220 for Perl ship a zipdetails CLI tool that crashes with undefined subroutine on Info-ZIP Unix Extra Field with 8-byte UID or GID. When decode_ux() in bin/zipdetails handles an Info-ZIP Unix Extra Field (tag 0x7875) with UID Size or GID Size set to 8, causing zipdetails to decode an 8-byte UID or GID value, it dispatches through decodeLitteEndian(), which calls a misnamed helper unpackValueQ. The actual function defined in the same file is unpackValue_Q (with underscore); the call raises 'Undefined subroutine &main::unpackValueQ' and the script exits with status 255. Library callers of IO::Compress and IO::Uncompress are not affected; the defect is in the bundled CLI tool.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-48961", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "7.3" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L" + }, + { + "key": "package_version", + "value": "5.40.1-6" + }, + { + "key": "package_name", + "value": "perl" + } + ] + }, + { + "name": "CVE-2026-13595", + "description": "A flaw was found in the libblkid library of util-linux. During nested partition probing, the BSD, Minix, Solaris x86, and UnixWare partition probers cache a raw pointer to a parent partition entry in a dynamically allocated array. When subsequent partition additions cause the array to be reallocated, this pointer becomes stale, leading to a heap use-after-free read. An attacker who can present a crafted block device image (for example, via USB insertion or a loop-mounted disk image) can trigger this flaw without user interaction, as libblkid is invoked automatically by udev/udisks as root on block-device hot-plug events. This could lead to limited information disclosure or denial of service.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-13595", + "severity": "MEDIUM", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "5.3" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:U/C:L/I:N/A:H" + }, + { + "key": "package_version", + "value": "2.41-5" + }, + { + "key": "package_name", + "value": "util-linux" + } + ] + }, + { + "name": "CVE-2026-66034", + "description": "libssh2 through 1.11.1, fixed in commit a13bb6c, contains a missing bounds check vulnerability that allows a malicious SSH server to trigger an arbitrary-length heap out-of-bounds read and a free of an uninitialized pointer via the publickey subsystem. In libssh2_publickey_list_fetch(), the version 1 response parser reads a server-controlled comment_len value and advances the parse pointer without verifying sufficient bytes remain in the buffer, causing the out-of-bounds read to leak heap pointers from adjacent allocations defeating ASLR, followed by heap allocator state corruption when the error cleanup path frees an uninitialized pointer from a non-zeroed realloc() region.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-66034", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS4_SCORE", + "value": "7.7" + }, + { + "key": "CVSS4_VECTOR", + "value": "CVSS:4.0/AV:N/AC:H/AT:N/PR:N/UI:P/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N/E:X/CR:X/IR:X/AR:X/MAV:X/MAC:X/MAT:X/MPR:X/MUI:X/MVC:X/MVI:X/MVA:X/MSC:X/MSI:X/MSA:X/S:X/AU:X/R:X/V:X/RE:X/U:X" + }, + { + "key": "package_version", + "value": "1.11.1-1+deb13u1" + }, + { + "key": "package_name", + "value": "libssh2" + } + ] + }, + { + "name": "CVE-2026-57432", + "description": "Perl versions through 5.43.10 have an integer overflow in S_measure_struct leading to an out-of-bounds heap read in pack and unpack. S_measure_struct adds each item's size times its repeat count to a running total with no overflow check, so a large repeat count in a pack or unpack template wraps the signed SSize_t total negative. The @, X, and x position codes then guard their moves with a signed length comparison that passes when the length is negative, advancing the buffer pointer out of bounds. A template derived from untrusted input can read heap memory past the buffer and return it to the caller.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-57432", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "8.4" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:L/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" + }, + { + "key": "package_version", + "value": "5.40.1-6" + }, + { + "key": "package_name", + "value": "perl" + } + ] + }, + { + "name": "CVE-2026-5450", + "description": "Calling the scanf family of functions with a %mc (malloc'd character match) in the GNU C Library version 2.7 to version 2.43 with a format width specifier with an explicit width greater than 1024 could result in a one byte heap buffer overflow.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-5450", + "severity": "CRITICAL", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "9.8" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" + }, + { + "key": "package_version", + "value": "2.41-12+deb13u3" + }, + { + "key": "package_name", + "value": "glibc" + } + ] + }, + { + "name": "CVE-2026-57433", + "description": "Storable versions before 3.41 for Perl have a signed integer overflow when deserializing a crafted SX_HOOK record. retrieve_hook_common reads a signed 32-bit item count from an SX_HOOK record and calls av_extend with that count plus one. A count of I32_MAX wraps the addition to a negative value. A crafted blob passed to thaw or retrieve triggers the overflow; av_extend receives the negative count and dies with a panic, terminating the deserialization.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-57433", + "severity": "CRITICAL", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "9.8" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" + }, + { + "key": "package_version", + "value": "5.40.1-6" + }, + { + "key": "package_name", + "value": "perl" + } + ] + }, + { + "name": "CVE-2026-7010", + "description": "HTTP::Tiny versions before 0.093 for Perl do not validate CRLF in HTTP request lines or control field header values. The unvalidated inputs are the method and URI in the request line, the URL host that becomes the `Host:` header, and HTTP/1.1 control data field values. An attacker who controls one of these inputs, for example a user supplied URL passed to a webhook or URL fetch endpoint, can inject additional headers and smuggle requests to the upstream server.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-7010", + "severity": "MEDIUM", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "6.5" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:N" + }, + { + "key": "package_version", + "value": "5.40.1-6" + }, + { + "key": "package_name", + "value": "perl" + } + ] + }, + { + "name": "CVE-2025-15649", + "description": "IO::Uncompress::Unzip versions before 2.215 for Perl propagate uncaught exception when parsing zip header with malformed DOS date. _dosToUnixTime() decodes the local-file-header last-modification date field and calls Time::Local::timelocal() without an eval guard. A header whose date field decodes to an out-of-range month, day, or hour causes timelocal() to die. The exception propagates out of IO::Uncompress::Unzip->new($file) where callers expect undef plus $UnzipError.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2025-15649", + "severity": "MEDIUM", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "5.5" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:N/I:N/A:H" + }, + { + "key": "package_version", + "value": "5.40.1-6" + }, + { + "key": "package_name", + "value": "perl" + } + ] + }, + { + "name": "CVE-2026-48962", + "description": "IO::Compress versions before 2.220 for Perl can execute arbitrary code in File::GlobMapper via an attacker-controlled output glob. _parseOutputGlob() wraps the caller-supplied output glob string in double quotes and stores it in the parser state; _getFiles() then runs the stored expression through eval STRING. A literal double quote in the output glob closes the dquote wrapper, and the characters that follow are evaluated as Perl. Arbitrary Perl in the output glob executes at the calling process's privilege.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-48962", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "7.8" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H" + }, + { + "key": "package_version", + "value": "5.40.1-6" + }, + { + "key": "package_name", + "value": "perl" + } + ] + }, + { + "name": "CVE-2026-66032", + "description": "libssh2 through 1.11.1, fixed in commit 5e47761, contains a double-free vulnerability in the sftp_open() function in src/sftp.c that allows a malicious SSH server to corrupt the heap of any authenticated client opening an SFTP session. When a server responds to SSH_FXP_OPEN with SSH_FXP_STATUS containing FX_OK, the response data buffer is freed, and if a subsequent sftp_packet_require() call returns a specific error such as LIBSSH2_ERROR_CHANNEL_PACKET_EXCEEDED, the same pointer is freed a second time, enabling tcache dup conditions on glibc systems that allow overlapping allocations and function pointer overwrites.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-66032", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS4_SCORE", + "value": "8.7" + }, + { + "key": "CVSS4_VECTOR", + "value": "CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:P/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N/E:X/CR:X/IR:X/AR:X/MAV:X/MAC:X/MAT:X/MPR:X/MUI:X/MVC:X/MVI:X/MVA:X/MSC:X/MSI:X/MSA:X/S:X/AU:X/R:X/V:X/RE:X/U:X" + }, + { + "key": "package_version", + "value": "1.11.1-1+deb13u1" + }, + { + "key": "package_name", + "value": "libssh2" + } + ] + }, + { + "name": "CVE-2026-13221", + "description": "Perl versions through 5.43.9 produce silently incorrect regular expression matches when an alternation of more than 65535 fixed string branches is compiled into a trie in Perl_study_chunk. When such branches are combined into a trie, the delta between the first branch and the shared tail is stored in a 16-bit field. A branch count above 65535 overflows the field, and the trie's match decision table is truncated with no warning or error. A pattern of this shape produces false positive matches (matching strings it should not) and false negative matches (failing to match strings it should). When such a pattern gates an access or filtering decision, the result is wrong.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-13221", + "severity": "CRITICAL", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "9.1" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:H" + }, + { + "key": "package_version", + "value": "5.40.1-6" + }, + { + "key": "package_name", + "value": "perl" + } + ] + }, + { + "name": "CVE-2026-7017", + "description": "HTTP::Tiny versions before 0.095 for Perl forward credential headers to cross-origin redirect targets. When the server returns a 3xx redirect, `_maybe_redirect` follows the `Location:` header and `_prepare_headers_and_cb` re-merges the caller's `headers` argument into the new request, without checking whether the redirect target shares an origin with the original URL. Caller-supplied `Authorization`, `Cookie` and `Proxy-Authorization` headers are therefore re-sent to whatever host the redirect names, across scheme, host or port boundaries, and including `https` to `http` downgrades that expose them in plaintext on the wire. The HTTP::Tiny POD note that \"Authorization headers will not be included in a redirected request\" applied only to the URL-userinfo Basic-auth path, not to headers passed explicitly by the caller.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-7017", + "severity": "HIGH", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "7.1" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:L/A:N" + }, + { + "key": "package_version", + "value": "5.40.1-6" + }, + { + "key": "package_name", + "value": "perl" + } + ] + }, + { + "name": "CVE-2026-18477", + "description": "A TOCTOU (Time-of-Check Time-of-Use) vulnerability in GNU tar's incremental dumpdir 'X' rename handling allows a local attacker with write access to a directory being backed up to influence the restore process if the attacker has access to the system where the restore is being performed. During restoration, files or directories may be created, renamed or overwritten outside the intended extraction directory. This could lead to unauthorized file modification or, in some cases, privilege escalation. Exploitation does not require the attacker to modify or craft the archive, and standard backup and restore workflows—including extracting into a newly created directory without using the -P option do not mitigate the issue.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-18477", + "severity": "MEDIUM", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "4.4" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:L/AC:H/PR:L/UI:R/S:U/C:N/I:H/A:N" + }, + { + "key": "package_version", + "value": "1.35+dfsg-3.1" + }, + { + "key": "package_name", + "value": "tar" + } + ] + }, + { + "name": "CVE-2026-59831", + "description": "GitHub CLI (gh) is GitHub’s official command line tool. From 2.10.0 through 2.95.0, connecting to a malicious Codespace with gh codespace jupyter can allow command execution because the command opens a JupyterLab URL supplied by a process inside the Codespace without validating that it is a loopback HTTP or HTTPS address, allowing a crafted vscode:// or vscode-insiders:// URL to be handed to VS Code. This issue is fixed in version 2.96.0.", + "uri": "https://nvd.nist.gov/vuln/detail/CVE-2026-59831", + "severity": "MEDIUM", + "attributes": [ + { + "key": "CVSS3_SCORE", + "value": "4.4" + }, + { + "key": "CVSS3_VECTOR", + "value": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:L/I:L/A:N" + }, + { + "key": "package_version", + "value": "2.46.0-3" + }, + { + "key": "package_name", + "value": "gh" + } + ] + } + ], + "imageScanCompletedAt": "2026-08-05T17:33:43-04:00", + "vulnerabilitySourceUpdatedAt": "2026-08-05T17:33:43-04:00", + "findingSeverityCounts": { + "HIGH": 12, + "MEDIUM": 6, + "LOW": 1, + "CRITICAL": 4 + } + }, + "registryId": "272493677981", + "repositoryName": "omninode-runtime", + "imageId": { + "imageDigest": "sha256:b829c56f547340363be347391275cc847b1f465c18fa275836dd098989f7ecb7" + }, + "imageScanStatus": { + "status": "COMPLETE", + "description": "The scan was completed successfully." + } +} diff --git a/docs/migrations/2026-08-06-omninode-internal-schema-transformation-receipt.md b/docs/migrations/2026-08-06-omninode-internal-schema-transformation-receipt.md new file mode 100644 index 0000000000..096e5ac9c5 --- /dev/null +++ b/docs/migrations/2026-08-06-omninode-internal-schema-transformation-receipt.md @@ -0,0 +1,111 @@ +# OMN-15359 — `omninode_internal` schema, physical build (P2 slice) + +Ticket: OMN-15359 (P2-P4 build classified schemas and migrate internal, +control-plane, catalog, and tenant targets). Parent: OMN-15354. Consumer this +PR unblocks: OMN-15426 (P5 cut internal projections to the `omninode_runtime` +identity). + +## What this PR built (additive only) + +1. **Physical schema.** `docker/migrations/forward/098_create_omninode_internal_schema.sql` + — `CREATE SCHEMA IF NOT EXISTS omninode_internal` inside `omnidash_analytics` + (the physical database backing the unified `application` topology + database). No table created, moved, or altered. Proven live against a real + ephemeral Postgres cluster: + `tests/integration/migrations/test_098_omninode_internal_schema_omn15359.py` + (schema created, zero tables land, idempotent re-apply, rollback drops the + empty schema, rollback fails closed — `RESTRICT` — once any table has + landed in it). +2. **Physical-schema bridge, extended to OMNINODE_INTERNAL.** + `src/omnibase_infra/topology/physical_schema_mapping.py` gains + `INTERNAL_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359`, mirroring the + already-shipped `TENANT_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359` + pattern. `physical_grant_schema_for_table` now resolves any of the 41 + listed tables to `public` (their real physical location) instead of the + schema the contract logically declares, so grant derivation and + `handler_wiring` stop targeting a relation that does not exist there. This + is the direct fix for the OMN-15426 live gap: *"handler_wiring.py issues + schema-qualified SQL against `omninode_internal` for the 41-table + `omninode_runtime` domain; physically all 41 still live in public... + Grant on public schema WITHHELD."* (rolling ledger, 2026-08-03T19:2xZ). +3. **Companion topology grant.** `omninode_runtime` did not hold `USAGE ON + SCHEMA public` in any shipped instance (only `USAGE ON SCHEMA + omninode_internal`) — a real, separate gap the bridge alone does not close, + since a table-level GRANT is inert without schema-level USAGE. Added to all + three source-of-truth instances + (`src/omnibase_infra/topology/instances/{local,onex-dev,onex-prod}.yaml`), + mirroring the grant `tenant_projection_writer` already holds. The 41 TABLE + grants for `omninode_runtime` were regenerated with + `scripts/generate_application_database_table_grants.py --write` (against + the live sibling `omnimarket` checkout — 57 `db_io.db_tables` declarations, + identical to what was already checked in) so they now read `schema: + public`, matching the bridge and the tenant precedent's shape. The 7 + `docker/catalog/database-topology/*.yaml` projections were regenerated with + `scripts/render_application_database_topology.py`. + +## Relation family covered + +**Family:** `omninode_runtime` internal-domain projections (41 tables: +`baselines_*` (6), registry/catalog (`node_service_registry`, +`contract_registry`, `capsule_store`, `mcp_tools`), evidence/deployment +(`deployment_evidence_projection`, `deployment_readiness_projection`, +`evidence_correlation_trace_projection`, `evidence_dashboard_projection`, +`evidence_readiness_aggregate_projection`), telemetry +(`llm_call_metrics`, `llm_delegation_daily_projection`, +`llm_routing_decisions`, `cost_by_repo_snapshots`, `gate_activity`, +`gate_metrics`, `generation_events`, `intent_classification_events`, +`live_events`, `traces`, `merge_state_transitions`, `pr_lifecycle_ledger_entries`, +`pr_merged_events`, `receipt_gate_rows`, `renderer_capability_projection`, +`sandbox_decisions`, `event_chain`), orchestration/session +(`nightly_loop_configs`, `nightly_loop_decisions`, `nightly_loop_iterations`, +`overnight_session_phases`, `overnight_sessions`, `session_outcomes`, +`session_replay_snapshots`, `swarm_runs`, `voice_sessions`, +`skill_execution_snapshots` is TENANT, excluded)). + +- **Owner:** `omnibase_infra` (schema owner `owner_omninode_internal` per + topology; not yet physically created — see Deferred). +- **Producer:** each table's own node migration under + `docker/migrations/forward/nodes//` (vendored from `omnimarket`). +- **Consumer:** `omninode_runtime` principal, binding + `omninode_runtime_service` (`OMNINODE_INTERNAL_DB_URL`). +- **Domain:** `OMNINODE_INTERNAL` (`docker/catalog/database-topology/*.yaml` + `schemas.omninode_internal.domain`). +- **Migration stream:** `omnibase_infra.application`. +- **Current physical location:** `public`, bridged (this PR does not move + data). +- **Target:** `omnidash_analytics.omninode_internal`, physically created and + empty as of this PR. + +`delegation_workflow_state` (R-q, OMN-15337) and the OMN-15423 residual +dispositions (`event_bus_events`, legacy `schema_migrations`) are **not** +part of this family and are untouched here — those are separate, still-open +classification lanes on OMN-15337/OMN-15423. + +## Deferred (explicitly, not silently) + +Per the ticket's own scope text — *"Preserve source relations until +family-level parity and migration proof complete. Do not use `ALTER TABLE +... SET SCHEMA` on sources and do not use blind indefinite dual-write"* — the +following are out of this PR and owned by the P5 cutover tickets +(OMN-15426/OMN-15360) via the OMN-15420 cutover-journal machinery +(`omnibase_infra.migration.cutover`): + +- **Physical per-table copy** of the 41 relations from `public` into + `omninode_internal`, each with its own transformation receipt (counts, key + sets, hashes, FKs, sequences, grants, owners, policies reconciled) — the + full acceptance-criteria proof this ticket's parent scope describes. +- **`owner_omninode_internal` / `omninode_runtime` role creation.** Neither + role is physically created anywhere in the migration corpus yet (verified: + zero `CREATE ROLE omninode_runtime` / `CREATE ROLE owner_omninode_internal` + hits across `docker/migrations/`). Creating them safely (RDS-compatible, + guarded against re-ALTER privilege demands) is its own change, matching the + care `094_create_app_dashboard_role.sql` took for `app_dashboard` — not + something to fold into a schema-creation PR. +- **Tenant-side physical move.** `TENANT_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359` + is untouched; the tenant `CREATE SCHEMA` target already exists via prior + work (OMN-14894/OMN-15655) and its own migration proof. +- **Grant regeneration re-run under the CI-pinned `omnimarket` checkout.** + This PR's `--write` ran against the live local sibling clone, not the pin + CI resolves; `scripts/generate_application_database_table_grants.py + --check --prove` in the OMN-15361 CI job re-verifies against the pinned + checkout and is the actual gate of record. diff --git a/docs/patterns/service_catalog.md b/docs/patterns/service_catalog.md new file mode 100644 index 0000000000..fe885e528d --- /dev/null +++ b/docs/patterns/service_catalog.md @@ -0,0 +1,140 @@ +> **Navigation**: [Home](../index.md) > [Patterns](README.md) > Service Catalog & Install Model + +# Service Catalog Architecture & Install Model + +Moved out of `CLAUDE.md` (OMN-15198). This document covers the two ways `omnibase_infra` +is consumed (pip package vs local clone) and the typed service-catalog system that +generates all Docker infrastructure. + +## Install Model + +`omnibase_infra` ships as both a **pip-installable package** and a **cloneable repository**. + +### Pip Package (library + runtime CLIs) + +Install via pip for using `omnibase_infra` as a library dependency in other ONEX services, +or for running the bundled runtime CLIs: + +```bash +pip install omnibase-infra +# or +uv add omnibase-infra +``` + +The bundled CLI entry points (`omni-infra`, `onex-runtime`, `onex-infra-test`, +`onex-git-hook-relay`, `onex-linear-relay`, `onex-status`, ...) are declared in +`pyproject.toml` under `[project.scripts]` — that table is the authoritative list. + +### Local Clone (operational scripts) + +A **local clone is required** to run the operational scripts in `scripts/`. These scripts +are **not bundled** in the pip package — they live only in the repository source tree: + +```bash +git clone https://github.com/OmniNode-ai/omnibase_infra.git +cd omnibase_infra +uv sync +``` + +**Why scripts require a clone:** they scan the repository source tree directly +(e.g., `seed-infisical.py` iterates over `src/omnibase_infra/nodes/*/contract.yaml`), +write back to `~/.omnibase/.env`, or depend on shell tooling co-located with the repo. +This applies to all of `scripts/` — including `seed-infisical.py`, +`bootstrap-infisical.sh`, `provision-infisical.py`, `setup-infisical-identity.sh`, +`create_kafka_topics.py`, and `validate.py`. + +### Decision Summary + +| Use Case | Install Method | +|----------|---------------| +| Add `omnibase_infra` as a library dependency | `pip install omnibase-infra` | +| Run ONEX runtime services | `pip install omnibase-infra` → `onex-runtime` | +| Bootstrap Infisical (first-time setup) | Clone + `scripts/bootstrap-infisical.sh` | +| Seed Infisical from contracts | Clone + `uv run python scripts/seed-infisical.py` | +| Provision machine identities | Clone + `uv run python scripts/provision-infisical.py` | +| Run CI validators | Clone + `uv run python scripts/validate.py` | +| Develop nodes and handlers | Clone (full dev environment) | + +> **Note on `sync-omnibase-env.py`**: This script is **not** part of +> `omnibase_infra`. Use the separately installed environment-sync tooling +> available in your workspace. + +## Service Catalog + +The service catalog is the authoritative source for all Docker infrastructure. +Every deployable unit is a typed YAML manifest; the compose file is generated, not hand-edited. + +### Concepts + +| Term | Description | +|------|-------------| +| **Manifest** | Typed YAML declaration of a single deployable service (`docker/catalog/services/.yaml`) | +| **Bundle** | Named group of manifests deployed together (`docker/catalog/bundles.yaml`) | +| **Resolver** | Loads manifests + bundles, resolves transitive `includes`, returns `ResolvedStack` | +| **Generator** | Renders `ResolvedStack` → `docker-compose.generated.yml` | +| **Validator** | Checks that all `required_env` vars are present before start | + +### Bundles + +`docker/catalog/bundles.yaml` is the single authoritative source for bundle names, +their service membership, `includes` composition, and env injection — read it rather +than any copied table (copied tables drift). Key structural facts: + +- **Transitive resolution**: bundles compose via `includes`; the resolver expands all + `includes` before collecting services (e.g. `runtime` pulls in `core` and its + sub-bundles; `tracing` pulls in `observability`). +- **Incremental rollout**: the `runtime` platform is split into sub-bundles so operators + can deploy `onex up runtime-core` (no new secret requirements) to pick up correctness + fixes without also enabling integrations that need new secrets. Once secrets are seeded + (Infisical or `~/.omnibase/.env`), bring up the remaining sub-bundles individually. +- **Env injection**: each bundle may declare `inject_env` (hardcoded values injected into + generated compose) and `inject_required_env` (vars that must be present in the operator + environment at start time). + +### onex CLI Commands + +The `onex` CLI (`src/omnibase_infra/docker/catalog/cli.py`) is the primary operator interface. + +```bash +# Generate compose file for one or more bundles +uv run python -m omnibase_infra.docker.catalog.cli generate core +uv run python -m omnibase_infra.docker.catalog.cli generate runtime memgraph + +# Validate env completeness before starting +uv run python -m omnibase_infra.docker.catalog.cli validate runtime + +# Start a bundle (generate + validate + docker compose up) +uv run python -m omnibase_infra.docker.catalog.cli up core +uv run python -m omnibase_infra.docker.catalog.cli up runtime memgraph tracing + +# Stop a running bundle +uv run python -m omnibase_infra.docker.catalog.cli down core +``` + +The shell functions `infra-up` (→ `onex up core`), `infra-up-runtime` (→ `onex up runtime`), +`infra-up-memory` (→ `onex up runtime memgraph`), and `infra-down` (defined in `~/.zshrc`) +are backwards-compatible wrappers around `onex up/down`. They remain the preferred operator +interface — do not bypass them with raw `docker compose -f `. + +### Adding a New Service + +1. Create `docker/catalog/services/.yaml` using an existing manifest as template. +2. Set `layer` to one of: `infrastructure`, `runtime`, `observability`, `auth`, `secrets`. +3. Declare all `required_env` vars that the container needs from the operator environment. +4. Add hardcoded container-internal addresses under `hardcoded_env` (never pass host-side env vars for internal addressing). +5. Add the service name to the appropriate bundle(s) in `docker/catalog/bundles.yaml`. +6. Run `uv run python -m omnibase_infra.docker.catalog.cli validate ` to confirm env contract. + +### Env Var Contract + +Three categories of env vars in the catalog: + +| Category | Location | Behavior | +|----------|----------|----------| +| `required_env` | Per-manifest YAML | Must be set in operator env; validated before start | +| `hardcoded_env` | Per-manifest YAML | Container-internal addresses; never overrideable | +| `inject_env` | Per-bundle in `bundles.yaml` | Injected only when that bundle is selected | + +**Rule**: Container-to-container addresses (e.g. `redpanda:9092`, `valkey:6379`) must live +in `hardcoded_env`, never in `required_env`. Operator-supplied secrets (`POSTGRES_PASSWORD`, +API keys) belong in `required_env`. diff --git a/docs/runbooks/application-database-cutover-receipts.md b/docs/runbooks/application-database-cutover-receipts.md new file mode 100644 index 0000000000..5d4bf7669c --- /dev/null +++ b/docs/runbooks/application-database-cutover-receipts.md @@ -0,0 +1,99 @@ +# Application database cutover receipts + +OMN-15420 provides proof mechanics for one-application-database family cutovers. +It does not activate a writer, change a DSN, apply a grant or policy, or authorize a +live cutover. Each coherent relation family is registered and stopped independently. + +## Contract and evidence boundary + +`ModelCutoverFamilyContract` declares: + +- a stable family ID and whether the family is a projection or control-plane family; +- secret-free source and target topology binding references; +- SHA-256 pins for the exact source and target evidence-query contracts; +- one post-checkpoint mode: proven reverse delta or explicit forward-fix-only; +- a hard maximum for any optional dual-write window (zero disables it); and +- the required observation-window duration. + +`ModelPostgresEvidenceQuerySet` is a complete, read-only query contract. All ten +queries are required and must return one text signature column. A source query may +perform an explicit transformation such as legacy slug to canonical UUID. The +collector binds each query set by SHA-256 and reads the source/target pair in one +read-only, repeatable-read PostgreSQL snapshot. + +The receipt evaluates every dimension in canonical order: + +1. transformed key set and row count; +2. transformation-aware row hashes; +3. foreign keys and sequence state/ownership; +4. owners and explicit grants; +5. policies and dependent views/functions; +6. projection versions/event offsets or control-plane snapshot/final-delta evidence; +7. transformation collision scans; and +8. dependency signatures. + +Every dimension is present even when it is expected to be empty. Any mismatch makes +the receipt `FAIL`, durably blocks only that family, and prevents further journal +transitions. A repair receipt must be a newly generated PASS that postdates the failed +receipt; an older PASS cannot be replayed to clear the block. + +## Durable journal + +`RepositoryPostgresCutoverJournal.initialize()` explicitly creates the proof tables +in `omninode_internal`. The bootstrap SQL is packaged with the library but is not in +the automatic forward-migration stream. Registration is immutable: changing a +same-ID family contract is rejected. + +The repository serializes each family with a row lock and appends SHA-256-linked +events for backfill, optional bounded dual-write, final delta, writer checkpoint, +real application-path write, reader cutover, observation window, quiescence, and the +declared post-checkpoint proof. Event times must be monotonic. Foreign keys bind every +receipt, journal event, and reverse-delta proof to the same family. +Observation completion is refused before the family-specific durable deadline. + +The journal records evidence references, not credentials or customer payloads. A +real application-path write proof names only the logical database, principal, schema, +and target sequence. Projection receipts retain versions and offsets. Control-plane +receipts retain source/target snapshot hashes, final-delta hashes, and watermarks. + +## Rollback decision + +| Family state | Direct DSN rollback | +| --- | --- | +| No target-only write | Allowed; source remains authoritative | +| Bounded dual-write still open | Refused until the window ends and writers quiesce | +| Target-only write, reverse delta incomplete | Refused | +| Target-only write, forward-fix-only | Refused; apply the declared forward fix | +| Writer quiesced, contiguous reverse delta applied, fresh reconciliation receipt and behavioral readback recorded | Allowed | + +Reverse-delta entries must cover every target sequence from the first target-only +write through the quiesced final sequence. Merely retaining the source or recording a +proof ID is insufficient. The proof must cite the exact quiescence event, a fresh PASS +reconciliation receipt, and a behavioral readback artifact. + +## Verification + +Focused local proof: + +```bash +uv run pytest \ + tests/unit/migration/cutover \ + tests/integration/migrations/cutover/test_cutover_receipts_postgres16.py \ + tests/ci/test_legacy_rds_fixture_contract.py -v +``` + +Rebuilt Docker proof (where Docker is available): + +```bash +docker compose --project-name omn15420-cutover-proof \ + -f docker/legacy-rds-fixture/compose.yml \ + up --build --abort-on-container-exit --exit-code-from proof +docker compose --project-name omn15420-cutover-proof \ + -f docker/legacy-rds-fixture/compose.yml \ + down --volumes --remove-orphans +``` + +The Docker harness is wholly synthetic. It seeds RED controls for a family-local +metadata mismatch, expired blind dual-write, post-write direct rollback, incomplete +reverse-delta coverage, and forward-fix-only rollback. It grants no live execution or +retirement authority. diff --git a/docs/runbooks/application-migration-ledger.md b/docs/runbooks/application-migration-ledger.md new file mode 100644 index 0000000000..f5339721e2 --- /dev/null +++ b/docs/runbooks/application-migration-ledger.md @@ -0,0 +1,166 @@ +# Application migration ledger + +**Ticket:** OMN-15413 +**Target database:** `omnidash_analytics` +**Canonical relation:** `platform_catalog.schema_migrations` + +This runbook defines the deterministic migration-history boundary for the +unified application database. It does not merge the separate +`omnibase_infra.public.schema_migrations` service ledger into the application +database. + +## Canonical model + +The selected application ledger is the existing checksum-capable +`public.node_schema_migrations` table. The bootstrap moves that table into +`platform_catalog` in one PostgreSQL transaction and keeps its relation OID, +rows, timestamps, and owner. It then adds the contract dimensions required by +the deployment topology: + +| Column | Meaning | +| --- | --- | +| `migration_stream` | Exact producer stream, such as `node:` | +| `owner` | Checked-in producer owner; never inferred from the login role | +| `domain` | `tenant` or `omninode_internal` for executable node migrations | +| `version` | Exact historical runner identity | +| `checksum` | Lowercase SHA-256 | +| `checksum_kind` | `content_sha256` or quarantined `legacy_attestation` | +| `applied_at` | Original application timestamp | +| `provenance` | Deterministic source relation/file identity | + +The primary key is `(migration_stream, domain, version)`. An active node row +must match the checked-in file SHA-256 exactly. A `legacy_attestation` row can +never satisfy an active migration probe. + +## Historical import policy + +Source ledgers remain intact. Import is additive to the selected canonical +ledger and rerunning it must produce an identical row set. + +| Source | Authority | Canonical treatment | +| --- | --- | --- | +| `public.node_schema_migrations(version, applied_at, checksum)` | Applied node set | Move in place; require exact manifest identity and file SHA-256 | +| `public.omnimarket_schema_migrations` | Applied projection set | Normalize `(node_name, filename)` to the exact node version; require manifest checksum; reject overlap with node history | +| Filename-only `public.schema_migrations(filename, applied_at)` | Applied legacy set without checksums | Preserve source and import a deterministic source-record attestation under `legacy:filename-only` | +| `public.schema_migrations(migration_id, applied_at, checksum, source_set)` with `source_set = 'node'` | Applied node set written by the pre-OMN-15413 runner | Preserve source and adopt under the manifest's exact stream/owner/domain; see OMN-15695 below | +| Cloud `public.schema_migrations(version, applied_at, checksum)` | Applied cloud set | Preserve source and import under exact producer stream `omninode-cloud` | +| Cloud `public.migrations_log` | Audit/attempt evidence only | Enrich matching applied rows through the checked-in alias map; a log-only alias is fatal | + +Filename-only and cloud records currently use domain +`legacy_unclassified`. This is an explicit non-executable quarantine, not a +claim that the migrations target `tenant`. OMN-15423 must provide +per-artifact topology domains before those rows can be promoted. Cloud SQL is +cross-domain, so a blanket domain default is prohibited. + +### OMN-15695 — adopting the predecessor `migration_id` node ledger + +**Operator ruling, 2026-08-04: ADOPT/CONVERT.** + +The four-column `public.schema_migrations(migration_id, applied_at, checksum, +source_set)` relation is ambiguous by column signature alone. The +pre-OMN-15413 runner created it in **both** databases: in the service database +it is the service-owned ledger (`source_set = 'docker'`, ids `docker/`), +and in the **application** database it is the predecessor **node** ledger +(`source_set = 'node'`, ids `node::.sql`). The original arm refused +the shape outright, which was a false negative for the application database and +left the dev lane unable to bootstrap. + +The bootstrap now partitions on row content before deciding: + +| Source row | Treatment | +| --- | --- | +| `source_set = 'node'` and id matches `node::.sql` | Adoptable | +| `source_set = 'docker'` and id matches `docker/…` | Ignored — service-owned | +| anything else | Fatal: `unknown migration ledger shape: … unrecognized migration_id rows` | + +A relation with zero adoptable rows is still refused with the original +`unknown migration stream: service-owned migration_id ledger cannot be selected +for the application database`. **The guard is narrowed for exactly this shape; +it is not weakened generally.** + +Adoption is additive and non-destructive, exactly like the filename-only +import: the source relation is never renamed, updated, deleted, or moved. Each +adoptable row inserts one canonical row with `migration_stream`, `owner`, +`domain` and `checksum` taken verbatim from the checked-in manifest, +`version` = the source `migration_id` verbatim, `applied_at` = the source +timestamp verbatim, and +`provenance = adopted::public.schema_migrations:migration_id::raw-checksum=`. + +**The one non-derivable point, stated plainly.** The historical runner wrote the +literal `applied-by-runner` in the `checksum` column — there is no byte evidence +in the database. `checksum_kind = 'legacy_attestation'` cannot be used, because +a legacy attestation can never satisfy an active node probe +(`run-forward-migrations.sh` FATALs with *"has only a legacy checksum +attestation"*), which would force re-application — the one thing the ruling +forbids. Adoption therefore writes `content_sha256` with the manifest checksum, +which **asserts** that the bytes applied historically equal today's checked-in +bytes. That assertion is the operator ruling made mechanical, not a derivation. +It is bounded three ways: only the exact `applied-by-runner` literal is +adoptable, a 64-hex source checksum that disagrees with the manifest is still +fatal, and `provenance` permanently records the raw source checksum under an +`adopted:` prefix so an adopted row is never confusable with a runner-verified +`file:nodes/…` row. + +Evidence basis for the dev lane specifically (`omnidash_analytics`, read-only, +2026-08-04): all 80 source rows are `checksum='applied-by-runner'`, +`source_set='node'`; all 80 ids are present in the 94-row manifest, zero +live-not-in-manifest; and `git log --since='2026-07-31T06:44:00Z' --name-only -- +docker/migrations/forward/nodes/` touches only files in the 14-entry +not-yet-applied set, so no applied artifact's bytes changed after the apply +timestamp. That corroboration is specific to this database and is **not** true +by construction for any other database with this shape. + +`scripts/run-forward-migrations.sh` is unchanged: `migration_is_applied()` +already probes on `(migration_stream, domain, version)` + owner + kind + +checksum, which is exactly what an adopted row carries, and `record_migration()` +stays `content_sha256`-only so no forward path can mint an adopted row. + +## Deterministic procedure + +1. Validate every vendored node SQL file against + `_ledger/application-migrations.tsv` or an explicit ticketed block in + `_ledger/application-migration-blocks.tsv`. +2. Stop before migration DDL if an unresolved block is not protected by the + existing operator fence. +3. Select and extend the checksum-capable node ledger in one transaction. +4. Import filename-only and omnimarket source rows without updating or deleting + either source relation. +5. Read cloud applied state and audit aliases under one repeatable-read source + snapshot; import it transactionally into the application ledger. +6. Probe active migrations by stream, domain, version, owner, checksum kind, + and checksum. Apply and record only an absent, fully declared file. +7. Run the same process a second time and require an identical ledger signature + with zero newly applied files. + +Never insert, update, or delete migration-ledger rows by hand. A missing row is +an indeterminate apply: stop the lane, retain the source catalogs/logs, and +repair through a reviewed deterministic import or a new migration. + +## Validation + +```bash +uv run python scripts/validation/validate_application_migration_manifest.py +uv run python scripts/validation/validate_application_migration_manifest.py --require-complete +uv run pytest tests/unit/scripts/validation/test_application_migration_manifest.py \ + tests/integration/migrations/test_application_migration_ledger_omn15413.py -q +docker compose -f docker/legacy-rds-fixture/compose.yml up \ + --build --abort-on-container-exit --exit-code-from proof +``` + +The first command validates the checked-in surface and reports explicit +blocks. The completion command remains RED while any block exists. The +PostgreSQL 16 integration suite proves fresh and sanitized legacy execution +twice plus checksum, unknown-stream, and double-declaration RED controls. The +Docker fixture must also use `--build` so it contains the changed runner and +ledger artifacts. + +## Landing holds + +- OMN-15423 still leaves `delegation_judge_verdict_events` unclassified. Its + unfenced `0016` migration makes the complete runner preflight RED. +- The Kubernetes runner still targets `public.node_schema_migrations`. This + change cannot land until its exact stacked update targets + `platform_catalog.schema_migrations`; otherwise it would recreate a second + ledger after the in-place move. +- No live database query, deployment, grant/RLS change, cutover, or destructive + action is part of this ticket. diff --git a/docs/runbooks/apply-migrations.md b/docs/runbooks/apply-migrations.md index 68d9003895..54fa89757c 100644 --- a/docs/runbooks/apply-migrations.md +++ b/docs/runbooks/apply-migrations.md @@ -217,22 +217,12 @@ This is a development error — rename one of the files and update the fingerpri ### Migration applied but `schema_migrations` row missing -If a migration applied successfully but was not tracked (e.g., the runner was killed mid-run), -manually insert the tracking row: - -```sql -INSERT INTO schema_migrations (migration_id, checksum, source_set) -VALUES ( - 'docker/035_another_migration.sql', - encode(sha256(pg_read_binary_file('...')::bytea), 'hex'), - 'docker' -) -ON CONFLICT DO NOTHING; -``` - -Or re-run `run-migrations.py` — it uses `ON CONFLICT DO NOTHING` so re-applying is safe -for already-applied migrations that have tracking rows. For migrations without tracking rows, -wrap in a transaction and check for idempotency before re-applying. +Treat this as an indeterminate apply and stop the lane. Do not hand-insert a +tracking row: that would claim content, owner, and apply provenance that the +runner did not durably record. Preserve the database catalogs and runner logs, +prove whether the DDL committed, then repair through a reviewed deterministic +import or a new idempotent migration. Application/node history follows +[the application migration ledger procedure](application-migration-ledger.md). ## Omnidash Read-Model Migrations diff --git a/docs/runbooks/gateway-lane-deploy.md b/docs/runbooks/gateway-lane-deploy.md new file mode 100644 index 0000000000..c60eb28275 --- /dev/null +++ b/docs/runbooks/gateway-lane-deploy.md @@ -0,0 +1,242 @@ +# Gateway lane deploy (`omninode-gateway` compose project on `.201`) + +This runbook documents the sanctioned deploy path for the `.201` operator-edge +gateway forwarder — the standalone process that bridges the cloud MSK bus to +the `.201` local Redpanda listener for cloud→local inference (OMN-12908 hybrid +gateway). It exists because, before OMN-15521, this lane had **no** repo- +resident deploy path at all: it was stood up on 2026-07-29 by hand-copying +`docker/docker-compose.gateway.yml` and `docker/gateway/beta-gateway-canary.yaml` +into a root-owned directory and running compose there directly. That left the +lane invisible to `deploy-runtime.sh` (whose `-p omnibase-infra` scope never +touches it), running an image with an **empty** +`org.opencontainers.image.revision` label, and with no recorded rollback +target. + +Ticket: **OMN-15521**. + +> Scope: this is the `.201` `omninode-gateway` lane ONLY (container +> `omninode-gateway-forwarder`, systemd unit `onex-gateway-forwarder`). It is +> unrelated to the `omnibase-infra` runtime lane `deploy-runtime.sh` manages — +> no migrations, no `RUNTIME_SERVICES` restart, no broker readiness preflight. +> See `docs/runbooks/cold-lane-full-bringup.md` for that lane instead. + +--- + +## What's already on the host (read this before you "fix" it) + +The `.201` box already has a working, restart-safe supervision layer for this +lane that this runbook does **not** replace: + +- systemd unit `/etc/systemd/system/onex-gateway-forwarder.service` (repo copy: + `docker/gateway/onex-gateway-forwarder.service`) — starts the container via + `docker compose ... up -d --no-build --wait`, and its `ExecStartPre` **hard + refuses to start** unless `/etc/omninode/gateway/gateway.env`'s + `GATEWAY_IMAGE=` line is a real `sha256:<64 hex>` digest. It never builds — + the image must already exist locally. +- `/etc/omninode/gateway/gateway.env` (root-owned, mode `0444`) — the AWS + Roles Anywhere / TPM / container-UID variables the compose file requires, + plus the pinned `GATEWAY_IMAGE` digest. +- `/opt/omninode/gateway/` (root-owned, mode `0444`) — the compose file + + canary config the systemd unit's `WorkingDirectory` points at. + +What was missing is everything **upstream** of that: a repeatable way to +build a new image from merged dev, stamp it with real provenance labels, +update the pinned digest, and sync the host files without hand-editing a +root-owned directory. `scripts/deploy-gateway.sh` is that path. + +--- + +## Procedure + +### 0. Pre-flight: sync the canonical clone to the merged-dev tip + +Run from the canonical `omnibase_infra` clone on `.201` (same convention as +`deploy-runtime.sh` and `refresh_stability_lane.sh` — this script is **not** +run from a worktree): + +```bash +cd /data/omninode/omni_home/omnibase_infra # or wherever the canonical clone lives on this host +git pull --ff-only +``` + +### 1. Preview (dry-run, the default) + +```bash +./scripts/deploy-gateway.sh +``` + +Dry-run prints the resolved version/git SHA and the exact compose build +command it would run, and lists every file/registry mutation it would +perform, without touching anything. + +### 2. Inspect the exact build command + +```bash +./scripts/deploy-gateway.sh --print-compose-cmd +``` + +This is the fix for the ticket's core finding: the printed command carries +`--build-arg VCS_REF= --build-arg RUNTIME_VERSION= --build-arg +COMPOSE_PROJECT=omninode-gateway --build-arg RUNTIME_SOURCE_HASH= +--build-arg PROMOTION_CLASS=clean-main --build-arg NON_MAIN_LINEAGE=false` — +the same OCI provenance build-args `deploy-runtime.sh`'s `build_images()` +passes for every `omnibase-infra` runtime container. The compose file's own +declared `build.args` block only ever carried `BUILD_SOURCE`/ +`EXPECTED_BUILD_SOURCE` (mirroring `docker-compose.infra.yml`'s split between +compose-declared and CLI-supplied build-args) — that split is why the +hand-built image had `rev=(empty)`. + +### 3. Execute the deploy + +```bash +./scripts/deploy-gateway.sh --execute +``` + +In order, this: + +1. Resolves `repo_root` / `version` (pyproject.toml) / `git_sha` (HEAD). +2. Resolves the CONTAINER's currently running image + (`docker inspect omninode-gateway-forwarder --format '{{.Image}}'`) as the + rollback target — **never** `gateway.env`'s `GATEWAY_IMAGE=` line, which can + go stale relative to what is actually running — and retags it durably as + `docker-gateway-forwarder:previous` so it survives the build below moving + the build tag onto the new image. If the previous image no longer resolves + locally (already pruned), no rollback target is recorded for this deploy + rather than recording a digest `docker image inspect` cannot find. +3. Sources the env file so the AWS/TPM/UID variables the compose file + requires resolve for the build. +4. If `BUILD_SOURCE=workspace` (default `release`): stages + `workspace/sibling-repos/` from `OMNI_HOME` via the SAME + `scripts/runtime_build/stage_workspace.sh` `deploy-runtime.sh` uses, then + runs the OMN-12987 sibling lock-pin preflight. `docker/Dockerfile.runtime` + unconditionally `COPY`s `workspace/sibling-repos/`, so skipping this step + in workspace mode silently built against the committed placeholder / + whatever staging happened to already be in the checkout, while still + stamping workspace-provenance labels. `BUILD_SOURCE=release` (the default) + skips this entirely. +5. Builds `docker-gateway-forwarder:build` with the provenance build-args from + step 2 above, **plus** `OMNIBASE_COMPAT_REF` / `OMNIMARKET_REF` / + `ONEX_CHANGE_CONTROL_REF` — the same sibling-ref args + `deploy-runtime.sh`'s `build_images()` passes unconditionally. Omitting + these silently falls back to the Dockerfile's hardcoded ARG defaults, + which is how the gateway image's `onex-change-control` pin drifted from + the `omnibase-infra` runtime image's pin on the same box. +6. Resolves the built image's digest (`docker image inspect --format='{{.Id}}'`). +7. Syncs `docker/docker-compose.gateway.yml` and + `docker/gateway/beta-gateway-canary.yaml` from **this checkout** into + `/opt/omninode/gateway/` (`sudo install -m 0444 -o root -g root`, matching + the existing file posture) — replacing the 2026-07-29 hand-copy as the + source of truth. Every deploy re-syncs, so the host copy can never drift + from a merged commit again. +8. Rewrites `/etc/omninode/gateway/gateway.env`'s `GATEWAY_IMAGE=` line to the + new digest, leaving every other key untouched. +9. Writes `~/.omnibase/gateway/registry.json` recording `active_digest`, + `previous_digest` (the rollback target), `git_sha`, `deployed_at`, and a + ready-to-run `rollback_command` — the same convention + `~/.omnibase/infra/registry.json` uses for the `omnibase-infra` lane. + `previous_digest` and `rollback_command` are both `null` when there is no + rollback target (first deploy, or the previous image had already been + pruned) — never a fabricated digest or a sed command built from an empty + value. +10. `sudo systemctl reload onex-gateway-forwarder` — the unit's existing + `ExecReload` force-recreates the container on the new digest. +11. Verifies: the container is actually running the digest just built (a + reload that silently fails to recreate the container is caught here + instead of reporting success), labels are non-empty, and the two + OMN-12912 files (`service_gateway_delivery.py`, `store_sqlite.py`) are + present inside the running container. + +Pass `--skip-reload` to build + sync + pin the new digest without recreating +the running container yet (the old container keeps running on the old +digest until a manual `sudo systemctl reload onex-gateway-forwarder`). + +### 4. Verify + +```bash +docker inspect omninode-gateway-forwarder \ + --format='rev={{index .Config.Labels "org.opencontainers.image.revision"}} src={{index .Config.Labels "com.omninode.build_source"}}' +# -> rev=<12-char sha> src=release (never empty) + +docker exec omninode-gateway-forwarder \ + ls /app/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/ +# -> must include service_gateway_delivery.py + +docker exec omninode-gateway-forwarder \ + ls /app/src/omnibase_infra/idempotency/ +# -> must include store_sqlite.py + +diff /opt/omninode/gateway/docker-compose.gateway.yml docker/docker-compose.gateway.yml +# -> empty + +cat ~/.omnibase/gateway/registry.json | jq . +``` + +--- + +## Rollback + +`~/.omnibase/gateway/registry.json`'s `previous_digest` names the last-known- +good digest — resolved from the container's own running state at deploy time +and retagged as `docker-gateway-forwarder:previous` so a routine +`docker image prune` cannot silently make it unresolvable before anyone needs +it. `registry.json`'s own `rollback_command` field carries the exact restore +command pre-filled with that digest — run it verbatim, do not reconstruct it +by hand: + +```bash +jq -r .rollback_command ~/.omnibase/gateway/registry.json +# -> sudo sed -i "s|^GATEWAY_IMAGE=.*|GATEWAY_IMAGE=sha256:<64 hex>|" /etc/omninode/gateway/gateway.env && sudo systemctl reload onex-gateway-forwarder + +# then either paste that command, or: +bash -c "$(jq -r .rollback_command ~/.omnibase/gateway/registry.json)" +``` + +**`rollback_command` is `null` when there is no rollback target** — the first +deploy ever run against this lane, or a deploy whose previous running image +had already been pruned before this script could retag it. There is nothing +to roll back to in that case; do not reconstruct a sed command from +`.previous_digest` by hand (a JSON `null` printed through `jq -r` renders as +the literal string `null`, which `sed`s straight into `gateway.env`'s +`GATEWAY_IMAGE=` line and corrupts it — the systemd unit's `ExecStartPre` +digest-format assertion then refuses to start on the next restart/reboot). +Deploy forward instead. + +This mirrors the `omnibase-infra` lane's own manual rollback-via- +`registry.json` pattern — `deploy-runtime.sh` has no automated `--rollback` +flag either; a prior digest is always restored by hand from the registry. + +--- + +## AC5 — the OMN-12912 restart/redelivery proof + +```bash +./scripts/gateway_restart_safety_proof.sh +``` + +Confirms the container is reachable via `docker exec` and records its +identity (`Id` + `State.StartedAt`), snapshots the running container's +durable idempotency store row count, reloads `onex-gateway-forwarder` (the +same mechanism `--execute` uses to recreate the container), waits for +Docker-healthy, then asserts the container's identity actually **changed** +(a reload that exits 0 and reports healthy without recreating anything is a +false-green a health check alone cannot catch), re-confirms reachability, +and re-snapshots — failing if any durable marker was lost across the +restart, the container never came back genuinely healthy, was never actually +recreated, or became unreachable. This is a real restart-durability smoke +proof, driven against the actual running container — it is **not** the full +cross-broker at-least-once/exactly-once redelivery proof (a deliberately +killed in-flight message never duplicating or dropping on the far side), +which needs a synthetic in-flight cloud MSK message and is OMN-12912's own +test suite's job. + +Per OMN-15521's own AC5 wording ("that receipt lands on OMN-12912, not this +ticket"): run the script, then paste its printed receipt into an OMN-12912 +comment. This runbook and `scripts/gateway_restart_safety_proof.sh` do not +file it anywhere themselves. + +## Related runbooks + +- `docs/runbooks/cold-lane-full-bringup.md` — the `omnibase-infra` runtime + lane's own cold bring-up (a different compose project, different scope). +- `docker/gateway/onex-gateway-forwarder.service` — the systemd unit this + script's `--execute` reloads. diff --git a/docs/runbooks/managed-staging-aug5-readiness-rollback-packet.md b/docs/runbooks/managed-staging-aug5-readiness-rollback-packet.md new file mode 100644 index 0000000000..117d4de3f0 --- /dev/null +++ b/docs/runbooks/managed-staging-aug5-readiness-rollback-packet.md @@ -0,0 +1,72 @@ +# Aug-5 managed-staging readiness / rollback — GO-NO-GO PACKET (template) + +**Ticket:** OMN-15125 · **Plan row:** rolling plan §3 B7 · **Parent epic:** OMN-14724 +**Field manifest (the seam):** [`docs/runbooks/managed-staging-proof-kit/fields.yaml`](managed-staging-proof-kit/fields.yaml) +**Seam test:** `tests/ci/test_managed_staging_proof_kit_seam.py` + +> **This is a template. It executes nothing and authorizes nothing.** Assembling it +> does not authorize the Aug-5 window. It is the artifact the operator reads *in order +> to decide*. + +## The standing rule this packet exists to satisfy + +Rolling plan §3 B7's proof column, verbatim: + +> August 5 (soak Aug 6) is a **target, not a forecast**, until the reconciled blocker +> graph, dated chain with slack, and T20 handoff are attached. + +So three rows below — `reconciled_blocker_graph`, `dated_chain_with_slack`, +`t20_handoff` — are not decoration. **Until all three carry real content, the correct +`go_no_go_decision` is NO-GO by construction**, regardless of how green everything +else looks. + +Section-level proof-class ceiling: every §3 cloud fact is `proof_class: receipt-bound` +(contractor self-report), never `live-readback`. Do not upgrade a row's proof class in +this packet beyond what its evidence source can actually support. + +## How to use it + +1. Copy to `docs/evidence/OMN-15125/-aug5-readiness-rollback-packet.md`. +2. Fill every row from its evidence source. `a6_thresholds_with_live_samples` needs a + **sub-table**: one row per signal — signal, numeric threshold, live sampled value, + sample timestamp. A threshold with no live sample counts as unloaded. +3. `executable_rollback` requires a **dry-run output**, not an assertion that a + rollback exists (ticket criterion: "proven by a named dry-run or readback"). +4. `source_digest` must equal the OMN-15123 freeze packet's `source_digest`, and + `previous_digest` must be captured **before** any promotion — after promotion the + rollback target is unrecoverable from the live surface. +5. Record the decision, the decider, and the UTC timestamp in `go_no_go_decision`, and + mirror the row into `omni_home:docs/tracking/ROLLING_WORK_LEDGER.md`. + + +## Fields + +Every row is required. `Value` is filled at run time from `Evidence source`; +an empty or prose-only value cell means the packet is not complete. + +| Field | What it is | Value (paste verbatim readback) | Evidence source (command / path) | +|---|---|---|---| +| `source_digest` | Source digest being promoted | _(unfilled)_ | `git -C rev-parse HEAD # must equal one_tenant_contract_freeze.source_digest` | +| `previous_digest` | Previous digest (the rollback target) | _(unfilled)_ | `kubectl -n rollout history deploy/ && kubectl -n get deploy/ -o jsonpath='{.spec.template.spec.containers[0].image}' # record the currently-serving digest BEFORE promotion` | +| `amd64_manifest` | linux/amd64 manifest present for the promoted digest | _(unfilled)_ | `docker manifest inspect @ \| jq '.manifests[].platform' # must include {"architecture":"amd64","os":"linux"}` | +| `config_hash` | Rendered config hash | _(unfilled)_ | `kubectl -n get cm -o json \| jq -S '.data' \| shasum -a 256 # must equal one_tenant_contract_freeze.config_digest` | +| `policy_hash` | IAM/broker policy hash | _(unfilled)_ | `aws iam get-role-policy --role-name --policy-name --query PolicyDocument \| jq -S . \| shasum -a 256` | +| `vulnerability_result` | Vulnerability scan result for the promoted digest | _(unfilled)_ | `aws ecr describe-image-scan-findings --repository-name --image-id imageDigest= --query 'imageScanFindingsSummary.findingSeverityCounts'` | +| `a6_thresholds_with_live_samples` | A6 numeric thresholds loaded, each with a live sample value | _(unfilled)_ | `B10 wiring readback (OMN-14735/OMN-14948) -- one row per threshold: signal, numeric threshold, live sampled value, timestamp. A threshold with no live sample is unloaded.` | +| `monitoring_owner_actions` | Staffed monitoring owner + the action each breach triggers | _(unfilled)_ | `docs/runbooks/managed-staging-canary-teardown-rollback.md §3 (abort path, owner + sequence); name the on-call human and the window they are staffed for` | +| `b12_psql_readback` | B12 psql readback proving the landing table exists | _(unfilled)_ | `docs/runbooks/managed-staging-canary-postgres-provisioning.md §3.4 -- paste the psql \d+ delivery_replay_canary_projection output` | +| `teardown_readback` | OMN-14772 teardown readback | _(unfilled)_ | `docs/runbooks/managed-staging-canary-teardown-rollback.md §2 T-4..T-8 -- paste the post-teardown steady-state assertion output (OMN-14772)` | +| `executable_rollback` | Executable rollback procedure, proven by dry-run (not asserted) | _(unfilled)_ | `docs/runbooks/managed-staging-canary-teardown-rollback.md §4.1 rollback tuple; dry-run: kubectl -n rollout undo deploy/ --to-revision= --dry-run=server` | +| `reconciled_blocker_graph` | Reconciled blocker graph (every open blocker, with its owner) | _(unfilled)_ | `Linear: children of OMN-14724 with statusType != completed, plus their blockedBy edges -- attach the query output, not a paraphrase` | +| `dated_chain_with_slack` | Dated critical-path chain with explicit slack | _(unfilled)_ | `one row per chained item: item, owner, start date, end date, slack days; slack computed against the Aug-5 window (soak Aug 6)` | +| `t20_handoff` | T20 handoff (final linux/amd64 build + digest handoff from the contractor lane) | _(unfilled)_ | `the contractor handoff artifact under docs/plans/ or docs/handoff/ naming the T20/B1 final build; cite path + commit` | +| `go_no_go_decision` | Go / no-go decision, decider, UTC timestamp | _(unfilled)_ | `operator decision recorded in this packet + the corresponding row in docs/tracking/ROLLING_WORK_LEDGER.md` | +| `plan_row_binding` | Rolling plan §3 B7 bound to OMN-15125 | _(unfilled)_ | `git -C $OMNI_HOME/omni_home log -p -- docs/plans/ROLLING_SEVEN_DAY_PLAN.md \| grep -n 'OMN-15125'` | + +## Related + +- `docs/runbooks/managed-staging-canary-postgres-provisioning.md` — B12 landing table + §3.4 readback +- `docs/runbooks/managed-staging-canary-teardown-rollback.md` — teardown / abort / rollback ownership (B13) +- `src/omnibase_infra/topics/managed_staging_canary_catalog.py` + `..._namespace.yaml` — B7 `onex.mstg1.` catalog, epoch, zero-collision readback (OMN-14727) +- `scripts/proof/e2e_cloud_workflow_harness.py` — OMN-10858 end-to-end proof harness (`--live` defaults OFF) +- `omni_home:docs/plans/2026-07-17-managed-staging-verified-state-and-task-split.md` — lane B task split diff --git a/docs/runbooks/managed-staging-candidate-isolation-compatibility-proof.md b/docs/runbooks/managed-staging-candidate-isolation-compatibility-proof.md new file mode 100644 index 0000000000..80dba54fb1 --- /dev/null +++ b/docs/runbooks/managed-staging-candidate-isolation-compatibility-proof.md @@ -0,0 +1,66 @@ +# Candidate-in-isolation compatibility proof (template) + +**Ticket:** OMN-15124 · **Plan row:** rolling plan §3 B5 · **Parent epic:** OMN-14724 +**Field manifest (the seam):** [`docs/runbooks/managed-staging-proof-kit/fields.yaml`](managed-staging-proof-kit/fields.yaml) +**Seam test:** `tests/ci/test_managed_staging_proof_kit_seam.py` + +> **This is a template. It executes nothing and authorizes nothing.** The isolation +> run itself is a live mutation of an **isolation lane only** — zero staging mutation, +> zero prod mutation — and is HELD FOR OPERATOR like every other live step in this +> lane. + +## What this proves (and what it deliberately does not) + +It proves the **candidate image** speaks MSK and RDS correctly *before* it is pointed +at the staging lane: real IAM/TLS signer, an observed token refresh, explicit topic +bootstrap against a broker with auto-create OFF, `verify-full` to RDS, config coming +from the typed authority, and — the load-bearing half — **negative controls** showing +the failure paths actually fail. It is *not* the cutover (OMN-14933) and *not* the +canary run (OMN-14736/B11). + +## Why the negative controls are the point + +Three of the rows below are negative controls +(`negative_control_out_of_catalog`, `no_raw_endpoint_fallback`, and the +`dashboard_zero_authority` readback). A green positive path with a silent plaintext +or auto-create fallback is a **false pass**: the candidate would "work" against the +wrong surface. Each negative control must show the *observed refusal* — pasted error +text, not "verified". + +## How to use it + +1. Copy to `docs/evidence/OMN-15124/-candidate-isolation-proof.md`. +2. Name the `isolation_lane` **before** the run, in the committed copy. +3. Run each evidence source; paste verbatim output (or link `probes/.txt`). +4. `candidate_image_digest` must equal `image_digest` in the OMN-15123 freeze packet. + If it does not, this proof does not cover the candidate being promoted. + + +## Fields + +Every row is required. `Value` is filled at run time from `Evidence source`; +an empty or prose-only value cell means the packet is not complete. + +| Field | What it is | Value (paste verbatim readback) | Evidence source (command / path) | +|---|---|---|---| +| `isolation_lane` | Named isolation lane/host (zero staging + zero prod mutation) | _(unfilled)_ | `hostname && kubectl config current-context # record both; the lane must be named in the packet before the run` | +| `candidate_image_digest` | Candidate image digest under test (must equal one_tenant_contract_freeze.image_digest) | _(unfilled)_ | `kubectl -n get pod -o jsonpath='{.status.containerStatuses[0].imageID}'` | +| `msk_iam_signer` | MSK IAM/TLS signer actually used (not plaintext, not a fallback) | _(unfilled)_ | `uv run python -c 'from omnibase_infra.event_bus.kafka_auth import build_aiokafka_auth_kwargs_from_env as f; print({k: (type(v).__name__ if k=="sasl_oauth_token_provider" else v) for k,v in f().items()})' # src/omnibase_infra/event_bus/kafka_auth.py:105` | +| `token_refresh_cycle` | At least one observed IAM token-refresh cycle (>=2 token mints, same session) | _(unfilled)_ | `kubectl -n logs --since= \| grep -iE 'token\|refresh\|expiry' # two distinct mint timestamps in one uninterrupted session` | +| `auto_create_off` | Broker auto-create is OFF | _(unfilled)_ | `aws kafka describe-configuration-revision --arn --revision --query ServerProperties --output text \| base64 -d \| grep auto.create.topics.enable` | +| `explicit_topic_bootstrap` | Explicit topic bootstrap succeeded (every catalog topic created deliberately) | _(unfilled)_ | `kafka-topics --bootstrap-server --command-config --list \| grep '^onex.mstg1.' # compare set-equal against one_tenant_contract_freeze.topic_catalog` | +| `negative_control_out_of_catalog` | NEGATIVE CONTROL: out-of-catalog topic/group name is denied | _(unfilled)_ | `kafka-topics --bootstrap-server --command-config --create --topic notonex.mstg1.denied.v1 # MUST fail with AccessDenied/TopicAuthorizationException; paste the error verbatim` | +| `broker_group_perms` | Broker + consumer-group permissions scoped to the catalog patterns | _(unfilled)_ | `aws iam get-role-policy --role-name --policy-name --query 'PolicyDocument.Statement[?contains(Action, `kafka-cluster:*`)]' # patterns must be onex.* / omninode.*` | +| `rds_verify_full` | RDS connection proven sslmode=verify-full | _(unfilled)_ | `psql "$CANARY_DSN" -Atc "select ssl, version, cipher from pg_stat_ssl join pg_stat_activity using (pid) where pid = pg_backend_pid()" # plus the DSN readback showing sslmode=verify-full and a pinned root CA` | +| `typed_config_authority` | Typed config authority (config comes from the typed model, not ad-hoc env reads) | _(unfilled)_ | `uv run python -c 'from omnibase_infra.event_bus.kafka_auth import build_aiokafka_auth_kwargs_from_env; print(build_aiokafka_auth_kwargs_from_env.__module__)' # plus scripts/check_required_env_vars.py output for the candidate's env contract` | +| `no_raw_endpoint_fallback` | NEGATIVE CONTROL: no raw-endpoint / plaintext fallback path was exercised | _(unfilled)_ | `unset the IAM env, start the candidate, and show it FAILS CLOSED rather than falling back; static half: scripts/check_no_cloud_bus_wrapper.sh and grep -rn 'PLAINTEXT' src/omnibase_infra/event_bus/` | +| `dashboard_zero_authority` | Dashboard holds zero broker/RDS authority in the candidate config | _(unfilled)_ | `kubectl -n get cm,secret -o yaml \| grep -nE 'omnidash' # must show no broker bootstrap or RDS DSN granted to any dashboard surface` | +| `plan_row_binding` | Rolling plan §3 B5 bound to OMN-15124 | _(unfilled)_ | `git -C $OMNI_HOME/omni_home log -p -- docs/plans/ROLLING_SEVEN_DAY_PLAN.md \| grep -n 'OMN-15124'` | + +## Related + +- `docs/runbooks/managed-staging-canary-postgres-provisioning.md` — B12 landing table + §3.4 readback +- `docs/runbooks/managed-staging-canary-teardown-rollback.md` — teardown / abort / rollback ownership (B13) +- `src/omnibase_infra/topics/managed_staging_canary_catalog.py` + `..._namespace.yaml` — B7 `onex.mstg1.` catalog, epoch, zero-collision readback (OMN-14727) +- `scripts/proof/e2e_cloud_workflow_harness.py` — OMN-10858 end-to-end proof harness (`--live` defaults OFF) +- `omni_home:docs/plans/2026-07-17-managed-staging-verified-state-and-task-split.md` — lane B task split diff --git a/docs/runbooks/managed-staging-one-tenant-contract-freeze.md b/docs/runbooks/managed-staging-one-tenant-contract-freeze.md new file mode 100644 index 0000000000..f87c1c2a4e --- /dev/null +++ b/docs/runbooks/managed-staging-one-tenant-contract-freeze.md @@ -0,0 +1,70 @@ +# Managed-staging one-tenant contract — FREEZE PACKET (template) + +**Ticket:** OMN-15123 · **Plan row:** rolling plan §3 B3 · **Parent epic:** OMN-14724 +**Field manifest (the seam):** [`docs/runbooks/managed-staging-proof-kit/fields.yaml`](managed-staging-proof-kit/fields.yaml) +**Seam test:** `tests/ci/test_managed_staging_proof_kit_seam.py` + +> **This is a template. It executes nothing and authorizes nothing.** Filling it in +> does not authorize any AWS / MSK / RDS / k8s mutation. Every live readback below is +> run by the operator (or by an agent with an explicit per-step GO), per +> `omni_home:docs/plans/2026-07-17-managed-staging-agent-driven-execution-plan.md`. + +## How to use it + +1. Copy this file to `docs/evidence/OMN-15123/-one-tenant-contract-freeze.md`. +2. Run each row's **evidence source** and paste the *verbatim output* into the value + cell (or into a linked `probes/.txt` next to the copy). Prose in a value + cell is not evidence. +3. Commit. **The commit that lands the filled packet IS the freeze event** — the + `freeze_signature` row records that commit. +4. After freeze, the tuple is immutable. A change does not edit this artifact; it + **bumps the epoch** (`mstg1` → `mstg2`) and mints a new freeze packet, per the + epoch rules in `docs/runbooks/managed-staging-canary-teardown-rollback.md` §4.2. + +## Acceptance gate (OMN-15123) + +- Every row below has a pasted readback, not a claim. +- `image_digest` / `config_digest` read back **equal to the live candidate at freeze + time** — this is the ticket's explicit "readback command output attached, not + asserted prose" criterion. +- `zero_prod_diff` returns no matches and `omnidash_exclusion` shows no omnidash + workload. +- The rolling plan §3 B3 row cites OMN-15123 (`plan_row_binding`), replacing its + "unverifiable by construction" tag. +- OMN-14736 (B11 canary execution) links this packet as its input surface. + + +## Fields + +Every row is required. `Value` is filled at run time from `Evidence source`; +an empty or prose-only value cell means the packet is not complete. + +| Field | What it is | Value (paste verbatim readback) | Evidence source (command / path) | +|---|---|---|---| +| `aws_account` | AWS account | _(unfilled)_ | `aws sts get-caller-identity --query Account --output text` | +| `aws_region` | AWS region | _(unfilled)_ | `aws configure get region # cross-check against the cluster ARN in msk_cluster_arn` | +| `k8s_namespace` | Kubernetes namespace (single canary namespace) | _(unfilled)_ | `kubectl get ns -o jsonpath='{.metadata.name}{"\t"}{.metadata.uid}'` | +| `msk_cluster_arn` | MSK cluster ARN | _(unfilled)_ | `aws kafka list-clusters-v2 --query 'ClusterInfoList[?ClusterName==`omninode-dev-msk`].ClusterArn' --output text` | +| `rds_instance_identifier` | RDS instance identifier | _(unfilled)_ | `aws rds describe-db-instances --db-instance-identifier omninode-dev-postgres --query 'DBInstances[0].[DBInstanceIdentifier,Endpoint.Address]' --output text` | +| `gateway_endpoint` | The one gateway (exactly one ingress path into the canary) | _(unfilled)_ | `kubectl -n get svc,ingress -o jsonpath='{range .items[*]}{.kind}/{.metadata.name}{"\n"}{end}' # must resolve to exactly one externally reachable gateway` | +| `synthetic_tenant_id` | The one synthetic tenant (UUID) | _(unfilled)_ | `psql "$CANARY_DSN" -Atc "select distinct tenant_id from delivery_replay_canary_projection" # see docs/runbooks/managed-staging-canary-postgres-provisioning.md §3.4` | +| `source_digest` | Candidate source digest (git commit the image was built from) | _(unfilled)_ | `git -C rev-parse HEAD # cross-check the image label: docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}'` | +| `image_digest` | Candidate image digest (immutable, by digest not tag) | _(unfilled)_ | `aws ecr describe-images --repository-name --image-ids imageTag= --query 'imageDetails[0].imageDigest' --output text` | +| `config_digest` | Rendered runtime config digest | _(unfilled)_ | `kubectl -n get cm -o json \| jq -S '.data' \| shasum -a 256` | +| `topic_catalog` | Approved onex.mstg1. topic/group catalog (full generated list) | _(unfilled)_ | `uv run python -c 'from omnibase_infra.topics.managed_staging_canary_catalog import build_canary_catalog_from_candidate as b; c=b(); print("\n".join(sorted(c.topics)))' # generator: src/omnibase_infra/topics/managed_staging_canary_catalog.py (OMN-14727)` | +| `zero_collision_readback` | Zero-collision readback against live topics + consumer groups | _(unfilled)_ | `uv run python -c 'from omnibase_infra.topics.managed_staging_canary_catalog import verify_zero_collision' # run against the live topic/group snapshot; see the module's "Zero-collision readback" docstring` | +| `msk_epoch` | Unique MSK epoch (namespace segment; bump to re-run) | _(unfilled)_ | `src/omnibase_infra/topics/managed_staging_canary_catalog_namespace.yaml -> epoch` | +| `group_start_reset_policy` | Signed consumer-group start / reset policy | _(unfilled)_ | `src/omnibase_infra/topics/managed_staging_canary_catalog_namespace.yaml -> group_start_policy (+ the operator signature line in the filled packet)` | +| `rollback_authority` | Named rollback authority (who may execute the revert) | _(unfilled)_ | `docs/runbooks/managed-staging-canary-teardown-rollback.md §0 ownership table + §4 rollback path` | +| `zero_prod_diff` | Zero-prod-diff assertion (no prod resource in the tuple) | _(unfilled)_ | `grep -nE 'omnibase-infra-prod\|:28085\|:28086' # must return no matches; prod lanes are named in CLAUDE.md's .201 lane table` | +| `omnidash_exclusion` | Omnidash exclusion (omnidash carries no canary authority) | _(unfilled)_ | `kubectl -n get deploy -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' # must contain no omnidash workload` | +| `freeze_signature` | Freeze signature (operator + UTC timestamp the tuple became immutable) | _(unfilled)_ | `git log -1 --format='%H %aI %an' -- # the commit that landed the packet IS the freeze event` | +| `plan_row_binding` | Rolling plan §3 B3 bound to OMN-15123 | _(unfilled)_ | `git -C $OMNI_HOME/omni_home log -p -- docs/plans/ROLLING_SEVEN_DAY_PLAN.md \| grep -n 'OMN-15123' # cite the plan diff that replaced the 'unverifiable by construction' tag` | + +## Related + +- `docs/runbooks/managed-staging-canary-postgres-provisioning.md` — B12 landing table + §3.4 readback +- `docs/runbooks/managed-staging-canary-teardown-rollback.md` — teardown / abort / rollback ownership (B13) +- `src/omnibase_infra/topics/managed_staging_canary_catalog.py` + `..._namespace.yaml` — B7 `onex.mstg1.` catalog, epoch, zero-collision readback (OMN-14727) +- `scripts/proof/e2e_cloud_workflow_harness.py` — OMN-10858 end-to-end proof harness (`--live` defaults OFF) +- `omni_home:docs/plans/2026-07-17-managed-staging-verified-state-and-task-split.md` — lane B task split diff --git a/docs/runbooks/managed-staging-proof-kit/fields.yaml b/docs/runbooks/managed-staging-proof-kit/fields.yaml new file mode 100644 index 0000000000..fa643a0c62 --- /dev/null +++ b/docs/runbooks/managed-staging-proof-kit/fields.yaml @@ -0,0 +1,214 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# Managed-staging proof kit -- machine-readable field manifest. +# +# Tickets: OMN-15123 (frozen one-tenant contract), OMN-15124 (candidate-in-isolation +# compatibility proof), OMN-15125 (Aug-5 readiness/rollback go/no-go packet), +# OMN-10858 (end-to-end cloud workflow proof harness). +# +# THIS FILE IS THE SEAM. Three surfaces are matched field-by-field against it: +# +# 1. the ticket acceptance criteria (pinned verbatim in +# tests/ci/test_managed_staging_proof_kit_seam.py::REQUIRED_FIELDS), +# 2. the markdown packet templates under docs/runbooks/ (one table row per field +# id, with a non-placeholder evidence source), and +# 3. the executable harness scripts/proof/e2e_cloud_workflow_harness.py +# (one stage function per harness stage id). +# +# A field added here without a template row, or a stage added here without a +# harness stage function, fails the seam test. Placeholder evidence sources +# (TBD / TODO / FIXME / ??? / N/A) fail the seam test: the whole point of the +# packet is that every field names the surface that proves it BEFORE the run. +# +# `value` is deliberately absent from this manifest. The manifest owns WHAT must +# be proven and HOW it is read back; the filled-in packet (a dated copy of the +# template committed under docs/evidence/) owns the observed values. +schema_version: "1.0.0" +packets: + one_tenant_contract_freeze: + ticket: "OMN-15123" + plan_row: "rolling plan §3 B3" + template: "docs/runbooks/managed-staging-one-tenant-contract-freeze.md" + description: >- + The immutable canary tuple. Frozen once, before the B11 canary run; any change mints a new epoch rather than editing the frozen artifact. + fields: + - id: aws_account + label: "AWS account" + evidence_source: "aws sts get-caller-identity --query Account --output text" + - id: aws_region + label: "AWS region" + evidence_source: "aws configure get region # cross-check against the cluster ARN in msk_cluster_arn" + - id: k8s_namespace + label: "Kubernetes namespace (single canary namespace)" + evidence_source: "kubectl get ns -o jsonpath='{.metadata.name}{\"\\t\"}{.metadata.uid}'" + - id: msk_cluster_arn + label: "MSK cluster ARN" + evidence_source: "aws kafka list-clusters-v2 --query 'ClusterInfoList[?ClusterName==`omninode-dev-msk`].ClusterArn' --output text" + - id: rds_instance_identifier + label: "RDS instance identifier" + evidence_source: "aws rds describe-db-instances --db-instance-identifier omninode-dev-postgres --query 'DBInstances[0].[DBInstanceIdentifier,Endpoint.Address]' --output text" + - id: gateway_endpoint + label: "The one gateway (exactly one ingress path into the canary)" + evidence_source: "kubectl -n get svc,ingress -o jsonpath='{range .items[*]}{.kind}/{.metadata.name}{\"\\n\"}{end}' # must resolve to exactly one externally reachable gateway" + - id: synthetic_tenant_id + label: "The one synthetic tenant (UUID)" + evidence_source: "psql \"$CANARY_DSN\" -Atc \"select distinct tenant_id from delivery_replay_canary_projection\" # see docs/runbooks/managed-staging-canary-postgres-provisioning.md §3.4" + - id: source_digest + label: "Candidate source digest (git commit the image was built from)" + evidence_source: "git -C rev-parse HEAD # cross-check the image label: docker image inspect --format '{{index .Config.Labels \"org.opencontainers.image.revision\"}}'" + - id: image_digest + label: "Candidate image digest (immutable, by digest not tag)" + evidence_source: "aws ecr describe-images --repository-name --image-ids imageTag= --query 'imageDetails[0].imageDigest' --output text" + - id: config_digest + label: "Rendered runtime config digest" + evidence_source: "kubectl -n get cm -o json | jq -S '.data' | shasum -a 256" + - id: topic_catalog + label: "Approved onex.mstg1. topic/group catalog (full generated list)" + evidence_source: "uv run python -c 'from omnibase_infra.topics.managed_staging_canary_catalog import build_canary_catalog_from_candidate as b; c=b(); print(\"\\n\".join(sorted(c.topics)))' # generator: src/omnibase_infra/topics/managed_staging_canary_catalog.py (OMN-14727)" + - id: zero_collision_readback + label: "Zero-collision readback against live topics + consumer groups" + evidence_source: "uv run python -c 'from omnibase_infra.topics.managed_staging_canary_catalog import verify_zero_collision' # run against the live topic/group snapshot; see the module's \"Zero-collision readback\" docstring" + - id: msk_epoch + label: "Unique MSK epoch (namespace segment; bump to re-run)" + evidence_source: "src/omnibase_infra/topics/managed_staging_canary_catalog_namespace.yaml -> epoch" + - id: group_start_reset_policy + label: "Signed consumer-group start / reset policy" + evidence_source: "src/omnibase_infra/topics/managed_staging_canary_catalog_namespace.yaml -> group_start_policy (+ the operator signature line in the filled packet)" + - id: rollback_authority + label: "Named rollback authority (who may execute the revert)" + evidence_source: "docs/runbooks/managed-staging-canary-teardown-rollback.md §0 ownership table + §4 rollback path" + - id: zero_prod_diff + label: "Zero-prod-diff assertion (no prod resource in the tuple)" + evidence_source: "grep -nE 'omnibase-infra-prod|:28085|:28086' # must return no matches; prod lanes are named in CLAUDE.md's .201 lane table" + - id: omnidash_exclusion + label: "Omnidash exclusion (omnidash carries no canary authority)" + evidence_source: "kubectl -n get deploy -o jsonpath='{range .items[*]}{.metadata.name}{\"\\n\"}{end}' # must contain no omnidash workload" + - id: freeze_signature + label: "Freeze signature (operator + UTC timestamp the tuple became immutable)" + evidence_source: "git log -1 --format='%H %aI %an' -- # the commit that landed the packet IS the freeze event" + - id: plan_row_binding + label: "Rolling plan §3 B3 bound to OMN-15123" + evidence_source: "git -C $OMNI_HOME/omni_home log -p -- docs/plans/ROLLING_SEVEN_DAY_PLAN.md | grep -n 'OMN-15123' # cite the plan diff that replaced the 'unverifiable by construction' tag" + candidate_isolation_compatibility: + ticket: "OMN-15124" + plan_row: "rolling plan §3 B5" + template: "docs/runbooks/managed-staging-candidate-isolation-compatibility-proof.md" + description: >- + Proves the candidate image speaks MSK/RDS correctly IN ISOLATION, with negative controls, before it is pointed at the staging lane. + fields: + - id: isolation_lane + label: "Named isolation lane/host (zero staging + zero prod mutation)" + evidence_source: "hostname && kubectl config current-context # record both; the lane must be named in the packet before the run" + - id: candidate_image_digest + label: "Candidate image digest under test (must equal one_tenant_contract_freeze.image_digest)" + evidence_source: "kubectl -n get pod -o jsonpath='{.status.containerStatuses[0].imageID}'" + - id: msk_iam_signer + label: "MSK IAM/TLS signer actually used (not plaintext, not a fallback)" + evidence_source: "uv run python -c 'from omnibase_infra.event_bus.kafka_auth import build_aiokafka_auth_kwargs_from_env as f; print({k: (type(v).__name__ if k==\"sasl_oauth_token_provider\" else v) for k,v in f().items()})' # src/omnibase_infra/event_bus/kafka_auth.py:105" + - id: token_refresh_cycle + label: "At least one observed IAM token-refresh cycle (>=2 token mints, same session)" + evidence_source: "kubectl -n logs --since= | grep -iE 'token|refresh|expiry' # two distinct mint timestamps in one uninterrupted session" + - id: auto_create_off + label: "Broker auto-create is OFF" + evidence_source: "aws kafka describe-configuration-revision --arn --revision --query ServerProperties --output text | base64 -d | grep auto.create.topics.enable" + - id: explicit_topic_bootstrap + label: "Explicit topic bootstrap succeeded (every catalog topic created deliberately)" + evidence_source: "kafka-topics --bootstrap-server --command-config --list | grep '^onex.mstg1.' # compare set-equal against one_tenant_contract_freeze.topic_catalog" + - id: negative_control_out_of_catalog + label: "NEGATIVE CONTROL: out-of-catalog topic/group name is denied" + evidence_source: "kafka-topics --bootstrap-server --command-config --create --topic notonex.mstg1.denied.v1 # MUST fail with AccessDenied/TopicAuthorizationException; paste the error verbatim" + - id: broker_group_perms + label: "Broker + consumer-group permissions scoped to the catalog patterns" + evidence_source: "aws iam get-role-policy --role-name --policy-name --query 'PolicyDocument.Statement[?contains(Action, `kafka-cluster:*`)]' # patterns must be onex.* / omninode.*" + - id: rds_verify_full + label: "RDS connection proven sslmode=verify-full" + evidence_source: "psql \"$CANARY_DSN\" -Atc \"select ssl, version, cipher from pg_stat_ssl join pg_stat_activity using (pid) where pid = pg_backend_pid()\" # plus the DSN readback showing sslmode=verify-full and a pinned root CA" + - id: typed_config_authority + label: "Typed config authority (config comes from the typed model, not ad-hoc env reads)" + evidence_source: "uv run python -c 'from omnibase_infra.event_bus.kafka_auth import build_aiokafka_auth_kwargs_from_env; print(build_aiokafka_auth_kwargs_from_env.__module__)' # plus scripts/check_required_env_vars.py output for the candidate's env contract" + - id: no_raw_endpoint_fallback + label: "NEGATIVE CONTROL: no raw-endpoint / plaintext fallback path was exercised" + evidence_source: "unset the IAM env, start the candidate, and show it FAILS CLOSED rather than falling back; static half: scripts/check_no_cloud_bus_wrapper.sh and grep -rn 'PLAINTEXT' src/omnibase_infra/event_bus/" + - id: dashboard_zero_authority + label: "Dashboard holds zero broker/RDS authority in the candidate config" + evidence_source: "kubectl -n get cm,secret -o yaml | grep -nE 'omnidash' # must show no broker bootstrap or RDS DSN granted to any dashboard surface" + - id: plan_row_binding + label: "Rolling plan §3 B5 bound to OMN-15124" + evidence_source: "git -C $OMNI_HOME/omni_home log -p -- docs/plans/ROLLING_SEVEN_DAY_PLAN.md | grep -n 'OMN-15124'" + aug5_readiness_rollback: + ticket: "OMN-15125" + plan_row: "rolling plan §3 B7" + template: "docs/runbooks/managed-staging-aug5-readiness-rollback-packet.md" + description: >- + The immutable go/no-go packet assembled BEFORE the Aug-5 window. Aug 5 is a target, not a forecast, until the blocker graph + dated chain + T20 handoff are attached. + fields: + - id: source_digest + label: "Source digest being promoted" + evidence_source: "git -C rev-parse HEAD # must equal one_tenant_contract_freeze.source_digest" + - id: previous_digest + label: "Previous digest (the rollback target)" + evidence_source: "kubectl -n rollout history deploy/ && kubectl -n get deploy/ -o jsonpath='{.spec.template.spec.containers[0].image}' # record the currently-serving digest BEFORE promotion" + - id: amd64_manifest + label: "linux/amd64 manifest present for the promoted digest" + evidence_source: "docker manifest inspect @ | jq '.manifests[].platform' # must include {\"architecture\":\"amd64\",\"os\":\"linux\"}" + - id: config_hash + label: "Rendered config hash" + evidence_source: "kubectl -n get cm -o json | jq -S '.data' | shasum -a 256 # must equal one_tenant_contract_freeze.config_digest" + - id: policy_hash + label: "IAM/broker policy hash" + evidence_source: "aws iam get-role-policy --role-name --policy-name --query PolicyDocument | jq -S . | shasum -a 256" + - id: vulnerability_result + label: "Vulnerability scan result for the promoted digest" + evidence_source: "aws ecr describe-image-scan-findings --repository-name --image-id imageDigest= --query 'imageScanFindingsSummary.findingSeverityCounts'" + - id: a6_thresholds_with_live_samples + label: "A6 numeric thresholds loaded, each with a live sample value" + evidence_source: "B10 wiring readback (OMN-14735/OMN-14948) -- one row per threshold: signal, numeric threshold, live sampled value, timestamp. A threshold with no live sample is unloaded." + - id: monitoring_owner_actions + label: "Staffed monitoring owner + the action each breach triggers" + evidence_source: "docs/runbooks/managed-staging-canary-teardown-rollback.md §3 (abort path, owner + sequence); name the on-call human and the window they are staffed for" + - id: b12_psql_readback + label: "B12 psql readback proving the landing table exists" + evidence_source: "docs/runbooks/managed-staging-canary-postgres-provisioning.md §3.4 -- paste the psql \\d+ delivery_replay_canary_projection output" + - id: teardown_readback + label: "OMN-14772 teardown readback" + evidence_source: "docs/runbooks/managed-staging-canary-teardown-rollback.md §2 T-4..T-8 -- paste the post-teardown steady-state assertion output (OMN-14772)" + - id: executable_rollback + label: "Executable rollback procedure, proven by dry-run (not asserted)" + evidence_source: "docs/runbooks/managed-staging-canary-teardown-rollback.md §4.1 rollback tuple; dry-run: kubectl -n rollout undo deploy/ --to-revision= --dry-run=server" + - id: reconciled_blocker_graph + label: "Reconciled blocker graph (every open blocker, with its owner)" + evidence_source: "Linear: children of OMN-14724 with statusType != completed, plus their blockedBy edges -- attach the query output, not a paraphrase" + - id: dated_chain_with_slack + label: "Dated critical-path chain with explicit slack" + evidence_source: "one row per chained item: item, owner, start date, end date, slack days; slack computed against the Aug-5 window (soak Aug 6)" + - id: t20_handoff + label: "T20 handoff (final linux/amd64 build + digest handoff from the contractor lane)" + evidence_source: "the contractor handoff artifact under docs/plans/ or docs/handoff/ naming the T20/B1 final build; cite path + commit" + - id: go_no_go_decision + label: "Go / no-go decision, decider, UTC timestamp" + evidence_source: "operator decision recorded in this packet + the corresponding row in docs/tracking/ROLLING_WORK_LEDGER.md" + - id: plan_row_binding + label: "Rolling plan §3 B7 bound to OMN-15125" + evidence_source: "git -C $OMNI_HOME/omni_home log -p -- docs/plans/ROLLING_SEVEN_DAY_PLAN.md | grep -n 'OMN-15125'" +harness: + ticket: "OMN-10858" + script: "scripts/proof/e2e_cloud_workflow_harness.py" + description: >- + End-to-end cloud workflow proof: login -> tenant -> submit -> terminal readback -> cross-tenant denial. Every assertion is stubbed against the real endpoint/topic named below and is gated behind --live, which defaults OFF. + stages: + - id: login + label: "Authenticate to the gateway and obtain a scoped token" + surface: "gateway: POST {gateway_base_url}/auth/token (Keycloak-backed); tenant-scoped claim required" + - id: tenant + label: "Resolve/assert the synthetic tenant context" + surface: "gateway: GET {gateway_base_url}/v1/tenants/{tenant_id} -- must match one_tenant_contract_freeze.synthetic_tenant_id" + - id: submit + label: "Submit the reference workflow command" + surface: "topic: {topic_prefix}onex.cmd.omnimarket.alpha-text-analysis-requested.v1 (payload: prompt, max_tokens, tenant_id, correlation_id)" + - id: terminal_readback + label: "Read back the terminal event + projection row, correlation-linked" + surface: "topic: {topic_prefix}onex.evt.omnimarket.alpha-text-analysis-completed.v1 (or ...-failed.v1); table: alpha_workflow_results" + - id: cross_tenant_denial + label: "NEGATIVE CONTROL: tenant B cannot read tenant A's workflow status or stream" + surface: "gateway: GET {gateway_base_url}/v1/workflows/{workflow_id} with tenant B's token -- MUST be 403/404, never 200" diff --git a/docs/runbooks/market-node-deployment.md b/docs/runbooks/market-node-deployment.md index e17f8bb49a..e7819e3b25 100644 --- a/docs/runbooks/market-node-deployment.md +++ b/docs/runbooks/market-node-deployment.md @@ -44,38 +44,40 @@ branch protection. **What happens:** On a merged PR, the `runtime-rebuild-trigger.yml` workflow fires. It evaluates whether the PR touched runtime-relevant files and, if so, publishes a -signed Kafka event to the cloud data-plane bus. +strict `redeploy-start` command to the development control bus. The runtime-owned +redeploy orchestrator validates and gates the request before its deploy effect +publishes a separately signed command to the deploy agent. **Trigger owner:** GitHub Actions — `on: pull_request: types: [closed]` on the -`omnimarket` repo, gated by `if: github.event.pull_request.merged == true`. +runtime repository, gated by `if: github.event.pull_request.merged == true`. **Code location:** -- Workflow: `omnimarket/.github/workflows/runtime-rebuild-trigger.yml` -- Decision script: `omnimarket/scripts/trigger_rebuild_on_merge.py` -- Topic published: `onex.cmd.deploy.rebuild-requested.v1` (via cloud SASL/SSL Kafka) -- Required secrets: `KAFKA_BOOTSTRAP_SERVERS`, `KAFKA_SASL_USERNAME`, - `KAFKA_SASL_PASSWORD`, `DEPLOY_AGENT_HMAC_SECRET` +- Workflow: `.github/workflows/runtime-rebuild-trigger.yml` +- Decision script: `scripts/trigger_rebuild_on_merge.py` +- Broker authority: `omnimarket/config/ci_bus_lanes.yaml` +- Topic published: `onex.cmd.omnimarket.redeploy-start.v1` +- Required GitHub secrets: none for the development control bus **Trigger conditions** (`trigger_rebuild_on_merge.py::should_trigger`): - PR had the `runtime_change` label, **OR** - Any changed file matches `src/omnimarket/*` or `src/omnibase_infra/nodes/*` -**Payload:** `ModelRebuildRequested` (from `deploy_agent/events.py`): +**Payload:** the strict `ModelRedeployStartCommand` shape: ```text -correlation_id, requested_by, scope="runtime", git_ref="origin/main" +correlation_id, scope, git_ref, runtime_lane, build_source, requested_by ``` -**HMAC signing:** the payload is signed with `DEPLOY_AGENT_HMAC_SECRET` before -publishing. The deploy-agent verifies the signature on receipt. - -**Deploy-agent Kafka control bus:** The trigger script publishes to -`KAFKA_BOOTSTRAP_SERVERS`, and the deploy-agent now requires the same explicit -Kafka config for command consumption, completion publishing, and rejection -publishing. There is intentionally no localhost fallback. A prior fix eliminated the -code-level split-brain path where the deploy-agent could silently consume or -publish on a different bus from the trigger publisher. Post-merge validation must -still verify the runtime host user service environment contains the intended -`KAFKA_BOOTSTRAP_SERVERS` value before relying on automated rebuilds. +**HMAC signing:** the upstream `redeploy-start` command is not a deploy-agent +command and is not HMAC-signed. `node_redeploy_deploy_effect` signs the downstream +`onex.cmd.deploy.rebuild-requested.v1` command with `DEPLOY_AGENT_HMAC_SECRET`. +Both `runtime-effects` and `deploy-agent.service` receive that secret from the +runtime host environment; the deploy agent verifies it on receipt. + +**Control-bus routing:** trusted self-hosted CI selects the checked-in `dev` lane +and publishes over the local plaintext Redpanda listener. The selected broker is +reviewable configuration, not a secret; any injected broker value is accepted only +when it exactly matches the overlay. The deploy agent separately consumes the +downstream signed command on the target lane selected by the orchestrator. --- diff --git a/docs/runbooks/node-skill-package-install.md b/docs/runbooks/node-skill-package-install.md index e9282f0715..5393c9bcc9 100644 --- a/docs/runbooks/node-skill-package-install.md +++ b/docs/runbooks/node-skill-package-install.md @@ -114,7 +114,7 @@ Fact (1) is a separate release-pipeline lane (OMN-14064). This runbook covers the venv-side guard for fact (2), split detect/repair per CLAUDE.md's "enforcement, not detection" rule: -- **Pre-flight (hot path, every `onex skill` dispatch):** +- **Pre-flight (hot path, every dispatch):** `src/omnibase_infra/cli/omnimarket_drift_guard.py` — cheap and LOCAL ONLY (compares the current interpreter's installed omnimarket commit against the already-checked-out `$OMNI_HOME/omnimarket` clone's HEAD; no network). Fails @@ -123,6 +123,15 @@ the venv-side guard for fact (2), split detect/repair per CLAUDE.md's the `$OMNI_HOME` convention doesn't apply. On a real mismatch it raises a `click.ClickException` naming both commits and pointing at the repair command below — it never re-installs anything itself. + + Wired on **all three** dispatch surfaces that can reach an omnimarket-provided + node: `onex skill` (OMN-14531), `onex node` / `onex run` (OMN-14560), and + `onex delegate` (OMN-13930 — `node_delegate_skill_orchestrator` is + omnimarket-provided, and this surface previously had no guard at all, so a + drifted venv surfaced as a bare contract-resolution failure). Each surface + binds `--omni-home` to the `$OMNI_HOME` envvar; that binding is load-bearing, + since nothing passes the flag explicitly and an unbound guard silently + receives `omni_home=None` and never fires. - **Repair (session/cron tick, or run by hand):** `scripts/check-omnimarket-venv-drift.sh [--repair] [PYTHON]` — refreshes the canonical clone from `origin/dev` (network), compares against the target @@ -140,3 +149,35 @@ scripts/check-omnimarket-venv-drift.sh --repair /path/to/venv/bin/python Wire the repair invocation to a session/cron tick so drift self-heals instead of waiting on the next operator to hit the pre-flight error. + +`--repair` re-runs `install-node-skill-package.sh`, whose step 3 is the +post-sync smoke: it asserts the mapped operator-skill nodes +(`node_pr_lifecycle_orchestrator`, `node_session_orchestrator`, plus +`node_aislop_sweep` as a broad canary) actually resolve from the `onex.nodes` +entry points afterward. A repair that installs but leaves nodes unresolvable +exits non-zero rather than reporting success. + +### Overriding the refusal (`ONEX_ALLOW_OMNIMARKET_DRIFT`) + +Refusal is default-ON and fail-closed. The one supported way past it is: + +```bash +ONEX_ALLOW_OMNIMARKET_DRIFT=1 onex skill ... +``` + +The variable is named in every refusal message, so the escape hatch is +discoverable from the failure itself. Semantics: + +- Only explicit affirmatives (`1`, `true`, `yes`, `on`, case-insensitive) + override. Empty, `0`, `false`, and anything unrecognized still refuse — + ambiguity fails closed, and a variable left exported from a previous session + must not silently disable the guard. +- An override that actually suppresses a refusal logs a WARNING on every + dispatch. A silent bypass would recreate the invisible-drift failure the + guard exists to end. +- Output produced under an override comes from an unverified omnimarket build + and is **not** evidence. Repair, don't override, for anything that will be + cited. + +Do not "work around" the guard by unsetting `$OMNI_HOME`: that disables it +globally and silently on every surface, with no warning and no record. diff --git a/docs/runbooks/runner-fleet-listener-liveness.md b/docs/runbooks/runner-fleet-listener-liveness.md index 80379c0081..fc55727bdb 100644 --- a/docs/runbooks/runner-fleet-listener-liveness.md +++ b/docs/runbooks/runner-fleet-listener-liveness.md @@ -1,12 +1,20 @@ # Runner fleet listener liveness (OMN-13915) **Status:** active runbook -**Ticket:** OMN-13915 (incident 2026-07-03) — related: OMN-12433 (egress healthcheck), OMN-13109 (silent wedge / crash loop monitor) +**Ticket:** OMN-13915 (incident 2026-07-03) — related: OMN-12433 (egress healthcheck), OMN-13109 (silent wedge / crash loop monitor), OMN-15233 (2026-07-27 threshold recalibration + orphan/crash-loop detection), OMN-15255 (composite readiness + quarantine gate), OMN-15776 (broker-dispatch/reconnect race — a distinct GitHub-side failure class, targeted rerun remediation) ## The rule that changed > **"All runner containers are `Up (healthy)`" is NOT sufficient evidence that the fleet is serving jobs.** > The GitHub org runner registry (`GET /orgs/OmniNode-ai/actions/runners`) is the authoritative signal, and the `runner-fleet-canary` scheduled workflow is the enforced surface that watches it. +> +> **The converse is equally true (OMN-15233): "Docker says unhealthy" is NOT sufficient evidence that a runner is degraded.** +> **Cross-check the GitHub runner registry before ANY restart sweep. If the runners are online, the flag is the bug — do not restart.** +> ```bash +> gh api /orgs/OmniNode-ai/actions/runners --jq \ +> '[.runners[]|select(.status=="online")]|length' +> ``` +> Never restart-sweep off the Docker-unhealthy flag alone. On 2026-07-27 that count went 13 → 37 → 59 while the registry reported **64/64 online throughout**; 59 → 4 resolved with only 8 restarts and the untouched control group self-healed. The "growth" was measurement phase, not fleet degradation. ## Incident summary (2026-07-03) @@ -27,11 +35,132 @@ A point-in-time process/container check cannot prove a runner is serving jobs: | Layer | Surface | What it proves | Latency | |-------|---------|----------------|---------| -| 1 | `docker/runners/healthcheck.sh` (in-container) | `bin/Runner.Listener` process alive AND `_diag` heartbeat fresh (`RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS`, default 900s) AND github.com egress | ≤ ~17 min (staleness + 3×30s retries) | -| 2 | `docker/runners/entrypoint.sh` watchdog | listener process exists while `run.sh` runs; recycles the wrapper tree after 5×60s consecutive misses (bounded by `LISTENER_RESTART_MAX=50`, then container exit → restart policy). **OMN-14564:** also recycles (with an explicit listener kill) when the listener process is alive but its `_diag` heartbeat is older than `LISTENER_HEARTBEAT_MAX_AGE_SECONDS` (3600s) for `LISTENER_HEARTBEAT_MISSES` (3) consecutive 60s ticks — never while a `Runner.Worker` job is executing | ≤ ~6 min (dead) / ≤ ~63 min (hung: 3600s staleness + 3×60s) | +| 1 | `docker/runners/healthcheck.sh` (in-container) | `bin/Runner.Listener` process alive AND exactly one listener with a non-1 PPID (OMN-15233) AND `_diag` heartbeat fresh (`RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS`, default **4500s** since OMN-15233) AND ≤ `RUNNER_HEALTH_MAX_LOG_STARTS_PER_HOUR` (6) listener starts in the last hour AND the GitHub **broker session** is not persistently broken (`RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS`, default **900s** since OMN-15311) AND github.com egress | ≤ ~77 min (staleness + 3×30s retries); orphan/crash-loop layers fire in ≤ ~2 min; broken-session layer in ≤ ~17 min | +| 2 | `docker/runners/entrypoint.sh` watchdog | listener process exists while `run.sh` runs; recycles the wrapper tree after 5×60s consecutive misses (bounded by `LISTENER_RESTART_MAX=50`, then container exit → restart policy). **OMN-14564:** also recycles (with an explicit listener kill) when the listener process is alive but its `_diag` heartbeat is older than `LISTENER_HEARTBEAT_MAX_AGE_SECONDS` (3600s) for `LISTENER_HEARTBEAT_MISSES` (3) consecutive 60s ticks — never while a `Runner.Worker` job is executing. **OMN-15233:** reaps any surviving listener (TERM → KILL after `LISTENER_REAP_TIMEOUT_SECONDS`) BEFORE spawning a replacement | ≤ ~6 min (dead) / ≤ ~63 min (hung: 3600s staleness + 3×60s) | | 3 | `runner-monitor.sh` cron on `.201` (OMN-13109) | Docker vs GitHub divergence, silent wedge, crash loop → Slack | 3 min cadence, shares fate with host | | 4 | **`runner-fleet-canary` GHA workflow (authoritative)** | GitHub org registry online count vs `config/runner_fleet.yaml` `expected_count`; fails the run when offline+missing > `RUNNER_CANARY_MAX_OFFLINE` (5) | 15 min cadence, GitHub-hosted — survives total `.201` loss | +## Composite readiness — the adjudicating surface (OMN-15255) + +Layers 1–4 above each answer one question and none of them adjudicates when two +disagree. On 2026-07-27T16:40Z the registry read `{total: 64, online: 64, busy: 0}` +while **53 of 64 containers read docker-unhealthy**. Deciding which surface was +right meant a human diffing three outputs by hand — `gh api .../actions/runners`, +`docker ps`, and per-container `_diag` mtimes. + +`node_runner_fleet_health_compute` now emits one composite verdict per runner. +Readiness is a **conjunction** over six independently-probed signals; a runner is +`READY` only when every one PASSes: + +| Signal | PASS when | +|---|---| +| `github_registration` | registry reports `online` | +| `docker_health` | container state `running` and health `healthy` (or `none` — image declares no healthcheck) | +| `diag_heartbeat` | newest `_diag/*.log` age ≤ `RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS` (**4500s**, brackets the ~50-min token-refresh cycle — OMN-15233) | +| `listener_topology` | exactly one `Runner.Listener`, zero at PPID 1 | +| `container_stability` | `RestartCount` ≤ `CRASHLOOP_RESTART_THRESHOLD` (5) | +| `disk_capacity` | runner-host disk used < 90% (module constant, deliberately not a new env read — see OMN-15234) | + +Read the verdict, not the individual surfaces: + +- `ready_count` — usable capacity. **This, not `online_count`.** +- `quarantined_runners` — a signal was probed and FAILed. +- `bounce_eligible_runners` — the strict subset a force-recreate can actually fix. +- `readiness_signal_rollups` — which surface disagrees, and about how many runners. + +**Two fail directions, on purpose.** Readiness fails CLOSED: `UNKNOWN` is not +`READY`, so an unprobeable runner is not counted as capacity. The bounce gate +fails SAFE: no restart on an indeterminate source, never on a busy runner, and +never for a cause a recreate cannot fix. Concretely — a GitHub-offline runner +whose local listener is single and non-orphaned with a fresh heartbeat is +quarantined but **never bounced** (OMN-14057 status-lag corroboration), and a +full host disk is quarantined but never bounced (a recreate frees no disk). + +`state` (the precedence classification) and `readiness` legitimately disagree, and +neither is wrong: `state` answers "what is the most severe single thing wrong", +first-match-wins; `readiness` answers "may this runner take work". A runner that +is GitHub-online with a fresh heartbeat, an unhealthy container and two listeners +is `state=HEALTHY` and `readiness=NOT_READY`. + +**Retired by this:** the four state-keyed `RESTART_RUNNER` branches +(`CRASH_LOOPING` 0.9 / `LISTENER_ZOMBIE` 0.85 / `OFFLINE_IDLE` 0.6 / `WEDGED` 0.5) +are **deleted** — bounce-eligibility is now the single producer of a restart +recommendation. Four independently-tunable confidence heuristics over the same +facts is how one misread threshold (the retired 900s heartbeat window) became a +fleet-wide restart storm with nothing able to veto it. + +**Not yet retired — rollout-gated.** The manual three-surface cross-check in +"The rule that changed" above remains the operator procedure until the extended +probe is deployed: `docker_health`, listener topology and host disk are gathered +by `node_runner_health_snapshot_effect`, and until that runs against the fleet +those signals report `UNKNOWN`. UNKNOWN quarantines nothing and bounces nothing, +so the code is inert before rollout by construction — verify `ready_count` is +non-zero before trusting the view. + +## Broker-dispatch/reconnect race — a DIFFERENT failure class (OMN-15776) + +Layers 1–4 and the composite readiness verdict above all detect state the +*local runner process* can observe: a dead listener, a hung listener, a +container that is unhealthy or offline. **This class is invisible to every one +of them**, because the failure happens on the GitHub side of the wire before +any local process this repo controls ever runs. + +**Mechanism (proven, 2026-08-09 — direct evidence on 4/4 independently-checked +runners: omninode-runner-2/17/35/48):** GitHub's Actions broker dispatches a +job to a self-hosted runner within 2-7s of that SAME runner finishing its +*previous, unrelated* job — exactly while the runner's `Runner.Listener` is +mid-reconnect on its broker long-poll (every job completion triggers a +`TaskCanceledException`/`IOException`/`SocketException(125)` retry storm on +that connection, with an observed 5-12s exponential backoff). The new dispatch +lands in that reconnect gap and is **never delivered** to the runner's active +message loop — no `Runner.Worker` process is ever spawned locally, so the +runner's own `_diag/Runner_*.log` has **zero** `"Running job: "` entry +for that job (this is not a crashed step 1 — it is a dispatch that never +arrived) — while GitHub's server side records the assignment, sets +`started_at`, and independently times the orphaned assignment out at a +**fixed ~10m0-1s**, unrelated to any declared `timeout-minutes` and unrelated +to this repo's `LISTENER_HEARTBEAT_MAX_AGE_SECONDS` (3600s) watchdog. + +**Ruled out** (2026-08-09 investigation): host resource contention (`docker +events` = zero container-level events in every kill window, `RestartCount=0` +throughout), kernel/NIC faults (`journalctl -k` shows only routine veth +churn), host cron/`runner-monitor.sh` auto-bounce (zero bounces on any +implicated runner in any kill window), and the OMN-14564 heartbeat watchdog +(fires on these runners routinely but always 50min-3h offset from the actual +kill windows — its detection surface, idle-listener silence, has zero overlap +with this failure's signature: an *actively chattering* listener silently +dropping one specific dispatch, with no Worker for the watchdog's +Worker-running guard to ever observe). + +**Why no entrypoint.sh/watchdog fix applies.** The drop occurs in the GitHub +Actions client/broker protocol path, strictly before local process state +diverges from normal — there is nothing to `pgrep`, no heartbeat to go stale, +no wrapper tree to recycle. A local fix cannot close this gap. + +**Remediation layer 5 — targeted, signature-keyed rerun +(`runner-broker-dispatch-wedge-rerun` GHA workflow, 10-min cadence, +GitHub-hosted — same isolation rationale as layer 4):** +`scripts/ci/runner_broker_dispatch_wedge_rerun.sh` queries the Jobs API +(never log-text grepping — there is no log content to grep) for jobs matching +the exact structural fingerprint and reissues only the matched job: + +| Signal | Match condition | +|---|---| +| `runner_name` | set (self-hosted only — GitHub-hosted jobs are never touched) | +| `conclusion` | `failure` or `cancelled` | +| `steps` | empty array (no Worker ever spawned) | +| duration | `completed_at - started_at` within a tight band (default 595-605s) around the proven fixed ~10m0-1s server-side timeout | + +This is deliberately narrow and additive to the existing +`scripts/infra-signature-rerun.sh` (OMN-13040), which matches known infra +**log-content** signatures (disk casualty, network wedge) — this class has no +log content to match against, so it needed a structural (Jobs-API-shape) +matcher instead. A job outside the duration band, or with any recorded steps, +is a genuine failure and is never rerun by this layer. + +Coverage: `tests/unit/nodes/node_runner_fleet_maintain/test_runner_readiness_composite_omn15255.py` +and `..._facts_effect_omn15255.py`. + ## Hung-listener mode (OMN-14564, incident 2026-07-16..23) A second zombie variant the OMN-13915 process-existence watchdog cannot catch: @@ -52,8 +181,10 @@ listener ignores the wrapper-tree TERM and would collide with the respawned listener's session). Restarts remain bounded by `LISTENER_RESTART_MAX`. **The kill threshold (`LISTENER_HEARTBEAT_MAX_AGE_SECONDS`, 3600s) is -deliberately HIGHER than the healthcheck alert threshold -(`RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS`, 900s).** Live readback +deliberately DECOUPLED from the healthcheck alert threshold +(`RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS`, 900s at the time of this readback; 4500s +since OMN-15233 — see the orphan section below, which explains why the alert +threshold now sits ABOVE the kill threshold).** Live readback 2026-07-23T05:25–06:02Z: a fleet-wide broker-quiet window silenced `_diag` on 53/64 listeners for 35–50 min while GitHub kept all of them online — two docker-"unhealthy" runners were actively executing jobs, and runners 2 and 45 @@ -64,8 +195,235 @@ kills once staleness clears the observed benign ceiling with margin, which still recovers a true AAD-deadlock zombie in ~1 h instead of the 6 days the 2026-07-16..23 incident took. +## Orphan / session-conflict mode (OMN-15233, incident 2026-07-27) + +The zombie shape the layer-1 heartbeat check was built to catch was **scoring +HEALTHY**, and the same check was **manufacturing false unhealthy on idle +runners**. Both defects were in the same layer. + +**(a) False positive by arithmetic.** `RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS=900` +sat far below the **~50-minute IDLE `_diag` write cadence** — when a runner has +no job, only the OAuth/AAD token refresh writes `_diag`; the minutes-scale +cadence holds only while jobs run. An idle runner therefore read unhealthy for +**~35 of every 50 minutes** with nothing degraded. The threshold is now **4500s** +(75 min), which clears the observed idle cadence with 50% margin and is +justified inline in `healthcheck.sh`. The watchdog KILL threshold +(`LISTENER_HEARTBEAT_MAX_AGE_SECONDS`, 3600s) stays decoupled and is now BELOW +the alert threshold — remediation is bounded by `LISTENER_HEARTBEAT_MISSES` +(3×60s) and guarded by the `Runner.Worker` job check, so it is the narrower, +better-guarded signal. + +**(b) Inversion.** An orphaned `Runner.Listener` reparented to **PPID 1** keeps +holding the GitHub broker session. The watchdog spawns a replacement, which +crash-loops every ~5 min on `TaskAgentSessionConflictException` because the +orphan still owns the session — and **every crash mints a fresh +`Runner_*.log`**, which keeps the `_diag` mtime fresh, so the check read HEALTHY +forever. Four such zombies (runners **1, 43, 55, 57**; **88–234** `Runner_*.log` +files vs **3–7** on normal runners) were found only by process scan. + +Three fixes, all in `docker/runners/`: + +1. **Process topology** — `healthcheck.sh` fails on **duplicate** + `Runner.Listener` processes and on any listener with **PPID 1**. A healthy + listener's chain is `entrypoint.sh(PID 1) → run.sh → run-helper.sh → + Runner.Listener`, so PPID 1 is unambiguously an orphan. +2. **Rate-based crash-loop signal** — `healthcheck.sh` counts `Runner_*.log` + files touched inside `RUNNER_HEALTH_LOG_RATE_WINDOW_MINUTES` (60) and fails + above `RUNNER_HEALTH_MAX_LOG_STARTS_PER_HOUR` (6). A ~5-min crash cadence is + ~12/hour. The threshold is a **rate**, normalized to the window before the + comparison (`ceil(per_hour × window_minutes / 60)`), so retuning the window + alone does not silently retune the threshold — a 30m window allows 3 starts, + not 6. Both tunables fail closed if set to anything but a non-negative + integer. **This is deliberately NOT cumulative:** a cumulative count grows + monotonically with container uptime, so any long-lived healthy container + would eventually red-line forever, and a permanently-red check is a disabled + check. +3. **Reap before respawn** — `entrypoint.sh` kills any surviving listener + (TERM, then KILL after `LISTENER_REAP_TIMEOUT_SECONDS`) and confirms it is + gone before spawning a replacement. Spawn-without-reap is what manufactures + the `TaskAgentSessionConflictException` in the first place; if a listener + survives SIGKILL the entrypoint exits so the restart policy replaces the whole + PID namespace rather than looping silently. + +Coverage: `tests/ci/test_runner_listener_liveness.py` — +`TestHealthcheckIdleThresholdRecalibration`, `TestHealthcheckOrphanInversion`, +`TestHealthcheckCrashLoopRate`, `TestEntrypointOrphanReap`. + +## Broken broker session — the FOURTH state (OMN-15311, measured 2026-07-27) + +Three container-local failure states were already modelled: **listener dead** +(layer 1 `pgrep`), **PPID-1 orphan / duplicate listener** (OMN-15233 topology), +**listener permanently silent** (heartbeat staleness). During the OMN-15233 +fan-out a fourth appeared, and every existing surface called it healthy. + +A transient host↔GitHub network fault hit at ~17:26Z. The registry-offline spike +mostly self-healed, but runners **36, 38 and 56 stayed registry-OFFLINE for ~20 +minutes** with, at the same time: + +- a single, non-orphaned, live `Runner.Listener` (PPID chain intact); +- `_diag` **fresh** — the listener's own reconnect **retry traffic** is itself a + `_diag` write, so staleness never accrues; +- a normal listener start rate; +- `github.com` reachable; +- and `healthcheck.sh` returning **exit 0**. + +The layers above assert that a listener exists, is singular, is parented, is +writing, and has egress. None of them asserts the property that actually decides +whether a runner can take work: **that it holds a live GitHub broker session.** +A state-4 runner is counted as capacity by Docker *and* is suppressed from +`runner-monitor.sh` auto-bounce by the local-listener evidence rule, so it +silently absorbs zero jobs until something else restarts it. All three cleared on +restart. + +**Detection (layer 3b in `healthcheck.sh`).** The container has no GitHub +credential to query the registry, and 64 unauthenticated pollers would +rate-limit themselves. The listener's own newest `Runner_*.log` is the local +projection of the same fact: + +- the session is **broken** when the **last** session marker in that log is an + error (`Runner connect error`, `TaskAgentSessionConflictException`, + `A session for this runner already exists`, `Unable to connect to the server`, + `Failed to create session`) with **no** re-establish + (`Listening for Jobs`, `Runner reconnected`, `Job message received`) after it. + Marker **order**, not presence — a healthy long-lived runner has connect errors + somewhere in its log, and a presence check would red-line the fleet. +- **`SocketException` is deliberately NOT a broken marker** (removed 2026-07-28, + adversarial fleet probe). `BrokerServer` writes + `System.Net.Sockets.SocketException (125): Operation canceled` ~45–150x per + listener log as ordinary long-poll cancellation, immediately followed by + `Get messages has been cancelled using local token source. Continue to get + messages with new status.` — the session is still up. Ordering does **not** + rescue this one, because the connected markers only fire at session + establishment / job assignment, so on any runner idle >15 min the retry noise + is the last marker. Measured across all 64 live listeners on `omninode-pc` + (all `Up (healthy)`, all registry-**online**): with `SocketException` in the + set, **64/64** classified broken; without it, **0/64**. Shipping it would have + flipped the whole fleet Docker-unhealthy 15 min after the bind-mount swap — + a permanently-red check is a disabled check. + `tests/ci/fixtures/runner_diag_real_tail.log.gz` (a byte-faithful contiguous + tail of `omninode-runner-10`'s live `Runner_20260727-170542-utc.log`, captured + 2026-07-28: last connected marker at line 9, 45 `SocketException` lines after + it) pins the vocabulary against real data — the synthetic 3-line fixtures + exercise the artifact that runs but not the input distribution that runs. +- reconnects are routine and fast, so the layer gates on **persistence**: + `${RUNNER_HOME}/_diag/.session_broken_since` is stamped on first observation + and the check only fails once that stamp is older than + `RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS` (**900s**). Recovery deletes the + stamp, so a later blip restarts the clock instead of inheriting an old one. + 900s sits above every recovery observed in the fault and below the ~20-minute + dwell of the cohort that never recovered. +- a live listener with **no** `Runner_*.log` at all fails closed (a registered + listener always mints one) — same class as the missing-`_diag` branch. +- `RUNNER_HEALTH_SESSION_STATE_CHECK=0` disarms the layer fleet-wide by env, + without a bind-mount file swap, if it ever misfires. + +**Operator reading.** `unhealthy: GitHub broker session broken for more than …` +means the registry almost certainly reports this runner OFFLINE while the +container looks fine. This is the one docker-unhealthy signature that is **not** +covered by the OMN-15233 interim rule ("if the registry says online, the flag is +the bug") — here the flag and the registry agree. A bounce is the known +remediation; all three affected runners cleared on restart. + +Coverage: `tests/ci/test_runner_listener_liveness.py` — +`TestHealthcheckBrokerSessionState`. + +## Reconnect-gap churn — why `offline` flaps, and why it is NOT capacity loss (OMN-16030, measured 2026-08-14) + +The `offline` count in the org registry is **not** a liveness signal on this +fleet, and a red `runner-fleet-canary` is not by itself evidence of an outage. +This section supersedes any reading of layer 4 as authoritative-on-its-own; it +is the same conclusion OMN-15255 already reached ("`ready_count` — usable +capacity. **This, not `online_count`**") and OMN-14057 recorded as status-lag +corroboration, now with a measured mechanism. + +**Measurement (72-runner fleet, 2026-08-14, 02:00–10:30Z, 7136 jobs):** + +| Observation | Value | +|---|---| +| Runners reporting `offline` at any instant | 11–16 of 72, membership rotating between 20s polls | +| `missing` (lost registrations), every sample | **0** | +| Docker `RestartCount`, all 72 containers | **0** | +| Jobs served by the 13 persistently-offline-labelled runners (2h40m) | **153** (mean 11.8/runner vs 14.7 online — ~80% of nominal) | +| Correlation of offline count with concurrent job count | Pearson **r = +0.55** (n=7) | + +Direct disproof of "offline = dead": `runner-51` completed a job at 10:24:22Z +and `runner-67` at 10:25:20Z **while both were labelled offline**; `runner-30` +held an in-progress job while labelled offline; `ps` inside `runner-23` showed +`Runner.Worker` running `uv sync` while the registry reported it offline. + +**Mechanism — the same reconnect gap as OMN-15776.** Every job completion +triggers a `TaskCanceledException`/`SocketException(125)` retry storm on the +listener's broker long-poll, with 5–12s backoff. During that gap the runner has +no active broker session, so the registry reports it `offline`. This is the same +window in which OMN-15776 dispatches are dropped — the two are one phenomenon +observed from two sides: + +> Of the 13 runners labelled offline at 10:17Z, **10** had an OMN-15776 +> dispatch-wedge hit in the same window (expected 3.1 if independent; +> **P(≥10 by chance) = 7.5e-06**). + +So `offline` tracks **reconnect churn**, which rises with job turnover — hence +the *positive* correlation with fleet load. The runners that flap offline most +are the ones cycling jobs fastest, i.e. the healthiest-utilised ones. + +**Operational consequence: do not bounce on this signal.** A force-recreate +forces *more* reconnects, kills in-flight jobs, and wipes the warm tool cache +the C2 git-mirror pre-seed depends on. On 2026-08-14 a proposed fleet +"recovery" targeted 12 runners that were actively serving jobs, on the strength +of a red canary alone; the claimed dead core (runners 4/18/19/24/27/59/61) was +verified `online` **and** `busy` at that moment. Two landing sweeps were halted +for nothing. + +**Residual real cost (this is the part worth fixing).** The wedge itself is +still occurring and is *not* fixed by the git mirror: 18 jobs matched the exact +OMN-15776 fingerprint (zero steps, 600–601s) in the 8h sample, 13 of them after +the mirror went live at 07:30Z, spread across 17 distinct runners with almost no +repeats — a fleet-wide GitHub-side race, not a per-runner defect. Example +citations: `omnibase_infra` job 94704437780 (runner-19, 07:32:28Z), +`onex_change_control` job 94729303340 (runner-28, 09:30:38Z), `omnibase_infra` +job 94726673643 (runner-55, 09:20:47Z). Layer-5 +(`runner-broker-dispatch-wedge-rerun`, running ~every 30 min, all green) reruns +them so they do not block, but it is remediation, not prevention — the standing +cost is ~2 wasted job slots/hour plus rerun latency. + +### Triage: check throughput, not status labels + +```bash +# THE DECIDING CHECK — are jobs completing on self-hosted runners right now? +gh api "repos/OmniNode-ai/omnibase_infra/actions/runs?per_page=20" --jq '.workflow_runs[].id' \ + | while read -r id; do + gh api "repos/OmniNode-ai/omnibase_infra/actions/runs/$id/jobs?per_page=100" \ + --jq '.jobs[] | select(.runner_name|startswith("omninode-runner")) + | "\(.completed_at // "RUNNING")\t\(.runner_name)\t\(.conclusion // .status)"' + done | sort -r | head -30 +``` + +Healthy baseline (2026-08-14, post-mirror): ~77 jobs completed per 30 min, ~40 +in progress at any instant, ~44 distinct runners active per 30 min — *while +11–16 runners were labelled offline.* Declare a real outage only if job +completions have collapsed **and** `missing > 0`. + +### Ruled out (do not re-litigate without new evidence) + +- **DNS.** OMN-15736 proposes a local DNS cache on the premise of "no local + resolver cache" and a single-upstream chokepoint. Measured on the host + 2026-08-14: 60 concurrent lookups of `files.pythonhosted.org` in **5ms**; + `pypi.org` 1ms and `files.pythonhosted.org` 0ms (cache hits — `systemd-resolved` + is already caching); zero resolution failures under burst. The premise is + falsified as stated, and DNS is upstream of nothing in the reconnect-gap path. +- **Egress bandwidth saturation** as a checkout-failure cause. The C2 git-mirror + pre-seed took checkout failures from 21/5111 jobs (0.41%) pre-07:30Z to + **0/1873 (0.00%)** after; queue-wait median 166s → 130s, p90 1009s → 859s. +- **Container crash-looping.** `RestartCount == 0` fleet-wide, `missing == 0` in + every sample. + ## Operator response to a canary failure +0. **First: is this a real outage?** Run the throughput check in the + reconnect-gap section above. A red canary with jobs still completing is a + status-flap, not an outage — do not bounce anything. Since OMN-16030 the + canary only *fails* on `missing > 0` or offline-and-idle ≥ 50% of fleet; the + old advisory band now WARNs on a green run. 1. Read the failed `runner-fleet-canary` run summary — it lists offline runner names. 2. Do **NOT** `docker restart` runners (crash-loops: cached creds + expired baked token — OMN-13109). 3. Safe bounce, named services only, fresh token, detached: diff --git a/docs/runbooks/stability-test-runtime-lane.md b/docs/runbooks/stability-test-runtime-lane.md index a31d6ed8f7..1409389c73 100644 --- a/docs/runbooks/stability-test-runtime-lane.md +++ b/docs/runbooks/stability-test-runtime-lane.md @@ -128,8 +128,10 @@ uv run pytest tests/integration/infra/test_stability_test_runtime_compose_render ## Worker Replica Census -The base compose sets the runtime-worker deploy replicas to -`${WORKER_REPLICAS:-0}` — a soft default of **zero**. The stability lane's +The base compose used to set the runtime-worker deploy replicas to a bare +`${WORKER_REPLICAS:-0}` — a soft default of **zero** that no surface exported +(OMN-14968 replaced it with the lane-prefixed, fail-closed +`${DEV_WORKER_REPLICAS:?...}`). The stability lane's required state includes a running worker (`GATE_ZERO_PROOF.md`: 4 runtime containers — main, effects, worker, projection-api). Any plain `docker compose up`/`recreate` that does not supply the worker replica count diff --git a/docs/standards/INCIDENT_REPLAY_COVERAGE.md b/docs/standards/INCIDENT_REPLAY_COVERAGE.md new file mode 100644 index 0000000000..459fca0df3 --- /dev/null +++ b/docs/standards/INCIDENT_REPLAY_COVERAGE.md @@ -0,0 +1,78 @@ +# Incident-Replay Coverage + + + + +> **The enforcing definition is `scripts/ci/check_incident_replay_coverage.py`, not this +> page.** This page exists so the other repos have something to cite while they adopt it. +> If the two ever disagree, the script wins — a convention that lives only in prose is the +> thing this convention exists to replace. + +## The rule + +**Every enforcement guard carries at least one regression case sourced from a real +incident.** A guard that has never been run against the real thing it exists to catch is +decorative. + +On 2026-07-30 that failed three times in one day, with an identical shape every time — +the guard was tested against a synthetic input that *could not exhibit the failure*: + +| Guard | Synthetic input | What it missed | +|---|---|---| +| workflow-pin validators (5 repos) | any `[0-9a-f]{40}` | `879d6fc6`, the pin that wedged every open `omnibase_infra` PR for ~2.5h (OMN-15536). Re-running the validator against it returns `2 passed`. | +| `.201` system-health alert | a 63-byte `HEALTHY_BODY` | a 180-byte pre-parse truncation. Real bodies are 2079–2644 bytes, so `jq` died on every lane and the alert paged CRITICAL against a healthy fleet (OMN-15525). | +| merge-hold falsifier | validated under `bash -c` / `sh -c` | the runner reads *command position*, so a probe wrapped in `python3 -c "…"` classified `NOT_EXECUTED` — the check never ran at all (OMN-15484). | + +In all three the guard was green, the enforcement was zero, and nothing in CI could tell +the difference. + +## What counts as "real" + +A fixture is real only if it **came from an actual failure**, not from a hand-typed +approximation of one. That distinction has to survive contact with a machine, so it is +five checkable rules rather than an intention: + +| Rule | Requirement | Why a machine can apply it | +|---|---|---| +| **R1** | `artifact.fixture` is a committed FILE **ending in `.captured`**, and `sha256(bytes)` matches `artifact.sha256` | A literal inside a test cannot be hashed or diffed against an origin. Editing a capture breaks the claim, so it must break the build. The suffix is load-bearing, not cosmetic: these very fixtures were rewritten by `end-of-file-fixer` on their first commit because they still ended in `.json`, and R1 is what caught it. | +| **R2** | `capture.source` matches a locator grammar: `gh-api:`, `git-object:`, `host-file:`, `live-http:`, `ci-artifact:` | **This is the whole discriminator.** An invented payload has no locator that resolves. Free-text provenance ("same shape as prod") is rejected on purpose — that prose is exactly what the OMN-15525 repair wrote above a fixture it had typed by hand. | +| **R3** | `incident` is `OMN-` or `/#` | A replay case has to name the failure it replays. | +| **R4** | `test` exists and references the fixture path | A registry entry nobody reads is paperwork. | +| **R5** | `regression_class` (`false_green` → verdict `reject`; `false_red` → verdict `accept` **plus** a `discriminator` test) | Pins the verdict the buggy guard got *wrong*. A `false_red` proof alone cannot tell a working guard from one stuck open, so it must be paired. | + +## Coverage is ratcheted, and new guards default-deny + +- `scope.required_guards` — each must have a valid case. **Append-only.** An entry whose + file does not exist yet reports `PENDING`: that is how a requirement is armed *before* + the guard lands (`check_pin_reachability.py` is pre-registered against OMN-15538). +- `scope.debt_baseline` — the wired guards with no case yet, enumerated so the debt is + countable. **May only shrink.** Being on it is a debt record, not permission. +- **DEFAULT-DENY** — a newly wired guard in neither list fails. This is the load-bearing + property: it is what stops the detection shelf growing faster than the proof behind it. + +The lint fired on itself the first time it was wired, and the case that resolved it +(`omn15547-handtyped-fixture-passed-as-proof`) replays the verbatim dev blob that shipped +a hand-typed fixture as proof. That is the intended experience. + +## Adding a case + +1. **Get the real bytes.** `gh api` the run/PR, `git cat-file` the object, `curl` the live + endpoint, `scp` the host file. Do not retype them from a report. +2. **Commit them verbatim** under `tests/fixtures//`, with a `.captured` infix so + header and format hooks leave the bytes alone — a reformatted artifact is no longer the + artifact that failed. +3. **Record** the sha256 and a re-fetchable `capture.source`. +4. **Write the test** that drives the real guard with those bytes and asserts the verdict + the buggy guard got wrong. + +## Status + +Adopted in `omnibase_infra` (OMN-15547): 99 wired guards, 2 covered, 97 baselined. +Rollout to `omnibase_core`, `omnimarket`, `omniclaude` and `onex_change_control` is +registry + wiring only — the lint takes `--repo-root`. + +Fleet-wide starting point (2026-07-31 audit): of **361** wired guards across the five CI +repos, **14 (3.9%)** replay anything real; 206 are synthetic-only and 105 have no test at +all. The closest pre-existing exemplar is OCC's +`tests/fixtures/evidence_admissibility_cases.yaml` (OMN-15309) — 21 cases, each pinned to +a named defect class, missing only the locator and byte-parity this convention adds. diff --git a/docs/standards/ONEX_TERMINOLOGY.md b/docs/standards/ONEX_TERMINOLOGY.md index 1a87ceacb1..9ffc98d917 100644 --- a/docs/standards/ONEX_TERMINOLOGY.md +++ b/docs/standards/ONEX_TERMINOLOGY.md @@ -105,11 +105,14 @@ return ModelIntent(intent_type="extension", target="consul://service/...", paylo **Deprecated**: Do not call these patterns: - `intent_type="consul.register"` directly on `ModelIntent` (wrong — that belongs on the payload) -- `ModelIntentPayloadBase` (removed in omnibase_core 0.6.2 — extend `BaseModel` directly) +- `ModelIntentPayloadBase` as a base for infra payload DTOs (extend `BaseModel` directly; + the class itself was never removed — it still exists in + `omnibase_core.models.reducer.payloads` and bases core's closed-set intent payloads) ### Typed Payload Models -All typed intent payloads extend `pydantic.BaseModel` directly (not `ModelIntentPayloadBase`). +All infra typed intent payloads extend `pydantic.BaseModel` directly (not core's +`ModelIntentPayloadBase`, which remains the base for core's own closed-set intent payloads). | Canonical Pattern | Deprecated Pattern | |-------------------|--------------------| diff --git a/pyproject.toml b/pyproject.toml index 4b8dbf4c20..d88d377204 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "omnibase_infra" -version = "0.38.4" +version = "0.38.6" description = "ONEX Infrastructure - Service integration and database infrastructure tools" authors = [{ name = "OmniNode.ai", email = "contact@omninode.ai" }] license = {text = "MIT"} @@ -55,10 +55,13 @@ dependencies = [ # - SHAs provide audit trail and tamper-evidence # Core dependencies "pydantic>=2.11.7,<3.0.0", - "fastapi>=0.120.1,<0.140.0", - "uvicorn>=0.32.0,<0.52.0", + "fastapi>=0.120.1,<0.142.0", + "uvicorn>=0.32.0,<0.53.0", "pyyaml>=6.0.2,<7.0.0", - "aiohttp>=3.9.0,<4.0.0", + # OMN-15688 (2026-08-04): floor raised to 3.14.3 to clear CVE-2026-69244 + # (HIGH, out-of-bounds heap read in the C HTTP response parser error path + # on malformed responses). 3.9.0 was reachable and the lock resolved 3.14.1. + "aiohttp>=3.14.3,<4.0.0", # Database dependencies (for PostgreSQL connection manager) "asyncpg>=0.29.0,<0.32.0", "psycopg2-binary>=2.9.10,<3.0.0", @@ -82,7 +85,20 @@ dependencies = [ # Constraint: >=46.0.3,<47.0.0 (compatible release within the 46.x series). # Security: CVE-2024-26130 (42.0.4), CVE-2024-12797 (44.0.1) - both addressed by 46.x. # See docs/decisions/adr-cryptography-upgrade-46.md for full rationale. - "cryptography>=46.0.3,<50.0.0", + # OMN-15688 (2026-08-04): this ceiling is KNOWINGLY one major BELOW what the + # runtime image ships. Trivy flags cryptography 48.0.1 for CVE-2026-69247 + # (PKCS#7 EnvelopedData Bleichenbacher oracle, fixed 50.0.0) and + # CVE-2026-69249 (exponential cert path-building, fixed 49.0.0), so + # docker/Dockerfile.runtime force-installs cryptography>=50.0.0 after the + # plugin installs. Raising the declared ceiling HERE is blocked on a + # published omnibase-core release: omnibase-core==0.46.8 on PyPI declares + # cryptography>=46.0.3,<51.0.0, and `uv lock` fails closed -- + # Because omnibase-core>=0.46.8 depends on cryptography>=46.0.3,<51.0.0 + # and your project depends on cryptography>=50.0.0,<51.0.0 [...] + # your project's requirements are unsatisfiable. + # The core-side raise is omnibase_core PR (OMN-15688); this line moves to + # >=50.0.0,<51.0.0 in the same commit that bumps omnibase-core off 0.46.8. + "cryptography>=46.0.3,<51.0.0", "sigstore>=4.0.0,<5.0.0", # packaging: PEP 440 version parsing/comparison for the release-identity # fitness node (node_release_identity_compute, OMN-14471). Previously a @@ -94,18 +110,18 @@ dependencies = [ # Tracing and observability dependencies "sqlparse>=0.4.4,<0.6.0", # For secure SQL query sanitization in tracing "structlog>=23.2.0,<27.0.0", # Structured logging - "prometheus-client>=0.19.0,<0.26.0", # Metrics collection + "prometheus-client>=0.19.0,<0.27.0", # Metrics collection "opentelemetry-api>=1.27.0,<2.0.0", # OpenTelemetry API "opentelemetry-sdk>=1.27.0,<2.0.0", # OpenTelemetry SDK "opentelemetry-exporter-otlp-proto-http>=1.27.0,<2.0.0", # OTLP HTTP exporter (Phoenix uses HTTP, not gRPC) "opentelemetry-instrumentation>=0.48b0,<0.66", # Base instrumentation - "opentelemetry-instrumentation-fastapi>=0.48b0,<0.64", # FastAPI tracing + "opentelemetry-instrumentation-fastapi>=0.48b0,<0.66", # FastAPI tracing "opentelemetry-instrumentation-asyncpg>=0.48b0,<0.66", # AsyncPG tracing "opentelemetry-instrumentation-aiohttp-client>=0.48b0,<0.66", # AIOHTTP tracing "opentelemetry-instrumentation-kafka-python>=0.48b0,<0.66", # Kafka tracing "opentelemetry-instrumentation-redis>=0.48b0,<0.66", # Redis tracing # MCP (Model Context Protocol) integration - "mcp>=1.25.0,<2.0.0", # Anthropic MCP SDK for streamable HTTP transport + "mcp>=1.25.0,<3.0.0", # Anthropic MCP SDK for streamable HTTP transport # Additional infrastructure dependencies "httpx>=0.28.1,<0.29.0", # Modern HTTP client (>=0.27.1 required by MCP SDK) "tenacity>=9.0.0,<10.0.0", # Retry and resilience patterns @@ -114,7 +130,7 @@ dependencies = [ "pydantic-settings>=2.2.1,<3.0.0", # Settings management "jsonschema>=4.20.0,<5.0.0", # JSON schema validation "watchdog>=4.0.0", # Filesystem event monitoring (HandlerContractFileWatcher, OMN-3940) - "a2a-sdk>=0.3.4,<1.0.0", # Real A2A client/server protocol support for remote-agent invocation + "a2a-sdk>=0.3.4,<2.0.0", # Real A2A client/server protocol support for remote-agent invocation # Security: CVE-2026-32597 (HIGH) - PyJWT <2.12.0 accepts unknown `crit` header extensions. # Security: CVE-2026-48526 (HIGH) - PyJWT <2.13.0 authentication bypass via forged tokens. # Transitive dependency (via mcp/httpx/authlib). Explicitly declared to enforce minimum version. @@ -133,6 +149,8 @@ packages = ["src/omnibase_infra"] [project.scripts] omni-infra = "omnibase_infra.cli.commands:cli" onex-runtime = "omnibase_infra.runtime.kernel:main" +onex-gateway-forwarder = "omnibase_infra.runtime.gateway_forwarder:main" +onex-gateway-canary-probe = "omnibase_infra.runtime.gateway_canary_probe:main" onex-infra-test = "omnibase_infra.cli.infra_test.cli:cli" onex-git-hook-relay = "omnibase_infra.cli.git_hook_relay:main" onex-linear-relay = "omnibase_infra.cli.linear_relay:main" @@ -163,7 +181,7 @@ dev = [ "pytest-xdist>=3.5.0,<4.0.0", # Parallel test execution "pytest-timeout>=2.3.0,<3.0.0", # Test timeout support "mypy>=1.13.0,<3.0.0", - "ruff>=0.14.7,<0.16.0", # Ruff handles both formatting and linting (replaces black + isort) + "ruff>=0.14.7,<0.17.0", # Ruff handles both formatting and linting (replaces black + isort) "types-pyyaml>=6.0.12.20250822,<7.0.0", "pre-commit>=4.3.0,<5.0.0", "deepdiff>=8.0.0,<10.0.0", # OMN-12623: direct use in tests/replay topic-migration equivalence harness @@ -201,29 +219,18 @@ onex-change-control = { git = "https://github.com/OmniNode-ai/onex_change_contro # omnibase-core/omnibase-spi were DELETED here and the version pins bumped to # the released exact versions, restoring a fully PyPI-sourced reproducible lock. # -# OMN-14758 (epic OMN-14717, S6): TEMPORARY dev-build pin advanced from S3's -# aca32f24 to core dev HEAD (a2562165), a strict descendant (verified -# `git merge-base --is-ancestor`). S6 wires the delegation command spine onto -# S4 RuntimeDispatch, which is DEV-ONLY / UNRELEASED (latest core release is -# 0.46.7). This pin carries S1 transport protocols + ModelTransportMessage -# (#1463), S2 InMemoryTransport + TransportConformanceSuite (#1464), S4 -# RuntimeDispatch + DispatchRoute + runtime_envelope_router + EnumDeliveryDisposition -# (#1466), and S3 conformance (#1467). a2562165 is version 0.46.8, so the -# ==0.46.8 pins above stay consistent. Not on PyPI. -# -# OMN-15186: bumped a2562165 -> 3f2998b3 (core dev tip at bump time), a strict -# descendant (verified `git merge-base --is-ancestor`). a2562165 predates core -# commit 9449083a, which adds -# omnibase_core/models/delegation/wire/model_delegation_completed.py; the -# co-installed omnimarket node-skill package imports that module, so any -# omnibase_infra venv pinned at a2562165 hard-crashes with ModuleNotFoundError -# on `onex node node_delegate_skill_orchestrator` before the handler even -# constructs (blocks OMN-15170's live delegation driver). 3f2998b3 is still -# version 0.46.8 (core has not cut a new release across this range), so the -# ==0.46.8 pins above stay consistent. Not on PyPI. -# REMOVE WHEN RELEASED: once core publishes a version containing S1/S2/S4, delete -# this override and `uv lock`. -omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", rev = "3f2998b3337e4050b4758e0dd2a0fe1061ce0d98" } # raw-override-ok: OMN-14758 +# OMN-14628 (2026-08-01): the omnibase-core git-rev override is DELETED. Its own +# exit condition ("REMOVE WHEN RELEASED: once core publishes a version containing +# S1/S2/S4") is met — omnibase-core 0.46.8 is published on PyPI and is a content +# SUPERSET of the last override rev (3d51b047): it carries every module that rev +# carried, plus enum_delegation_terminal_failure_cause and +# enum_quality_score_comparison, plus the typed execution_scope field. Verified by +# tree diff of the published wheel against `git archive 3d51b047 -- src/omnibase_core` +# (nothing existed only in the rev). The lock is once again fully PyPI-sourced. +# The seam is now held mechanically, not by this comment: +# tests/unit/runtime/test_seam_released_core_pin.py asserts the frozen symbol and +# field-TYPE surface in tests/fixtures/seams/core_release/. +# Do NOT reintroduce a git-rev or branch pin here. [tool.ruff] target-version = "py312" @@ -345,6 +352,8 @@ ignore = [ "tests/integration/runtime/db/test_postgres_repository_runtime_integration.py" = ["S608"] # Migration discovery integration test - controlled view/id names (OMN-12559) "tests/integration/migrations/test_node_migration_discovery_applies.py" = ["S608"] +# Domain-adapter proof script - module-level constant table name, no external input (OMN-15421) +"docker/domain-adapter-proof/prove.py" = ["S608"] # S106: Hardcoded password in function arg - test fixtures only # Production secrets managed via Infisical/Vault (OMN-2287, OMN-2736) # T201: Print statements OK in tests @@ -383,7 +392,26 @@ split-on-trailing-comma = true [tool.pytest.ini_options] pythonpath = ["src", "."] -testpaths = ["tests"] +# OMN-15410: `testpaths` is the SINGLE SOURCE OF TRUTH for what CI collects. +# The full-suite job in .github/workflows/ci.yml deliberately passes NO +# positional path to pytest so it inherits this list verbatim; previously it +# hardcoded `pytest tests/`, which meant a root added here would still never +# run. scripts/validation/validate_test_root_collection.py fails closed on +# both halves of that seam (a tests/ dir absent from this list, and a ci.yml +# full-suite step that re-introduces a positional path). +# +# The four non-`tests/` entries are collocated roots that were uncollected by +# every CI job until OMN-15410 (the OMN-15378 class). They cannot be folded +# into tests/ as-is: scripts/tests/ and the agent_actions root declare their +# own packages, and moving them would break their relative imports for no +# collection benefit now that this list governs. +testpaths = [ + "tests", + "scripts/ci/tests", + "scripts/tests", + "scripts/runtime_build/tests", + "src/omnibase_infra/services/observability/agent_actions/tests", +] python_files = ["test_*.py", "*_test.py"] python_classes = ["Test*"] python_functions = ["test_*"] @@ -549,6 +577,7 @@ node_dlq_replay_effect = "omnibase_infra.nodes.node_dlq_replay_effect" node_emit_daemon_runtime = "omnibase_infra.nodes.node_emit_daemon_runtime" node_event_bus_wiring_effect = "omnibase_infra.nodes.node_event_bus_wiring_effect" node_event_forward_effect = "omnibase_infra.nodes.node_event_forward_effect" +node_gateway_attach_effect = "omnibase_infra.nodes.node_gateway_attach_effect" node_github_pr_poller_effect = "omnibase_infra.nodes.node_github_pr_poller_effect" node_gmail_archive_cleanup_effect = "omnibase_infra.nodes.node_gmail_archive_cleanup_effect" node_gmail_intent_poller_effect = "omnibase_infra.nodes.node_gmail_intent_poller_effect" diff --git a/scripts/audit-runner-routing.py b/scripts/audit-runner-routing.py index 56fd181aa4..575fd0b57c 100755 --- a/scripts/audit-runner-routing.py +++ b/scripts/audit-runner-routing.py @@ -23,6 +23,14 @@ ORG = "OmniNode-ai" DEFAULT_POLICY = Path("config/runner_routing_policy.yaml") +PUBLIC_PR_RUNNER_VARIABLE = "OMNI_PUBLIC_PR_RUNS_ON_JSON" +TRUSTED_CI_RUNNER_VARIABLE = "OMNI_TRUSTED_CI_RUNS_ON_JSON" +REQUIRED_CI_RUNNER_VARIABLE = "OMNI_REQUIRED_CI_RUNS_ON_JSON" +FORK_PR_PREDICATE = ( + "github.event_name=='pull_request'&&" + "github.event.pull_request.head.repo.full_name!=github.repository" +) +DEV_BASE_SHORTCUT = "github.event_name=='pull_request'&&github.base_ref=='dev'" @dataclass(frozen=True) @@ -130,6 +138,29 @@ def _workflow_paths(repo_root: Path) -> list[Path]: ) +def _normalized_expression(value: str) -> str: + return re.sub(r"\s+", "", value) + + +def runner_variable_for_event( + event_name: str, + head_repository: str | None, + repository: str, + *, + merge_group_variable: str | None = None, +) -> str: + """Return the runner variable allowed for a workflow event shape. + + The helper models the selector policy for focused regression tests. Workflow + expressions are audited separately below because Actions evaluates them. + """ + if event_name == "pull_request" and head_repository != repository: + return PUBLIC_PR_RUNNER_VARIABLE + if event_name == "merge_group" and merge_group_variable is not None: + return merge_group_variable + return TRUSTED_CI_RUNNER_VARIABLE + + def audit_local_workflows(policy: dict[str, Any], repo_root: Path) -> list[Finding]: allowlist = { str(item["path"]) @@ -143,16 +174,48 @@ def audit_local_workflows(policy: dict[str, Any], repo_root: Path) -> list[Findi for path in _workflow_paths(repo_root): rel = path.relative_to(repo_root).as_posix() text = path.read_text(encoding="utf-8") - if not bare_hosted.search(text): - continue - if rel in allowlist: - continue - findings.append( - Finding( - rel, - "bare runs-on: ubuntu-latest is not allowed; use OMNI_RUNNER_SELECTOR_V1 or add an explicit policy exception", + if "pull_request_target" in text: + findings.append( + Finding( + rel, + "pull_request_target is prohibited because untrusted fork code must never reach self-hosted runners", + ) + ) + if bare_hosted.search(text) and rel not in allowlist: + findings.append( + Finding( + rel, + "bare runs-on: ubuntu-latest is not allowed; use OMNI_RUNNER_SELECTOR_V1 or add an explicit policy exception", + ) ) - ) + + workflow = yaml.safe_load(text) + jobs = workflow.get("jobs", {}) if isinstance(workflow, dict) else {} + if not isinstance(jobs, dict): + continue + for job_name, job in jobs.items(): + if not isinstance(job, dict): + continue + runs_on = job.get("runs-on") + if not isinstance(runs_on, str): + continue + expression = _normalized_expression(runs_on) + if PUBLIC_PR_RUNNER_VARIABLE not in expression: + continue + if DEV_BASE_SHORTCUT in expression: + findings.append( + Finding( + f"{rel}:{job_name}", + "public PR runner selection must not use the pull_request/dev-base shortcut", + ) + ) + if FORK_PR_PREDICATE not in expression: + findings.append( + Finding( + f"{rel}:{job_name}", + "OMNI_PUBLIC_PR_RUNS_ON_JSON is allowed only for pull requests whose head repository differs from github.repository", + ) + ) return findings diff --git a/scripts/check-env-reads.sh b/scripts/check-env-reads.sh index 6f07f2ff6c..175c8b0180 100755 --- a/scripts/check-env-reads.sh +++ b/scripts/check-env-reads.sh @@ -48,10 +48,24 @@ APPROVED_INFIX_PATTERNS=( # env-var read for the same factory, not a new pattern introduced # elsewhere. "/runtime/models/model_postgres_pool_config.py" + # OMN-15234: the OMN-15233 entry for + # /nodes/node_runner_fleet_health_compute/handlers/handler_runner_fleet_health_evaluate.py + # was REMOVED here. It existed only because the old matcher could not tell + # "edit the default of a pre-existing read" from "introduce a new read"; + # the per-name grandfathering below handles that case on its own, so the + # allowlist no longer has to be widened to maintain an existing read. + # Rule 10: narrow the matcher, do not allowlist past it. ) ENV_READ_PATTERNS='os\.environ\[|os\.environ\.get|os\.getenv|from os import environ|from os import getenv' +# OMN-15234: name-extracting form of the patterns above. Only the literal +# forms carry an env-var name; `from os import environ` and any dynamic read +# (`os.environ.get(name_var)`) deliberately do NOT match, so they fall through +# to the fail-closed branch instead of being silently grandfathered. +_Q='["'"'"']' +ENV_NAME_READ_PATTERN="(os\.environ\[|os\.environ\.get\(|os\.getenv\()[[:space:]]*${_Q}[A-Za-z_][A-Za-z0-9_]*${_Q}" + is_approved() { local file="$1" for prefix in "${APPROVED_PREFIX_PATTERNS[@]}"; do @@ -94,6 +108,111 @@ get_diff() { esac } +# OMN-15234: the commit the diff above is computed against. This is what +# "already read in this file before the change" is resolved from, so it MUST +# match get_diff's endpoints exactly: +# --staged -> HEAD (the commit the index sits on top of) +# --base -> merge-base(BASE_REF, HEAD), because get_diff uses the +# three-dot `BASE_REF...HEAD` form. +# Empty (unborn HEAD, unfetched base) means "no base to grandfather from" and +# every added read is treated as new -- fail closed. +BASE_COMMIT="" +BASE_LABEL="" +resolve_base_commit() { + case "$MODE" in + --staged) + BASE_COMMIT="$(git rev-parse --verify --quiet HEAD || true)" + BASE_LABEL="HEAD" + ;; + --base) + BASE_COMMIT="$(git merge-base "$BASE_REF" HEAD 2>/dev/null || true)" + BASE_LABEL="merge-base($BASE_REF, HEAD)" + ;; + esac +} + +# OMN-15234: env-var names read from a blob of Python source on stdin. +# Deliberately literal-only -- see ENV_NAME_READ_PATTERN. +extract_env_names() { + grep -oE "$ENV_NAME_READ_PATTERN" \ + | grep -oE "${_Q}[A-Za-z_][A-Za-z0-9_]*${_Q}" \ + | tr -d "\"'" \ + | sort -u +} + +report_block() { + local file="$1" reason="$2" + echo "BLOCKED: $file introduces new os.environ/os.getenv read -- $reason" + echo " Use overlay-resolved config instead." + echo " Approved top-level dirs: ${APPROVED_PREFIX_PATTERNS[*]}" + echo " Approved path segments: ${APPROVED_INFIX_PATTERNS[*]}" +} + +# OMN-15234: the policy this gate enforces is "no NEW env read", but the +# pre-OMN-15234 matcher fired on ANY added line containing os.environ. Because +# a one-character edit to a read's default re-adds that whole line, editing a +# pre-existing, already-grandfathered read was indistinguishable from +# introducing a new one -- there is no formulation of "change 900 to 4500" +# that avoids re-adding the line. That made every grandfathered read's default +# permanently unmaintainable outside the allowlist, which is how the OMN-15233 +# allowlist entry (removed above) got added. +# +# Narrowed rule: an added read is allowed ONLY when the SAME env-var name is +# already read in the SAME file's base version. Consequences, all intended: +# - new name in an existing file -> BLOCK (name absent from base) +# - new file (base blob does not exist) -> BLOCK +# - read moved to a DIFFERENT file -> BLOCK (that file's base has no +# such read; the boundary moved) +# - read relocated within the same file -> ALLOW (same file, same name) +# - default/whitespace/format edit on an +# existing read -> ALLOW <- the OMN-15234 case +# - read deleted -> ALLOW (no added read at all) +# - non-literal or unnamed read form -> BLOCK (nothing to match on) +check_new_env_reads() { + local file="$1" + local added_reads base_names names name blocked=0 + + added_reads="$(get_diff "$file" \ + | grep -E "^\+" \ + | grep -v "^+++" \ + | grep -E "($ENV_READ_PATTERNS)" || true)" + [ -z "$added_reads" ] && return 0 + + base_names="" + if [ -n "$BASE_COMMIT" ]; then + base_names="$(git show "$BASE_COMMIT:$file" 2>/dev/null | extract_env_names || true)" + fi + + while IFS= read -r line; do + [ -z "$line" ] && continue + names="$(printf '%s\n' "$line" | extract_env_names || true)" + if [ -z "$names" ]; then + report_block "$file" \ + "added read has no literal env-var name to match against $BASE_LABEL (dynamic name, or a bare 'from os import environ/getenv' import)" + blocked=1 + continue + fi + while IFS= read -r name; do + [ -z "$name" ] && continue + if printf '%s\n' "$base_names" | grep -qx -- "$name"; then + # Grandfathered: this exact name is already read in this exact + # file at the base commit. Editing it is maintenance, not a new + # env-resolution surface. + continue + fi + report_block "$file" \ + "env-var name '$name' is not read in this file at $BASE_LABEL" + blocked=1 + done <`` token is exempt +from it *unconditionally and forever* — the token is validated only for +non-emptiness, never against whether the pinned rev's content actually matches +the PyPI version declared alongside it. Live incident this reproduces: +``omnibase_infra@dev`` declared ``omnibase-core==0.46.8`` while +``[tool.uv.sources]`` pinned git rev ``3d51b047`` (escaped via +``# raw-override-ok: OMN-15414``) whose ``src/`` tree measurably DIFFERED from +released tag ``v0.46.8`` (9 files, +381/-27 lines) — a version string that was a +label on a tree it did not describe, invisible to every existing gate. + +``find_lineage_violations`` closes that gap: for every forbidden package that +has BOTH a ``==X.Y.Z`` version constraint (in ``project.dependencies`` or +``[tool.uv] override-dependencies``) AND a ``[tool.uv.sources]`` git override, +it resolves the ``src/`` git tree object of the pinned ref and of released tag +``vX.Y.Z`` via the GitHub REST API (no local clone required — same technique as +``scripts/ci/check_pin_reachability.py``) and fails if the two tree SHAs +differ. This runs **regardless of a ``# raw-override-ok:`` token** — the token +exempts a line from the "forbid git source" rule only; it was never designed to +(and must not) exempt a pin from matching the content it claims to build. + +Because tree resolution requires network access, this is opt-in +(``--check-lineage``) and never runs from pre-commit (offline hook) or the +default no-args CLI invocation. It fails closed on network failure — an +unresolved rev/tag is reported as a lineage violation, not silently skipped — +unless ``--allow-undetermined-lineage`` is passed, which is refused when ``CI`` +is set (same posture as ``check_pin_reachability.py``). + +A non-exact declared constraint (a range like ``>=0.46.8,<0.47.0``, or no +declared constraint at all) sitting next to a git override is ALSO a lineage +violation, not a free pass: an earlier, single-operator-only version lookup +recognized only ``pkg==X.Y.Z``, so loosening the constraint from ``==`` to a +range was enough to make ``find_lineage_violations`` skip the package entirely +while ``find_violations`` still exempted the same line via its escape token — +a complete bypass of both checks. ``_declared_version_specs`` closes that gap +by capturing every requirement shape (see below), so a range/unversioned +constraint is now itself flagged rather than silently skipped. There is no +released tree that a range constraint unambiguously names, so it cannot be +lineage-verified; the constraint must be tightened to an exact pin (or the +override deleted) before this check can prove anything about it. + +Escape-token reconciliation (OMN-15604 AC3, opt-in via ``--check-token-expiry``) +--------------------------------------------------------------------------------- +The ``# raw-override-ok: `` escape token in ``find_violations`` never +expires and never checks whether ```` is still open. `` +find_escape_token_violations`` closes that gap two ways, either of which fails +the line: + +1. an explicit ``until=YYYY-MM-DD`` suffix on the token + (``# raw-override-ok: OMN-15414 until=2026-09-01``) that has passed, or +2. the cited ticket resolves (via the Linear API, ``LINEAR_API_KEY``) to a + Done/Cancelled/Duplicate status — a closed ticket is no longer a live + justification for an unconditional override. + +Like the escape-token check itself, LINEAR_API_KEY is optional infrastructure +(same graceful-degradation posture as ``scripts/validation/check_stale_todos.py`` +/ ``.github/workflows/stale-todo-gate.yml``): when it is unset, ticket-status +resolution is skipped (not failed) so a repo that has never provisioned the +secret is not permanently red. The ``until=`` date check requires no network +and always runs. + +**Mandatory-``until=`` residual (closed here).** ``LINEAR_API_KEY`` is not +provisioned as a repo *or* org secret anywhere in OmniNode-ai as of this +writing (verified via ``gh secret list`` / ``gh api orgs/.../actions/secrets``) +-- so condition 2 above never actually fires in any live enforcing +environment; it is dead code in production, not merely optional. Left as +originally shipped, a token with no ``until=`` suffix (e.g. the literal +``# raw-override-ok: OMN-15414`` from the live incident this ticket names) +would fall straight through both conditions and pass unconditionally and +forever -- the exact defect AC3 exists to close, reproduced under the exact +live incident token. A token that supplies neither a live ``until=`` date nor +a resolvable ticket status is therefore *itself* a violation: graceful +degradation on a missing ``LINEAR_API_KEY`` only applies when the ``until=`` +date is present and did the enforcing (see ``find_escape_token_violations``). + +Cascade-movability check (OMN-15604 AC4, ``--check-movable ``) +-------------------------------------------------------------------------- +``uv lock --upgrade-package ==`` cannot move a +``[tool.uv.sources]`` git-source override — uv always prefers an explicit +source override over registry resolution, so re-locking against the SAME +override typically re-resolves to a byte-identical ``uv.lock``. The automated +cascade in ``.github/workflows/dependency-cascade.yml`` reads that as +"no lockfile changes — already on latest", which is a false-positive SKIP: +the repo is not on latest, it is still stuck on the git pin. +``find_unmovable_cascade_targets`` gives that workflow (or any caller) an +explicit, actionable failure instead of a silent no-op: it fails when +``PACKAGE`` currently has an active ``[tool.uv.sources]`` git override, +regardless of a ``raw-override-ok`` token (the token was never designed to +exempt a cascade's ability to move the pin either). Usage:: uv run python scripts/check_dep_provenance.py uv run python scripts/check_dep_provenance.py --pyproject pyproject.toml + uv run python scripts/check_dep_provenance.py --check-lineage + uv run python scripts/check_dep_provenance.py --check-token-expiry + uv run python scripts/check_dep_provenance.py --check-movable omnibase-core """ from __future__ import annotations import argparse +import json +import os import re import sys import tomllib +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable +from datetime import UTC, date, datetime from pathlib import Path +_ResolveFn = Callable[[str, str], "tuple[str | None, str]"] +_TicketResolveFn = Callable[[str], "tuple[str | None, str]"] + # --------------------------------------------------------------------------- # First-party PyPI-published deps that must be resolved from PyPI, never git. # Names are stored in canonical hyphen form; underscore spellings are @@ -70,6 +178,44 @@ # Inline escape token: `# raw-override-ok: ` with a non-empty token. _ESCAPE_TOKEN_RE = re.compile(r"#\s*raw-override-ok:\s*(\S+)") +# Optional `until=YYYY-MM-DD` suffix on the escape token, e.g. +# `# raw-override-ok: OMN-15414 until=2026-09-01`. +_ESCAPE_TOKEN_UNTIL_RE = re.compile( + r"#\s*raw-override-ok:\s*\S+\s+until=(\d{4}-\d{2}-\d{2})" +) + +# A ticket identifier at the start of a token, e.g. `OMN-15414` out of a token +# that might carry trailing punctuation. +_TICKET_ID_RE = re.compile(r"^([A-Za-z]+-\d+)") + +# `pkg==X.Y.Z` inside a project.dependencies / override-dependencies string. +_DECLARED_VERSION_RE = re.compile(r"^([A-Za-z0-9_.-]+)==([A-Za-z0-9_.+-]+)$") + +# Any requirement string naming a package, regardless of operator/shape: +# captures the package name and the raw remainder (may be an exact `==` pin, +# a range, or empty for a bare unversioned name). +_REQUIREMENT_SPEC_RE = re.compile(r"^([A-Za-z0-9_.-]+)\s*(.*)$") + +# An exact-pin spec, e.g. `==0.46.8` (the remainder captured above). +_EXACT_PIN_SUFFIX_RE = re.compile(r"^==\s*([A-Za-z0-9_.+-]+)$") + +# GitHub org all first-party OmniNode repos live under, and the REST root. +_ORG = "OmniNode-ai" +_GITHUB_API = "https://api.github.com" # url-authority-ok: fixed public REST API, no ONEX routing authority +_LINEAR_API = "https://api.linear.app/graphql" # url-authority-ok: fixed public GraphQL API, no ONEX routing authority +_REQUEST_TIMEOUT_SECONDS = 10.0 + +# Linear issue-state names/types that mean "closed" -- same set as +# scripts/validation/check_stale_todos.py's done_statuses. +_TICKET_DONE_STATUSES = frozenset( + {"done", "completed", "canceled", "cancelled", "duplicate"} +) + +# `git = "https://github.com/OmniNode-ai/.git"` — extracts . +_SOURCE_URL_RE = re.compile( + r"\Ahttps://github\.com/" + _ORG + r"/(?P[\w.-]+?)(?:\.git)?/?\Z" +) + # --------------------------------------------------------------------------- # [tool.uv.sources] parsing. TOML is authoritative for source classification so # single quotes, indentation, and package subtables cannot hide a git override. @@ -180,6 +326,499 @@ def find_violations(text: str) -> list[str]: return violations +# --------------------------------------------------------------------------- +# Content-lineage check (OMN-15604) — network, opt-in via --check-lineage. +# --------------------------------------------------------------------------- + + +def _declared_version_specs(parsed: dict[str, object]) -> dict[str, str]: + """Return {normalized_pkg: raw_version_spec} for every requirement string + naming a package, regardless of operator/shape. + + Scans `project.dependencies` and `[tool.uv] override-dependencies` — the + two loci the live incident used (pyproject.toml:36 and :189). Later + entries win over earlier ones so override-dependencies (which is what uv + actually resolves against) takes precedence over a project.dependencies + entry for the same package, if they ever disagree. + + Captures EVERY shape referencing the package -- an exact pin + (`==0.46.8`), a range (`>=0.46.8,<0.47.0`), or a bare unversioned name + (empty spec) -- so a loosened constraint cannot silently evade + comparison the way an exact-only, single-operator lookup would (OMN-15604: + this was the exact bypass a ranged constraint produced against + `find_lineage_violations` before this function replaced that lookup). + Callers that need only the exact `==` pins filter this dict's values + through `_EXACT_PIN_SUFFIX_RE` themselves (see `find_lineage_violations`). + """ + specs: dict[str, str] = {} + + project = parsed.get("project", {}) + dependencies = project.get("dependencies", []) if isinstance(project, dict) else [] + + tool = parsed.get("tool", {}) + uv = tool.get("uv", {}) if isinstance(tool, dict) else {} + overrides = uv.get("override-dependencies", []) if isinstance(uv, dict) else [] + + for requirement_list in (dependencies, overrides): + if not isinstance(requirement_list, list): + continue + for requirement in requirement_list: + if not isinstance(requirement, str): + continue + match = _REQUIREMENT_SPEC_RE.match(requirement.strip()) + if match: + specs[_normalize(match.group(1))] = match.group(2).strip() + + return specs + + +def _repo_from_git_url(git_url: str) -> str | None: + """Extract the bare `` name from an OmniNode-ai github.com URL.""" + match = _SOURCE_URL_RE.match(git_url.strip()) + return match.group("repo") if match else None + + +def _api_get(url: str) -> tuple[int | None, dict[str, object] | None, str]: + """GET a GitHub REST endpoint. Returns ``(status, body, detail)``. + + ``status is None`` means the request could not be performed at all. Same + shape/posture as ``scripts/ci/check_pin_reachability.py``'s helper of the + same name (not imported directly — that module's helper is private, and + duplicating ~15 lines of stdlib HTTP plumbing is cheaper than coupling two + independently-evolving CI gates through a private symbol). + """ + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "omnibase-infra-dep-provenance-lineage-gate (OMN-15604)", + } + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + request = urllib.request.Request(url, headers=headers) # noqa: S310 - fixed https host + try: + with urllib.request.urlopen( # noqa: S310 - fixed https host + request, timeout=_REQUEST_TIMEOUT_SECONDS + ) as response: + payload = json.loads(response.read().decode("utf-8", errors="replace")) + body = payload if isinstance(payload, dict) else None + return response.status, body, f"HTTP {response.status}" + except urllib.error.HTTPError as exc: + detail = f"HTTP {exc.code}" + try: + body = json.loads(exc.read().decode("utf-8", errors="replace")) + message = body.get("message", "") if isinstance(body, dict) else "" + if message: + detail = f"HTTP {exc.code}: {message}" + except (ValueError, OSError): + pass + return exc.code, None, detail + except (urllib.error.URLError, OSError, TimeoutError, ValueError) as exc: + return None, None, f"transport error: {exc}" + + +def resolve_src_tree_sha(repo: str, ref: str) -> tuple[str | None, str]: + """Resolve the git tree SHA of ``:src`` in ``OmniNode-ai/``. + + Two REST calls, no local clone: (1) resolve ``ref`` to its commit and read + the commit's root tree SHA, (2) list that root tree's top-level entries and + return the ``sha`` of the entry named ``src`` (type ``tree``). That `sha` + IS the tree object GitHub/git would produce for ``git rev-parse :src`` + — trees are addressed by content, so identical trees hash identically + regardless of which commit/ref reached them. + + Returns ``(tree_sha, "ok")`` on success, or ``(None, detail)`` describing + why resolution failed (network failure, missing ref, no top-level `src/`). + """ + status, body, detail = _api_get( + f"{_GITHUB_API}/repos/{_ORG}/{repo}/commits/{urllib.parse.quote(ref, safe='')}" + ) + if status != 200 or body is None: + return None, f"could not resolve commit for {ref!r}: {detail}" + commit = body.get("commit") + root_tree = commit.get("tree") if isinstance(commit, dict) else None + root_tree_sha = root_tree.get("sha") if isinstance(root_tree, dict) else None + if not isinstance(root_tree_sha, str): + return None, f"commit response for {ref!r} missing commit.tree.sha" + + status, body, detail = _api_get( + f"{_GITHUB_API}/repos/{_ORG}/{repo}/git/trees/{root_tree_sha}" + ) + if status != 200 or body is None: + return None, f"could not list root tree for {ref!r}: {detail}" + entries = body.get("tree") + if not isinstance(entries, list): + return None, f"tree response for {ref!r} missing 'tree' entries" + for entry in entries: + if ( + isinstance(entry, dict) + and entry.get("path") == "src" + and entry.get("type") == "tree" + and isinstance(entry.get("sha"), str) + ): + return entry["sha"], "ok" + return None, f"no top-level 'src' tree entry found at {ref!r}" + + +def find_lineage_violations( + text: str, + *, + resolve: _ResolveFn = resolve_src_tree_sha, +) -> list[str]: + """RED when a git-pinned override's `src/` tree differs from the released + tree of the version declared alongside it (OMN-15604). + + Applies to every forbidden package (omnibase-core / omnibase-spi / + omnibase-compat) that has a `[tool.uv.sources]` git override — regardless + of a `# raw-override-ok:` escape token, which exempts a line from + `find_violations` only. A package with a git override but NO requirement + string referencing it anywhere is skipped (nothing to compare against — + that shape is a `find_violations` failure, not a lineage failure). A + package that IS referenced but not with an exact `pkg==X.Y.Z` pin (a + range, or a bare unversioned name) is a VIOLATION, not a skip: a range + constraint does not unambiguously name one released tree to compare + against, and treating it as "nothing to compare" is precisely the bypass + that let a `>=0.46.8,<0.47.0`-style loosening evade this check entirely + while `find_violations` still exempted the same line via its token. + + `resolve` is injectable for hermetic unit tests; it defaults to the live + `resolve_src_tree_sha`, which calls the GitHub REST API. + """ + resolver = resolve + + try: + parsed = tomllib.loads(text) + except tomllib.TOMLDecodeError as exc: + return [f"invalid TOML: {exc}"] + + declared_specs = _declared_version_specs(parsed) + sources = _parse_uv_source_entries(text) + + violations: list[str] = [] + for pkg, attrs in sources.items(): + if pkg not in _FORBIDDEN_PACKAGES: + continue + git_url = attrs.get("git") + if not isinstance(git_url, str): + continue + ref = attrs.get("rev") or attrs.get("tag") or attrs.get("branch") + if not isinstance(ref, str) or not ref: + continue + + spec = declared_specs.get(pkg) + if spec is None: + # Never referenced by any dependency/override-dependency entry -- + # nothing to compare against, and out of this check's scope. + continue + + exact_m = _EXACT_PIN_SUFFIX_RE.match(spec) + if exact_m is None: + violations.append( + f"{pkg}: git-pinned override (rev={ref!r}) sits alongside a " + f"non-exact declared constraint ({spec!r}) instead of a " + f"single `{pkg}==X.Y.Z` pin. A range/loosened/unversioned " + "constraint cannot be lineage-verified against one released " + "tree -- this is the exact shape that bypasses this check " + "by construction (loosen the pin, keep the escape token). " + "Pin an exact `pkg==X.Y.Z` version, or delete the " + "[tool.uv.sources] override, so lineage can be proven." + ) + continue + version = exact_m.group(1) + + repo = _repo_from_git_url(git_url) + if repo is None: + violations.append( + f"{pkg}: cannot resolve a repo name from git url {git_url!r} to " + "verify lineage against declared version " + f"{version!r}" + ) + continue + + pinned_sha, pinned_detail = resolver(repo, ref) + released_sha, released_detail = resolver(repo, f"v{version}") + + if pinned_sha is None or released_sha is None: + violations.append( + f"{pkg}: UNDETERMINED lineage for rev={ref!r} vs declared " + f"version {version!r} (tag v{version}) — pinned: {pinned_detail}; " + f"released: {released_detail}" + ) + continue + + if pinned_sha != released_sha: + violations.append( + f"{pkg}: pinned rev {ref!r} src/ tree ({pinned_sha}) differs from " + f"released v{version} src/ tree ({released_sha}) — the declared " + f"version {version!r} does not describe what this override " + "actually builds. Either delete the [tool.uv.sources] override " + "and re-lock from the declared version, or correct the declared " + "version to match what the pin actually builds." + ) + + return violations + + +# --------------------------------------------------------------------------- +# Escape-token reconciliation (OMN-15604 AC3) — network (Linear), opt-in via +# --check-token-expiry. The `until=` date half never needs network. +# --------------------------------------------------------------------------- + + +def _parse_escape_token(raw_line: str) -> tuple[str, str | None] | None: + """Return `(ticket, until_date)` from a raw source line's + `# raw-override-ok: [until=YYYY-MM-DD]` comment, or `None` if no + valid non-empty token is present on the line. + """ + token_m = _ESCAPE_TOKEN_RE.search(raw_line) + if not token_m or not token_m.group(1).strip(): + return None + ticket = token_m.group(1).strip() + until_m = _ESCAPE_TOKEN_UNTIL_RE.search(raw_line) + until_date = until_m.group(1) if until_m else None + return ticket, until_date + + +def resolve_ticket_status(ticket_id: str) -> tuple[str | None, str]: + """Resolve a Linear ticket's status name via the Linear GraphQL API. + + Returns `(status_name, "ok")` on success, or `(None, detail)` if the + ticket could not be resolved: `LINEAR_API_KEY` unset, transport failure, + or the ticket not found. `detail == "LINEAR_API_KEY not set"` is the + specific sentinel `find_escape_token_violations` checks to apply the same + graceful-degradation posture as this org's other already-shipped, + LINEAR_API_KEY-gated Linear check (the stale ticket-tag scanner under + `scripts/validation/`, wired as its own required CI gate): without a + credential the check cannot run at all, so it is skipped rather than + failing every PR in a repo that never provisioned the secret. + + Uses `issue(id: "")`, not the `issueSearch` filter shape that + scanner uses — that filter shape (`identifier: { eq: ... }`) is rejected + by the live Linear schema (`GRAPHQL_VALIDATION_FAILED`); `issue(id:)` + accepts a human-readable identifier directly and was verified live + against the real OMN-15414 ticket during this ticket's own build. + """ + api_key = os.environ.get("LINEAR_API_KEY", "") + if not api_key: + return None, "LINEAR_API_KEY not set" + + query = { + "query": ( + 'query { issue(id: "' + + ticket_id.replace('"', "") + + '") { identifier state { name type } } }' + ) + } + request = urllib.request.Request( # noqa: S310 - fixed https host + _LINEAR_API, + data=json.dumps(query).encode("utf-8"), + headers={"Authorization": api_key, "Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen( # noqa: S310 - fixed https host + request, timeout=_REQUEST_TIMEOUT_SECONDS + ) as response: + payload = json.loads(response.read().decode("utf-8", errors="replace")) + except urllib.error.HTTPError as exc: + return None, f"HTTP {exc.code}" + except (urllib.error.URLError, OSError, TimeoutError, ValueError) as exc: + return None, f"transport error: {exc}" + + if not isinstance(payload, dict): + return None, "malformed Linear API response" + if payload.get("errors"): + return None, f"Linear API error: {payload['errors']}" + issue = ( + payload.get("data", {}).get("issue") + if isinstance(payload.get("data"), dict) + else None + ) + if not isinstance(issue, dict): + return None, f"ticket {ticket_id!r} not found in Linear" + state = issue.get("state") if isinstance(issue.get("state"), dict) else {} + name = state.get("name") if isinstance(state, dict) else None + if not isinstance(name, str) or not name: + return None, f"ticket {ticket_id!r} has no resolvable state" + return name, "ok" + + +def find_escape_token_violations( + text: str, + *, + resolve_ticket: _TicketResolveFn = resolve_ticket_status, + today: date | None = None, +) -> list[str]: + """RED when a `# raw-override-ok: ` escape token has expired + (OMN-15604 AC3). + + The token exempted a forbidden git-source override unconditionally and + forever in the original OMN-13873 design. This closes that gap with + either of the two conditions the ticket accepts as sufficient: + + 1. an explicit `until=YYYY-MM-DD` suffix has passed `today` + (`# raw-override-ok: OMN-15414 until=2026-09-01`), or + 2. the cited ticket resolves (via Linear) to a Done/Cancelled/Duplicate + status — the override's justification is closed, so the override + itself is stale. + + `LINEAR_API_KEY` is not provisioned anywhere in this org today, so + condition 2 never actually resolves in the live enforcing environment -- + a token with no `until=` suffix would otherwise fall through both + conditions and pass forever, exactly reproducing the AC3 defect against + the live incident token. A token supplying neither a live `until=` date + nor a resolvable ticket status is therefore a violation in its own right: + graceful degradation on a missing `LINEAR_API_KEY` only applies once an + `until=` date has already provided a live, network-free enforcement path + for this line. + + A line with NO escape token is out of scope here — `find_violations` + already fails it unconditionally; this function only reconciles tokens + that `find_violations` currently treats as a permanent pass. + + `resolve_ticket` is injectable for hermetic unit tests; it defaults to + the live `resolve_ticket_status`, which calls the Linear API. + """ + if today is None: + today = datetime.now(tz=UTC).date() + + block = _uv_sources_block(text) or text + try: + entries = _parse_uv_source_entries(text) + except ValueError as exc: + return [str(exc)] + + violations: list[str] = [] + for pkg, attrs in entries.items(): + if pkg not in _FORBIDDEN_PACKAGES: + continue + if not (_GIT_SOURCE_KEYS & set(attrs)): + continue + + raw_line = _line_for_package(block, pkg) + if raw_line is None: + continue + parsed_token = _parse_escape_token(raw_line) + if parsed_token is None: + continue + raw_ticket, until_date = parsed_token + + if until_date is not None: + try: + expiry = date.fromisoformat(until_date) + except ValueError: + violations.append( + f"{pkg}: raw-override-ok token for {raw_ticket} has an " + f"unparseable until= date ({until_date!r}); expected " + "YYYY-MM-DD" + ) + continue + if today > expiry: + violations.append( + f"{pkg}: raw-override-ok token for {raw_ticket} EXPIRED " + f"on {until_date} (today: {today.isoformat()}) — the " + "override is no longer exempt from the forbid-git-source " + "gate. Renew with a new until= date, or resolve the " + "override." + ) + continue + + ticket_m = _TICKET_ID_RE.match(raw_ticket) + ticket_id = ticket_m.group(1) if ticket_m else raw_ticket + status_name, detail = resolve_ticket(ticket_id) + if status_name is None: + if detail == "LINEAR_API_KEY not set": + if until_date is None: + # Neither reconciliation condition is live: there is no + # until= date to fail closed on, and ticket-status + # resolution cannot run without LINEAR_API_KEY (unset + # everywhere in this org today). Falling through here + # would reproduce the exact OMN-13873 unconditional- + # forever pass this function exists to close, so the + # token itself is the violation. + violations.append( + f"{pkg}: raw-override-ok token for {raw_ticket} has " + "no until= expiry date, and Linear ticket-status " + "resolution is unavailable (LINEAR_API_KEY not set) " + "-- neither reconciliation condition is live, so the " + "override would otherwise be exempt unconditionally " + "and forever. Add an explicit `until=YYYY-MM-DD` " + "suffix to the token." + ) + continue + # An until= date IS present (and, since we reached this line, + # already checked above as not yet expired) -- that date is + # itself a live, network-free fail-closed condition, so a + # missing credential for this secondary ticket-status check + # is not fatal. Graceful degradation, same posture as + # check_stale_todos.py. + continue + violations.append( + f"{pkg}: could not resolve Linear status for cited ticket " + f"{ticket_id} to verify the raw-override-ok token is still " + f"valid: {detail}" + ) + continue + if status_name.strip().lower() in _TICKET_DONE_STATUSES: + violations.append( + f"{pkg}: raw-override-ok token cites {ticket_id}, whose " + f"Linear status is {status_name!r} — a closed ticket is no " + "longer a valid justification for an unconditional " + "git-source override. File a reconciliation ticket, or " + "resolve the override." + ) + + return violations + + +# --------------------------------------------------------------------------- +# Cascade-movability check (OMN-15604 AC4) — offline, `--check-movable`. +# --------------------------------------------------------------------------- + + +def find_unmovable_cascade_targets(text: str, package: str) -> list[str]: + """Return a violation message if `package` cannot be moved by a + dependency cascade (OMN-15604 AC4). + + `uv lock --upgrade-package ==` re-resolves the dependency + graph, but a `[tool.uv.sources]` entry for that package takes precedence + over registry resolution regardless — uv re-resolves against the SAME + pinned git ref and typically produces a byte-identical `uv.lock`, which + `.github/workflows/dependency-cascade.yml` currently reads as "no + lockfile changes — already on latest" (a false-positive SKIP: the repo is + not on latest, it is still stuck on the git pin). + + Returns a non-empty list (one message) when `package` has an active + `[tool.uv.sources]` git override — regardless of a `raw-override-ok` + escape token, since the token only ever exempted the forbid-git-source + rule, never a cascade's ability to move the pin. Returns `[]` when + `package` has no such override (movable), including when `package` isn't + tracked at all (`onex-change-control`-style git pins are legitimate and + out of scope for this check, same as `find_violations`/ + `find_lineage_violations`). + """ + pkg = _normalize(package) + try: + sources = _parse_uv_source_entries(text) + except ValueError as exc: + return [str(exc)] + + attrs = sources.get(pkg) + if attrs is None: + return [] + git_keys = sorted(_GIT_SOURCE_KEYS & set(attrs)) + if not git_keys: + return [] + + keys_desc = ", ".join(f"{k}={attrs[k]!r}" for k in git_keys) + return [ + f"{pkg}: pinned via [tool.uv.sources] git override ({keys_desc}). " + "'uv lock --upgrade-package' cannot move a [tool.uv.sources] git " + "pin -- it re-resolves against the SAME override and will silently " + "report no lockfile change, masking a no-op cascade. Remove the " + "[tool.uv.sources] override for this package before running a " + "dependency cascade against it." + ] + + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -195,8 +834,65 @@ def main(argv: list[str] | None = None) -> int: default="pyproject.toml", help="Path to pyproject.toml (default: pyproject.toml)", ) + parser.add_argument( + "--check-lineage", + action="store_true", + help=( + "Additionally run the content-lineage check (OMN-15604): fail if a " + "[tool.uv.sources] git-pinned rev's src/ tree differs from the " + "released tree of the version declared alongside it. Applies even " + "to lines carrying a valid raw-override-ok token. Makes GitHub " + "REST API calls — off by default; never invoked from pre-commit." + ), + ) + parser.add_argument( + "--allow-undetermined-lineage", + action="store_true", + help=( + "Downgrade an UNDETERMINED lineage resolution (network failure) " + "from failure to a warning. Local/offline use only — REFUSED when " + "CI is set, matching check_pin_reachability.py's posture." + ), + ) + parser.add_argument( + "--check-token-expiry", + action="store_true", + help=( + "Additionally run the escape-token reconciliation check " + "(OMN-15604 AC3): fail if a raw-override-ok token has no " + "until= date, if its until= date has passed, or if its cited " + "ticket resolves (via the Linear API) to a closed status. " + "Ticket-status resolution requires LINEAR_API_KEY and is " + "gracefully skipped (not failed) when unset -- but ONLY once " + "a live until= date is present; a token with neither is a " + "violation, matching scripts/validation/check_stale_todos.py's " + "graceful-degradation posture for the secondary check only." + ), + ) + parser.add_argument( + "--check-movable", + metavar="PACKAGE", + default=None, + help=( + "Standalone check (OMN-15604 AC4): fail if PACKAGE has an active " + "[tool.uv.sources] git override, which a `uv lock " + "--upgrade-package` dependency cascade cannot move and will " + "silently no-op against. Runs instead of the default " + "find_violations check; does not combine with --check-lineage " + "or --check-token-expiry." + ), + ) args = parser.parse_args(argv) + in_ci = bool(os.environ.get("CI")) + if args.allow_undetermined_lineage and in_ci: + print( + "error: --allow-undetermined-lineage is refused under CI. The " + "enforcing surface must fail closed on an unresolvable pin.", + file=sys.stderr, + ) + return 2 + pyproject_path = Path(args.pyproject) if not pyproject_path.exists(): print( @@ -205,7 +901,25 @@ def main(argv: list[str] | None = None) -> int: ) return 1 - violations = find_violations(pyproject_path.read_text()) + if args.check_movable: + text = pyproject_path.read_text() + movable_violations = find_unmovable_cascade_targets(text, args.check_movable) + if movable_violations: + print( + f"FAIL: {args.check_movable} cannot be moved by a dependency cascade:", + file=sys.stderr, + ) + for msg in movable_violations: + print(f" - {msg}", file=sys.stderr) + return 1 + print( + f"OK: {args.check_movable} has no [tool.uv.sources] git override " + "-- movable by a dependency cascade." + ) + return 0 + + text = pyproject_path.read_text() + violations = find_violations(text) if violations: print( @@ -230,7 +944,65 @@ def main(argv: list[str] | None = None) -> int: f"OK: no forbidden first-party git-source override in " f"{pyproject_path} [tool.uv.sources]." ) - return 0 + + if not args.check_lineage and not args.check_token_expiry: + return 0 + + failed = False + + if args.check_lineage: + lineage_violations = find_lineage_violations(text) + undetermined = [v for v in lineage_violations if "UNDETERMINED lineage" in v] + diverged = [v for v in lineage_violations if v not in undetermined] + + if diverged: + print( + "\nFAIL: git-pinned override content diverges from the released " + f"tree of its declared version in {pyproject_path}:", + file=sys.stderr, + ) + for msg in diverged: + print(f" - {msg}", file=sys.stderr) + failed = True + elif undetermined and not args.allow_undetermined_lineage: + print( + f"\nFAIL: {len(undetermined)} pin(s) could not be " + "lineage-resolved. An unresolvable pin is not a passing pin.", + file=sys.stderr, + ) + for msg in undetermined: + print(f" - {msg}", file=sys.stderr) + failed = True + elif undetermined: + print( + f"\nWARNING: {len(undetermined)} pin(s) UNDETERMINED and " + "--allow-undetermined-lineage was passed. This run proved " + "nothing about lineage for those pins; CI is the enforcing " + "surface." + ) + else: + print( + "OK: no git-pinned override content diverges from its declared version." + ) + + if args.check_token_expiry: + token_violations = find_escape_token_violations(text) + if token_violations: + print( + "\nFAIL: raw-override-ok escape token(s) need reconciliation " + f"in {pyproject_path}:", + file=sys.stderr, + ) + for msg in token_violations: + print(f" - {msg}", file=sys.stderr) + failed = True + else: + print( + "OK: no raw-override-ok escape token is expired or cites a " + "closed ticket." + ) + + return 1 if failed else 0 if __name__ == "__main__": diff --git a/scripts/check_release_identity.py b/scripts/check_release_identity.py index c26bdf3655..ba20e2ecc2 100644 --- a/scripts/check_release_identity.py +++ b/scripts/check_release_identity.py @@ -54,6 +54,7 @@ import subprocess import sys import tomllib +import warnings from pathlib import Path _REPO_ROOT = Path(__file__).resolve().parents[1] @@ -77,10 +78,16 @@ sys.path[:] = [p for p in sys.path if p != _src_str] sys.path.insert(0, _src_str) -from omnibase_infra.nodes.node_release_identity_compute import ( - HandlerReleaseIdentity, - ModelReleaseIdentityRequest, -) +with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r'Field name "schema" in .* shadows an attribute in parent "BaseModel"', + category=UserWarning, + ) + from omnibase_infra.nodes.node_release_identity_compute import ( + HandlerReleaseIdentity, + ModelReleaseIdentityRequest, + ) def _git(args: list[str]) -> str: diff --git a/scripts/check_required_env_vars.py b/scripts/check_required_env_vars.py index a72ba29608..50b2597e66 100644 --- a/scripts/check_required_env_vars.py +++ b/scripts/check_required_env_vars.py @@ -13,6 +13,7 @@ from __future__ import annotations import argparse +import os import re import sys from pathlib import Path @@ -79,7 +80,10 @@ def main(argv: list[str] | None = None) -> int: return 1 required_vars = _parse_compose_vars(compose_path) - set_vars: set[str] = set() + # Docker Compose resolves the invoking process environment before env files. + # Honor the same input surface so CI can provide short-lived render values + # without persisting real runtime secrets on a build machine. + set_vars: set[str] = {key for key, value in os.environ.items() if value} for env_path in env_paths: set_vars.update(_parse_env_file(env_path)) @@ -94,7 +98,8 @@ def main(argv: list[str] | None = None) -> int: env_path_text = ", ".join(str(path) for path in env_paths) print( - f"ERROR: {len(missing)} env var(s) referenced in {compose_path} are missing from {env_path_text}:", + f"ERROR: {len(missing)} env var(s) referenced in {compose_path} are " + f"missing from the process environment and {env_path_text}:", file=sys.stderr, ) for var in missing: diff --git a/scripts/check_subscribe_wiring_health.py b/scripts/check_subscribe_wiring_health.py index e9e7e28ad8..aee506fb84 100644 --- a/scripts/check_subscribe_wiring_health.py +++ b/scripts/check_subscribe_wiring_health.py @@ -87,6 +87,15 @@ # not a contract-declared node publisher. The script publishes to this topic to trigger the # node_baselines_batch_compute effect node. (OMN-11177) "onex.cmd.omnibase-infra.baselines-batch-compute.v1": "Published by scripts/run_baselines_batch_compute.py CLI trigger, not a contract-declared node | owner: jonah | expiry: 2026-12-01", + # Gateway attach control-plane command topics (OMN-15750) — published by the + # edge-side dialer (customer-premise / .201 test-lane connector, + # docker/docker-compose.gateway-attach-test-lane.yml + + # scripts/proof/gateway_attach_e2e_proof.py), never by another + # contract-declared node. Same external-CLI-publisher shape as the + # baselines-batch-compute entry above. + "onex.cmd.omnibase-infra.gateway-attach-request.v1": "Published by the edge-side attach dialer (customer-premise/.201 test-lane connector), not a contract-declared node | owner: jonah | expiry: 2026-12-01", + "onex.cmd.omnibase-infra.gateway-heartbeat-request.v1": "Published by the edge-side attach dialer (customer-premise/.201 test-lane connector), not a contract-declared node | owner: jonah | expiry: 2026-12-01", + "onex.cmd.omnibase-infra.gateway-detach-request.v1": "Published by the edge-side attach dialer (customer-premise/.201 test-lane connector), not a contract-declared node | owner: jonah | expiry: 2026-12-01", # Runner-fleet-maintain tick — triggered by the reused OMN-13915 # runner-fleet-canary 15-min GitHub-hosted schedule (OMN-13942 Increment 1), # not a contract-declared Kafka publisher. diff --git a/scripts/ci/application_database_sql_baseline.yaml b/scripts/ci/application_database_sql_baseline.yaml new file mode 100644 index 0000000000..e4bb0bd645 --- /dev/null +++ b/scripts/ci/application_database_sql_baseline.yaml @@ -0,0 +1,628 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# OMN-15361: frozen baseline for the application-database SQL gate. +# +# WHY THIS FILE EXISTS: the gate lints SQL *changed against the PR base*. On a +# dev PR that is a handful of files, so each file was only ever scanned in its +# own small PR. At the dev->main promotion boundary the base is main, so the +# entire accumulated migration corpus counts as changed and every latent +# violation fires at once -- none newly authored, all already deployed. +# Rewriting deployed migrations to satisfy a gate at release time is the more +# dangerous path, so those pre-existing violations are recorded here and +# soft-passed. This mirrors the OMN-14443 deploy-gate grandfather ratchet. +# +# THE RATCHET: a violation NOT listed here is held to the full bar and fails +# closed. An entry whose file is deleted, or whose violation stops firing on a +# file the run actually linted, is STALE and FAILS the gate -- so the list can +# only shrink. +# +# BURN-DOWN ONLY: never hand-add an entry. Fix violations, then regenerate with +# scripts/ci/generate_application_database_sql_baseline.py to shrink the list. +# Widening it defeats the ratchet. +# +# Entries are keyed by sha256 of the ": " violation line -- content, +# never line numbers, so unrelated SQL edits cannot silently re-key an entry. +generated_at: "2026-08-15" +count: 200 +violations: + - key: "d253a5eec3857233d66f04c8ff8b30e9b7807ca0e7fad2df3095640e1e017faf" + path: "docker/application-acl-proof/seed.sql" + violation: "application relation target 'all' must be schema-qualified" + - key: "87513822c4c19c1c97e72a782ddd20ba1fa8c33a5808a7bf7bc4aaa70ab3b842" + path: "docker/application-acl-proof/seed.sql" + violation: "application relation target 'false' must be schema-qualified" + - key: "af6044a4b757ce804d0612ed0332c04f45167f99a6a628cf8537caef1fda84ed" + path: "docker/application-acl-proof/seed.sql" + violation: "application relation target 'omninode_internal' must be schema-qualified" + - key: "2abbab0a3e3d10807812f6ff96a28f289df03233419edb28e6ba53e613b22214" + path: "docker/application-acl-proof/seed.sql" + violation: "application relation target 'platform_catalog' must be schema-qualified" + - key: "bdde09dc72c65af5ed96c1c9dd893b2669700265ae92202e9168d820cca6a344" + path: "docker/application-acl-proof/seed.sql" + violation: "application relation target 'public.legacy_scaffold_data' is prohibited in public" + - key: "504420a2df287548117ddee80d176cf4d5a2fbd841073ff0d205721219612bc0" + path: "docker/application-acl-proof/seed.sql" + violation: "application relation target 'true' must be schema-qualified" + - key: "d3d3a4ec16674640bcf885cd5f53b59677fa6e03e21ba457679fdda49ba77378" + path: "docker/application-acl-proof/seed.sql" + violation: "application target omninode_internal.runtime_code requires exactly one ownership declaration" + - key: "434d3794c87552c8304bd9dda71b1ce3b4f119731e8aae5ce031e89091568d01" + path: "docker/application-acl-proof/seed.sql" + violation: "application target omninode_internal.runtime_state requires exactly one ownership declaration" + - key: "9ce04613f581475cc27fce43543016e4f225cb3d8289d0314493813507aa107a" + path: "docker/application-acl-proof/seed.sql" + violation: "application target omninode_internal.runtime_status requires exactly one ownership declaration" + - key: "54c0682897eca23178fcfd81f85f5ba902f7f4fe7468fa321c3140c5cef592b2" + path: "docker/application-acl-proof/seed.sql" + violation: "application target platform_catalog.plan_tier_snapshot requires exactly one ownership declaration" + - key: "27e5de6f8c6a6bf8ceeaf5e6a402ca49d5303d293ffb4f8a25dddaacc93290b8" + path: "docker/application-acl-proof/seed.sql" + violation: "application target public.legacy_scaffold_data requires exactly one ownership declaration" + - key: "bd99e120f11de5305d312b4b8e09fbe11912d448e329e472397c08e5db226ee5" + path: "docker/application-acl-proof/seed.sql" + violation: "application target tenant.account_id_span requires exactly one ownership declaration" + - key: "2716719682f221c0dfb0af93fb38b8b2057d597c734946656ee85581bd470fc5" + path: "docker/application-acl-proof/seed.sql" + violation: "application target tenant.account_id_span_set requires exactly one ownership declaration" + - key: "22937e85291ac9db4ce4404c6d57064edcd63d1dd6c6e0e8e6188a45aa6d202c" + path: "docker/application-acl-proof/seed.sql" + violation: "application target tenant.account_ref requires exactly one ownership declaration" + - key: "680a63e0c4d134d05ca6cac42bfeda28712681cbecef62aec53f869a704fb64e" + path: "docker/application-acl-proof/seed.sql" + violation: "application target tenant.delegation_event_count requires exactly one ownership declaration" + - key: "a17124ac3336bd523432df20473e51b961957568ca5bb864f5453f982118a4a5" + path: "docker/application-acl-proof/seed.sql" + violation: "application target tenant.delegation_events_id_seq requires exactly one ownership declaration" + - key: "5a79f54d045600be3165b4d008e10ae58dea1cc4b1fc838c0ff5c553c0f935b7" + path: "docker/application-acl-proof/seed.sql" + violation: "application target tenant.hostile_signature requires exactly one ownership declaration" + - key: "e9ce42dc9061dd3b80c9b3b7a8dbd04404a35d801e8a3ac39b22768474816ace" + path: "docker/application-acl-proof/seed.sql" + violation: "application target tenant.partitioned_events requires exactly one ownership declaration" + - key: "a6792baa3ac2c2ab60e26d4f5f6de6bfe64b11931c67ffdc8c3c91a13b9ae1aa" + path: "docker/application-acl-proof/seed.sql" + violation: "application target tenant.record_delegation requires exactly one ownership declaration" + - key: "9377cb33ed82ae176b1cc6059535dcab03626421f95cadd43f60c4c6009d83cc" + path: "docker/application-acl-proof/seed.sql" + violation: "application target tenant.tenant_account_names requires exactly one ownership declaration" + - key: "337d9e82745a83d38c38ed372d8b245158f379b736788ac035a9bef9cc9a0d09" + path: "docker/application-acl-proof/seed.sql" + violation: "application target tenant.tenant_accounts requires exactly one ownership declaration" + - key: "ecbaa6b8d3cfd2d1d7218d48c1aa214729044c64319034fb526cc87b4fe3f116" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('omninode_internal', 'runtime_code', , None) lacks an authoritative ownership declaration" + - key: "5e734a22961976708849ddb8633ea8556d17460837007cb4a8bc9cde5fd499e3" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('omninode_internal', 'runtime_state', , None) lacks an authoritative ownership declaration" + - key: "ee1813bd8de52e397fc993dcc41a2d16d9817a230616cf6e514adaf32a62de3d" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('omninode_internal', 'runtime_status', , None) lacks an authoritative ownership declaration" + - key: "b17fc3296dcdb99101a01e572eea5d20de76bb8c2c7e13ac2c18dd40f7d87eef" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('platform_catalog', 'plan_tier_snapshot', , None) lacks an authoritative ownership declaration" + - key: "1e0fb2308495cccbc011c8c57be23340fd630af763a4d139e3a69df0c38b6915" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('tenant', 'account_id_span', , None) lacks an authoritative ownership declaration" + - key: "6394e7ce6865ce8505856af85adf7bb5d4589426c75083e627221b703626e968" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('tenant', 'account_id_span_set', , None) lacks an authoritative ownership declaration" + - key: "dd9945d4eab92dcb57784f312ec6a8ce239d602711a4eeb0d1f3cdb3a57df984" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('tenant', 'account_ref', , None) lacks an authoritative ownership declaration" + - key: "d911f18a17b8b554f0083610a88fe1921bd08f34282602490bedc6715359f977" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('tenant', 'delegation_event_count', , '()') lacks an authoritative ownership declaration" + - key: "b4227f36d16ed87bdb4de6fc7874ab3d246bcbd46bff5397a10043c560a3e02d" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('tenant', 'delegation_events_id_seq', , None) lacks an authoritative ownership declaration" + - key: "701e046ac1790fa67544526020324d1aff2b70929d2965ff2704d35738504464" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('tenant', 'hostile_signature', , '(IN \"arg\\'name%\" INTEGER)') lacks an authoritative ownership declaration" + - key: "651acaf72fbc497e16fe80650bb596bd0ae4d4b0f76cbafb4d4325d8bca4b7ac" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('tenant', 'partitioned_events', , None) lacks an authoritative ownership declaration" + - key: "ba024bb7af5f99c1e6457cfa1927e5e8bc1d85fe8e9e5317088c699b30aa33ca" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('tenant', 'record_delegation', , '(p_payload TEXT)') lacks an authoritative ownership declaration" + - key: "444b758d0cb9c0a97c938f399f5f332a075c1a896b42227d9c7fa2362912522c" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('tenant', 'tenant_account_names', , None) lacks an authoritative ownership declaration" + - key: "1e195aab90c6bc64468ff400b51cc8f6696fe29edb70b54eec8a7220d7186003" + path: "docker/application-acl-proof/seed.sql" + violation: "created application object ('tenant', 'tenant_accounts', , None) lacks an authoritative ownership declaration" + - key: "c130c9ad660610794dd57ef0c7fbd4ad80e85d838ff4b8d720dcdd15678de9d6" + path: "docker/migrations/forward/094_create_app_dashboard_role.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "cb78e18274d22dea9c720dce9c6988bb4ab5c66d5970f3c6e5f490f4b2b424b5" + path: "docker/migrations/forward/095_add_attach_readiness_to_runtime_manifests.sql" + violation: "application relation target 'runtime_manifests' must be schema-qualified" + - key: "33813563a022029725183d86341730df4b4c6c23ad6546af426b42b1d500ec82" + path: "docker/migrations/forward/095_add_attach_readiness_to_runtime_manifests.sql" + violation: "application relation target 'runtime_manifests.attach_not_ready_contracts' uses unknown topology schema" + - key: "77ac1194808cce99dfc58e595f34de8adadb13091830dee214e225235a8be114" + path: "docker/migrations/forward/095_add_attach_readiness_to_runtime_manifests.sql" + violation: "application relation target 'runtime_manifests.attach_state' uses unknown topology schema" + - key: "440d47ba713d0ad53b2e2b7b0101d03a80f10ad1c8be723ca0ed44eebcc9ce66" + path: "docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql" + violation: "application relation target 'capability_scores' must be schema-qualified" + - key: "2e8c60849d1606c6b8583acf4bcc991e53f859cc4d86b61af901f99d6b7131d4" + path: "docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql" + violation: "application relation target 'public.capability_scores' is prohibited in public" + - key: "f31a87b9a464e3fd2cebed360577e1282b1af25772d4682cdf0e4d72dd4c8e24" + path: "docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql" + violation: "application target public.capability_scores requires exactly one ownership declaration" + - key: "cfb4c684917761e5a79af3d3ccb357268b4b9e857811a07f780296bafdc6dd8f" + path: "docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql" + violation: "created application object ('public', 'capability_scores', , None) lacks an authoritative ownership declaration" + - key: "f5127673c428394ec568eb25e93b487da5f4673e709351ef0f0cc89459aa8269" + path: "docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "d160eb4c6f3238cd4d43c1552a626665a28e3964123c66b45d1120eed14b25bb" + path: "docker/migrations/forward/nodes/node_canary_score_reducer/0002_capability_scores_tenant_id_and_rls.sql" + violation: "created application object ('public', 'capability_scores', , None) lacks an authoritative ownership declaration" + - key: "02b3c606241ea061638b890609e77965c545c959b01099240f25481e227ac55f" + path: "docker/migrations/forward/nodes/node_contract_registry/0000_create_contract_registry.sql" + violation: "application relation target 'contract_registry' must be schema-qualified" + - key: "2e2e3ca5e70a1af15611ee6f7e741d7efddfc17dfaefbfe3954881e88a3aabaf" + path: "docker/migrations/forward/nodes/node_contract_registry/0000_create_contract_registry.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "7152c24a5ab875a8e27a6e1a830ffda67cbb79724a0b5aac29399c7352c28a2c" + path: "docker/migrations/forward/nodes/node_deployment_evidence_reducer/0001_create_deployment_evidence_projection_tables.sql" + violation: "application relation target 'deployment_evidence_projection' must be schema-qualified" + - key: "9cb557ec96803eca18970a29a434209467f6c1406f716b415511fdcb9578d0a7" + path: "docker/migrations/forward/nodes/node_deployment_evidence_reducer/0001_create_deployment_evidence_projection_tables.sql" + violation: "application relation target 'deployment_readiness_projection' must be schema-qualified" + - key: "fe2b8cad9c43ad383710609dd9607933b7e5c302ef44c70b754b557669101fc5" + path: "docker/migrations/forward/nodes/node_deployment_evidence_reducer/0001_create_deployment_evidence_projection_tables.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "f9082087a2037ad44e7d0c5a6314babdd2b4ea9f4d7b269eec9448ece9502154" + path: "docker/migrations/forward/nodes/node_evidence_dashboard_reducer/0001_create_evidence_dashboard_projection_tables.sql" + violation: "application relation target 'evidence_correlation_trace_projection' must be schema-qualified" + - key: "415c007f8690ae8952aa2811127ccc05237d01cc1fa5fac7fc707a446fd30ee2" + path: "docker/migrations/forward/nodes/node_evidence_dashboard_reducer/0001_create_evidence_dashboard_projection_tables.sql" + violation: "application relation target 'evidence_dashboard_projection' must be schema-qualified" + - key: "bd1f01d35bdcc38abcb6f482ba6bdade5a78b33094c664d309db1826756ff110" + path: "docker/migrations/forward/nodes/node_evidence_dashboard_reducer/0001_create_evidence_dashboard_projection_tables.sql" + violation: "application relation target 'evidence_readiness_aggregate_projection' must be schema-qualified" + - key: "4c9b367dd4cc6ec02ec987e615db99f84d906b48516b5f1a7e0767c62c89b679" + path: "docker/migrations/forward/nodes/node_evidence_dashboard_reducer/0001_create_evidence_dashboard_projection_tables.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "c6f3fc7ba4d44e4bbf65c8c5885245249ac44fcd6e32d0d350d2617cedc152ec" + path: "docker/migrations/forward/nodes/node_llm_delegation_projection/0001_create_llm_delegation_daily_projection.sql" + violation: "application relation target 'llm_delegation_daily_projection' must be schema-qualified" + - key: "9a780212786b69ebd884256ec3f5be7dc381d4ddada3a10892527adfe9ed2fbc" + path: "docker/migrations/forward/nodes/node_llm_delegation_projection/0001_create_llm_delegation_daily_projection.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "55bbff3686b895100dbf850f11e1c538bcbe53321104856f8beab09c3d856206" + path: "docker/migrations/forward/nodes/node_merge_state_projection/0001_create_merge_state_transitions.sql" + violation: "application relation target 'merge_state_transitions' must be schema-qualified" + - key: "0016f693a081e4348b8d2bcc49e420253ff9b31d8ef503128a936b8559830128" + path: "docker/migrations/forward/nodes/node_merge_state_projection/0001_create_merge_state_transitions.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "e1177def346bdf16b2c74e7d4f01620c6561397cbde304631973282d32fd03bb" + path: "docker/migrations/forward/nodes/node_nightly_loop_controller/001_create_nightly_loop_tables.sql" + violation: "application relation target 'nightly_loop_decisions' must be schema-qualified" + - key: "f447cddf8743ccec38d813dc087ec8deeb1449df4a8406ef67f05b397fedb5f4" + path: "docker/migrations/forward/nodes/node_nightly_loop_controller/001_create_nightly_loop_tables.sql" + violation: "application relation target 'nightly_loop_iterations' must be schema-qualified" + - key: "5b95ed3ddae3f7582e8f6e21553d9d77cf1569f2dd9cf4489c8a4cde4e3a1def" + path: "docker/migrations/forward/nodes/node_nightly_loop_controller/001_create_nightly_loop_tables.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "421e42273cdb9a0c3a617dba91a5fa768a5241e7f6bc9299428023721da1cea8" + path: "docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql" + violation: "application relation target 'gate_activity' must be schema-qualified" + - key: "a3faf3afcbf9b2d96e1cfdfd8198c27c9bef3dc369c3d153acf7442429e20c15" + path: "docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql" + violation: "application relation target 'gate_metrics' must be schema-qualified" + - key: "eb1dec84c4970fd0762fe71a3f4537f56e994d90df580d7d4262402f780d3a12" + path: "docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "5d8b2bb02f442d7374023069ce5247703321183d4e0bb471c6d184f7a55743c0" + path: "docker/migrations/forward/nodes/node_pr_lifecycle_state_reducer/0001_create_pr_lifecycle_ledger_entries.sql" + violation: "application relation target 'public.pr_lifecycle_ledger_entries' is prohibited in public" + - key: "646af4463e314d2276bdc1b187d7105744c816e80334e4c0e62d9877a0481dc0" + path: "docker/migrations/forward/nodes/node_pr_lifecycle_state_reducer/0001_create_pr_lifecycle_ledger_entries.sql" + violation: "application target public.pr_lifecycle_ledger_entries requires exactly one ownership declaration" + - key: "4b6397574931f0aeef203011a92caaf14c54ba11495a6be9cffcd18e8ad58b83" + path: "docker/migrations/forward/nodes/node_pr_lifecycle_state_reducer/0001_create_pr_lifecycle_ledger_entries.sql" + violation: "created application object ('public', 'pr_lifecycle_ledger_entries', , None) lacks an authoritative ownership declaration" + - key: "75ac28b14c902d87166b8823fd7e999a5fbbe8685c88ccdf226c4a5d135c7b44" + path: "docker/migrations/forward/nodes/node_pr_lifecycle_state_reducer/0001_create_pr_lifecycle_ledger_entries.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "ccea3a97f15918ab240514c15f8017f73b268ea13e7490eac9cd536537c52b8b" + path: "docker/migrations/forward/nodes/node_pr_merged_projection/0001_create_pr_merged_events.sql" + violation: "application relation target 'pr_merged_events' must be schema-qualified" + - key: "02ed3b6f2f223e8bd2b6246c6527050b5f0a3d5fa2d0bbe9d3ef2d2f07d972a4" + path: "docker/migrations/forward/nodes/node_pr_merged_projection/0001_create_pr_merged_events.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "1b72f7c27e79c1767fb30e12a4952726742f3e31c97742f19961f7ff592c3cee" + path: "docker/migrations/forward/nodes/node_projection_baselines/0001_create_baselines_tables.sql" + violation: "application relation target 'baselines_breakdown' must be schema-qualified" + - key: "99966af5d4caa70fc40e57b6d6de97fd5d120b5a1e8af8683698c775d04a36da" + path: "docker/migrations/forward/nodes/node_projection_baselines/0001_create_baselines_tables.sql" + violation: "application relation target 'baselines_comparisons' must be schema-qualified" + - key: "d855da0f276d11389512d924104cc1504c3be42fa17ff4fe232e2ad1d61dc376" + path: "docker/migrations/forward/nodes/node_projection_baselines/0001_create_baselines_tables.sql" + violation: "application relation target 'baselines_snapshots' must be schema-qualified" + - key: "403b4cf4dce96e4f73ac0d91a803f369df01aa1eeb3a2e3b5ae5e74c4b281abf" + path: "docker/migrations/forward/nodes/node_projection_baselines/0001_create_baselines_tables.sql" + violation: "application relation target 'baselines_trend' must be schema-qualified" + - key: "991c02b5b4caae6ab820be00a910f0145ebeea593addc5add8d88189469a5a56" + path: "docker/migrations/forward/nodes/node_projection_baselines/0001_create_baselines_tables.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "ef5758e94c575bf1ba620d4fd81e012825b193890b00da5f2d5bfeb24a01cc44" + path: "docker/migrations/forward/nodes/node_projection_baselines/0002_realign_child_tables_to_producer_schema.sql" + violation: "application relation target 'baselines_breakdown' must be schema-qualified" + - key: "ccbe4e458f4b5272172612597f3a672ce61e3baf8eda74e0877e9efe9129d432" + path: "docker/migrations/forward/nodes/node_projection_baselines/0002_realign_child_tables_to_producer_schema.sql" + violation: "application relation target 'baselines_comparisons' must be schema-qualified" + - key: "7560683383e10937eb0be94371caf3953659b5d4b522d7cccd928ab2c796c086" + path: "docker/migrations/forward/nodes/node_projection_baselines/0002_realign_child_tables_to_producer_schema.sql" + violation: "application relation target 'baselines_trend' must be schema-qualified" + - key: "80484285980093c76a9963c01d764fc02a5c5f3ea922c5a89b8958829870dbc1" + path: "docker/migrations/forward/nodes/node_projection_baselines/0002_realign_child_tables_to_producer_schema.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "a6764ea2aba4a0967e48ec7724aeba6579730e00b9a10f98f826ce3e2382a028" + path: "docker/migrations/forward/nodes/node_projection_baselines_quality/001_create_baselines_quality_snapshots.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "6541e5a4f009e3c1323db08baa703e73f06d73bd0f486cfb64f7d6fb7a6eae2e" + path: "docker/migrations/forward/nodes/node_projection_baselines_quality/001_create_baselines_quality_snapshots.sql" + violation: "application relation target 'baselines_quality_snapshots' must be schema-qualified" + - key: "8c5231ed50a69ca055e180c9be5e7c03a80e970df7defa1d43b56eba863ae5b0" + path: "docker/migrations/forward/nodes/node_projection_baselines_quality/001_create_baselines_quality_snapshots.sql" + violation: "application relation target 'refresh_baselines_quality_snapshots_projected_at' must be schema-qualified" + - key: "d029bd7afde246f96de2a446f8006f8cc3422c4853df58372c04049563df668b" + path: "docker/migrations/forward/nodes/node_projection_baselines_quality/001_create_baselines_quality_snapshots.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "47edf4d7de597670493f917184c6100caeb587a30a7022fe54e2840bef1a3853" + path: "docker/migrations/forward/nodes/node_projection_baselines_quality/002_realign_quality_snapshots_to_producer_schema.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "ba2fd98ff13aee7d84cc351f23c4c28b36df3bee5808118b5d5655cd9e133a44" + path: "docker/migrations/forward/nodes/node_projection_baselines_quality/002_realign_quality_snapshots_to_producer_schema.sql" + violation: "application relation target 'baselines_quality_snapshots' must be schema-qualified" + - key: "c21957e4d1037f8f484f32d264c70b5b0143376af066cd88faa50a22d659e5ee" + path: "docker/migrations/forward/nodes/node_projection_baselines_quality/002_realign_quality_snapshots_to_producer_schema.sql" + violation: "application relation target 'refresh_baselines_quality_snapshots_projected_at' must be schema-qualified" + - key: "cb1c696a01feddfe0cdc9a88b4f6168bf8ff303edd10afd3a236ab3ce6bafd1c" + path: "docker/migrations/forward/nodes/node_projection_baselines_quality/002_realign_quality_snapshots_to_producer_schema.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "8190c8cf7fa6b5ac90d5acf08382bcf8c8173171e9462575a51454dea0ad81a4" + path: "docker/migrations/forward/nodes/node_projection_baselines_roi/001_create_baselines_roi_snapshots.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "8fb6f41c472e7917b37bf66bd695915a8a433339b5b793276c85ec76b6180c16" + path: "docker/migrations/forward/nodes/node_projection_baselines_roi/001_create_baselines_roi_snapshots.sql" + violation: "application relation target 'baselines_roi_snapshots' must be schema-qualified" + - key: "8192bcb020c7ff28e9c8f36564b89202f92d1cb9e3720d0b69bcf1fc4d764942" + path: "docker/migrations/forward/nodes/node_projection_baselines_roi/001_create_baselines_roi_snapshots.sql" + violation: "application relation target 'refresh_baselines_roi_snapshots_projected_at' must be schema-qualified" + - key: "7577ac5e1256a10211014a9a5c4d32960bef249c4cc27d2cd3361426dca27f49" + path: "docker/migrations/forward/nodes/node_projection_baselines_roi/001_create_baselines_roi_snapshots.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "804723c5da9fce0ce97bb0218b5f729d4b1a4cd4a5613cf8b41ee3f86face6b9" + path: "docker/migrations/forward/nodes/node_projection_baselines_roi/002_realign_roi_snapshots_to_producer_schema.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "ed367a456c788c510de8589a6bbbec2fec751bffdc6925d9ac81c92321b43c4d" + path: "docker/migrations/forward/nodes/node_projection_baselines_roi/002_realign_roi_snapshots_to_producer_schema.sql" + violation: "application relation target 'baselines_roi_snapshots' must be schema-qualified" + - key: "db4168bce0c4f0e0a3232e7dad90795a5e22cefa3ddcc6355f97913c6d6a85b5" + path: "docker/migrations/forward/nodes/node_projection_baselines_roi/002_realign_roi_snapshots_to_producer_schema.sql" + violation: "application relation target 'refresh_baselines_roi_snapshots_projected_at' must be schema-qualified" + - key: "edafe4440614d1618af84369b1440d57632ee468978ef061e6b8f6004f74a50e" + path: "docker/migrations/forward/nodes/node_projection_baselines_roi/002_realign_roi_snapshots_to_producer_schema.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "c9dd4b2813d131aeebc467a889f51c40a18a561a33c532c2b86a56e5421e13b9" + path: "docker/migrations/forward/nodes/node_projection_capsule_store/078_create_capsule_store.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "d37a2fcd84685dfae307f9f69b6744920853ef935f6f12dd61536670d376636d" + path: "docker/migrations/forward/nodes/node_projection_capsule_store/078_create_capsule_store.sql" + violation: "application relation target 'capsule_store' must be schema-qualified" + - key: "0181522a32680721ee117cdf724f2e50c50deb51115a883f517323801db1bf3c" + path: "docker/migrations/forward/nodes/node_projection_capsule_store/078_create_capsule_store.sql" + violation: "application relation target 'refresh_capsule_store_updated_at' must be schema-qualified" + - key: "a7f87e68e8afa96733571e7f98da795e30f42cc8410ca4cdac251ef2b04f5a89" + path: "docker/migrations/forward/nodes/node_projection_capsule_store/078_create_capsule_store.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "87d2def1d7fc69388f4829eae6cffc2eff9a33a536aa1f8cc8f26b6087ff9479" + path: "docker/migrations/forward/nodes/node_projection_context_roi/001_create_context_roi_scores.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "d632dd3dd353740d3c317d1d3a8f5d2456b3ca5258be20c80ee58bce18109548" + path: "docker/migrations/forward/nodes/node_projection_context_roi/001_create_context_roi_scores.sql" + violation: "application relation target 'context_roi_scores' must be schema-qualified" + - key: "db4d2ee48b205a6512791f11db227fbaf5bc7713c568ba2ac00db01030b4cce4" + path: "docker/migrations/forward/nodes/node_projection_context_roi/001_create_context_roi_scores.sql" + violation: "application relation target 'refresh_context_roi_scores_updated_at' must be schema-qualified" + - key: "6049cc5338764cb27ef61b905a6a341856096fd85a0d3d6ac9feb7fcdbe16451" + path: "docker/migrations/forward/nodes/node_projection_context_roi/001_create_context_roi_scores.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "c451c9a8ae4dcee5034c848186e1185621b1d4f4cb55b60090aa489e8c7df26b" + path: "docker/migrations/forward/nodes/node_projection_cost_by_repo/0001_create_cost_by_repo_snapshots.sql" + violation: "application relation target 'cost_by_repo_snapshots' must be schema-qualified" + - key: "bf6f17756e294f237985b90b8178b89bbe5f83411b6846ad01779bf3c04fa167" + path: "docker/migrations/forward/nodes/node_projection_cost_by_repo/0001_create_cost_by_repo_snapshots.sql" + violation: "application relation target 'update_cost_by_repo_snapshots_updated_at' must be schema-qualified" + - key: "1b03873bfb76c7891b9cde24e717903f380f9df2e286cabf117e4b0d362bb7bb" + path: "docker/migrations/forward/nodes/node_projection_cost_by_repo/0001_create_cost_by_repo_snapshots.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "629767098a55bbcfe55cf57c783af7076109ec1fd2a01ba0c89caff2ffc9e4db" + path: "docker/migrations/forward/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql" + violation: "application relation target 'llm_cost_aggregates' must be schema-qualified" + - key: "e5489ee6968c182337d74e9167d7d35d039260b5480313e70a4e0bb777aa51a6" + path: "docker/migrations/forward/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql" + violation: "application relation target 'update_llm_cost_aggregates_updated_at' must be schema-qualified" + - key: "0ab89df4ba42339b2a043f8b6b8684716285c278dea73145a57249550a70ab8d" + path: "docker/migrations/forward/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "a6c4da633707fbc573e30a476dadde0df0f5fb36c5186a5b20986919137f4f77" + path: "docker/migrations/forward/nodes/node_projection_delegation/0007_delegation_events.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "46d05aea130271a659774bdb2bf5481cc2f1dbb543cc618fa47f8a9c4c34482a" + path: "docker/migrations/forward/nodes/node_projection_delegation/0007_delegation_events.sql" + violation: "application relation target 'delegation_events' must be schema-qualified" + - key: "706ae9dc64a6ed13f8eefb785c48f64adf7e7c41be2385e1918b2137cb4db54e" + path: "docker/migrations/forward/nodes/node_projection_delegation/0007_delegation_events.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "653c13acabaccb198fbcc9777b280ac597e3d38d3f8978d5f50eed875e9c7036" + path: "docker/migrations/forward/nodes/node_projection_delegation/0008_generation_events.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "811344970cf859eef864841501841c185b7538fc2236d47074fcf432b945b781" + path: "docker/migrations/forward/nodes/node_projection_delegation/0008_generation_events.sql" + violation: "application relation target 'generation_events' must be schema-qualified" + - key: "adb3ae275dc2b091217f4533fbac9820e41df55de5134b8f54f14b5a05bccb9d" + path: "docker/migrations/forward/nodes/node_projection_delegation/0008_generation_events.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "46d3a30e176af2521ef8b4f3af2d0eba81b52cc9541b12f92fba012c7c660423" + path: "docker/migrations/forward/nodes/node_projection_delegation/0009a_delegation_events_legacy_schema_reconcile.sql" + violation: "application relation target 'delegation_events' must be schema-qualified" + - key: "d9260a95d63c40499970227b08364e7587b14de5279e64f06bf698c5add08b7c" + path: "docker/migrations/forward/nodes/node_projection_delegation/0009a_delegation_events_legacy_schema_reconcile.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "f838bc341dc8e8ad110e515b6a974fa5c922b192ca795aabe3863cc9cc7e93a7" + path: "docker/migrations/forward/nodes/node_projection_delegation/0016_delegation_judge_verdict_events.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "e718e88aff96816e1f5b336c12b741fa232b0c653580c682557c98bee55bead1" + path: "docker/migrations/forward/nodes/node_projection_delegation/0016_delegation_judge_verdict_events.sql" + violation: "application relation target 'delegation_judge_verdict_events' must be schema-qualified" + - key: "e204ee1b70c118bc7f06c07fe3ac48df3336965e8ab7d6b0db6fbe9ff4e745dd" + path: "docker/migrations/forward/nodes/node_projection_delegation/0016_delegation_judge_verdict_events.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "08a00876b32ac3b8eee8fd56c79616162631f8eaff9665ccb66af6774a12a412" + path: "docker/migrations/forward/nodes/node_projection_delegation/0019_delegation_budget_state.sql" + violation: "application relation target 'delegation_budget_state' must be schema-qualified" + - key: "04b0af4e911912a7ee9fe0956e05a417469553f454a3b87c3bf22f3793f1c308" + path: "docker/migrations/forward/nodes/node_projection_delegation/0019_delegation_budget_state.sql" + violation: "application relation target 'refresh_delegation_budget_state_updated_at' must be schema-qualified" + - key: "874a61b4380cb8dbebcccf5ac75ecf43c12661b0552a4e7f6f31ea06e6d25f00" + path: "docker/migrations/forward/nodes/node_projection_delegation/0019_delegation_budget_state.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "b1d3aee9ec3ff75f8cae4159d943c48121995855f48773619e49d7b0744f827b" + path: "docker/migrations/forward/nodes/node_projection_delegation/0027_generation_events_tenant_rls.sql" + violation: "application relation target 'generation_events' must be schema-qualified" + - key: "b767802804cb534e2ac628513e0da4e15a084eb449bf72ffc46cb84da76c4915" + path: "docker/migrations/forward/nodes/node_projection_delegation/0027_generation_events_tenant_rls.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "21b2ae69b67a7bdbc43e03dded197b258f4c25fd39cd5a8f2ef911c426a39a66" + path: "docker/migrations/forward/nodes/node_projection_delegation/0028_reconcile_delegation_observability_views.sql" + violation: "application relation target 'created_at' must be schema-qualified" + - key: "ae8e121a72a2e2d589dd500a3b35d40cb33f3522eeb361e6fc35aafcda5b898d" + path: "docker/migrations/forward/nodes/node_projection_delegation/0028_reconcile_delegation_observability_views.sql" + violation: "application relation target 'delegation_events' must be schema-qualified" + - key: "f0a7f0995a3264da5e820c5425d8d58e2f2a8c01e253208f6c0699ca12ab05e5" + path: "docker/migrations/forward/nodes/node_projection_delegation/0028_reconcile_delegation_observability_views.sql" + violation: "application relation target 'projection_delegation_model_routing' must be schema-qualified" + - key: "75f1125ee99133467f7ec5d5cc29fd21bf1922cf1f59f2bd9891337ee7badd6c" + path: "docker/migrations/forward/nodes/node_projection_delegation/0028_reconcile_delegation_observability_views.sql" + violation: "application relation target 'projection_delegation_quality_gate' must be schema-qualified" + - key: "a52bf0473353caf285f6cacb341efa80898b9e982e094cf9c045965538fb07af" + path: "docker/migrations/forward/nodes/node_projection_delegation/0028_reconcile_delegation_observability_views.sql" + violation: "application relation target 'projection_delegation_summary' must be schema-qualified" + - key: "dc6043b8d90b85d55df3625772db32a34c735521ab90f0b16ace2461b1d6d26f" + path: "docker/migrations/forward/nodes/node_projection_delegation/0028_reconcile_delegation_observability_views.sql" + violation: "application relation target 'projection_delegation_token_usage' must be schema-qualified" + - key: "edafc899a25a7e5fc5b2da837bf083b4c99ffaf2d45aac7a3e0a79358698f262" + path: "docker/migrations/forward/nodes/node_projection_delegation_inference_response/0001_create_projection_delegation_inference_response_text.sql" + violation: "application relation target 'projection_delegation_inference_response_text' must be schema-qualified" + - key: "24b56d73bc1b9cc5f7f300172ab66b9f986c89a6f6f7148138fc89d691fa58fc" + path: "docker/migrations/forward/nodes/node_projection_delegation_inference_response/0001_create_projection_delegation_inference_response_text.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "a423255250a14b2f83d79d071f43b53dbdbfdaf3d842ce44b80d65d9e99296be" + path: "docker/migrations/forward/nodes/node_projection_dep_health/001_create_dep_health_findings.sql" + violation: "application relation target 'dep_health_findings' must be schema-qualified" + - key: "86cf5dd1a373f22728e1437c6931672491fbca384594d14f53c1484e1026d7e1" + path: "docker/migrations/forward/nodes/node_projection_dep_health/001_create_dep_health_findings.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "79d8581a990c2bbf276405320beb9a97d41b10f0e5f38f886db054b60fff9cd3" + path: "docker/migrations/forward/nodes/node_projection_event_chain/0001_create_event_chain.sql" + violation: "application relation target 'event_chain' must be schema-qualified" + - key: "230fa8b4706b62f3f3a321d131c9257ec58407fc167719f98a0b587143feafd0" + path: "docker/migrations/forward/nodes/node_projection_event_chain/0001_create_event_chain.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "1e6d95de57d42205211720a77fc50da7143cd5511cabaf1f4cf2cfdb1a636e23" + path: "docker/migrations/forward/nodes/node_projection_instruction_eval/0001_create_instruction_eval_aggregate_snapshots.sql" + violation: "application relation target 'instruction_eval_aggregate_snapshots' must be schema-qualified" + - key: "286e095561a6c90c0c554cf256207a1bbac5f28dee93065989057de7e40e3a54" + path: "docker/migrations/forward/nodes/node_projection_instruction_eval/0001_create_instruction_eval_aggregate_snapshots.sql" + violation: "application relation target 'update_instruction_eval_aggregate_snapshots_updated_at' must be schema-qualified" + - key: "dcdee082e02a045c4f96b70189d314c96f161896e97af40a2c6d6316d71f2301" + path: "docker/migrations/forward/nodes/node_projection_instruction_eval/0001_create_instruction_eval_aggregate_snapshots.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "2b0285353f66ff3b1549070f7637495cda987153e9097fcefc99b41a4cd96066" + path: "docker/migrations/forward/nodes/node_projection_intent_classification/0000_create_intent_classification_events.sql" + violation: "application relation target 'intent_classification_events' must be schema-qualified" + - key: "87463f7cbceeda01f0a9d2f8cd9f4e2a88a2e4aae6326633d8046cf43de9e208" + path: "docker/migrations/forward/nodes/node_projection_intent_classification/0000_create_intent_classification_events.sql" + violation: "application relation target 'refresh_intent_classification_events_updated_at' must be schema-qualified" + - key: "874af56544248e2e6d2855730dd59fc79e7a83f0ddeecf5ace326784cd778469" + path: "docker/migrations/forward/nodes/node_projection_intent_classification/0000_create_intent_classification_events.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "65d772abd060aaee92e93465a404d662cfbb99926d8fa21b21b1f5f8a9e87f25" + path: "docker/migrations/forward/nodes/node_projection_live_events/0000_create_live_events.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "b827ef14b94a17a716a03bbe5f9c9f04f740c79676cdad52744e2661d8ace8af" + path: "docker/migrations/forward/nodes/node_projection_live_events/0000_create_live_events.sql" + violation: "application relation target 'live_events' must be schema-qualified" + - key: "cc04528969e134942672b32ab7dd1b3e69abdaa5f9536a959d1aca242037bc41" + path: "docker/migrations/forward/nodes/node_projection_live_events/0000_create_live_events.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "edda7150bb73c15410826527e9a123d80d2eb044da66f06d3a2dde3ce3d4140e" + path: "docker/migrations/forward/nodes/node_projection_live_events/0001_reclassify_event_lifecycle_types.sql" + violation: "application relation target 'classified.canonical_type' uses unknown topology schema" + - key: "5a43ba1ba5cd6e6723e34355ea4aa2af9af73411cfa15076ac5eb1d2a14f190f" + path: "docker/migrations/forward/nodes/node_projection_live_events/0001_reclassify_event_lifecycle_types.sql" + violation: "application relation target 'live_events' must be schema-qualified" + - key: "11e0276b3eeb323ca9c22731def2b6633a40fb682344cdd502c5ca84f095e6eb" + path: "docker/migrations/forward/nodes/node_projection_live_events/0001_reclassify_event_lifecycle_types.sql" + violation: "application relation target 'or' must be schema-qualified" + - key: "22b572fd4bf78d656909e5c045aa38769ebadb55d85d884e747e84dbb721027f" + path: "docker/migrations/forward/nodes/node_projection_live_events/0001_reclassify_event_lifecycle_types.sql" + violation: "application relation target 'then' must be schema-qualified" + - key: "6871b504bb84f4f11518926de889321064820bfa37d5282bea6a8b5d47ea2492" + path: "docker/migrations/forward/nodes/node_projection_llm_cost/0001_create_llm_call_metrics.sql" + violation: "application relation target 'llm_call_metrics' must be schema-qualified" + - key: "b939a558c9dbc2ed6cc271c6e175d93d401a2eb89cf0dd08d350824013845666" + path: "docker/migrations/forward/nodes/node_projection_llm_cost/0001_create_llm_call_metrics.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "fd2c0ee7ccc6ee3a16a0260d042ad5be8a603d74e188f7eab29124d772229971" + path: "docker/migrations/forward/nodes/node_projection_llm_routing/0000_create_llm_routing_decisions.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "9d7b3ad7892bb9316c6a664adfe3ca3582f8b43fe41241d166339e0cd713ded9" + path: "docker/migrations/forward/nodes/node_projection_llm_routing/0000_create_llm_routing_decisions.sql" + violation: "application relation target 'llm_routing_decisions' must be schema-qualified" + - key: "6e80197240be87e11d2a4530154b9084250e04f91afeb80fd2a9384b99a2c535" + path: "docker/migrations/forward/nodes/node_projection_llm_routing/0000_create_llm_routing_decisions.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "7d402d9461c4b988246b7094ff94bf505a7adf63eaf543e1534f5d84e96b6856" + path: "docker/migrations/forward/nodes/node_projection_mcp_tools/0001_create_mcp_tools.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "c4adeae68e674e6854505784c6cce1d1c37d201d590fa79f37e389f376cf3b85" + path: "docker/migrations/forward/nodes/node_projection_mcp_tools/0001_create_mcp_tools.sql" + violation: "application relation target 'mcp_tools' must be schema-qualified" + - key: "c69d1f2b6a81572c293d1e84b5aa661d67ba34009cc50f8b3c927c9b89ee030e" + path: "docker/migrations/forward/nodes/node_projection_mcp_tools/0001_create_mcp_tools.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "1fae171c8554f90fe90712e36b6060bac48e8a1810063e7092f0354e70ffad11" + path: "docker/migrations/forward/nodes/node_projection_overnight/0000_create_overnight_sessions_tables.sql" + violation: "application relation target 'overnight_session_phases' must be schema-qualified" + - key: "f9684b1cba4e949e251b268921880c5f0e592e5cc72ea4e91dbf2fe0f818549e" + path: "docker/migrations/forward/nodes/node_projection_overnight/0000_create_overnight_sessions_tables.sql" + violation: "application relation target 'overnight_sessions' must be schema-qualified" + - key: "4fb860c31dbdbe11ba83f907762b6356a375dc3baae0dd53c463c1ced7e44c8f" + path: "docker/migrations/forward/nodes/node_projection_overnight/0000_create_overnight_sessions_tables.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "1752b3303c4fb07cd4b96f6f409a034cb9321991710229be69e052c4e2d8876a" + path: "docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql" + violation: "application relation target 'pattern_learning_artifacts' must be schema-qualified" + - key: "aaca1cb1a9b8017047133460b494b8e15f7288d6f1a0dbab7997e6985670fcf7" + path: "docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql" + violation: "application relation target 'pattern_learning_artifacts.correlation_id' uses unknown topology schema" + - key: "8322d53268a847b97b0e57bee26e7b9465ecaf342c511ebb0110a9e8a65155b0" + path: "docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql" + violation: "application relation target 'pattern_learning_artifacts.pattern_id' uses unknown topology schema" + - key: "0800a9ec11bc30f398c7122a1fde87a8927b6283348b2c6d0f5cff582ca4d5da" + path: "docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "a47146167495b67c70d3b7385cd3f17e8ad886be2f8679e7b4067b2b94b33ced" + path: "docker/migrations/forward/nodes/node_projection_receipt_gate/0000_create_receipt_gate_projection_table.sql" + violation: "application relation target 'receipt_gate_rows' must be schema-qualified" + - key: "11a5c093b7285cf96de69706faf937f714620e6a5c551a279a672e3bb5a66c60" + path: "docker/migrations/forward/nodes/node_projection_receipt_gate/0000_create_receipt_gate_projection_table.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "3af523380de2179adc7ed5cb64a26bfd1e52da445e4508dfaf514c236fa565bd" + path: "docker/migrations/forward/nodes/node_projection_registration/0003_reconcile_heartbeat_observability.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "22946b89431b0a500917b957c3da0f7bfea7589465c1c1a19ba19b8446f5d895" + path: "docker/migrations/forward/nodes/node_projection_routing_decision/0021_create_agent_routing_decisions.sql" + violation: "application relation target 'agent_routing_decisions' must be schema-qualified" + - key: "d8b85254027f2bbfce53e5b8f7c02ec36f3d711327b5cf8b76308b58bdc4503a" + path: "docker/migrations/forward/nodes/node_projection_routing_decision/0021_create_agent_routing_decisions.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "5d7f81565dd14fc61ae0264af27a63088c9729058ebc58959b0a5049e3871f06" + path: "docker/migrations/forward/nodes/node_projection_sandbox_decisions/0001_create_sandbox_decisions.sql" + violation: "application relation target 'sandbox_decisions' must be schema-qualified" + - key: "9f741a29957f17c142529f4db2dc472a5547785903dd26c09164e89d9de711bc" + path: "docker/migrations/forward/nodes/node_projection_sandbox_decisions/0001_create_sandbox_decisions.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "5e55fffd3324dcec673d67970531a132865801679dd0e77be88c8e8ed7853959" + path: "docker/migrations/forward/nodes/node_projection_savings/074_create_savings_estimates.sql" + violation: "CREATE EXTENSION must be schema-qualified with an explicit topology WITH SCHEMA target" + - key: "addae29519776b763a2d3bc4e4cf3e7c0998cb07649b223d6b2afcd0e78e02ba" + path: "docker/migrations/forward/nodes/node_projection_savings/074_create_savings_estimates.sql" + violation: "application relation target 'refresh_savings_estimates_updated_at' must be schema-qualified" + - key: "98c94d83b24c33ae16e82c8b3c9f3c5f47e900c927b7b56b3b685c018204fb94" + path: "docker/migrations/forward/nodes/node_projection_savings/074_create_savings_estimates.sql" + violation: "application relation target 'savings_estimates' must be schema-qualified" + - key: "6a97baff9de6f84b5b1234a5098905de78333fe2f767fedd25211518057cf710" + path: "docker/migrations/forward/nodes/node_projection_savings/074_create_savings_estimates.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "28fc5864b43b2e21912114e86412c80d3687cd7383e8d3f87cba3ae126ca2d97" + path: "docker/migrations/forward/nodes/node_projection_session_outcome/0021_session_outcomes.sql" + violation: "application relation target 'refresh_session_outcomes_updated_at' must be schema-qualified" + - key: "75cc04905921e0603874bfbbad768aa67afe023153a919c6c0e150b27b00b68e" + path: "docker/migrations/forward/nodes/node_projection_session_outcome/0021_session_outcomes.sql" + violation: "application relation target 'session_outcomes' must be schema-qualified" + - key: "c9466e367a4bc2efd2da787f45ebffcf480fe41bba355f5126cdfb1b41734191" + path: "docker/migrations/forward/nodes/node_projection_session_outcome/0021_session_outcomes.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "b3e0f8de008da7959fde1ec616d67bd41b1380b967402e7be35b91afe099fb56" + path: "docker/migrations/forward/nodes/node_projection_session_replay/0001_create_session_replay_snapshots.sql" + violation: "application relation target 'public.session_replay_snapshots' is prohibited in public" + - key: "b2d5ecdc6a37399739ff8ef7509bb08a17234596e004f1afc0bd340ecc9e2b22" + path: "docker/migrations/forward/nodes/node_projection_session_replay/0001_create_session_replay_snapshots.sql" + violation: "application target public.session_replay_snapshots requires exactly one ownership declaration" + - key: "f39e524922e0f060ec46ce79432bd83052c6c28946c74f5b0df79235413a5edb" + path: "docker/migrations/forward/nodes/node_projection_session_replay/0001_create_session_replay_snapshots.sql" + violation: "created application object ('public', 'session_replay_snapshots', , None) lacks an authoritative ownership declaration" + - key: "ba4a9cf25679ed524342e968e5e1f11271231336b703b5859c46e3be1e653488" + path: "docker/migrations/forward/nodes/node_projection_session_replay/0001_create_session_replay_snapshots.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "361d68ad4aa19911e837a39dfc7ed3e2f36a8da91e5574d5fa06239336d0b318" + path: "docker/migrations/forward/nodes/node_projection_skill_executions/0001_create_skill_execution_snapshots.sql" + violation: "application relation target 'skill_execution_snapshots' must be schema-qualified" + - key: "5894d9e25bb014b8e73d5b6b1d76473a6d15d08c2dec4eb553eb86299e29f750" + path: "docker/migrations/forward/nodes/node_projection_skill_executions/0001_create_skill_execution_snapshots.sql" + violation: "application relation target 'update_skill_execution_snapshots_updated_at' must be schema-qualified" + - key: "af4d118a3a44d0cf412512d253e76b0a287adf3fc2efed10c832a7f105e611a7" + path: "docker/migrations/forward/nodes/node_projection_skill_executions/0001_create_skill_execution_snapshots.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "3fe8a15542cdb2495a61b6582212ddd5735ad7f8d126abcabcd4690335ed1b5d" + path: "docker/migrations/forward/nodes/node_projection_swarm/0001_create_swarm_runs.sql" + violation: "application relation target 'swarm_runs' must be schema-qualified" + - key: "73e9a00d7bc7e3b2c7999dc888ce354ef0d982141cfc892965f99f408df1e4fc" + path: "docker/migrations/forward/nodes/node_projection_swarm/0001_create_swarm_runs.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "2f582e5ef156a0dccc61df66c76a31f5b5f6ad0c6e7eef31666f1da363a5ba0e" + path: "docker/migrations/forward/nodes/node_projection_traces/0001_create_traces.sql" + violation: "application relation target 'traces' must be schema-qualified" + - key: "8a72c648b614fd1e390c36e182efdf030b96c68466165e3b6cd71b2a16015281" + path: "docker/migrations/forward/nodes/node_projection_traces/0001_create_traces.sql" + violation: "application relation target 'update_traces_updated_at' must be schema-qualified" + - key: "f7fc0183f1555d2066841bf4f89fdab09075f357cbed079444295bebb1455db4" + path: "docker/migrations/forward/nodes/node_projection_traces/0001_create_traces.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "25991e218a378bb7eff7179ff2314f7ee6920c0cf1ad332fff50293f796d9169" + path: "docker/migrations/forward/nodes/node_projection_voice_sessions/0001_create_voice_sessions.sql" + violation: "application relation target 'refresh_voice_sessions_updated_at' must be schema-qualified" + - key: "80bc273b8fdce40cf285be4d49d7e3527ab7ce88601bfc577b683b155b88839c" + path: "docker/migrations/forward/nodes/node_projection_voice_sessions/0001_create_voice_sessions.sql" + violation: "application relation target 'voice_sessions' must be schema-qualified" + - key: "e6cf29407aab4632bc494d42824e41bb2f46060dcad2df030e2e204be92265d3" + path: "docker/migrations/forward/nodes/node_projection_voice_sessions/0001_create_voice_sessions.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "1bba42638a5d19f3db191a19c482d88ad9cac7382024c643cedf2351859c6aba" + path: "docker/migrations/forward/nodes/node_renderer_capability_projection/0001_create_renderer_capability_projection.sql" + violation: "application relation target 'renderer_capability_projection' must be schema-qualified" + - key: "afe07e73f7b78fbe2c507ece2c0888231ba47e147d1b1f8ddff41f967eca0a1b" + path: "docker/migrations/forward/nodes/node_renderer_capability_projection/0001_create_renderer_capability_projection.sql" + violation: "procedural block contains dynamic SQL whose relation targets cannot be proven statically" + - key: "eecbe580dc08de48a6ca441aadd9191a901a1b7dc66e3efc8bf05c6e3d49cd1b" + path: "docker/migrations/intelligence/026_create_code_entities.sql" + violation: "application relation target 'code_entities' must be schema-qualified" + - key: "a5b1545729f4a20557951b252f91346b56366d3d887253054238f044e97f1902" + path: "docker/migrations/intelligence/026_create_code_entities.sql" + violation: "application relation target 'code_relationships' must be schema-qualified" + - key: "f6c6e998dd8567ac04ce66d9de4599726fee18e896a180c0627319e56b38f87d" + path: "docker/migrations/rollback/rollback_096_grant_role_omnidash_omnidash_analytics.sql" + violation: "application relation target 'all' must be schema-qualified" + - key: "9d54fdd9dd6f098f4e51562cceee3e4c38f25b2b48124c001fff560e4e243548" + path: "docker/migrations/rollback/rollback_096_grant_role_omnidash_omnidash_analytics.sql" + violation: "application relation target 'role_omnidash' must be schema-qualified" diff --git a/scripts/ci/assert_image_config_paths.py b/scripts/ci/assert_image_config_paths.py new file mode 100644 index 0000000000..bb939f605e --- /dev/null +++ b/scripts/ci/assert_image_config_paths.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Assert every required config path exists INSIDE a built runtime image (OMN-15676). + +This is the anti-recurrence mechanism for the "tracked in repo, absent from the +image" class. It reads the single typed source +(``omnibase_infra.runtime.required_image_config_paths``) and runs one ``test -f`` +per entry *inside the image under test*, so a missing Dockerfile ``COPY`` fails +the build instead of reaching a registry and boot-crashing a deployed pod. + +A repo-working-tree check cannot substitute for this: all three prior incidents +(grants fixture, routing_tiers.yaml, runner_fleet.yaml) had a perfectly correct +working tree. + +Fail-closed in every direction: + +* any declared path missing -> exit 1, +* the probe container failing to run at all -> exit 1, +* a path the probe returned no verdict for -> exit 1 (never treated as present), +* an empty registry -> exit 1 (a silently-emptied registry must not look green). + +Usage:: + + python scripts/ci/assert_image_config_paths.py --image + python scripts/ci/assert_image_config_paths.py --image --docker-bin podman +""" + +from __future__ import annotations + +import argparse +import importlib.util +import shlex +import subprocess # nosec B404 - deliberate: probing a container image requires the docker CLI +import sys +from collections.abc import Sequence +from pathlib import Path +from types import ModuleType +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Import-for-types only. At runtime the registry is loaded by file path + # (see _load_registry_module) so an ambient/installed copy can never be + # substituted for the one in this checkout. + from omnibase_infra.runtime.required_image_config_paths import ( + ModelRequiredImageConfigPath, + ) + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_REGISTRY_PATH = ( + _REPO_ROOT / "src" / "omnibase_infra" / "runtime" / "required_image_config_paths.py" +) + + +def _load_registry_module() -> ModuleType: + """Load the registry from THIS checkout by file path, never by package name. + + A plain ``from omnibase_infra.runtime... import`` resolves through whatever + ``omnibase_infra`` the ambient environment already binds -- an installed + wheel, or a canonical clone on ``PYTHONPATH``. Observed while building this + guard: with ``PYTHONPATH`` pointing at the canonical clone, the import + raised ModuleNotFoundError for a module that exists right here in the + checkout. The failure mode that does NOT announce itself is the other one: + an ambient copy resolving to an OLDER registry, so the guard silently + asserts fewer paths than the branch declares and reports green. + Path-loading pins the guard to the tree it ships in. + """ + spec = importlib.util.spec_from_file_location( + "_omn15676_required_image_config_paths", _REGISTRY_PATH + ) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load required-config registry: {_REGISTRY_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_registry = _load_registry_module() +REQUIRED_IMAGE_CONFIG_PATHS: tuple[ModelRequiredImageConfigPath, ...] = ( + _registry.REQUIRED_IMAGE_CONFIG_PATHS +) + +_PRESENT = "PRESENT" +_MISSING = "MISSING" +_PROBE_TIMEOUT_SECONDS = 300 + + +def _validate_paths(entries: Sequence[ModelRequiredImageConfigPath]) -> list[str]: + """Reject entries that are not safe, absolute, unambiguous in-image paths.""" + problems: list[str] = [] + seen: set[str] = set() + for entry in entries: + path = entry.image_path + if not path.startswith("/"): + problems.append(f"{path}: image_path must be absolute") + if path != shlex.quote(path): + problems.append( + f"{path}: image_path contains characters requiring shell quoting" + ) + if path in seen: + problems.append(f"{path}: duplicate registry entry") + seen.add(path) + return problems + + +def _build_probe_script(paths: Sequence[str]) -> str: + """Emit a POSIX-sh probe printing one ` ` line per path. + + The probe never exits non-zero on a missing file -- the verdict lines are the + channel, so a non-zero container exit unambiguously means the probe itself + could not run. + """ + lines = ["set -u"] + for path in paths: + quoted = shlex.quote(path) + lines.append( + f"if [ -f {quoted} ]; then echo '{_PRESENT} {path}'; " + f"else echo '{_MISSING} {path}'; fi" + ) + return "\n".join(lines) + + +def _run_probe(*, docker_bin: str, image: str, script: str) -> tuple[int, str, str]: + command = [ + docker_bin, + "run", + "--rm", + "--entrypoint", + "sh", + image, + "-c", + script, + ] + try: + completed = subprocess.run( # nosec B603 - fixed argv, no shell, paths validated above + command, + capture_output=True, + text=True, + timeout=_PROBE_TIMEOUT_SECONDS, + check=False, + ) + except FileNotFoundError: + return 127, "", f"{docker_bin}: not found on PATH" + except subprocess.TimeoutExpired: + return 124, "", f"probe timed out after {_PROBE_TIMEOUT_SECONDS}s" + return completed.returncode, completed.stdout, completed.stderr + + +def _parse_verdicts(stdout: str) -> dict[str, str]: + verdicts: dict[str, str] = {} + for raw in stdout.splitlines(): + line = raw.strip() + if not line: + continue + verdict, _, path = line.partition(" ") + if verdict in (_PRESENT, _MISSING) and path: + verdicts[path] = verdict + return verdicts + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Assert every path in the required-image-config registry exists " + "inside the given built image (OMN-15676)." + ) + ) + parser.add_argument( + "--image", + required=True, + help="Image ref to probe. Must already be present locally (docker build --load).", + ) + parser.add_argument( + "--docker-bin", + default="docker", + help="Container CLI to use (default: docker).", + ) + args = parser.parse_args(argv) + + entries = REQUIRED_IMAGE_CONFIG_PATHS + if not entries: + print( + "FAIL: required-image-config registry is EMPTY -- refusing to report " + "green. An empty registry asserts nothing (OMN-15676).", + file=sys.stderr, + ) + return 1 + + problems = _validate_paths(entries) + if problems: + for problem in problems: + print(f"FAIL: invalid registry entry -- {problem}", file=sys.stderr) + return 1 + + paths = [entry.image_path for entry in entries] + print(f"Asserting {len(paths)} required config path(s) inside image: {args.image}") + + returncode, stdout, stderr = _run_probe( + docker_bin=args.docker_bin, + image=args.image, + script=_build_probe_script(paths), + ) + if returncode != 0: + print( + f"FAIL: probe container exited {returncode} -- cannot prove any path " + "is present, failing closed.", + file=sys.stderr, + ) + if stdout.strip(): + print(f"stdout: {stdout.strip()}", file=sys.stderr) + if stderr.strip(): + print(f"stderr: {stderr.strip()}", file=sys.stderr) + return 1 + + verdicts = _parse_verdicts(stdout) + missing: list[ModelRequiredImageConfigPath] = [] + unobserved: list[ModelRequiredImageConfigPath] = [] + for entry in entries: + verdict = verdicts.get(entry.image_path) + if verdict == _PRESENT: + print(f" OK {entry.image_path}") + elif verdict == _MISSING: + print(f" MISSING {entry.image_path}") + missing.append(entry) + else: + print(f" NO-VERDICT {entry.image_path}") + unobserved.append(entry) + + if not missing and not unobserved: + print(f"PASS: all {len(paths)} required config path(s) present in the image.") + return 0 + + print("", file=sys.stderr) + for entry in missing: + print( + f"FAIL: {entry.image_path} is NOT in the image.\n" + f" resolved by: {entry.resolved_by}\n" + f" impact: {entry.why_required}\n" + f" ticket: {entry.ticket}\n" + f" fix: add a COPY for this path to docker/Dockerfile.runtime " + f"(a bind-mount does not satisfy this -- the deployed pod has no such mount).", + file=sys.stderr, + ) + for entry in unobserved: + print( + f"FAIL: {entry.image_path} returned no verdict from the probe -- " + "treated as absent (fail-closed).", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/check_application_database_sql.py b/scripts/ci/check_application_database_sql.py new file mode 100644 index 0000000000..c3e219da01 --- /dev/null +++ b/scripts/ci/check_application_database_sql.py @@ -0,0 +1,502 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Fail CI when changed deployable SQL uses unsafe application targets.""" + +from __future__ import annotations + +import argparse +import hashlib +import subprocess +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from pathlib import Path + +from omnibase_infra.topology.application_database import load_topology_profile +from omnibase_infra.validation.application_database_domain_enforcement import ( + application_database_created_catalog_identities, + application_database_sql_target_requirements, + lint_application_database_sql, + load_application_database_ownership_identities, +) + +_NON_DEPLOYABLE_SQL_EXACT_PATHS = frozenset( + { + Path("docker/application-domain-enforcement/seed.sql"), + Path("docker/migrations/forward/_ledger/bootstrap.sql"), + Path("src/omnibase_infra/migration/cutover/sql/bootstrap.sql"), + } +) +_NON_DEPLOYABLE_SQL_PREFIXES = (Path("docker/legacy-rds-fixture"),) +_LEGACY_DEFAULT_SCHEMA_SQL_EXACT_PATHS = frozenset( + { + # OMN-15503 adds one migration to the existing node_projection_delegation + # stream. That stream still creates and mutates delegation_events in the + # legacy default schema; qualifying only this new ALTER would target a + # table that does not exist in the migration runner. + Path( + "docker/migrations/forward/nodes/node_projection_delegation/" + "0029_delegation_terminal_failure_cause.sql" + ), + # OMN-15655 adds the house-tenant tenant_id/RLS tranche for relations + # whose physical tables intentionally remain in public until the + # governed OMN-15359 schema cutover moves the full classified set. + Path( + "docker/migrations/forward/nodes/node_canary_score_reducer/" + "0002_capability_scores_tenant_id_and_rls.sql" + ), + # OMN-15732: 0003 is the direct sequel to 0002 above -- it continues + # the OMN-15356 tenant_id TEXT->UUID conversion on the SAME physical + # public.capability_scores table 0002 already exempted for the + # identical reason (physical table intentionally remains in public + # until the governed OMN-15359 schema cutover). OMN-15732 AC2 + # adjudication (2026-08-08) rejected a standalone, schema-qualified + # mapping function as inadmissible for a node migration stream (no + # `node:%` domain -- {tenant, omninode_internal} -- admits + # platform_catalog); the mapping is now inlined into this same file's + # `ALTER COLUMN ... USING` clause, so there is no separate object to + # qualify. node-migration-sync (OMN-13332) forces this file to be + # vendored verbatim from omnimarket dev regardless of this gate, so + # exempting it here (not editing the SQL) is the canonical fix -- see + # docs/tracking/ROLLING_WORK_LEDGER.md 2026-08-08 + # [mergesweep-0808-deadlock] for the full deadlock analysis. + Path( + "docker/migrations/forward/nodes/node_canary_score_reducer/" + "0003_capability_scores_tenant_id_to_uuid.sql" + ), + # OMN-15732: node_service_registry has been unqualified (default/ + # public schema) since its 0000 CREATE TABLE; 0004 only flips its + # FORCE ROW LEVEL SECURITY posture (OMN-15336 item 4 / operator + # ruling R-q, 2026-08-05) inside a DO block that references the same + # already-unqualified table -- it introduces no new physical-schema + # authority. Same node-migration-sync-forced-vendoring rationale as + # above. + Path( + "docker/migrations/forward/nodes/node_projection_registration/" + "0004_node_service_registry_no_force_rls.sql" + ), + Path( + "docker/migrations/forward/nodes/node_projection_context_roi/" + "003_context_roi_scores_tenant_id_and_rls.sql" + ), + Path( + "docker/migrations/forward/nodes/node_projection_cost_summary/" + "0002_llm_cost_aggregates_tenant_id_and_rls.sql" + ), + Path( + "docker/migrations/forward/nodes/node_projection_delegation/" + "0030_delegation_budget_state_house_tenant_rekey.sql" + ), + Path( + "docker/migrations/forward/nodes/node_projection_dep_health/" + "002_dep_health_findings_tenant_id_and_rls.sql" + ), + Path( + "docker/migrations/forward/nodes/node_projection_instruction_eval/" + "0002_instruction_eval_aggregate_snapshots_tenant_id_and_rls.sql" + ), + Path( + "docker/migrations/forward/nodes/node_projection_pattern_learning/" + "0001_pattern_learning_artifacts_tenant_id_and_rls.sql" + ), + Path( + "docker/migrations/forward/nodes/node_projection_routing_decision/" + "0022_agent_routing_decisions_tenant_id_and_rls.sql" + ), + Path( + "docker/migrations/forward/nodes/node_projection_skill_executions/" + "0002_skill_execution_snapshots_tenant_id_and_rls.sql" + ), + # OMN-15655 also reconciles historical root migration shapes for + # fixture parity. These are legacy default-schema repair paths, not new + # application-database authority, and they must retain compatibility with + # the existing migration runner until the governed schema cutover lands. + Path( + "docker/migrations/forward/031_create_llm_call_metrics_and_cost_aggregates.sql" + ), + Path("docker/migrations/forward/050_create_baselines_tables.sql"), + # OMN-15717: node_pr_review_bot's 001_create_review_bot_bypass_log.sql is + # a legacy-declared, vendor-synced migration (node-migration-sync, + # OMN-13332) restored byte-identical to the original omnimarket commit -- + # sha256-verified against the historical file and cross-checked against + # the live omnidash_analytics.platform_catalog.schema_migrations row + # this ticket exists to declare. Its unqualified `review_bot_bypass_log` + # target predates this gate; qualifying it now would break both the + # byte-identity proof and the checksum this migration's own declaration + # row binds to. Same node-migration-sync-forced-vendoring rationale as + # the node_canary_score_reducer / node_projection_registration entries + # above -- exempting here (not editing the SQL) is the canonical fix. + Path( + "docker/migrations/forward/nodes/node_pr_review_bot/" + "001_create_review_bot_bypass_log.sql" + ), + # OMN-15359: 099 performs the governed physical-schema cutover itself -- + # it creates NEW omninode_internal-domain authority + # (omninode_internal.live_events, ownership declared in omnimarket's + # application-relation-ownership.yaml via omnimarket#2031). Still + # exempted after OMN-15838 removed 099's transform-copy/reconciliation + # block (the original reason this file first needed the exemption -- it + # read unqualified `live_events` / `public.live_events` as its + # migration source, which the schema-qualification scanner has no + # accepting form for): this file's 3 remaining role/grant DO blocks + # (guarded CREATE ROLE, fail-loud existence check, CONNECT grant) are + # untouched by OMN-15838 and still trip `_requires_dynamic_sql_rejection` + # on their own. `changed_sql_paths()` only lints files an actual PR + # diff touches, so 098/096/094's own DO blocks have never needed an + # entry here -- but 099 IS touched by both OMN-15359 and this OMN-15838 + # follow-up, so it still needs one. Removing this file from the + # exemption list is therefore NOT safe post-OMN-15838 -- it would newly + # fail the lint on the untouched DO blocks, not clear it. + Path("docker/migrations/forward/099_create_omninode_internal_live_events.sql"), + # OMN-15846: this file is TOMBSTONED (undeliverable via the k8s Job -- + # cross-DB \connect, no execution path -- see the file's own header + # and docker/migrations/forward/cross-database-flat-migrations.yaml). + # The tombstone convention requires the SQL body stay byte-unchanged + # below the header (migration files are append-only ledgered + # history), so its pre-existing unqualified `log_entries` target + # cannot be qualified in place -- the node-owned replacement + # (docker/migrations/forward/nodes/node_log_persistence_effect/) is + # what physically creates the schema-qualified + # omninode_internal.log_entries table now. This entry only exists + # because OMN-15846 is the first PR to touch this file (a header-only + # comment addition) since this gate started linting changed files -- + # its own DDL was never in scope before. + Path("docker/migrations/forward/083_create_log_entries.sql"), + # OMN-15846: this file is TOMBSTONED (undeliverable via the k8s Job, + # and unneeded on RDS -- role_omnidash already owns the schema there; + # see the file's own header). Same byte-unchanged-below-header + # append-only convention as 083 above -- its `GRANT ... ON ALL TABLES + # IN SCHEMA public` (unqualified `all` target) and role-DDL DO blocks + # predate this gate's scope and are not being re-authored. First PR + # to touch this file (header-only) since this gate started linting + # changed files. + Path( + "docker/migrations/forward/096_grant_role_omnidash_omnidash_analytics.sql" + ), + # OMN-15846: the OMN-15376 shape-reconciliation blocks in this NEW + # node-owned migration (NOT NULL convergence + primary-key guard, + # added for CodeRabbit finding 3754510314 on omnimarket#2046) use + # `EXECUTE format(...)` for per-column dynamic SQL -- the exact same + # idiom `nodes/node_projection_live_events/0000_create_live_events.sql` + # already uses for its own NOT NULL/PK reconciliation (OMN-15376 + # original). This gate cannot prove a dynamic relation target + # statically regardless of how well-guarded/idempotent the block is; + # 099's own exemption entry above establishes this is an accepted, + # already-precedented limitation for legitimate reconciliation DO + # blocks in this corpus, not something unique to this file. + Path( + "docker/migrations/forward/nodes/node_log_persistence_effect/" + "0000_create_log_entries.sql" + ), + } +) + + +def _is_non_deployable_sql_path(relative_path: Path) -> bool: + if relative_path in _NON_DEPLOYABLE_SQL_EXACT_PATHS: + return True + return any( + relative_path == prefix or relative_path.is_relative_to(prefix) + for prefix in _NON_DEPLOYABLE_SQL_PREFIXES + ) + + +def _is_legacy_default_schema_sql_path(relative_path: Path) -> bool: + return relative_path in _LEGACY_DEFAULT_SCHEMA_SQL_EXACT_PATHS + + +def changed_sql_paths( + repository: Path, + base_revision: str, + head_revision: str, +) -> tuple[Path, ...]: + """Return changed deployable SQL, excluding the exact ephemeral proof seed.""" + result = subprocess.run( + [ + "git", + "diff", + "--name-only", + "--diff-filter=ACMR", + "-z", + f"{base_revision}...{head_revision}", + "--", + "*.sql", + ], + cwd=repository, + capture_output=True, + check=False, + ) + if result.returncode != 0: + detail = result.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(f"cannot resolve changed SQL range: {detail}") + relative_paths = tuple( + Path(raw.decode("utf-8")) for raw in result.stdout.split(b"\0") if raw + ) + resolved: list[Path] = [] + repository_root = repository.resolve() + for relative_path in relative_paths: + if _is_non_deployable_sql_path(relative_path): + # These SQL files initialize proof fixtures or internal control + # ledgers. Their synthetic authority universes are validated by + # dedicated gates and must not be composed with production owners. + continue + candidate = (repository_root / relative_path).resolve() + if not candidate.is_relative_to(repository_root): + raise RuntimeError(f"changed SQL path escapes repository: {relative_path}") + if candidate.is_file(): + resolved.append(candidate) + return tuple(resolved) + + +# --------------------------------------------------------------------------- +# OMN-15361 frozen baseline ratchet, mirroring the OMN-14443 deploy-gate +# grandfather pattern. +# +# WHY: this gate lints SQL *changed against the PR base*. On a dev PR that is a +# handful of files. At the dev->main promotion boundary the base is main, so the +# whole accumulated migration corpus counts as changed and every latent +# violation in it fires at once -- none of it newly authored, all of it already +# deployed. Rewriting deployed migrations to satisfy a gate at release time is +# the more dangerous path, so pre-existing violations are recorded in a frozen +# snapshot and soft-passed. +# +# SHRINK-ONLY: a violation NOT in the snapshot is held to the full bar and fails +# closed. An entry whose file is gone, or whose violation no longer fires on a +# file this run actually linted, is a STALE entry and FAILS -- the baseline may +# only shrink. Never widen it by hand; regenerate a shrunk one with +# scripts/ci/generate_application_database_sql_baseline.py. +# --------------------------------------------------------------------------- +_BASELINE_PATH = Path(__file__).parent / "application_database_sql_baseline.yaml" + + +@dataclass(frozen=True) +class SqlGateOutcome: + """Effective violations plus how many the frozen baseline absorbed.""" + + violations: tuple[str, ...] + grandfathered: int + + +def violation_key(violation: str) -> str: + """Content hash of one ``: `` violation line. + + Keyed on content, never on a line number: SQL edits that shift lines must + not silently re-key an entry and grandfather a violation that is actually + new. + """ + return hashlib.sha256(violation.encode("utf-8")).hexdigest() + + +def load_sql_baseline(baseline_path: Path) -> dict[str, str]: + """Load the frozen snapshot as ``{violation_key: recorded_relative_path}``. + + A missing or unparseable snapshot fails CLOSED to an EMPTY mapping, which + holds every violation to the full bar. A broken baseline must never + silently grandfather anything. + + The path is explicit and has no default: grandfathering is opt-in at the + call site, so a caller that forgets it gets the full bar rather than a + silent soft-pass. + """ + if not baseline_path.exists(): + return {} + import yaml + + try: + data = yaml.safe_load(baseline_path.read_text(encoding="utf-8")) + except (yaml.YAMLError, OSError): + return {} + if not isinstance(data, dict): + return {} + entries = data.get("violations", []) + if not isinstance(entries, list): + return {} + loaded: dict[str, str] = {} + for entry in entries: + if not isinstance(entry, dict): + continue + key = entry.get("key") + path = entry.get("path") + if isinstance(key, str) and isinstance(path, str): + loaded[key] = path + return loaded + + +def _stale_baseline_entries( + repository: Path, + baseline: dict[str, str], + linted_paths: Iterable[str], + fired_keys: Iterable[str], +) -> tuple[str, ...]: + """Report baseline entries that no longer describe a live violation. + + Only files this run actually linted can be judged: an entry for a file + outside the changed set is unobservable, not stale. A deleted file is + always stale -- it can never fire again. + """ + linted = set(linted_paths) + fired = set(fired_keys) + stale: list[str] = [] + for key, path in baseline.items(): + if not (repository / path).exists(): + stale.append( + f"{path}: stale baseline entry {key[:12]} -- the recorded file no " + f"longer exists; drop it from {_BASELINE_PATH.name} " + "(the baseline is shrink-only)" + ) + elif path in linted and key not in fired: + stale.append( + f"{path}: stale baseline entry {key[:12]} -- the recorded violation " + f"no longer fires; drop it from {_BASELINE_PATH.name} " + "(the baseline is shrink-only)" + ) + return tuple(sorted(stale)) + + +def validate_changed_sql( + repository: Path, + base_revision: str, + head_revision: str, + *, + ownership_manifest_paths: Sequence[Path], + baseline_path: Path | None = None, +) -> SqlGateOutcome: + """Lint every changed deployable SQL file against typed topology authority. + + ``baseline_path`` is opt-in. Omitting it means no grandfathering at all -- + every observed violation is returned. Production wiring passes + ``_BASELINE_PATH`` explicitly from ``main``. + """ + topology = load_topology_profile("local") + linted_paths: list[str] = [] + violations: list[str] = [] + try: + ownership_identities = load_application_database_ownership_identities( + ownership_manifest_paths + ) + except ValueError as exc: + ownership_identities = () + violations.append(f"ownership manifests: {exc}") + declared_identities = {identity.identity for identity in ownership_identities} + for path in changed_sql_paths(repository, base_revision, head_revision): + relative_path = path.relative_to(repository.resolve()) + # A legacy-default-schema exemption narrows to (1) the schema-qualification + # LINT and (2) the target-requirement ownership check for ALTER/DROP/ + # GRANT/etc against relations the exemption's own justification names as + # already-existing legacy tables (docker/migrations/forward/031 and 050's + # own `public.legacy_shape` ALTERs are exactly this case -- modifying a + # pre-existing legacy table is not new authority and was never required + # to carry an ownership declaration). CREATED-OBJECT ownership validation + # stays active regardless of the exemption: an exempted file that CREATEs + # new schema-qualified authority (e.g. 099's `omninode_internal.live_events`) + # must still carry a declared owner -- narrowing this way (OMN-15359, + # CodeRabbit) closes the gap the prior blanket `continue` left open, where + # ownership was unchecked even for an exempted file's newly created objects. + is_legacy_exempt = _is_legacy_default_schema_sql_path(relative_path) + sql = path.read_text(encoding="utf-8") + if not is_legacy_exempt: + linted_paths.append(str(relative_path)) + for violation in lint_application_database_sql(sql, topology): + violations.append(f"{relative_path}: {violation}") + for requirement in application_database_sql_target_requirements( + sql, topology + ): + location_matches = tuple( + identity + for identity in ownership_identities + if identity.schema == requirement.schema + and identity.name == requirement.name + ) + kind_matches = tuple( + identity + for identity in location_matches + if identity.kind in requirement.allowed_kinds + ) + if not location_matches: + violations.append( + f"{relative_path}: application target " + f"{requirement.schema}.{requirement.name} requires exactly " + "one ownership declaration" + ) + elif not kind_matches: + violations.append( + f"{relative_path}: application target " + f"{requirement.schema}.{requirement.name} requires an exact " + "object-kind ownership declaration" + ) + elif requirement.function_signature is not None and all( + identity.function_signature != requirement.function_signature + for identity in kind_matches + ): + violations.append( + f"{relative_path}: application target " + f"{requirement.schema}.{requirement.name}" + f"{requirement.function_signature} requires an exact " + "routine ownership declaration" + ) + for identity in application_database_created_catalog_identities(sql): + if identity.identity not in declared_identities: + violations.append( + f"{relative_path}: created application object " + f"{identity.identity!r} lacks an authoritative ownership declaration" + ) + + observed = tuple(sorted(set(violations))) + baseline = load_sql_baseline(baseline_path) if baseline_path is not None else {} + if not baseline: + return SqlGateOutcome(violations=observed, grandfathered=0) + + fired = {violation_key(violation): violation for violation in observed} + unbaselined = tuple( + violation for key, violation in fired.items() if key not in baseline + ) + stale = _stale_baseline_entries(repository, baseline, linted_paths, fired) + return SqlGateOutcome( + violations=tuple(sorted(unbaselined + stale)), + grandfathered=sum(1 for key in fired if key in baseline), + ) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", type=Path, default=Path.cwd()) + parser.add_argument("--base-revision", required=True) + parser.add_argument("--head-revision", default="HEAD") + parser.add_argument( + "--ownership-manifest", + action="append", + type=Path, + required=True, + help="Typed ownership manifest; repeat for every authoritative source", + ) + return parser + + +def main() -> int: + args = _parser().parse_args() + outcome = validate_changed_sql( + args.repository, + args.base_revision, + args.head_revision, + ownership_manifest_paths=tuple( + path if path.is_absolute() else args.repository / path + for path in args.ownership_manifest + ), + baseline_path=_BASELINE_PATH, + ) + # Announced every run, green or red: a silent grandfather count is how a + # frozen baseline quietly becomes permanent. + print( + f"application_database_sql_gate grandfathered={outcome.grandfathered} " + f"(frozen baseline {_BASELINE_PATH.name}; shrink-only)" + ) + if outcome.violations: + print("application_database_sql_gate=FAIL") + for violation in outcome.violations: + print(violation) + return 1 + print("application_database_sql_gate=PASS") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/check_flat_migration_foreign_connect.py b/scripts/ci/check_flat_migration_foreign_connect.py new file mode 100644 index 0000000000..45cefdfc5d --- /dev/null +++ b/scripts/ci/check_flat_migration_foreign_connect.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Fail CI on a NEW flat migration with an unledgered cross-DB `\\connect` (OMN-15819). + +The k8s Job that applies `docker/migrations/forward/*.sql` (flat, -maxdepth 1 +-- `omninode_infra` repo, `k8s/migrations/omnibase-infra-migrate.yaml`) owns +exactly one database, `omnibase_infra`. Its flat loop's `psql -f` apply is +gated on the file's first `\\connect` directive naming that same database; +a file whose `\\connect` names anything else has NO execution path in that +Job, in that loop or any other. OMN-15819 found two files (098/099) that had +silently accreted a false "applied" ledger row for exactly that reason. + +This is the STATIC, pre-merge half of the fix (the runner-side companion +lives in the `omninode_infra` repo). Every existing cross-DB flat file is +listed in ``docker/migrations/forward/cross-database-flat-migrations.yaml`` +with a citation -- see that file's own docstring for the two dispositions +(``undeliverable`` / ``grandfathered``) and why the distinction matters. A +NEW cross-DB flat file that is not in the manifest is a hard, fail-closed +reject: the fix is a node-owned migration under +``docker/migrations/forward/nodes//``, which connects to the target +database directly as its own role. + +Both directions are enforced -- a live cross-DB file missing from the +manifest, and a manifest entry that no longer matches live reality (file +gone, or no longer cross-DB) -- exactly the AC2 pattern +``tests/ci/test_flat_node_migration_shape_parity.py`` (OMN-15384) already +uses for its own ledger. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +FORWARD_DIR = REPO_ROOT / "docker" / "migrations" / "forward" +MANIFEST_PATH = FORWARD_DIR / "cross-database-flat-migrations.yaml" + +# The one database the k8s Job's flat loop ever \connects to for its own +# `psql -f` apply (DB_NAME in omninode_infra's +# k8s/migrations/omnibase-infra-migrate.yaml; the identical default in this +# repo's own scripts/run-forward-migrations.sh POSTGRES_DB and every +# docker-compose.infra.yml POSTGRES_DB binding for this service). +RUNNER_OWN_DATABASE = "omnibase_infra" + +_VALID_DISPOSITIONS = frozenset({"undeliverable", "grandfathered"}) + +# The manifest is a CLOSED ledger as of OMN-15819 (this gate's own +# authorship, commit 6083b76b4): "a NEW cross-DB flat file is a hard, +# fail-closed reject" (module docstring) means what it says regardless of +# disposition or citation -- a manifest entry alone must never be able to +# authorize a brand-new cross-DB flat migration, because the manifest is +# ordinary repo-tracked YAML a PR can edit in the same diff that adds the +# offending file. Without this, `check()` verified only that a live +# cross-DB file's manifest entry *exists and its connect_target matches* -- +# it never asked whether the entry was itself new, so a PR could add both +# the file and a plausible-looking manifest entry (any disposition) in one +# shot and pass clean (found in review, OMN-15819 CodeRabbit thread +# r3749990788). Pinning the exact filenames the manifest was seeded with at +# gate-authorship time closes that regardless of what the YAML says. +# +# `check()` takes this as an explicit, opt-in `frozen_seed` kwarg (default +# None = unrestricted) rather than a hardcoded default so every OTHER test +# in this suite -- which exercises stale-entry / target-drift / malformed +# scenarios against synthetic, non-production filenames -- is unaffected; +# only `main()` (the real CI entrypoint) and the tests that specifically +# cover this closed-ledger property pass it. +MANIFEST_FROZEN_SEED: frozenset[str] = frozenset( + { + "098_create_omninode_internal_schema.sql", + "099_create_omninode_internal_live_events.sql", + "083_create_log_entries.sql", + "096_grant_role_omnidash_omnidash_analytics.sql", + "097_grant_app_dashboard_connect_omnidash_analytics.sql", + } +) + +# Same directive shape the k8s Job's own awk one-liner recognizes: +# `awk '$1 == "\\connect" { print $2; exit }'` -- first line whose first +# whitespace-delimited field is the literal token `\connect`, first match +# wins. awk's default field splitting strips leading horizontal whitespace, +# so a line like ` \connect other_db` still has `$1 == "\\connect"` in the +# runner. `[^\S\r\n]*` mirrors that (leading spaces/tabs, not newlines) so +# an indented directive cannot silently read as "no \connect" to this gate +# while the runner still executes it as a real cross-DB connect. Mirrored +# here (not shelled out to awk) so this gate has no bash dependency and is +# independently testable. +_CONNECT_DIRECTIVE = re.compile(r"^[^\S\r\n]*\\connect\s+(\S+)", re.MULTILINE) + + +@dataclass(frozen=True) +class ManifestEntry: + file: str + connect_target: str + disposition: str + citation: str + + +@dataclass(frozen=True) +class Violation: + file: str + reason: str + + def describe(self) -> str: + return f"{self.file}: {self.reason}" + + +def flat_migration_connect_target(sql_path: Path) -> str | None: + """The file's first `\\connect ` directive target, or None if it has none.""" + match = _CONNECT_DIRECTIVE.search(sql_path.read_text(encoding="utf-8")) + return match.group(1) if match else None + + +def flat_migration_files(forward_dir: Path = FORWARD_DIR) -> list[Path]: + """Top-level (-maxdepth 1) `*.sql` files -- excludes `nodes/` deliberately. + + Matches the k8s Job's own discovery expression + (`find "$MIGRATION_DIR" -maxdepth 1 -name '*.sql' -type f`): a file under + `forward/nodes//` is a DIFFERENT corpus, applied by a DIFFERENT + loop that connects directly to its own target database, and is never in + scope for this gate. + """ + return sorted(forward_dir.glob("*.sql")) + + +def load_manifest(manifest_path: Path = MANIFEST_PATH) -> dict[str, ManifestEntry]: + raw = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + if not isinstance(raw, dict) or not isinstance(raw.get("entries"), list): + msg = f"{manifest_path} must be a mapping with a top-level `entries:` list" + raise AssertionError(msg) + entries: dict[str, ManifestEntry] = {} + for raw_entry in raw["entries"]: + entry = ManifestEntry( + file=raw_entry["file"], + connect_target=raw_entry["connect_target"], + disposition=raw_entry["disposition"], + citation=raw_entry["citation"], + ) + if entry.disposition not in _VALID_DISPOSITIONS: + msg = ( + f"{manifest_path}: {entry.file} has unknown disposition " + f"{entry.disposition!r} (expected one of {sorted(_VALID_DISPOSITIONS)})" + ) + raise AssertionError(msg) + if not entry.citation.strip(): + msg = f"{manifest_path}: {entry.file} has an empty citation" + raise AssertionError(msg) + if entry.file in entries: + msg = f"{manifest_path}: duplicate entry for {entry.file}" + raise AssertionError(msg) + entries[entry.file] = entry + return entries + + +def check( + forward_dir: Path = FORWARD_DIR, + manifest_path: Path = MANIFEST_PATH, + runner_own_database: str = RUNNER_OWN_DATABASE, + *, + frozen_seed: frozenset[str] | None = None, +) -> list[Violation]: + """Fail-closed, both directions. Empty return == gate passes. + + ``frozen_seed``, when provided, additionally rejects any manifest entry + for a filename outside that set -- see ``MANIFEST_FROZEN_SEED`` above. + ``None`` (the default) skips that check entirely, which is what every + synthetic-fixture test in this suite that is not specifically about the + closed-ledger property wants. + """ + manifest = load_manifest(manifest_path) + live_cross_db: dict[str, str] = {} + for sql_path in flat_migration_files(forward_dir): + target = flat_migration_connect_target(sql_path) + if target is not None and target != runner_own_database: + live_cross_db[sql_path.name] = target + + violations: list[Violation] = [] + try: + manifest_display = manifest_path.relative_to(REPO_ROOT) + except ValueError: + manifest_display = manifest_path + + for filename, target in sorted(live_cross_db.items()): + entry = manifest.get(filename) + if entry is None: + violations.append( + Violation( + file=filename, + reason=( + f"flat migration carries `\\connect {target}` (foreign to " + f"the runner's own database {runner_own_database!r}) but has " + f"NO entry in {manifest_display}. This " + "file has no execution path in the k8s Job (OMN-15819) -- " + "author a node-owned replacement under " + "docker/migrations/forward/nodes// instead of adding " + "a flat cross-DB migration." + ), + ) + ) + elif frozen_seed is not None and filename not in frozen_seed: + violations.append( + Violation( + file=filename, + reason=( + f"flat migration carries `\\connect {target}` and has a " + f"matching entry in {manifest_display} (disposition=" + f"{entry.disposition!r}), but {filename!r} is not part of " + "the frozen OMN-15819 seed set -- a NEW cross-DB flat " + "migration is a hard reject regardless of disposition or " + "citation; a manifest entry alone cannot authorize one. " + "Author a node-owned replacement under " + "docker/migrations/forward/nodes// instead." + ), + ) + ) + elif entry.connect_target != target: + violations.append( + Violation( + file=filename, + reason=( + f"manifest says connect_target={entry.connect_target!r} but " + f"the file now targets {target!r} -- update the manifest " + "entry to match live reality" + ), + ) + ) + + for filename, entry in sorted(manifest.items()): + if filename not in live_cross_db: + reason = ( + "manifest entry has no live counterpart -- the file is gone, or no " + "longer carries a foreign \\connect. Remove the stale entry." + ) + violations.append(Violation(file=filename, reason=reason)) + + return violations + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--forward-dir", + type=Path, + default=FORWARD_DIR, + help="docker/migrations/forward directory to scan", + ) + parser.add_argument( + "--manifest", + type=Path, + default=MANIFEST_PATH, + help="cross-database-flat-migrations.yaml path", + ) + args = parser.parse_args(argv) + + violations = check( + forward_dir=args.forward_dir, + manifest_path=args.manifest, + frozen_seed=MANIFEST_FROZEN_SEED, + ) + if not violations: + print("OK: no un-ledgered cross-DB flat migrations (OMN-15819)") + return 0 + + print("FAIL: cross-DB flat migration gate (OMN-15819)", file=sys.stderr) + for violation in violations: + print(f" - {violation.describe()}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/check_incident_replay_coverage.py b/scripts/ci/check_incident_replay_coverage.py new file mode 100644 index 0000000000..71f2afc8b8 --- /dev/null +++ b/scripts/ci/check_incident_replay_coverage.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Every enforcement guard must replay a REAL incident (OMN-15547). + +WHY THIS EXISTS +--------------- +A guard that has never been run against the real thing it exists to catch is +decorative. On 2026-07-30 that failed three times in one day, with an identical +shape every time -- *the guard was tested against a synthetic input that cannot +exhibit the failure*: + +1. The workflow-pin validators are shape-only. Re-running + ``tests/ci/test_merge_hold_gate_omn15484.py`` against ``879d6fc6`` -- the + exact pin that wedged every ``omnibase_infra`` PR for ~2.5h (OMN-15536) -- + returns ``2 passed``. Any 40-hex string satisfies them. +2. ``tests/unit/scripts/test_omninode_system_slack_report.py`` drove the health + reporter with a **63-byte** ``HEALTHY_BODY`` while every real ``/health`` body + on ``.201`` is 2079-2644 bytes. The reporter truncated to 180 bytes before + ``jq``; under the short fixture the truncation never bit, so a 654-line suite + stayed green while the deployed artifact reported CRITICAL for all three + lanes against a fully healthy fleet (OMN-15525). +3. A merge-hold falsifier was validated under ``bash -c``/``sh -c`` rather than + the runner that executes it. The DoD runner's admissibility predicate reads + *command position*, so the probe -- wrapped in ``python3 -c "..."`` -- was + classified ``NOT_EXECUTED``: the check never ran at all (OMN-15484). + +In all three the guard was green, the enforcement was zero, and nothing in CI +could tell the difference. This lint makes that difference machine-visible. + +WHAT IT ENFORCES +---------------- +``tests/incident_replays/registry.yaml`` declares **incident replay cases**. A +case is REAL only if all five rules hold -- these are the honest bar, written so +a machine can apply them: + + R1 COMMITTED BYTES, UNMODIFIED. ``artifact.fixture`` names a file in the + repo (never a literal inside a test) and ``sha256(bytes)`` equals + ``artifact.sha256``. Editing a captured artifact after the fact breaks the + "this is what actually happened" claim, so it must break the build. + R2 RE-FETCHABLE, NON-AUTHORED ORIGIN. ``capture.source`` matches one of the + locator grammars in :data:`LOCATOR_GRAMMARS`. This is the discriminator + against a hand-typed approximation: an invented payload has no locator + that resolves. Free-text provenance ("copied from the live body", "same + shape as prod") is REJECTED -- that prose is exactly what the OMN-15525 + fix wrote above a fixture it had typed by hand. + R3 REAL INCIDENT. ``incident`` is an ``OMN-`` ticket or an + ``/#`` pull request. + R4 THE CASE IS ACTUALLY CONSUMED. ``test`` names a file that exists and that + references the fixture path. A registry entry nobody reads is paperwork. + R5 WOULD HAVE CAUGHT IT. The case pins the verdict the buggy guard got + WRONG, and ``regression_class`` says which direction it got wrong: + ``false_green`` -- the guard said OK on a real BAD input, so the case's + ``guard_verdict_on_artifact`` must be ``reject`` (the pin outage: the + validator passed ``879d6fc6``). + ``false_red`` -- the guard said FAIL on a real GOOD input, so the + verdict must be ``accept`` (the health alert: CRITICAL on all three + lanes against a healthy fleet). A ``false_red`` case must also name a + ``discriminator`` test, because an accept-only proof cannot tell a + working guard from one that is stuck open. + +Coverage is enforced by two ratchets plus a default-deny: + + * ``scope.required_guards`` -- each entry MUST have >=1 valid case. Append-only. + An entry whose guard file does not exist yet is reported ``PENDING`` and does + not fail: that is how a requirement is armed *before* the guard lands. + * ``scope.debt_baseline`` -- the wired guards that have no case yet. This list + may only shrink. A covered guard left in the baseline fails (the list must + stay truthful); a baseline entry that is no longer wired fails (delete it). + * DEFAULT-DENY -- any newly wired guard in neither list and with no case fails. + **This is the load-bearing property**: a new gate cannot ship without a real + replay case, which is what stops the shelf growing faster than the proof. + +Exit codes: ``0`` every rule holds, ``1`` any violation. + +Related: OMN-15547 (this), OMN-15536/OMN-15538 (pins), OMN-15525/OMN-15509 +(health alert), OMN-15484/OMN-15483 (merge hold), OMN-15309 (the OCC +admissibility corpus, the closest pre-existing exemplar of the practice). +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any + +import yaml + +REGISTRY_REL = "tests/incident_replays/registry.yaml" + +# -------------------------------------------------------------------------- +# Wired-guard inventory +# -------------------------------------------------------------------------- +# A guard counts as WIRED when a repo-local script under one of these roots is +# named by a pre-commit hook entry or by a workflow `run:` line. Restricting to +# these roots keeps the inventory to things that are plausibly enforcement and +# keeps it deterministic -- it reads only committed files. +GUARD_ROOTS: tuple[str, ...] = ("scripts/", "deploy/") +GUARD_SUFFIXES: tuple[str, ...] = (".py", ".sh") +# Paths inside a guard root that are not themselves guards. +GUARD_EXCLUDE_PARTS: tuple[str, ...] = ("/tests/", "/__pycache__/", "/test_") + +_PATH_RE = re.compile(r"(?:scripts|deploy)/[A-Za-z0-9_.@/-]+\.(?:py|sh)") + +# -------------------------------------------------------------------------- +# R2 -- locator grammars. Each names an origin somebody else can re-fetch. +# -------------------------------------------------------------------------- +LOCATOR_GRAMMARS: dict[str, re.Pattern[str]] = { + # A REST path that pins ONE specific resource -- either a numeric id + # (run/PR/check/job) or a 40-hex object. A path that only names a + # collection ("…/actions/runs") is refused: it does not identify what was + # captured, so it cannot be re-fetched to compare. + # gh-api:repos/OmniNode-ai/omnibase_infra/actions/runs/30574058377/jobs + # gh-api:repos/OmniNode-ai/omnimarket/compare/dev...879d6fc6825f8764… + "gh-api": re.compile( + r"^gh-api:[A-Za-z0-9._/-]*(?:/\d{2,}|[0-9a-f]{40})[A-Za-z0-9._/?=&,-]*$" + ), + # git-object:OmniNode-ai/omnimarket@879d6fc6...:.github/workflows/x.yml + "git-object": re.compile( + r"^git-object:[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@[0-9a-f]{40}:[A-Za-z0-9_./-]+$" + ), + # host-file:omni-201-ts:/data/maintenance/bin/report.sh + "host-file": re.compile(r"^host-file:[A-Za-z0-9_.-]+:/[A-Za-z0-9_./+-]+$"), + # live-http:omni-201-ts:8085/health + "live-http": re.compile(r"^live-http:[A-Za-z0-9_.-]+:\d{2,5}/[A-Za-z0-9_./?=&-]*$"), + # ci-artifact:30574058377/coverage-shard-3 + "ci-artifact": re.compile(r"^ci-artifact:\d{6,}/[A-Za-z0-9_.-]+$"), + # container-probe:ghcr.io/omninode-ai/omnibase-infra-runtime@sha256:<64hex>:/app/config + # + # Added by OMN-15676, which could not otherwise be replayed at all. That + # incident's failing artifact lives INSIDE a built image: runner_fleet.yaml + # was tracked, valid and referenced, and absent only from the image, so + # every locator above points at a surface that was correct at the time. + # The image MUST be digest-pinned -- a tag is mutable, so a tag-locator + # would let the bytes behind a case silently change and is exactly the + # "capture that stops being the capture" R1 exists to prevent. Re-fetch: + # docker run --rm --entrypoint sh -c 'ls -R ' + "container-probe": re.compile( + r"^container-probe:[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}:/[A-Za-z0-9_./,+-]*$" + ), +} + +INCIDENT_RE = re.compile(r"^(?:OMN-\d+|[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+#\d+)$") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +# R1 -- a capture must end in this suffix. Formatting hooks key off the +# trailing extension, so a fixture left as `.json`/`.yaml`/`.py` is silently +# rewritten by end-of-file-fixer or a formatter and stops being the bytes +# that failed. Learned by execution: that is exactly what happened to these +# fixtures on their first commit here, and R1 is what caught it. +CAPTURED_SUFFIX = ".captured" +TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2})?Z$") +VALID_VERDICTS = frozenset({"reject", "accept"}) +# R5 -- which direction the buggy guard got wrong, and the verdict that pins it. +REGRESSION_CLASSES: dict[str, str] = { + "false_green": "reject", # guard said OK on a real BAD input + "false_red": "accept", # guard said FAIL on a real GOOD input +} + + +@dataclass +class Finding: + rule: str + subject: str + detail: str + + def render(self) -> str: + return f" [{self.rule}] {self.subject}\n {self.detail}" + + +@dataclass +class Result: + findings: list[Finding] = field(default_factory=list) + pending: list[str] = field(default_factory=list) + covered: set[str] = field(default_factory=set) + wired: set[str] = field(default_factory=set) + + def fail(self, rule: str, subject: str, detail: str) -> None: + self.findings.append(Finding(rule, subject, detail)) + + +def _read(path: Path) -> str: + try: + return path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + + +def _is_guardish(rel: str) -> bool: + if not rel.startswith(GUARD_ROOTS): + return False + if not rel.endswith(GUARD_SUFFIXES): + return False + probe = "/" + rel + return not any(part in probe for part in GUARD_EXCLUDE_PARTS) + + +def wired_guards(repo_root: Path) -> set[str]: + """Repo-relative paths of scripts wired as pre-commit or workflow enforcement.""" + found: set[str] = set() + + def consider(text: str) -> None: + for match in _PATH_RE.finditer(text): + rel = match.group(0) + if not _is_guardish(rel): + continue + if (repo_root / rel).is_file(): + found.add(rel) + + consider(_read(repo_root / ".pre-commit-config.yaml")) + workflows = repo_root / ".github" / "workflows" + if workflows.is_dir(): + for wf in sorted(workflows.iterdir()): + if wf.suffix in {".yml", ".yaml"}: + consider(_read(wf)) + return found + + +# -------------------------------------------------------------------------- +# Case validation (R1-R5) +# -------------------------------------------------------------------------- +def _validate_case(repo_root: Path, idx: int, case: Any, result: Result) -> str | None: + """Validate one registry case. Returns the guard path if the case is REAL.""" + label = f"cases[{idx}]" + if not isinstance(case, dict): + result.fail("SCHEMA", label, "case must be a mapping") + return None + + case_id = case.get("id") or label + label = f"case {case_id}" + + guard = case.get("guard") + if not isinstance(guard, str) or not guard: + result.fail("SCHEMA", label, "missing required key `guard`") + return None + if not (repo_root / guard).exists(): + result.fail( + "SCHEMA", + label, + f"`guard: {guard}` does not exist in this repo -- a case cannot " + "cover a guard that is not here", + ) + return None + + ok = True + + # R3 -- real incident id. + incident = case.get("incident") + if not isinstance(incident, str) or not INCIDENT_RE.match(incident): + result.fail( + "R3", + label, + f"`incident: {incident!r}` must be `OMN-` or `/#`; " + "a replay case has to name the failure it replays", + ) + ok = False + + # R1 -- committed bytes, unmodified. + artifact = case.get("artifact") + if not isinstance(artifact, dict): + result.fail("R1", label, "missing `artifact:` mapping") + ok = False + else: + fixture = artifact.get("fixture") + declared = artifact.get("sha256") + if not isinstance(fixture, str) or not fixture: + result.fail( + "R1", + label, + "`artifact.fixture` must name a committed FILE. A literal typed " + "into the test is not a capture -- that is the OMN-15525 defect.", + ) + ok = False + elif not fixture.endswith(CAPTURED_SUFFIX): + result.fail( + "R1", + label, + f"`artifact.fixture: {fixture}` must end in `{CAPTURED_SUFFIX}` so " + "no formatter claims it. A capture left as `.json`/`.yaml`/`.py` " + "gets rewritten by end-of-file-fixer or a formatter and quietly " + "stops being the bytes that failed.", + ) + ok = False + else: + fpath = repo_root / fixture + if not fpath.is_file(): + result.fail( + "R1", label, f"`artifact.fixture: {fixture}` does not exist" + ) + ok = False + elif not isinstance(declared, str) or not SHA256_RE.match(declared): + result.fail( + "R1", label, "`artifact.sha256` must be 64 lowercase hex digits" + ) + ok = False + else: + actual = hashlib.sha256(fpath.read_bytes()).hexdigest() + if actual != declared: + result.fail( + "R1", + label, + f"{fixture} has been modified since capture: " + f"declared {declared}, actual {actual}. An edited artifact " + "is no longer the thing that failed.", + ) + ok = False + + # R2 -- re-fetchable, non-authored origin. + capture = case.get("capture") + if not isinstance(capture, dict): + result.fail("R2", label, "missing `capture:` mapping") + ok = False + else: + source = capture.get("source") + if not isinstance(source, str) or not any( + pattern.match(source) for pattern in LOCATOR_GRAMMARS.values() + ): + result.fail( + "R2", + label, + f"`capture.source: {source!r}` is not a re-fetchable locator. " + f"Allowed grammars: {', '.join(sorted(LOCATOR_GRAMMARS))}. " + "Free-text provenance is rejected on purpose -- a hand-typed " + "fixture has no locator that resolves.", + ) + ok = False + # PyYAML resolves an unquoted ISO-8601 scalar to a datetime, so accept + # both spellings rather than failing an author for YAML's type coercion. + captured_at = capture.get("captured_at") + if isinstance(captured_at, datetime): + captured_at = captured_at.strftime("%Y-%m-%dT%H:%M:%SZ") + if not isinstance(captured_at, str) or not TIMESTAMP_RE.match(captured_at): + result.fail( + "R2", label, "`capture.captured_at` must be an ISO-8601 UTC timestamp" + ) + ok = False + + # R5 -- would-have-caught, in the direction the guard actually got wrong. + verdict = case.get("guard_verdict_on_artifact") + regression_class = case.get("regression_class") + if verdict not in VALID_VERDICTS: + result.fail( + "R5", + label, + f"`guard_verdict_on_artifact` must be one of {sorted(VALID_VERDICTS)}", + ) + ok = False + if regression_class not in REGRESSION_CLASSES: + result.fail( + "R5", + label, + f"`regression_class` must be one of {sorted(REGRESSION_CLASSES)}: " + "which direction did the buggy guard get wrong?", + ) + ok = False + elif verdict != REGRESSION_CLASSES[regression_class]: + result.fail( + "R5", + label, + f"`regression_class: {regression_class}` requires " + f"`guard_verdict_on_artifact: {REGRESSION_CLASSES[regression_class]}`, " + f"got {verdict!r}", + ) + ok = False + elif regression_class == "false_red": + discriminator = case.get("discriminator") + if not isinstance(discriminator, str) or not discriminator: + result.fail( + "R5", + label, + "a `false_red` case proves only that the guard ACCEPTS a real " + "good input, which a stuck-open guard also does. Name a " + "`discriminator:` test that proves the same guard still says NO.", + ) + ok = False + elif not (repo_root / discriminator.split("::", 1)[0]).is_file(): + result.fail( + "R5", + label, + f"`discriminator: {discriminator}` -- file does not exist", + ) + ok = False + + # R4 -- the case is actually consumed by a test. + test_ref = case.get("test") + if not isinstance(test_ref, str) or not test_ref: + result.fail("R4", label, "missing `test:` node id") + ok = False + else: + test_file = test_ref.split("::", 1)[0] + tpath = repo_root / test_file + if not tpath.is_file(): + result.fail( + "R4", label, f"`test: {test_ref}` -- {test_file} does not exist" + ) + ok = False + elif isinstance(artifact, dict) and isinstance(artifact.get("fixture"), str): + fixture = artifact["fixture"] + body = _read(tpath) + needle = Path(fixture).name + if needle not in body and fixture not in body: + result.fail( + "R4", + label, + f"{test_file} never references {needle}: the registry claims " + "a replay the test does not perform", + ) + ok = False + + return guard if ok else None + + +# -------------------------------------------------------------------------- +def evaluate(repo_root: Path) -> Result: + result = Result() + registry_path = repo_root / REGISTRY_REL + + if not registry_path.is_file(): + result.fail("REGISTRY", REGISTRY_REL, "registry file is missing") + return result + + try: + registry = yaml.safe_load(registry_path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as exc: + result.fail("REGISTRY", REGISTRY_REL, f"unparseable: {exc}") + return result + + scope = registry.get("scope") or {} + required = list(scope.get("required_guards") or []) + baseline = list(scope.get("debt_baseline") or []) + cases = list(registry.get("cases") or []) + + for idx, case in enumerate(cases): + guard = _validate_case(repo_root, idx, case, result) + if guard: + result.covered.add(guard) + + result.wired = wired_guards(repo_root) + + # required_guards -- hard requirement, with a PENDING escape only for a + # guard that does not exist yet (arming a requirement before the guard lands). + for guard in required: + if not (repo_root / guard).exists(): + result.pending.append(guard) + continue + if guard not in result.covered: + result.fail( + "COVERAGE", + guard, + "listed in scope.required_guards but has no valid incident replay " + f"case. Add one to {REGISTRY_REL} (see the module docstring for " + "R1-R5).", + ) + + # The baseline must stay truthful in both directions. + baseline_set = set(baseline) + for guard in sorted(baseline_set & result.covered): + result.fail( + "RATCHET", + guard, + "is covered by a replay case but still listed in scope.debt_baseline. " + "Remove the line -- the baseline may only shrink, and it has to be " + "readable as the real outstanding debt.", + ) + for guard in sorted(baseline_set - result.wired): + if (repo_root / guard).exists(): + result.fail( + "RATCHET", + guard, + "is in scope.debt_baseline but is no longer wired as enforcement. " + "Delete the line.", + ) + else: + result.fail( + "RATCHET", + guard, + "is in scope.debt_baseline but does not exist. Delete the line.", + ) + + # DEFAULT-DENY -- the load-bearing rule. + known = baseline_set | set(required) | result.covered + for guard in sorted(result.wired - known): + result.fail( + "DEFAULT-DENY", + guard, + "is wired as enforcement but carries no incident replay case and is " + f"not in scope.debt_baseline. A NEW guard must ship with a real " + f"regression case sourced from an actual failure -- add it to " + f"{REGISTRY_REL}. Baselining a genuinely new guard instead of " + "replaying it is the behaviour this rule exists to stop.", + ) + + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--repo-root", + default=str(Path(__file__).resolve().parents[2]), + help="repository root to audit (default: this script's repo)", + ) + parser.add_argument( + "--json", action="store_true", help="emit machine-readable JSON" + ) + parser.add_argument( + "filenames", + nargs="*", + help="ignored; accepted so the check can run as a pre-commit hook", + ) + args = parser.parse_args(argv) + + repo_root = Path(args.repo_root).resolve() + result = evaluate(repo_root) + + if args.json: + print( + json.dumps( + { + "ok": not result.findings, + "wired_guards": len(result.wired), + "covered_guards": sorted(result.covered), + "pending_guards": sorted(result.pending), + "findings": [ + {"rule": f.rule, "subject": f.subject, "detail": f.detail} + for f in result.findings + ], + }, + indent=2, + sort_keys=True, + ) + ) + return 1 if result.findings else 0 + + print( + f"incident-replay coverage: {len(result.covered)} guard(s) covered, " + f"{len(result.wired)} wired, {len(result.pending)} pending" + ) + for guard in sorted(result.pending): + print(f" PENDING {guard} (required, guard not present yet)") + if result.findings: + print(f"\nFAIL -- {len(result.findings)} violation(s):\n") + for finding in result.findings: + print(finding.render()) + print( + "\nA guard that has never been replayed against the real incident it " + "exists to catch is decorative (OMN-15547)." + ) + return 1 + print("OK -- every required guard replays a real incident.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/check_occ_companion_merged.py b/scripts/ci/check_occ_companion_merged.py new file mode 100644 index 0000000000..bf15db66a6 --- /dev/null +++ b/scripts/ci/check_occ_companion_merged.py @@ -0,0 +1,374 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""OCC companion-merged gate (OMN-15214) — a STRICT ``CI Summary`` gate. + +Why this exists +--------------- +On 2026-07-26 an automated "OCC queue hygiene" pass closed five OPEN +onex_change_control evidence companions (#5012-#5016) whose product PRs had +already MERGED, destroying three evidence chains (OMN-15199 / OMN-15200 / +OMN-15203) with no successor. The sweep's trigger state is exactly +"OPEN companion + MERGED product PR". + +This gate makes that state unreachable at the merge boundary for this repo: +a product PR's ``CI Summary`` (the sole required branch-protection context on +omnibase_infra) cannot go green until the OCC evidence cited by the PR's +``Evidence-Source:`` line is DURABLE — i.e. the cited companion PR is MERGED, +or the cited commit SHA is an ancestor of an onex_change_control durable +branch (dev/main). Because the product PR cannot merge before its companion +does, "merged product + open companion" can no longer arise via the merge +path, and a companion-closing sweep has nothing load-bearing to destroy. + +Deliberately NOT a new required status check: this job is registered in +:data:`scripts.ci.ci_summary_gate.STRICT_GATE_JOBS` and enforced through the +existing fail-closed ``CI Summary`` umbrella poller. Adding a new top-level +required context that does not report on every PR shape wedges merges +indefinitely (see CLAUDE.md, deploy-gate section); the umbrella pattern has no +such failure mode because its check-run always instantiates. + +Verdict model (mirrors ci_summary_gate exit codes) +-------------------------------------------------- +* ``PASS`` (0) — evidence is durable, or the gate does not apply + (non-PR event; trusted dependency-bot author, mirroring occ-preflight's + OMN-13762 exemption). +* ``PENDING`` (2) — evidence may still become durable without a new commit: + Evidence-Source not yet PATCHed onto the body by occ-autobind, companion + still OPEN (auto-merge in flight), or a transient API error. The runner + entrypoint polls; at the deadline PENDING converts to FAIL (fail-closed). +* ``FAIL`` (1) — evidence can never become durable in this state: + companion CLOSED without merging (the incident state), cited SHA not an + ancestor of an OCC durable branch (squash-only merges guarantee a + feature-branch head SHA never becomes one — the OMN-15216 defect), or a + malformed Evidence-Source value. + +The companion-must-merge-first ordering is safe: onex_change_control PRs have +no reverse dependency on product-PR merge state (occ-preflight validates OCC's +own PRs from their in-tree diff), and repo-level auto-merge is enabled there. +The incident's canary lane proved the ordering live: companion OCC#5008 merged +40+ minutes before its product PR. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess # fixed argv, no shell, trusted gh binary +import sys +import time +from dataclasses import dataclass + +OCC_REPO_DEFAULT = "OmniNode-ai/onex_change_control" + +# Branches on which an OCC commit SHA counts as durable evidence. +OCC_DURABLE_BRANCHES: tuple[str, ...] = ("dev", "main") + +# Mirrors occ-preflight's OMN-13762 dependency-bot exemption +# (validator_receipt_gate.DEPENDENCY_BOT_AUTHORS): bot-authored dependency +# bumps structurally cannot cite OCC evidence. +DEPENDENCY_BOT_AUTHORS: frozenset[str] = frozenset( + { + "dependabot[bot]", + "app/dependabot", + "dependabot", + "renovate[bot]", + "app/renovate", + "renovate", + } +) + +# Events on which the gate enforces (mirrors occ-preflight's event scope). +ENFORCED_EVENTS: frozenset[str] = frozenset({"pull_request", "merge_group"}) + +EVIDENCE_SOURCE_RE = re.compile( + r"^Evidence-Source:\s+(\S.*)$", re.IGNORECASE | re.MULTILINE +) +OCC_PR_REF_RE = re.compile(r"^OCC#(\d+)$", re.IGNORECASE) +HEX_SHA_RE = re.compile(r"^[0-9a-f]{7,40}$") +MERGE_GROUP_PR_RE = re.compile(r"/pr-(\d+)-") + +EXIT_PASS = 0 +EXIT_FAIL = 1 +EXIT_PENDING = 2 + +_VERDICT_NAMES = {EXIT_PASS: "PASS", EXIT_FAIL: "FAIL", EXIT_PENDING: "PENDING"} + + +@dataclass(frozen=True) +class Verdict: + """Terminal or poll-again outcome of a single gate evaluation.""" + + code: int # EXIT_PASS | EXIT_FAIL | EXIT_PENDING + reason: str + + @property + def name(self) -> str: + return _VERDICT_NAMES[self.code] + + +class GhFetcher: + """Live GitHub reads via the ``gh`` CLI. Every failure returns ``None`` + so the caller decides between PENDING (retryable) and FAIL (terminal).""" + + def _run(self, argv: list[str]) -> str | None: + try: + result = subprocess.run( # fixed argv, no shell + argv, capture_output=True, text=True, timeout=60, check=False + ) + except (OSError, subprocess.TimeoutExpired) as exc: + print(f"::warning::gh invocation failed: {exc}", file=sys.stderr) + return None + if result.returncode != 0: + print( + f"::warning::{' '.join(argv[:4])}... exited " + f"{result.returncode}: {result.stderr.strip()[:300]}", + file=sys.stderr, + ) + return None + return result.stdout + + def pr_view(self, repo: str, number: str, fields: str) -> dict[str, object] | None: + raw = self._run( + ["gh", "pr", "view", str(number), "--repo", repo, "--json", fields] + ) + if raw is None: + return None + try: + data = json.loads(raw) + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + def compare_status(self, repo: str, base: str, head_sha: str) -> str | None: + """``identical``/``behind`` ⇒ ``head_sha`` is an ancestor of ``base``.""" + raw = self._run( + [ + "gh", + "api", + f"repos/{repo}/compare/{base}...{head_sha}", + "--jq", + ".status", + ] + ) + return raw.strip() if raw is not None else None + + +def parse_evidence_source(body: str) -> str | None: + """First ``Evidence-Source:`` value in the PR body, or ``None``.""" + match = EVIDENCE_SOURCE_RE.search(body or "") + return match.group(1).strip() if match else None + + +def resolve_pr_number( + event_name: str, pr_number: str, merge_group_head_ref: str +) -> str: + """PR number for pull_request or merge_group events ('' if unresolvable).""" + if pr_number: + return pr_number + if event_name == "merge_group" and merge_group_head_ref: + match = MERGE_GROUP_PR_RE.search(merge_group_head_ref) + if match: + return match.group(1) + return "" + + +def evaluate_once( + fetcher: GhFetcher, + *, + event_name: str, + repo: str, + pr_number: str, + occ_repo: str = OCC_REPO_DEFAULT, + evidence_source_override: str | None = None, +) -> Verdict: + """One poll iteration. PENDING means the state may still resolve itself + (poll again); FAIL means it never can (terminal).""" + + if event_name not in ENFORCED_EVENTS: + return Verdict( + EXIT_PASS, + f"event '{event_name}' is not a merge-gating event; gate not applicable", + ) + + if not pr_number: + return Verdict( + EXIT_FAIL, + "could not resolve a PR number for this run — failing closed", + ) + + if evidence_source_override is None: + # Live body, never the event payload: occ-autobind PATCHes + # Evidence-Source onto the body AFTER the triggering event fired. + pr_data = fetcher.pr_view(repo, pr_number, "body,author") + if pr_data is None: + return Verdict( + EXIT_PENDING, f"could not fetch {repo}#{pr_number} (retryable)" + ) + + author_raw = pr_data.get("author") + author = "" + if isinstance(author_raw, dict): + author = str(author_raw.get("login") or "") + if author in DEPENDENCY_BOT_AUTHORS: + return Verdict( + EXIT_PASS, + f"trusted dependency-bot author '{author}' — occ-preflight OMN-13762 " + "exemption mirrored; no OCC evidence applicable", + ) + + evidence_source = parse_evidence_source(str(pr_data.get("body") or "")) + else: + evidence_source = evidence_source_override + + if not evidence_source: + return Verdict( + EXIT_PENDING, + f"{repo}#{pr_number} body has no 'Evidence-Source:' line yet " + "(occ-autobind mint may still be in flight)", + ) + + occ_ref = OCC_PR_REF_RE.match(evidence_source) + if occ_ref: + occ_pr = occ_ref.group(1) + occ_data = fetcher.pr_view(occ_repo, occ_pr, "state,mergeCommit") + if occ_data is None: + return Verdict( + EXIT_PENDING, + f"could not fetch companion {occ_repo}#{occ_pr} (retryable)", + ) + state = str(occ_data.get("state") or "").upper() + if state == "MERGED": + merge_commit = occ_data.get("mergeCommit") + merge_oid = "" + if isinstance(merge_commit, dict): + merge_oid = str(merge_commit.get("oid") or "") + return Verdict( + EXIT_PASS, + f"companion OCC#{occ_pr} is MERGED (merge commit {merge_oid or 'unknown'}) " + "— evidence is durable", + ) + if state == "OPEN": + return Verdict( + EXIT_PENDING, + f"companion OCC#{occ_pr} is still OPEN — the companion must MERGE " + "before this product PR may merge (OMN-15214). Land the companion " + "on onex_change_control, then re-run this job.", + ) + # CLOSED without merging: the exact state the 2026-07-26 hygiene sweep + # minted — the evidence was destroyed. Never poll; fail loudly. + return Verdict( + EXIT_FAIL, + f"companion OCC#{occ_pr} is {state or 'UNRESOLVED'} without merging — " + "the cited evidence no longer exists. Re-cut the companion (bind it to " + "this PR) and update Evidence-Source before merging.", + ) + + if HEX_SHA_RE.match(evidence_source.lower()): + sha = evidence_source.lower() + saw_api_error = False + for branch in OCC_DURABLE_BRANCHES: + status = fetcher.compare_status(occ_repo, branch, sha) + if status is None: + saw_api_error = True + continue + if status in ("identical", "behind"): + return Verdict( + EXIT_PASS, + f"Evidence-Source SHA {sha} is an ancestor of {occ_repo}@{branch} " + "— evidence is durable", + ) + if saw_api_error: + return Verdict( + EXIT_PENDING, + f"could not resolve Evidence-Source SHA {sha} against " + f"{occ_repo} durable branches (retryable)", + ) + # onex_change_control is squash-only: a feature-branch head SHA can + # NEVER become an ancestor of dev/main, so this is terminal — it is the + # strandable pre-merge pin OMN-15216 describes. + return Verdict( + EXIT_FAIL, + f"Evidence-Source SHA {sha} is not an ancestor of any durable " + f"{occ_repo} branch {OCC_DURABLE_BRANCHES} — cite 'OCC#' (which " + "must be MERGED) or a merged OCC commit SHA, never a feature-branch head.", + ) + + return Verdict( + EXIT_FAIL, + f"Evidence-Source value '{evidence_source}' is neither 'OCC#' nor a " + "hex commit SHA — fix the PR body.", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", default=os.environ.get("GH_REPO", "")) + parser.add_argument("--pr-number", default=os.environ.get("PR_NUMBER", "")) + parser.add_argument( + "--event-name", default=os.environ.get("GITHUB_EVENT_NAME", "pull_request") + ) + parser.add_argument( + "--merge-group-head-ref", default=os.environ.get("MERGE_GROUP_HEAD_REF", "") + ) + parser.add_argument( + "--occ-repo", default=os.environ.get("OCC_REPO", OCC_REPO_DEFAULT) + ) + parser.add_argument( + "--evidence-source", + default=None, + help="Override: evaluate this Evidence-Source value directly instead of " + "reading the PR body (diagnostics / dry-run).", + ) + parser.add_argument( + "--once", + action="store_true", + help="Single evaluation, no polling; exits 0/1/2 (PASS/FAIL/PENDING).", + ) + parser.add_argument( + "--deadline-seconds", + type=int, + default=int(os.environ.get("DEADLINE_SECONDS", "1500")), + ) + parser.add_argument( + "--poll-interval-seconds", + type=int, + default=int(os.environ.get("POLL_INTERVAL_SECONDS", "30")), + ) + args = parser.parse_args(argv) + + pr_number = resolve_pr_number( + args.event_name, args.pr_number, args.merge_group_head_ref + ) + fetcher = GhFetcher() + deadline = time.monotonic() + args.deadline_seconds + + while True: + verdict = evaluate_once( + fetcher, + event_name=args.event_name, + repo=args.repo, + pr_number=pr_number, + occ_repo=args.occ_repo, + evidence_source_override=args.evidence_source, + ) + print(f"occ-companion-merged gate: {verdict.name} — {verdict.reason}") + + if verdict.code != EXIT_PENDING or args.once: + if verdict.code == EXIT_FAIL: + print(f"::error::{verdict.reason}") + return verdict.code + + if time.monotonic() >= deadline: + print( + f"::error::occ-companion-merged gate: poll deadline " + f"({args.deadline_seconds}s) reached while still PENDING — failing " + f"closed. Last state: {verdict.reason}" + ) + return EXIT_FAIL + + time.sleep(args.poll_interval_seconds) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/check_pin_reachability.py b/scripts/ci/check_pin_reachability.py new file mode 100644 index 0000000000..6b96836c0b --- /dev/null +++ b/scripts/ci/check_pin_reachability.py @@ -0,0 +1,1043 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Every cross-repo pin must name a commit REACHABLE FROM A PROTECTED BRANCH (OMN-15538). + +The confusion this exists to kill: **immutable is not reachable.** +------------------------------------------------------------------- +A feature-branch head SHA is immutable -- nothing will ever change what that +object contains. It is also *unreferenced*: GitHub deletes the branch when the +PR squash-merges, the object becomes reachable from nothing, and every consumer +that pinned it is permanently broken. Shape validation ("it's 40 hex chars, so +it's immutable, so it's safe") is precisely the reasoning that produced two +production incidents in a single day on 2026-07-30: + +* **Workflow pin.** ``omnibase_infra@dev .github/workflows/ci.yml`` pinned + ``OmniNode-ai/omnimarket/.github/workflows/merge-hold-gate-reusable.yml@879d6fc6`` + -- the head of an omnimarket PR branch. The branch was deleted on merge, + ``ci.yml`` began failing at *parse* time, ``CI Summary`` (infra dev's sole + required context) never reported, and every open PR in the repo became + unmergeable for ~2.5h (OMN-15536). +* **Dependency pin.** ``omnimarket@dev pyproject.toml`` pins + ``omnibase-core`` at ``5a907b71`` -- the live head of the still-open + ``jonah/omn-15392-evidence-execution-scope`` branch. Five lines of prose + directly above it say the branch "MUST NOT merge with a feature-branch pin". + It merged anyway. A prose comment is not a mechanism. + +Both passed every gate, because no gate checked reachability. + +Why "reachable from a PROTECTED branch", not "reachable at all" +--------------------------------------------------------------- +The pre-existing platform check (``omnimarket/scripts/ci/ +check_uv_lock_pin_reachability.py``, OMN-14449) asks ``git branch -r --contains +`` and accepts a non-empty answer -- i.e. *reachable from ANY remote +branch*. ``5a907b71`` IS contained by its own live feature branch, so that +oracle returns OK on it today and only turns RED once the branch is deleted: +it reports the bug **after** the outage rather than before it. The invariant +that discriminates a durable pin from a time bomb is membership in the history +of a branch nobody can delete: + + reachable <=> compare(...).status in {"behind", "identical"} + +``behind`` the pinned ref is an ANCESTOR of the protected branch -> durable +``identical`` the pinned ref IS the protected branch head -> durable +``ahead`` the protected branch is an ancestor of the ref -> NOT durable + (a descendant commit: an unlanded branch head) +``diverged`` no ancestry either way -> NOT durable + +``dev`` OR ``main`` both count. That union is load-bearing, not defensive: +``omnibase_infra`` legitimately pins ``onex_change_control@2dd26ade``, which is +``diverged`` from OCC ``dev`` and ``behind`` OCC ``main``. A dev-only oracle +would false-RED a correct pin, and a gate that cries wolf gets bypassed. + +Existence is not the same probe, and it gives the wrong answer +-------------------------------------------------------------- +``GET /repos/{o}/{r}/commits/{sha}`` **still serves unreferenced objects**, so +an existence probe PASSES on a broken pin. That is why this module uses the +compare endpoint, and only falls back to the commits endpoint in the failure +path -- to say *which* kind of dead a dead pin is ("exists but unreferenced" +vs "gone entirely"), never to grant a pass. + +Fail-closed posture +------------------- +Rate-limit, transport failure, or any non-definitive HTTP status yields +``UNDETERMINED``, which is a FAILURE. ``--allow-undetermined`` downgrades that +to a loud warning for offline local use and is **refused outright when ``CI`` +is set**, so the enforcing surface can never be weakened into a silent skip +(the optional-input-means-the-check-does-not-exist trap). + +Auth: all OmniNode-ai repos referenced here are public, so the compare endpoint +works unauthenticated (60 req/hr). ``GH_TOKEN``/``GITHUB_TOKEN`` is attached as +a bearer token when present purely for rate-limit headroom (5000 req/hr); in +GitHub Actions the job passes the ambient ``secrets.GITHUB_TOKEN``. Probes are +deduplicated by ``(repo, ref)``, so pinning the same SHA twice (``uses:`` plus +``vocabulary_ref``) costs one request. ``gh`` is deliberately NOT used: the CI +python steps carry no ``gh`` login and ``gh`` hard-refuses unauthenticated use, +which would make the gate permanently red for transport reasons rather than for +broken pins (same finding as OMN-14941). + +Exit codes: ``0`` every pin durable | ``1`` at least one pin not durable, or +undetermined | ``2`` misuse (e.g. ``--allow-undetermined`` under CI). +""" + +from __future__ import annotations + +import argparse +import http.client +import json +import os +import re +import sys +import time +import tomllib +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any, NamedTuple + +import yaml + +_ORG = "OmniNode-ai" +_ORG_PREFIX = f"{_ORG}/" +# The reachability oracle is a property of github.com itself, so there is no +# ONEX routing authority to resolve it from; wiring a CI-only build-hygiene +# gate through runtime service discovery would couple it to infrastructure it +# must be able to run without. Same posture as the PyPI index literal in +# scripts/ci/publish_with_retry.py. +_GITHUB_API = "https://api.github.com" # url-authority-ok: fixed public REST API, no ONEX routing authority +_DEFAULT_PROTECTED: tuple[str, ...] = ("dev", "main") +# Raised from 10.0 (2026-08-06, defect fix): a single 10s-timeout, zero-retry +# GET was redding this gate on a pin that was verifiably reachable live (the +# onex_change_control @ 2dd26ade... compare call) -- 10s is not a realistic +# GitHub REST compare-endpoint budget under transient load. 30s per attempt. +_REQUEST_TIMEOUT_SECONDS = 30.0 +# Bounded retry for transient transport failures (timeout/connection errors, +# HTTP 5xx, HTTP 429, and a rate-limit-signaled 403). A definitive HTTP 4xx is +# never retried -- it will not change on a second try, and this job runs on +# every PR so added latency is never free. Worst case for one fully-exhausted +# call has TWO ceilings depending on backoff class (corrected 2026-08-06, +# terminal adversarial verify round 2 -- the fixed-schedule figure below was +# previously mis-cited as THE worst case; it is only the cheaper of the two): +# fixed-schedule (non-rate-limit 5xx, no server backoff header): +# 3 * _REQUEST_TIMEOUT_SECONDS + sum(_API_RETRY_BACKOFF_SECONDS) = 96s +# rate-limited (429 / rate-limited 403 with a server Retry-After header, +# which is PREFERRED over the fixed schedule and capped at +# _MAX_RATE_LIMIT_BACKOFF_SECONDS -- see ``_rate_limit_backoff_seconds``): +# 3 * _REQUEST_TIMEOUT_SECONDS + 2 * _MAX_RATE_LIMIT_BACKOFF_SECONDS = 150s +# +# That 150s (the true ceiling, not 96s) bounds ONE call, not the run. The +# circuit breaker below resets its consecutive-failure counter on ANY +# HTTP-status-bearing response -- including a retried-and-still-failing +# 503/429 -- so a run whose transport failures are intermittent rather than +# sustained can pay the full per-call ceiling on every pin without the +# breaker ever tripping (defect found 2026-08-06: measured max 944s / 1174s +# across 21-pin trials with 19-23/30 exceeding the 600s CI job timeout). +# ``_RUN_DEADLINE_SECONDS`` below is the actual run-wide bound; the breaker +# remains a fast-path for the sustained-outage case it was built for, but is +# no longer the thing standing between this gate and the job timeout. +_API_MAX_ATTEMPTS = 3 +_API_RETRY_BACKOFF_SECONDS: tuple[float, ...] = (2.0, 4.0) +_TRANSIENT_HTTP_STATUS = frozenset({429, 500, 502, 503, 504}) +# A server-requested rate-limit backoff (``Retry-After`` or +# ``x-ratelimit-reset``) is honored up to this cap, never unbounded -- an +# hour-long primary-rate-limit reset must not turn one pin into an hour-long +# CI job. +_MAX_RATE_LIMIT_BACKOFF_SECONDS = 30.0 +# Stop probing after this many consecutive transport (non-HTTP-status) failures +# and report the remainder as undetermined -- a fast path for a SUSTAINED +# outage. It does not, by itself, bound a run with INTERMITTENT failures; see +# ``_RUN_DEADLINE_SECONDS``. +_TRANSPORT_FAILURE_CIRCUIT_BREAKER = 3 +# Run-wide wall-clock budget across every pin resolved by one ``_Resolver``. +# Checked before each per-branch compare call AND before ``_explain``'s own +# commits-endpoint lookup (defect found 2026-08-06: ``_explain`` used to run +# an unguarded second call after the last passing check, doubling the +# post-deadline tail to 192s). With both call sites guarded, the actual +# worst case is this deadline plus at most one already-in-flight call's tail. +# +# Lowered from 480.0 to 220.0 (2026-08-06, terminal adversarial verify round +# 2, PR #2679 comment 5209929069): the prior ``480 + 96 = 576s`` bound used +# the WRONG per-call ceiling and ignored job setup entirely. +# +# * The 96s figure only covered the fixed-schedule backoff class +# (2.0 + 4.0 = 6s of sleep). It is NOT the worst case: a 429 / rate-limited +# 403 with a server ``Retry-After`` prefers the server-provided delay over +# the fixed schedule, capped at ``_MAX_RATE_LIMIT_BACKOFF_SECONDS`` (30s) +# -- see ``_rate_limit_backoff_seconds``. One fully-exhausted call in that +# class costs ``_API_MAX_ATTEMPTS`` (3) timeouts of +# ``_REQUEST_TIMEOUT_SECONDS`` (30s) each, PLUS a capped 30s server backoff +# after each of the first two attempts: +# 3 * 30.0 + 2 * 30.0 = 90 + 60 = 150s (measured/derived from code, not +# the 96s this comment used to say) +# * 600s is the CI **job**'s ``timeout-minutes`` (.github/workflows/ci.yml), +# not the script's own budget. Measured on this PR's own prior CI head +# (job 92716571663): checkout + ``uv`` setup (cache disabled) costs ~196s +# BEFORE the script starts, leaving the script itself only ~404s of the +# 600s job budget -- not 600s. The old ``480 + 96 = 576s`` "margin" never +# accounted for that ~196s prefix, so real worst case was +# ``196 + 480 + 150 = 826s``: the job gets cancelled by GitHub at 600s and +# the script's own fail-closed UNDETERMINED print never happens -- the +# exact wedged-red-without-a-message failure mode this file exists to +# remove. +# +# [SUPERSEDED bound below] ~200s measured setup + 220s deadline + 150s max +# post-deadline tail = ~570s < 600s job timeout (~30s margin). This bound was +# itself falsified before it ever landed (PR #2679 comment 5211687650): the +# ``~196s`` / ``~200s`` setup figure above came from ONE below-median sample +# (job 92716571663), not the fleet. A 38-run fleet measurement of this same +# job found setup alone had median 317s and max 613s -- 6/38 runs were +# killed by ``timeout-minutes: 10`` DURING setup, before this deadline ever +# had a chance to fire. No value of ``_RUN_DEADLINE_SECONDS`` fixes a job +# whose setup sometimes eats the whole 600s budget by itself; the job +# timeout itself had to move. See ``.github/workflows/ci.yml``'s +# ``pin-reachability`` job comment for that half of the fix. +# +# Re-derived 2026-08-07 from the POST-FIX setup budget (workflow-level fix, +# same PR, same comment): ``timeout-minutes`` raised 10 -> 18 (1080s). +# ``cache-enabled: "false"`` is intentionally left unchanged for this job -- +# this repo's self-hosted-fleet CI already replaced the per-job uv cache with +# the shared host-local dependency-environment canary (see +# ``docs/ci/versioned-ci-env-canary.md``), which explicitly forbids enabling +# setup-uv cache save on jobs that run ``uv sync --no-cache``, and +# ``tests/ci/test_ci_workflow_resilience.py::test_short_gates_can_disable_uv_cache_cleanup`` +# enforces that uniformly across every ci.yml job. The job-timeout bound +# below is therefore built directly on the fleet-measured 613s setup MAX +# (not an assumed cache improvement that was never applied): +# 613s (measured max setup, fleet n=38) +# + 150s (this deadline, re-derived below) +# + 150s (max post-deadline tail, unchanged -- one already-in-flight +# rate-limited call: 3 * 30s timeouts + 2 * 30s capped backoffs) +# = 913s < 1080s job timeout (~167s / ~15% real margin) +# +# ``_RUN_DEADLINE_SECONDS`` itself is lowered 220.0 -> 150.0 to fit inside +# that budget with real margin rather than consuming nearly all of it: 150s +# still comfortably bounds a *sustained* cross-pin failure run (which is +# fail-closed UNDETERMINED anyway, not a false pass) -- it is exactly one +# worst-case per-call ceiling, i.e. this deadline alone guarantees at least +# one full retry-exhaustion cycle completes before the run is cut off -- +# while leaving the script room to print its own diagnostic before the job's +# hard cancellation. A binding test +# (``tests/ci/test_pin_reachability_omn15538.py::test_pin_reachability_job_timeout_covers_measured_budget``) +# parses ``timeout-minutes`` out of the real ci.yml and asserts this +# arithmetic in both directions (too-small AND absurdly-large timeout), so a +# future edit to either side of this budget that breaks the bound fails CI +# instead of silently drifting back into the pre-fix failure mode. +_RUN_DEADLINE_SECONDS = 150.0 + +_SHA40_RE = re.compile(r"\A[0-9a-f]{40}\Z") + +# `omnibase-core @ git+https://github.com/OmniNode-ai/omnibase_core.git@` +_PEP508_GIT_RE = re.compile( + r"git\+https://github\.com/" + + _ORG + + r"/(?P[\w.-]+?)(?:\.git)?@(?P[^\s#\]]+)" +) +# uv.lock: `git = "https://github.com/OmniNode-ai/.git?rev=#"` +_UV_LOCK_GIT_RE = re.compile( + r"https://github\.com/" + + _ORG + + r"/(?P[\w.-]+?)(?:\.git)?\?(?:rev|branch|tag)=(?P[^\"#&]+)" + r"(?:#(?P[0-9a-f]{40}))?" +) +# `git = "https://github.com/OmniNode-ai/.git"` in a [tool.uv.sources] table +_SOURCE_URL_RE = re.compile( + r"\Ahttps://github\.com/" + _ORG + r"/(?P[\w.-]+?)(?:\.git)?/?\Z" +) + + +class Verdict(str, Enum): + """Outcome of one reachability resolution.""" + + REACHABLE = "REACHABLE" + UNREACHABLE = "UNREACHABLE" + UNDETERMINED = "UNDETERMINED" + + +class PinRef(NamedTuple): + """One cross-repo pin found in a tracked file. + + ``source`` is the file it came from, ``locus`` the addressable position + inside it (so a failure names the exact line to edit), ``kind`` the pin + form, ``repo`` the target OmniNode-ai repo and ``ref`` the pinned git ref. + """ + + source: str + locus: str + kind: str + repo: str + ref: str + + +@dataclass(frozen=True) +class Resolution: + """A verdict plus the evidence that produced it.""" + + verdict: Verdict + detail: str + + +# --------------------------------------------------------------------------- +# Extraction +# --------------------------------------------------------------------------- + + +def _split_uses(value: str) -> tuple[str, str] | None: + """Split ``OmniNode-ai/[/]@`` into ``(repo, ref)``. + + Returns ``None`` for anything out of scope: local (``./``) pins, docker + pins, third-party actions, and any value with no ``@`` ref. + """ + if not value.startswith(_ORG_PREFIX) or "@" not in value: + return None + spec, _, ref = value.rpartition("@") + remainder = spec[len(_ORG_PREFIX) :] + repo, _, _path = remainder.partition("/") + if not repo or not ref: + return None + return repo, ref + + +def _is_literal(value: object) -> bool: + """True for a plain scalar; ``${{ ... }}`` expressions are not resolvable.""" + return isinstance(value, str) and "${{" not in value + + +def extract_workflow_pins(path: Path) -> list[PinRef]: + """Extract cross-repo pins from one GitHub Actions workflow file. + + Covers three surfaces: + + * ``jobs..uses`` -- reusable-workflow pins (the OMN-15536 shape), + * ``jobs..steps[i].uses`` -- step-level action pins, + * ``jobs..with.`` -- any 40-hex literal input on a job that calls + a cross-repo reusable workflow, attributed to that same repo. + + That third surface is why this is not just a ``uses:`` checker: the wedging + infra job pinned the identical dead SHA **twice**, once as ``uses:`` and + once as ``vocabulary_ref``, and repointing only the first would have left a + second dead reference behind (OMN-15538 AC-5). Keying on "40-hex literal + under ``with:``" rather than on the name ``vocabulary_ref`` means a renamed + or newly-added sibling input is covered without editing this file. + + YAML parsing (not a line regex) means commented-out ``uses:`` lines are + structurally invisible -- several infra workflows carry documentation + examples that a regex would extract and fail on. + """ + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(loaded, dict): + return [] + jobs = loaded.get("jobs") + if not isinstance(jobs, dict): + return [] + + pins: list[PinRef] = [] + for job_id, job in jobs.items(): + if not isinstance(job, dict): + continue + + job_uses = job.get("uses") + job_target: str | None = None + if _is_literal(job_uses): + assert isinstance(job_uses, str) # nosec B101 - narrowed by _is_literal + split = _split_uses(job_uses) + if split is not None: + repo, ref = split + job_target = repo + pins.append( + PinRef( + source=str(path), + locus=f"jobs.{job_id}.uses", + kind="workflow-uses", + repo=repo, + ref=ref, + ) + ) + + # Sibling ref inputs on a cross-repo caller (vocabulary_ref and kin). + job_with = job.get("with") + if job_target is not None and isinstance(job_with, dict): + for key, value in job_with.items(): + if _is_literal(value) and _SHA40_RE.match(str(value)): + pins.append( + PinRef( + source=str(path), + locus=f"jobs.{job_id}.with.{key}", + kind="workflow-with", + repo=job_target, + ref=str(value), + ) + ) + + steps = job.get("steps") + if isinstance(steps, list): + for index, step in enumerate(steps): + if not isinstance(step, dict): + continue + step_uses = step.get("uses") + if not _is_literal(step_uses): + continue + assert isinstance(step_uses, str) # nosec B101 - narrowed above + split = _split_uses(step_uses) + if split is None: + continue + repo, ref = split + pins.append( + PinRef( + source=str(path), + locus=f"jobs.{job_id}.steps[{index}].uses", + kind="workflow-step-uses", + repo=repo, + ref=ref, + ) + ) + return pins + + +def _pins_from_source_table( + source: Path, name: str, table: dict[str, Any] +) -> list[PinRef]: + """Pins from one ``[tool.uv.sources.]`` entry. + + A ``git`` URL with no ``rev``/``branch``/``tag`` tracks the target's default + branch, which is protected by construction and cannot evaporate -- nothing + to check. + """ + git_url = table.get("git") + if not isinstance(git_url, str): + return [] + match = _SOURCE_URL_RE.match(git_url.strip()) + if match is None: + return [] + repo = match["repo"] + pins: list[PinRef] = [] + for key in ("rev", "branch", "tag"): + ref = table.get(key) + if isinstance(ref, str) and ref: + pins.append( + PinRef( + source=str(source), + locus=f"tool.uv.sources.{name}.{key}", + kind="pyproject-source", + repo=repo, + ref=ref, + ) + ) + return pins + + +def _pep508_pins(source: Path, locus: str, requirements: object) -> list[PinRef]: + """Pins from a PEP 508 requirement list (``pkg @ git+https://...@``).""" + if not isinstance(requirements, list): + return [] + pins: list[PinRef] = [] + for index, requirement in enumerate(requirements): + if not isinstance(requirement, str): + continue + for match in _PEP508_GIT_RE.finditer(requirement): + pins.append( + PinRef( + source=str(source), + locus=f"{locus}[{index}]", + kind="pyproject-pep508", + repo=match["repo"], + ref=match["ref"], + ) + ) + return pins + + +def extract_pyproject_pins(path: Path) -> list[PinRef]: + """Extract cross-repo git pins from a ``pyproject.toml``. + + Two forms, both live on the platform today: + + * ``[tool.uv.sources]`` -- ``{ git = "...", rev|branch|tag = "..." }``. + This is the omnimarket instance. Note the SHA lives in a *separate key* + from the URL, which is exactly why the OMN-14449 uv.lock regex + (``...git?rev=`` in one string) cannot see it. + * PEP 508 direct references in ``project.dependencies``, + ``project.optional-dependencies.*`` and ``dependency-groups.*``. + """ + data = tomllib.loads(path.read_text(encoding="utf-8")) + pins: list[PinRef] = [] + + sources = data.get("tool", {}).get("uv", {}).get("sources", {}) + if isinstance(sources, dict): + for name, entry in sources.items(): + if isinstance(entry, dict): + pins.extend(_pins_from_source_table(path, name, entry)) + elif isinstance(entry, list): + # uv permits a list of conditional sources per package. + for element in entry: + if isinstance(element, dict): + pins.extend(_pins_from_source_table(path, name, element)) + + project = data.get("project", {}) + if isinstance(project, dict): + pins.extend( + _pep508_pins(path, "project.dependencies", project.get("dependencies")) + ) + optional = project.get("optional-dependencies") + if isinstance(optional, dict): + for extra, requirements in optional.items(): + pins.extend( + _pep508_pins( + path, f"project.optional-dependencies.{extra}", requirements + ) + ) + + groups = data.get("dependency-groups") + if isinstance(groups, dict): + for group, requirements in groups.items(): + pins.extend(_pep508_pins(path, f"dependency-groups.{group}", requirements)) + + return pins + + +def extract_uv_lock_pins(path: Path) -> list[PinRef]: + """Extract cross-repo git pins from a ``uv.lock``. + + Prefers the resolved commit in the URL fragment when present -- that is the + object a build actually fetches -- and falls back to the query-parameter ref. + """ + text = path.read_text(encoding="utf-8") + seen: set[tuple[str, str]] = set() + pins: list[PinRef] = [] + for match in _UV_LOCK_GIT_RE.finditer(text): + repo = match["repo"] + ref = match["resolved"] or match["ref"] + if (repo, ref) in seen: + continue + seen.add((repo, ref)) + pins.append( + PinRef( + source=str(path), + locus=f"package source for {repo}", + kind="uv-lock", + repo=repo, + ref=ref, + ) + ) + return pins + + +def _expand_targets(paths: Sequence[Path]) -> list[Path]: + """Expand directories to the pin-bearing files inside them.""" + expanded: list[Path] = [] + for path in paths: + if path.is_dir(): + expanded.extend(sorted(path.glob("*.yml"))) + expanded.extend(sorted(path.glob("*.yaml"))) + for name in ("pyproject.toml", "uv.lock"): + candidate = path / name + if candidate.is_file(): + expanded.append(candidate) + elif path.is_file(): + expanded.append(path) + return expanded + + +def extract_pins(paths: Iterable[Path]) -> list[PinRef]: + """Extract every cross-repo pin from the given files/directories.""" + pins: list[PinRef] = [] + for path in _expand_targets(list(paths)): + name = path.name + if name == "pyproject.toml": + pins.extend(extract_pyproject_pins(path)) + elif name == "uv.lock": + pins.extend(extract_uv_lock_pins(path)) + elif path.suffix in {".yml", ".yaml"}: + pins.extend(extract_workflow_pins(path)) + return pins + + +# --------------------------------------------------------------------------- +# Reachability resolution +# --------------------------------------------------------------------------- + + +def status_is_reachable(status: str) -> bool: + """Map a GitHub compare ``status`` to durable-reachability. + + ``behind`` (ancestor of the protected branch) and ``identical`` (it IS the + protected branch head) are durable. ``ahead`` -- the protected branch is an + ancestor of the pin, i.e. the pin is a descendant commit that has not + landed -- and ``diverged`` are not. + """ + return status in {"behind", "identical"} + + +def _is_transient_http_status(status: int) -> bool: + """True for an HTTP status that is unconditionally worth retrying: 429 + (rate limit) or any 5xx. + + 404, 400, 401, 410, ... are definitive regardless of headers: a second + identical GET will not produce a different answer, so retrying only adds + latency to a gate that runs on every PR. + + 403 is deliberately NOT in this set -- a plain 403 (bad/missing auth, + genuinely forbidden) is just as definitive as a 404. But GitHub also uses + 403 to signal BOTH the primary rate limit (``x-ratelimit-remaining: 0``) + and the secondary rate limit (a ``Retry-After`` header), and those two + ARE transient. Discriminating requires the response headers, which this + function -- status-code-only, by design, so it stays a trivial pure + predicate -- does not receive. See :func:`_is_rate_limited_403`, which + ``_api_get`` consults separately for the 403 case. + """ + return status in _TRANSIENT_HTTP_STATUS + + +def _is_rate_limited_403(headers: Any) -> bool: + """True when a 403's headers carry GitHub's rate-limit signal. + + Primary rate limit: ``x-ratelimit-remaining: 0``. Secondary rate limit: + a ``Retry-After`` header. A 403 with neither is a genuine authorization + failure -- definitive, not transient (see :func:`_is_transient_http_status`). + """ + if headers is None: + return False + if headers.get("x-ratelimit-remaining") == "0": + return True + return headers.get("Retry-After") is not None + + +def _rate_limit_backoff_seconds(headers: Any) -> float | None: + """Read a server-provided rate-limit backoff off response headers. + + Prefers ``Retry-After`` (seconds); falls back to ``x-ratelimit-reset`` + (unix epoch seconds), converted to a delta from now. Always capped at + ``_MAX_RATE_LIMIT_BACKOFF_SECONDS`` -- honoring the signal is better than + a blind fixed schedule, but honoring it UNBOUNDED would let one rate + limit turn a single pin into an hour-long CI job. Returns ``None`` when + neither header is present or parseable, so the caller falls back to the + fixed schedule. + """ + if headers is None: + return None + retry_after = headers.get("Retry-After") + if retry_after is not None: + try: + return max(0.0, min(float(retry_after), _MAX_RATE_LIMIT_BACKOFF_SECONDS)) + except ValueError: + pass + reset_at = headers.get("x-ratelimit-reset") + if reset_at is not None: + try: + delta = float(reset_at) - time.time() + except ValueError: + return None + if delta > 0: + return min(delta, _MAX_RATE_LIMIT_BACKOFF_SECONDS) + return None + + +def _api_get( + url: str, *, sleep: Callable[[float], None] | None = None +) -> tuple[int | None, dict[str, Any] | None, str]: + """GET a GitHub REST endpoint with bounded retry on transient failures. + + Returns ``(status, body, detail)``. ``status is None`` means the request + could not be performed at all after exhausting ``_API_MAX_ATTEMPTS``. + + Retries ONLY the transient failure classes: a transport-level failure + (timeout, connection error -- no HTTP status at all), an HTTP 429/5xx + (see :func:`_is_transient_http_status`), or a rate-limit-signaled 403 + (see :func:`_is_rate_limited_403`) -- up to ``_API_MAX_ATTEMPTS`` total + attempts. Backoff prefers the server-provided rate-limit signal + (:func:`_rate_limit_backoff_seconds`, capped) and otherwise falls back to + the short fixed schedule (``_API_RETRY_BACKOFF_SECONDS``, never + unbounded). A definitive HTTP 4xx returns immediately on the first + attempt -- no retry, no added latency. + + If every attempt fails on a transient class, the final failing result is + returned unchanged from today's single-shot behavior: the caller's + existing fail-closed UNDETERMINED mapping is untouched. This function + eliminates FALSE failures caused by one unlucky transient hiccup; it does + not, and must not, weaken the fail-closed posture for a genuinely + unresolvable pin. + """ + effective_sleep = sleep or time.sleep + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "omnibase-infra-pin-reachability-gate (OMN-15538)", + } + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + request = urllib.request.Request(url, headers=headers) # noqa: S310 - fixed https host + + last_status: int | None = None + last_body: dict[str, Any] | None = None + last_detail = "" + for attempt in range(1, _API_MAX_ATTEMPTS + 1): + server_backoff: float | None = None + try: + with urllib.request.urlopen( # noqa: S310 - fixed https host + request, timeout=_REQUEST_TIMEOUT_SECONDS + ) as response: + payload = json.loads(response.read().decode("utf-8", errors="replace")) + body = payload if isinstance(payload, dict) else None + return response.status, body, f"HTTP {response.status}" + except urllib.error.HTTPError as exc: + detail = f"HTTP {exc.code}" + try: + error_body = json.loads(exc.read().decode("utf-8", errors="replace")) + message = ( + error_body.get("message", "") + if isinstance(error_body, dict) + else "" + ) + if message: + detail = f"HTTP {exc.code}: {message}" + except (ValueError, OSError): + pass + rate_limited_403 = exc.code == 403 and _is_rate_limited_403(exc.headers) + if not _is_transient_http_status(exc.code) and not rate_limited_403: + return exc.code, None, detail + last_status, last_body, last_detail = exc.code, None, detail + if exc.code == 429 or rate_limited_403: + server_backoff = _rate_limit_backoff_seconds(exc.headers) + except ( + urllib.error.URLError, + OSError, + TimeoutError, + ValueError, + http.client.HTTPException, + ) as exc: + last_status, last_body, last_detail = None, None, f"transport error: {exc}" + + if attempt < _API_MAX_ATTEMPTS: + if server_backoff is not None: + delay = server_backoff + else: + # Clamp rather than index directly: if _API_MAX_ATTEMPTS is + # ever raised without extending _API_RETRY_BACKOFF_SECONDS to + # match, reuse the last known backoff instead of raising + # IndexError inside a required CI gate. + schedule_index = min(attempt - 1, len(_API_RETRY_BACKOFF_SECONDS) - 1) + delay = _API_RETRY_BACKOFF_SECONDS[schedule_index] + effective_sleep(delay) + + return last_status, last_body, last_detail + + +def _compare_url(repo: str, base: str, ref: str) -> str: + quoted_base = urllib.parse.quote(base, safe="/") + quoted_ref = urllib.parse.quote(ref, safe="/") + return f"{_GITHUB_API}/repos/{_ORG}/{repo}/compare/{quoted_base}...{quoted_ref}" + + +class _Resolver: + """Resolves pins against the live GitHub API, with dedup + a circuit breaker. + + Two independent bounds protect the CI job timeout, for two different + failure shapes: + + * ``tripped`` (the pre-existing consecutive-transport-failure breaker) is + a fast path for a SUSTAINED outage -- every call failing with no HTTP + status at all. + * ``_deadline_exceeded`` (the run-wide wall-clock budget) is what + actually bounds an INTERMITTENT-failure run: the breaker's counter + resets on any HTTP-status-bearing response (even a retried-and-still- + failing one), so a pattern that alternates success/failure across pins + never trips it, and every pin can independently pay the full per-call + retry ceiling. See ``_RUN_DEADLINE_SECONDS``. + """ + + def __init__( + self, protected: Sequence[str], *, now: Callable[[], float] | None = None + ) -> None: + self._protected = tuple(protected) + self._cache: dict[tuple[str, str], Resolution] = {} + self._consecutive_transport_failures = 0 + self._now = now or time.monotonic + self._deadline_at = self._now() + _RUN_DEADLINE_SECONDS + + @property + def tripped(self) -> bool: + return ( + self._consecutive_transport_failures >= _TRANSPORT_FAILURE_CIRCUIT_BREAKER + ) + + @property + def _deadline_exceeded(self) -> bool: + return self._now() >= self._deadline_at + + def resolve(self, repo: str, ref: str) -> Resolution: + key = (repo, ref) + cached = self._cache.get(key) + if cached is not None: + return cached + resolution = self._resolve_uncached(repo, ref) + self._cache[key] = resolution + return resolution + + def _resolve_uncached(self, repo: str, ref: str) -> Resolution: + # A pin that names a protected branch itself is durable by definition: + # the branch cannot be deleted, and no API call can make that truer. + if ref in self._protected: + return Resolution( + Verdict.REACHABLE, f"pins the protected branch {ref!r} directly" + ) + + if self.tripped: + return Resolution( + Verdict.UNDETERMINED, + "skipped: transport-failure circuit breaker already tripped", + ) + + not_found: list[str] = [] + for base in self._protected: + if self._deadline_exceeded: + return Resolution( + Verdict.UNDETERMINED, + f"skipped: run-wide {_RUN_DEADLINE_SECONDS:.0f}s deadline " + "exceeded -- remaining pins reported undetermined " + "(fail-closed) to stay within the CI job timeout", + ) + status, body, detail = _api_get(_compare_url(repo, base, ref)) + if status is None: + self._consecutive_transport_failures += 1 + if self.tripped: + return Resolution( + Verdict.UNDETERMINED, + f"{detail} (circuit breaker tripped after " + f"{_TRANSPORT_FAILURE_CIRCUIT_BREAKER} consecutive failures)", + ) + return Resolution(Verdict.UNDETERMINED, detail) + self._consecutive_transport_failures = 0 + if status == 200 and body is not None: + compare_status = str(body.get("status", "")) + if status_is_reachable(compare_status): + return Resolution( + Verdict.REACHABLE, + f"compare {base}...{ref[:12]} = {compare_status}", + ) + not_found.append(f"{base}...{ref[:12]} = {compare_status}") + continue + if status == 404: + not_found.append(f"{base}...{ref[:12]} = HTTP 404") + continue + # 403 / 429 (rate limit) and anything else: cannot determine. + return Resolution(Verdict.UNDETERMINED, detail) + + return Resolution(Verdict.UNREACHABLE, self._explain(repo, ref, not_found)) + + def _explain(self, repo: str, ref: str, observations: list[str]) -> str: + """Say WHICH kind of dead the pin is. + + The commits endpoint serves unreferenced objects, so a 200 here proves + the OMN-14447 shape exactly: the object exists and is reachable from + nothing protected. This call never grants a pass -- it only sharpens + the failure message. + + Guarded by the same run-wide deadline as the per-branch compare + calls above: without this check, this call could run AFTER the + deadline had already been exceeded by the last compare call, + doubling the documented 96s post-deadline tail to 192s (defect found + 2026-08-06). Once the deadline is exceeded, skip the lookup entirely + and return the plain observations -- the verdict is UNREACHABLE + either way; this only sharpens detail text, never a pass/fail + outcome, so skipping it costs nothing but explanation detail. + """ + joined = "; ".join(observations) + if self._deadline_exceeded: + return joined + status, _body, _detail = _api_get( + f"{_GITHUB_API}/repos/{_ORG}/{repo}/commits/" + f"{urllib.parse.quote(ref, safe='/')}" + ) + if status == 200: + return ( + f"{joined} -- the object EXISTS but is reachable from no protected " + "branch (unlanded or deleted feature-branch head)" + ) + if status == 404: + return f"{joined} -- the ref does not resolve at all (object gone or repo/ref typo)" + return joined + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +_FIX_GUIDANCE = """ + Fix: re-pin to the SQUASH-MERGED commit on dev (or main). Squash-merged + commits are reachable forever; feature-branch heads evaporate the moment + the PR lands, and an UNLANDED branch head is a time bomb that detonates + on merge -- it resolves right up until it doesn't. + + gh api repos/OmniNode-ai//compare/dev... --jq .status + behind | identical -> durable, safe to pin + ahead | diverged -> NOT durable, do not pin + + Do NOT "verify" a pin with `gh api .../commits/`: GitHub serves + unreferenced objects, so that probe passes on exactly the pins that break. +""" + + +def _default_targets(root: Path) -> list[Path]: + targets: list[Path] = [] + workflows = root / ".github" / "workflows" + if workflows.is_dir(): + targets.append(workflows) + for name in ("pyproject.toml", "uv.lock"): + candidate = root / name + if candidate.is_file(): + targets.append(candidate) + return targets + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Reject any cross-repo pin whose commit is not reachable from a " + "protected branch of the target repo." + ) + ) + parser.add_argument( + "paths", + nargs="*", + type=Path, + help=( + "Files or directories to scan. Defaults to .github/workflows/, " + "pyproject.toml and uv.lock under --root." + ), + ) + parser.add_argument( + "--root", type=Path, default=Path(), help="Repo root for default targets." + ) + parser.add_argument( + "--protected", + action="append", + default=None, + metavar="BRANCH", + help=f"Protected branch to resolve against (repeatable; default: {' '.join(_DEFAULT_PROTECTED)}).", + ) + parser.add_argument( + "--min-pins", + type=int, + default=0, + help=( + "Fail if fewer than N pins were extracted. Guards the " + "vacuous-green case where a glob typo or schema drift silently " + "makes the gate check nothing." + ), + ) + parser.add_argument( + "--allow-undetermined", + action="store_true", + help=( + "Downgrade UNDETERMINED (offline / rate-limited) from failure to a " + "warning. Local use only -- REFUSED when CI is set." + ), + ) + args = parser.parse_args(argv) + + in_ci = bool(os.environ.get("CI")) + if args.allow_undetermined and in_ci: + print( + "error: --allow-undetermined is refused under CI. The enforcing " + "surface must fail closed on unresolvable pins; an undetermined " + "result is not a pass.", + file=sys.stderr, + ) + return 2 + + protected = tuple(args.protected) if args.protected else _DEFAULT_PROTECTED + targets = list(args.paths) if args.paths else _default_targets(args.root) + pins = extract_pins(targets) + + if len(pins) < args.min_pins: + print( + f"FAIL: extracted {len(pins)} pin(s) from {len(targets)} target(s), " + f"expected at least {args.min_pins}. The extractor is more likely " + "broken than the tree is clean -- a gate that checks nothing " + "reports green forever.", + file=sys.stderr, + ) + return 1 + + if not pins: + print(f"no cross-repo {_ORG} pins found in {len(targets)} target(s)") + return 0 + + resolver = _Resolver(protected) + unreachable: list[tuple[PinRef, Resolution]] = [] + undetermined: list[tuple[PinRef, Resolution]] = [] + + print( + f"Resolving {len(pins)} cross-repo pin(s) against protected branches " + f"{'/'.join(protected)}:" + ) + for pin in sorted(pins): + resolution = resolver.resolve(pin.repo, pin.ref) + marker = { + Verdict.REACHABLE: " OK ", + Verdict.UNREACHABLE: " UNREACHABLE ", + Verdict.UNDETERMINED: " UNDETERMINED", + }[resolution.verdict] + print( + f"{marker} {pin.repo} @ {pin.ref[:12]} [{pin.kind}] " + f"{pin.source}::{pin.locus} ({resolution.detail})" + ) + if resolution.verdict is Verdict.UNREACHABLE: + unreachable.append((pin, resolution)) + elif resolution.verdict is Verdict.UNDETERMINED: + undetermined.append((pin, resolution)) + + # Keep the human-readable per-pin listing above the failure block in a + # merged 2>&1 CI log; unflushed stdout otherwise lands after stderr. + sys.stdout.flush() + + if undetermined and not args.allow_undetermined: + print( + f"\nFAIL: {len(undetermined)} pin(s) could not be resolved. An " + "unresolvable pin is not a passing pin -- this gate fails closed so " + "a network blip or rate limit can never mint a false green.\n", + file=sys.stderr, + ) + for pin, resolution in undetermined: + print( + f" {pin.repo} @ {pin.ref} {pin.source}::{pin.locus} " + f"({resolution.detail})", + file=sys.stderr, + ) + print( + "\n Set GH_TOKEN/GITHUB_TOKEN for rate-limit headroom. Offline, " + "re-run with --allow-undetermined (local only).", + file=sys.stderr, + ) + return 1 + + if undetermined: + print( + f"\nWARNING: {len(undetermined)} pin(s) UNDETERMINED and " + "--allow-undetermined was passed. This run PROVED NOTHING about " + "those pins; CI is the enforcing surface." + ) + + if not unreachable: + print(f"\nPin reachability OK: {len(pins)} cross-repo pin(s) checked") + return 0 + + print( + f"\nFAIL: {len(unreachable)} pin(s) are NOT reachable from any protected " + f"branch ({'/'.join(protected)}) of their target repo:\n", + file=sys.stderr, + ) + for pin, resolution in unreachable: + print(f" {pin.source}::{pin.locus}", file=sys.stderr) + print(f" {pin.repo} @ {pin.ref}", file=sys.stderr) + print(f" {resolution.detail}", file=sys.stderr) + print(_FIX_GUIDANCE, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/ci_summary_gate.py b/scripts/ci/ci_summary_gate.py index e0c74ae79b..7e3c4379b2 100644 --- a/scripts/ci/ci_summary_gate.py +++ b/scripts/ci/ci_summary_gate.py @@ -55,6 +55,28 @@ deadline, PENDING is converted to FAILURE (fail-closed): the required context always reaches a terminal state. +4. **External context assertion (OMN-15496).** Checks 1-3 all read + ``actions/runs/${RUN_ID}/jobs`` — *this* workflow run's job list. Any check + produced by a **different workflow file** is structurally invisible to them, + and ``omnibase_infra``'s ``dev`` requires exactly one context (``CI Summary``, + ``strict=false``), so such checks were enforced by **neither** layer: + 59 distinct cross-workflow check-run names on a real merged PR head + (#2567 / ``0fca3b5e``) versus 40 inside this run's suite. + + :data:`EXPECTED_EXTERNAL_CONTEXTS` closes that hole *without* re-fanning 59 + required contexts (which would discard the deliberate single-umbrella design + of OMN-4497/OMN-14127 — and a context that does not report on every PR shape + wedges the branch indefinitely). Each named context is resolved from the PR + head's ``commits/{sha}/check-runs`` and must be **present**, **completed**, + and conclude ``success``; missing or still-running is PENDING, which the + caller's deadline converts to FAILURE. This is the presence assertion + OMN-14456 AC4 asked for. + + *Why this was load-bearing:* PR #2555 merged 2026-07-30T04:25:09Z with + ``CI Summary`` = **success** (all 53 in-run jobs green) while + ``deploy-gate / deploy-gate`` = **failure** on the same head SHA. The + required context was green because the failing check was in another run. + Exit codes: ``0`` success, ``1`` failure, ``2`` pending. """ @@ -88,8 +110,59 @@ "Arch Invariants (OMN-3343)", # arch-invariants "Kafka Schema Handshake (OMN-3411)", # schema-handshake "Writer-Migration Coupling Check", # migration-required-check + "Node Migration Declaration Check", # node-migration-declaration-check (OMN-15717) "no-noncanonical-lifecycle-classes", # OMN-14350 non-canonical lifecycle-class ratchet "Effect-Assertion Gate (RT-5)", # OMN-14467 deploy-trigger fails closed on zero output + "OCC Companion Merged Gate (OMN-15214)", # occ-companion-merged — cited OCC evidence must be MERGED before product merge + # OMN-15378 AC3: scripts/deploy-agent's standalone pytest root. ci.yml's + # `deploy-agent-tests` job CALLS .github/workflows/deploy-agent-tests.yml, + # so the inner job surfaces as " / " + # (same shape as "occ-preflight / eligibility"). Registering it here is what + # makes those 201 tests GATE merge: while they lived in a separately- + # triggered workflow this poller could not observe them at all (different + # run_id), so a RED run left "CI Summary" — the sole required context on + # dev — green. + "Deploy Agent Tests (OMN-15378) / deploy-agent-tests", + # OMN-15484: the Merge Hold Gate, fanned out from OMN-15483 (omnibase_infra + # carries incident §C, #2560, and had zero coverage). THIS LINE IS THE + # MECHANISM — not the job's existence in ci.yml. The default-deny sweep + # below already catches a hold job that FAILS, but an unregistered job that + # is `skipped` or `absent` yields CI Summary SUCCESS, so a held PR would be + # required-green and the sweep would land it. Measured against this very + # evaluator on omnimarket#1973: unregistered, `skipped` -> SUCCESS and + # `absent` -> SUCCESS; registered, `skipped` -> FAILURE and `absent` -> + # PENDING. The job is unconditional (no needs/if), so a skip is anomalous, + # never a legitimate opt-out. Same " / " shape as the two entries above; renaming either half makes this + # gate permanently PENDING. Pinned by tests/ci/test_merge_hold_gate_omn15484.py. + "merge-hold-gate / evaluate", + # OMN-15538: every cross-repo pin must resolve to a commit reachable from a + # protected branch of the target repo. THIS LINE IS THE MECHANISM, not the + # job's presence in ci.yml — `CI Summary` is dev's sole required context, so + # an unregistered job that fails still yields SUCCESS here and the PR lands. + # The gate it replaces the absence of: on 2026-07-30 a `uses:` pin to a + # deleted omnimarket branch head made ci.yml startup-fail for ~2.5h + # (OMN-15536), and a pyproject rev pinned to an unlanded omnibase_core + # branch head merged past a comment forbidding it — both accepted by + # SHAPE-only 40-hex validators. The job is unconditional (`if: always()`), + # so a skip is anomalous and correctly fails closed here. + "Pin Reachability (OMN-15538)", + # OMN-15361: one unconditional source+Docker gate executes the classification, + # schema/RLS, role, adapter, one-database, and topology assertions together + # with their seeded RED controls. Registering the plain job display name here + # makes the source contract and rebuilt PostgreSQL 16 proofs part of the sole + # required CI Summary context rather than a separately-triggered advisory run. + "Application Database Domain Enforcement (OMN-15361)", + # OMN-15604: a [tool.uv.sources] git-pinned rev must build the SAME src/ + # tree as the released tag its declared `pkg==X.Y.Z` version names, even + # on a line carrying a `# raw-override-ok:` escape token (that token only + # exempts the separate, pre-existing `Dep Provenance Gate` -- the + # forbid-git-source rule -- never a content-lineage claim). The job is + # unconditional (`if: always()`), so a skip is anomalous and correctly + # fails closed here. Registered directly (not via EXPECTED_EXTERNAL_ + # CONTEXTS) because it is a job inside ci.yml's own run, observable + # without the external-context admission rule's historical measurement. + "Dep Provenance Lineage Gate (OMN-15604)", ) # Gates the old ci-summary accepted as ``success`` OR ``skipped``. Each carries @@ -131,9 +204,124 @@ } ) +# --------------------------------------------------------------------------- +# OMN-15496 — cross-workflow ("external") required contexts. +# --------------------------------------------------------------------------- +# Contexts produced by OTHER workflow files on the SAME head SHA. They are not +# in `dev`'s required_status_checks (which is exactly ["CI Summary"]) and are +# invisible to the run-scoped checks above, so before this tuple existed they +# blocked nothing. +# +# ADMISSION RULE — do not add a name here from a workflow file alone. +# A context is admitted only after measuring its *merge-time* report rate over +# the last N merged `dev` PRs: for each PR, the check-runs on its head SHA whose +# `started_at <= mergedAt` (post-merge runs are a retrospective artifact — on the +# first pass they produced three phantom "failures" for contexts that were green +# at merge). A context that does not report on every PR shape MUST NOT be listed: +# a permanently-absent entry burns the poll deadline and then fails closed, i.e. +# it wedges the branch. Every name below was measured 16/16 present over the 16 +# `dev` PRs merged 2026-07-29T23:04Z → 2026-07-30T14:54Z (#2546…#2567). +# +# Replaying those 16 PRs' merge-time payloads through this resolver yields 15 +# green and exactly one block — #2555, `deploy-gate / deploy-gate` = failure, +# which is the real defect this gate exists to catch. Slowest seeded context +# finished 24.9 min after `CI Summary` started, well inside the caller's 90 min +# poll deadline, so waiting on these cannot time the poller out. +# Fixture + regression: tests/ci/fixtures/omn15496_merge_time_external_check_runs.json. +EXPECTED_EXTERNAL_CONTEXTS: tuple[str, ...] = ( + "deploy-gate / deploy-gate", # 16/16 present, 15/16 green (#2555 red AT MERGE) + "verify / verify", # Receipt Gate + "call-reject-skip-token / scan / reject-skip-gate-token", # CLAUDE.md rule 10 mechanism + "main-target-guard", + "non-dev-base-guard", + "pr-title / check-title", + "URL Authority Gate", + "imperative-contract-guard / Imperative Contract Guard", + "gate / CodeRabbit Thread Check", + "Canonical Inference Gate", + "Type Safety Validation", + "Omni Standards Gate", + "Duplication Sweep", + "Stale TODO Gate", + "dispatcher-route-coverage", + "CodeQL", + "required-check-skip-guard / check-skip-vectors", +) + +# Contexts that were MEASURED and deliberately NOT enforced. Recorded as data — +# not silently omitted — so the exclusion is auditable and has to be re-argued +# with numbers rather than rediscovered. Pinned by test_ci_summary_gate.py. +MEASURED_NOT_ENFORCED_CONTEXTS: dict[str, str] = { + "Enforce clean + promoted build source": ( + "1/16 present — path-filtered; requiring it would wedge every PR that " + "does not touch its paths (the exact never-reports failure mode)." + ), + "occ-companion-effect / Publish occ-companion-effect command": ( + "16/16 present but only 10/16 green — a flaky publisher EFFECT, not a " + "validator. The substantive requirement it stands in for is already " + "enforced in-run by the STRICT gate 'OCC Companion Merged Gate " + "(OMN-15214)'." + ), + "Hostile Review Gate": ( + "16/16 present, 14/16 green — an adversarial-judgment gate. A 12.5% red " + "rate needs per-red root-cause before it may block merges; admitting it " + "blind would convert review opinion into a merge outage." + ), + "occ-preflight / eligibility": ( + "Already a STRICT_GATE_JOBS entry, and the ONE name observed both inside " + "and outside this run's check suite (duplicate producers: ci.yml and " + "hostile-reviewer.yml). Asserting it on both surfaces would double-count " + "an ambiguous name — see OMN-15112." + ), +} + +# OMN-15532 — contexts whose PRODUCER structurally does not report for a given +# PR author, so "absent" carries no information and must not burn the poll +# deadline. This is an *applicability* rule, not a bypass: the context stays +# fail-closed for every author not named here. +# +# ADMISSION RULE — an entry is justified only by a producer-side condition that +# makes the check-run impossible to create, quoted with the workflow file and +# the live readback that shows it absent. "It was red and I wanted it green" is +# never a reason. Keys must be members of EXPECTED_EXTERNAL_CONTEXTS and actors +# must be concrete logins (no wildcards) — both pinned by tests. +ACTOR_CONDITIONAL_CONTEXTS: dict[str, tuple[str, ...]] = { + # .github/workflows/cr-thread-gate-caller.yml gates the `gate` job on + # `(github.event_name == 'pull_request' && github.actor != 'dependabot[bot]')`. + # The context name is the `caller-job / reusable-job` form, so when the + # caller job is skipped the reusable's inner job never materialises and NO + # check-run is created — the context is absent, not `skipped`. + # + # Live readback 2026-07-30, infra dev Dependabot batch: + # #2522 2cdf352d actor=dependabot[bot] CR-gate run conclusion=skipped -> ABSENT + # #2521 841c292f actor=dependabot[bot] CR-gate run conclusion=skipped -> ABSENT + # #2520 feb6627b actor=jonahgabriel -> present, success + # #2519 08e356cc actor=jonahgabriel -> present, success + # #2518 e2d38605 actor=jonahgabriel -> present, success + # All five are `pull_request`, run_attempt=1: the actor is the discriminator. + # + # The OMN-15496 seed measured this context 16/16 present over #2546…#2567 — + # a window containing NO Dependabot PR, which is exactly how a 16/16 context + # can still be absent in production. + # + # Fixed consumer-side, not producer-side: OMN-10276 removed this actor skip + # on omnimemory, but omnimemory calls a LOCAL reusable and passes no secrets, + # whereas this caller invokes omniclaude's reusable with `secrets: + # CROSS_REPO_PAT`. Dependabot `pull_request` runs do not receive regular repo + # secrets, so dropping the skip here risks trading an absent-wedge for a + # red-wedge. See OMN-15532. + "gate / CodeRabbit Thread Check": ("dependabot[bot]",), +} + # Conclusions that count as "provably passed". GOOD_CONCLUSIONS: frozenset[str] = frozenset({"success", "skipped"}) +# External contexts are held to the STRICT bar: `skipped` fails closed. Every +# name above was measured `success` on all 16 sampled PRs (never skipped), so +# this costs nothing today and closes the skip-vector fail-open that OMN-15057 / +# OMN-14854 exist to prevent. +EXTERNAL_GOOD_CONCLUSIONS: frozenset[str] = frozenset({"success"}) + EXIT_SUCCESS = 0 EXIT_FAILURE = 1 EXIT_PENDING = 2 @@ -204,6 +392,99 @@ def dedup_latest( return latest +def latest_check_run_by_name( + check_runs: list[dict[str, object]], +) -> dict[str, JobState]: + """Collapse ``commits/{sha}/check-runs`` to one entry per context name. + + Resolution is **latest wins** by ``(started_at, id)`` — deliberately the same + rule GitHub itself applies when deciding a required status check from several + same-named check-runs on one SHA. + + A stricter "most-blocking across all same-named runs" rule was measured and + **rejected**: replayed over the 16 sampled merged PRs it blocks 6, of which 5 + are transient-red-then-rerun-green. Because check-runs accumulate on a SHA + forever, most-blocking makes any transient red permanent and removes re-run + as a recovery path — it manufactures merge outages instead of catching + defects. Latest-wins blocks 1/16, and that one is a real red at merge. + + Known bounded residual: when two workflow files emit the same context name, a + red from the earlier producer followed by a green from the later one resolves + green. That ANY-vs-ALL ambiguity is tracked in OMN-15112 and is why + ``occ-preflight / eligibility`` — the one name observed on both sides — is + excluded here (see :data:`MEASURED_NOT_ENFORCED_CONTEXTS`). + """ + + latest: dict[str, JobState] = {} + ordering: dict[str, tuple[str, int]] = {} + for raw in check_runs: + name = str(raw.get("name") or "") + if not name: + continue + try: + run_id = int(str(raw.get("id") or 0)) + except (TypeError, ValueError): + run_id = 0 + key = (str(raw.get("started_at") or ""), run_id) + if name in ordering and key <= ordering[name]: + continue + conclusion = raw.get("conclusion") + ordering[name] = key + latest[name] = JobState( + name=name, + status=str(raw.get("status") or ""), + conclusion=None if conclusion is None else str(conclusion), + run_attempt=1, + ) + return latest + + +def applicable_external_contexts( + expected: tuple[str, ...], + pr_author: str | None, +) -> tuple[str, ...]: + """Drop contexts whose producer cannot report for ``pr_author`` (OMN-15532). + + Order preserved. An unknown/empty ``pr_author`` drops NOTHING — the fail- + closed default — so a missing ``--pr-author`` argument enforces the full set + rather than silently exempting it. + """ + + if not pr_author: + return expected + return tuple( + context + for context in expected + if pr_author not in ACTOR_CONDITIONAL_CONTEXTS.get(context, ()) + ) + + +def evaluate_external_contexts( + check_runs: list[dict[str, object]] | None, + expected: tuple[str, ...], +) -> tuple[list[str], list[str]]: + """Return ``(failures, missing_or_pending)`` for the declared external contexts. + + ``check_runs is None`` means the caller could not fetch the head SHA's + check-runs. That is treated as **every** expected context being unobserved — + PENDING, never success — so a transient API failure retries and a permanent + one fails closed at the deadline. It must never read as green. + """ + + if not expected: + return [], [] + latest = latest_check_run_by_name(check_runs or []) + failures: list[str] = [] + unresolved: list[str] = [] + for context in expected: + state = latest.get(context) + if state is None or state.status != "completed": + unresolved.append(context) + elif state.conclusion not in EXTERNAL_GOOD_CONCLUSIONS: + failures.append(context) + return sorted(failures), sorted(unresolved) + + def _is_allowlisted(name: str, allowlist: frozenset[str]) -> bool: """Prefix-aware allowlist check. @@ -226,9 +507,23 @@ def evaluate( strict_gates: tuple[str, ...] = STRICT_GATE_JOBS, skippable_gates: tuple[str, ...] = SKIPPABLE_GATE_JOBS, allowlist: frozenset[str] = SOFT_ALLOWLIST, + check_runs: list[dict[str, object]] | None = None, + external_contexts: tuple[str, ...] = (), + pr_author: str | None = None, ) -> tuple[int, str]: - """Return ``(exit_code, human_report)`` for the current job snapshot.""" + """Return ``(exit_code, human_report)`` for the current job snapshot. + ``external_contexts`` defaults to empty (assert nothing) so non-PR callers — + ``merge_group`` / ``workflow_dispatch``, where no PR-scoped context set + exists — are not wedged. The CLI supplies + :data:`EXPECTED_EXTERNAL_CONTEXTS` and its ``--event-name`` defaults to + ``pull_request``, so a *forgotten* argument enforces rather than skips. + + ``pr_author`` drops only the contexts that :data:`ACTOR_CONDITIONAL_CONTEXTS` + marks unreportable for that author (OMN-15532). ``None`` drops nothing. + """ + + external_contexts = applicable_external_contexts(external_contexts, pr_author) latest = dedup_latest(jobs, run_attempt=run_attempt) gate_names = frozenset(strict_gates) | frozenset(skippable_gates) @@ -272,22 +567,19 @@ def evaluate( if (latest.get(g) is None or latest[g].status != "completed") ] - all_failures = strict_failures + skippable_failures + sweep_failures + # (4) OMN-15496 external contexts: cross-workflow checks on the PR head. + external_failures, external_unresolved = evaluate_external_contexts( + check_runs, external_contexts + ) - if all_failures: - return EXIT_FAILURE, _report( - "FAILURE", - latest, - strict_gates, - skippable_gates, - strict_failures, - skippable_failures, - sweep_failures, - gate_missing_or_pending, - ) - if gate_missing_or_pending: - return EXIT_PENDING, _report( - "PENDING", + all_failures = ( + strict_failures + skippable_failures + sweep_failures + external_failures + ) + all_unresolved = gate_missing_or_pending + external_unresolved + + def _verdict(label: str) -> str: + return _report( + label, latest, strict_gates, skippable_gates, @@ -295,17 +587,16 @@ def evaluate( skippable_failures, sweep_failures, gate_missing_or_pending, + external_contexts, + external_failures, + external_unresolved, ) - return EXIT_SUCCESS, _report( - "SUCCESS", - latest, - strict_gates, - skippable_gates, - strict_failures, - skippable_failures, - sweep_failures, - gate_missing_or_pending, - ) + + if all_failures: + return EXIT_FAILURE, _verdict("FAILURE") + if all_unresolved: + return EXIT_PENDING, _verdict("PENDING") + return EXIT_SUCCESS, _verdict("SUCCESS") def _report( @@ -317,6 +608,9 @@ def _report( skippable_failures: list[str], sweep_failures: list[str], gate_missing_or_pending: list[str], + external_contexts: tuple[str, ...] = (), + external_failures: list[str] | None = None, + external_unresolved: list[str] | None = None, ) -> str: lines = [f"CI Summary verdict: {verdict}", f" jobs observed: {len(latest)}"] lines.append(" strict gates:") @@ -343,6 +637,14 @@ def _report( lines.append(f" default-deny sweep failures: {', '.join(sweep_failures)}") if gate_missing_or_pending: lines.append(f" gates missing/pending: {', '.join(gate_missing_or_pending)}") + if external_contexts: + lines.append(f" external contexts asserted: {len(external_contexts)}") + if external_failures: + lines.append(f" external-context failures: {', '.join(external_failures)}") + if external_unresolved: + lines.append( + f" external contexts missing/pending: {', '.join(external_unresolved)}" + ) return "\n".join(lines) @@ -363,6 +665,35 @@ def _load_jobs(path: str | None) -> list[dict[str, object]]: return jobs +def _load_check_runs(path: str | None) -> list[dict[str, object]] | None: + """Load ``commits/{sha}/check-runs``; return ``None`` when unavailable. + + ``None`` is the fail-closed signal: :func:`evaluate_external_contexts` reads + it as "no context observed" → PENDING → FAILURE at the caller's deadline. A + missing, empty, or malformed payload must never green the gate, so every + failure path here returns ``None`` rather than an empty list. + """ + + if not path: + return None + try: + with open(path, encoding="utf-8") as handle: + raw = handle.read() + except OSError: + return None + if not raw.strip(): + return None + try: + data = json.loads(raw) + except json.JSONDecodeError: + return None + if isinstance(data, dict): + data = data.get("check_runs", []) + if not isinstance(data, list): + return None + return [row for row in data if isinstance(row, dict)] + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -382,10 +713,42 @@ def main(argv: list[str] | None = None) -> int: default=None, help="Evaluate only rows for this GitHub Actions run_attempt.", ) + parser.add_argument( + "--check-runs-file", + default=None, + help="Path to the PR head SHA's commits/{sha}/check-runs JSON, used to " + "assert EXPECTED_EXTERNAL_CONTEXTS (OMN-15496). A missing/unreadable " + "file is PENDING, never success.", + ) + parser.add_argument( + "--event-name", + default="pull_request", + help="GitHub event name. External contexts are asserted on " + "'pull_request' only — merge_group/workflow_dispatch have no PR-scoped " + "context set. Defaults to 'pull_request' so a FORGOTTEN argument " + "enforces rather than silently skips.", + ) + parser.add_argument( + "--pr-author", + default=None, + help="Login of the PR author. Drops ONLY the ACTOR_CONDITIONAL_CONTEXTS " + "entries that this author's PRs structurally cannot produce (OMN-15532). " + "Omitted/empty drops nothing, so a forgotten argument enforces the full " + "set rather than exempting it.", + ) args = parser.parse_args(argv) jobs = _load_jobs(args.jobs_file) - code, report = evaluate(jobs, run_attempt=args.run_attempt) + external_contexts = ( + EXPECTED_EXTERNAL_CONTEXTS if args.event_name == "pull_request" else () + ) + code, report = evaluate( + jobs, + run_attempt=args.run_attempt, + check_runs=_load_check_runs(args.check_runs_file), + external_contexts=external_contexts, + pr_author=args.pr_author, + ) print(report) if args.report_only: return EXIT_SUCCESS diff --git a/scripts/ci/detect_test_paths.py b/scripts/ci/detect_test_paths.py index eaa19fb376..d431029153 100644 --- a/scripts/ci/detect_test_paths.py +++ b/scripts/ci/detect_test_paths.py @@ -24,12 +24,94 @@ REPO_ROOT = Path(__file__).resolve().parents[2] TEST_UNIT_PREFIX = "tests/unit/" TEST_INTEGRATION_PREFIX = "tests/integration/" +TESTS_PREFIX = "tests/" +SCRIPTS_PREFIX = "scripts/" +# The two directories that actually exercise `scripts/`: the hermetic script +# tests (tests/scripts/) and the unit-tree mirror (tests/unit/scripts/). +SCRIPTS_TEST_PREFIXES = ("tests/scripts/", "tests/unit/scripts/") CI_PROCESS_TEST_PATHS = ( ".github/workflows/", "scripts/ci/", "config/runner_routing_policy.yaml", ) +# OMN-15336 item 4 repair follow-up: the vendored node-migration tree lives +# under neither src/, scripts/, nor tests/, so a change there (a new +# migration .sql, its _ledger row, or the FORCE-RLS fence/grandfather +# manifests) produced NO selection at all and fell through to the +# conservative tests/unit/ fallback -- which does not contain +# tests/scripts/test_node_migration_fence_parity.py. That test is the ratchet +# guarding against a future FORCE-RLS migration being laundered onto the +# grandfather snapshot; it is unreachable by the everyday change-aware +# selector on exactly the class of change that would breach it (verified: +# a grandfather-manifest + new .sql + ledger-row diff selected only +# tests/unit/ before this mapping existed). +# +# ADDITIVE, not a swap (2026-08-05 fix-forward). The first cut of this mapping +# added ONLY "tests/scripts/". Because `compute_selection`'s conservative +# fallback (`if not selected: selected = ["tests/unit/"]`) only fires when +# `_resolve()` returns nothing at all, giving migration-tree changes their own +# non-empty selection SUPPRESSED that fallback -- an ordinary migration diff +# (new .sql + ledger row, no YAML) went from selecting the whole tests/unit/ +# tree to selecting tests/scripts/ ONLY. That is a real coverage regression, +# not a narrowing-to-something-equivalent swap like the scripts/ mapping +# above: tests/unit/migrations/, tests/unit/topology/, test_schema_fingerprint, +# test_db_ownership, and test_adversarial_fingerprint_drift all live under +# tests/unit/ (outside tests/unit/scripts/) and genuinely exercise migration +# .sql/ledger changes -- unlike scripts/, where tests/unit/ never covered the +# code plain-blanket-fallback was standing in for. So this branch selects +# BOTH tests/scripts/ (the fence-parity ratchet) AND tests/unit/ (the +# pre-existing real coverage) rather than trading one for the other. Any +# future prefix branch added here must make the same "does the blanket +# tests/unit/ fallback carry real coverage for this path class?" check before +# assuming a narrower, targeted selection is safe to swap in -- the +# fallback-suppression trap in `compute_selection` (a non-empty `_resolve()` +# result silently defeats the safety net for the whole diff, not just the +# part the new branch understands) is structural, not specific to migrations. +MIGRATION_TREE_PREFIX = "docker/migrations/forward/" + +# OMN-15410: pytest roots that live NEXT TO the code they cover instead of +# under tests/. They are collected by the full suite (pyproject.toml +# `testpaths`), but the full suite is only one of two pytest steps — a +# NARROWED smart-selection run reaches nothing it is not explicitly told to +# reach. Without these mappings the four roots would be "collected" in the +# weakest possible sense: exercised only when something else escalated the +# job to full suite. Keys are source prefixes, values are the test roots a +# change under that prefix must run. Over-selection here is safe (extra tests +# run); under-selection is the OMN-15378 false-green class. +# +# Every value MUST also appear in pyproject.toml `testpaths`, and every +# non-`tests` testpaths entry MUST appear as a value here — both directions +# are asserted by scripts/validation/validate_test_root_collection.py. +COLLOCATED_TEST_ROOTS: dict[str, str] = { + # Broadest first is irrelevant (all matches apply), but note scripts/tests/ + # covers the seed/keycloak scripts that live directly under scripts/, so it + # is mapped from the whole scripts/ tree, matching SCRIPTS_TEST_PREFIXES. + "scripts/": "scripts/tests/", + "scripts/ci/": "scripts/ci/tests/", + "scripts/runtime_build/": "scripts/runtime_build/tests/", + "src/omnibase_infra/services/observability/agent_actions/": ( + "src/omnibase_infra/services/observability/agent_actions/tests/" + ), +} + +# Test families the change-aware pytest job structurally cannot run, so +# selecting one can never make it execute -- it would only make pytest exit 5 +# ("no tests ran") when it is the sole selected path, reddening the gate without +# running anything. This is NOT a narrowing carve-out: the FULL suite excludes +# these identically, and each has its own dedicated gate. +# * tests/integration/docker/ -- `--ignore`d by BOTH pytest steps in +# .github/workflows/ci.yml; covered by docker-build.yml, whose paths filter +# includes tests/integration/docker/**. +# * tests/chaos/ and tests/performance/ -- deselected by the job's marker +# expression (-m "not slow and not chaos and not kafka and not performance"), +# which applies to the full suite too. +UNRUNNABLE_TEST_PREFIXES = ( + "tests/integration/docker/", + "tests/chaos/", + "tests/performance/", +) + # Positive-evidence documentation classification (OMN-14753). A path matching # either of these can never contain executable code or fixture data, so it # cannot influence any test outcome. This is narrower and stronger than "no @@ -45,6 +127,62 @@ def _is_docs_only_path(path: str) -> bool: return path.endswith(DOCS_ONLY_SUFFIXES) or path.startswith(DOCS_ONLY_PREFIXES) +def _is_covered_by(selected: set[str] | list[str], path: str) -> bool: + """True when pytest, given `selected`, would collect `path`.""" + return any(path.startswith(prefix) for prefix in selected) + + +def _changed_test_paths(changed_files: list[str]) -> list[str]: + """Changed paths under tests/ that the selector is obliged to cover. + + Excludes documentation (provably inert, OMN-14753) and the families the + pytest job structurally cannot run (`UNRUNNABLE_TEST_PREFIXES`). + """ + return [ + path + for path in changed_files + if path.startswith(TESTS_PREFIX) + and not _is_docs_only_path(path) + and not path.startswith(UNRUNNABLE_TEST_PREFIXES) + ] + + +def _requires_unnarrowable_full_suite(changed_files: list[str]) -> bool: + """True when a changed test path cannot be narrowed below `tests/` itself. + + A test module sitting directly in the tests/ root (no subdirectory) has no + containing directory other than `tests/`. Emitting `tests/` as a *smart* + selection would run the whole suite under the smart step's split count and + timeouts; the honest answer is the real full-suite escalation. + """ + return any( + path.count("/") == 1 and path.endswith(".py") + for path in _changed_test_paths(changed_files) + ) + + +def _uncovered_changed_test_dirs( + changed_files: list[str], + selected: set[str], +) -> set[str]: + """Directories that must be added so every changed test path is collected. + + Additive only: a changed test path already covered by an existing selection + contributes nothing. Root-level test modules are handled by the + `_requires_unnarrowable_full_suite` escalation in `compute_selection`, so + they are skipped here rather than emitting `tests/` as a smart selection. + """ + extra: set[str] = set() + for path in _changed_test_paths(changed_files): + parent = path.rsplit("/", 1)[0] + "/" + if parent == TESTS_PREFIX: + continue + if _is_covered_by(selected | extra, path): + continue + extra.add(parent) + return extra + + FULL_SUITE_BRANCHES = {"main"} # Full suite uses 15 splits (infra CI split count) @@ -55,14 +193,17 @@ def resolve_test_paths( changed_files: list[str], adjacency_path: Path, ) -> list[str]: - """Map changed file paths to deterministic UNIT test directories. + """Map changed file paths to deterministic test directories. Behavior: - Source changes under src/omnibase_infra/: include tests/unit//. - - Test-only changes under tests/unit/: include the changed unit-test directory. - - Test-only changes under tests/integration/: ignored (integration runs always). - - Files outside src/ and tests/unit/: no contribution; caller decides + - Changes under scripts/: include tests/scripts/ + tests/unit/scripts/ + (scripts/ci/ additionally keeps its tests/ci/ CI-process mapping). + - ANY changed path under tests/ is covered by the returned selection -- + its own directory at minimum (OMN-15245). Narrowing may add tests; it + may never drop a test file the diff itself touched. + - Files outside src/, scripts/ and tests/: no contribution; caller decides whether to escalate to full suite. Adjacency expansion maps each changed module to its reverse dependents, @@ -95,6 +236,45 @@ def _resolve( ): selected.add("tests/ci/") + if path.startswith(SCRIPTS_PREFIX): + # OMN-15245: scripts/ holds deploy-path and governance-guard code + # whose tests live in tests/scripts/ and tests/unit/scripts/. Before + # this mapping a scripts/ change reached neither: it produced no + # selection at all and fell through to the blanket tests/unit/ + # fallback, which exercises none of it (recorded live on OMN-15218 / + # omnibase_infra#2493). Note this is an `if`, not an `elif`: + # scripts/ci/ keeps its tests/ci/ CI-process mapping AND gains these. + selected.update(SCRIPTS_TEST_PREFIXES) + + if path.startswith(MIGRATION_TREE_PREFIX): + # OMN-15336 item 4 repair follow-up: see MIGRATION_TREE_PREFIX's + # own comment above. Deliberately NOT routed through + # COLLOCATED_TEST_ROOTS -- tests/scripts/ is already collected via + # the plain "tests" testpaths entry, so adding it as a + # COLLOCATED_TEST_ROOTS value would trip + # check_collocated_selector_coverage's parity assertion in + # scripts/validation/validate_test_root_collection.py (that check + # is scoped to roots requiring their OWN testpaths entry, which + # tests/scripts/ does not). + selected.add("tests/scripts/") + # ADDITIVE fix-forward (see MIGRATION_TREE_PREFIX comment): also + # keep the blanket tests/unit/ coverage this path class relied on + # via the `if not selected` fallback before this mapping existed. + # Unlike scripts/ above, tests/unit/ genuinely exercises migration + # .sql/ledger changes (tests/unit/migrations/, tests/unit/topology/, + # test_schema_fingerprint.py, test_db_ownership.py, + # test_adversarial_fingerprint_drift.py), so giving this branch its + # own non-empty selection must not silently drop that coverage by + # suppressing the fallback. + selected.add(TEST_UNIT_PREFIX) + + # OMN-15410: collocated roots (tests living beside their code rather + # than under tests/). Independent of every branch above — a path can + # legitimately map to a tests/ directory AND to its collocated root. + for source_prefix, collocated_root in COLLOCATED_TEST_ROOTS.items(): + if path.startswith(source_prefix): + selected.add(collocated_root) + expanded: set[str] = set(direct_modules) for module in direct_modules: expanded.update(config.adjacency[module].reverse_deps) @@ -102,6 +282,11 @@ def _resolve( for module in expanded: selected.add(f"{TEST_UNIT_PREFIX}{module}/") + # OMN-15245 fail-closed invariant, applied LAST so it sees everything the + # mappings above already cover: every CHANGED path under tests/ must be + # collected by the emitted selection. + selected.update(_uncovered_changed_test_dirs(changed_files, selected)) + # Drop selected directories that do not exist on disk. A module in the # adjacency map (e.g. `dlq`) may have source under src/ but no # corresponding tests/unit// directory; passing a missing path to @@ -140,6 +325,11 @@ def compute_selection( ): return _full_suite(EnumFullSuiteReason.TEST_INFRASTRUCTURE) + # 2b. Unnarrowable changed test (OMN-15245): a changed test module directly + # under tests/ has no containing directory below `tests/` itself. + if _requires_unnarrowable_full_suite(changed_files): + return _full_suite(EnumFullSuiteReason.CHANGED_TEST_UNNARROWABLE) + # 3. Shared module escalation. changed_modules = { path[len(SRC_PREFIX) :].split("/", 1)[0] diff --git a/scripts/ci/generate_application_database_sql_baseline.py b/scripts/ci/generate_application_database_sql_baseline.py new file mode 100644 index 0000000000..6f2c791215 --- /dev/null +++ b/scripts/ci/generate_application_database_sql_baseline.py @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Regenerate the frozen application-database SQL baseline (OMN-15361). + +The baseline records violations that already existed in deployed SQL when it was +frozen, so the dev->main promotion boundary does not re-litigate the whole +migration corpus. See the ratchet comment in ``check_application_database_sql``. + +Deterministic by construction: entries are content-keyed and emitted in sorted +order, so regenerating against an unchanged tree reproduces the same entry set +and the freeze is auditable in review. + +The baseline is SHRINK-ONLY. Regenerate to *remove* entries after fixing +violations; never to absorb new ones. Run with ``--check`` in CI or locally to +assert the committed file matches what the current tree produces -- that check +compares the entry KEY SET, not raw bytes, because the repo's yamlfmt hook +reflows this file on commit. + + uv run python scripts/ci/generate_application_database_sql_baseline.py \\ + --base-revision origin/main \\ + --ownership-manifest [--ownership-manifest ...] +""" + +from __future__ import annotations + +import argparse +import json +import sys +from datetime import UTC, datetime +from pathlib import Path + +# Importable both as `python -m scripts.ci.generate_...` and as a direct script +# path; the bootstrap must precede the package import, not sit under __main__. +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from scripts.ci.check_application_database_sql import ( + _BASELINE_PATH, + load_sql_baseline, + validate_changed_sql, + violation_key, +) + +_HEADER = """\ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# OMN-15361: frozen baseline for the application-database SQL gate. +# +# WHY THIS FILE EXISTS: the gate lints SQL *changed against the PR base*. On a +# dev PR that is a handful of files, so each file was only ever scanned in its +# own small PR. At the dev->main promotion boundary the base is main, so the +# entire accumulated migration corpus counts as changed and every latent +# violation fires at once -- none newly authored, all already deployed. +# Rewriting deployed migrations to satisfy a gate at release time is the more +# dangerous path, so those pre-existing violations are recorded here and +# soft-passed. This mirrors the OMN-14443 deploy-gate grandfather ratchet. +# +# THE RATCHET: a violation NOT listed here is held to the full bar and fails +# closed. An entry whose file is deleted, or whose violation stops firing on a +# file the run actually linted, is STALE and FAILS the gate -- so the list can +# only shrink. +# +# BURN-DOWN ONLY: never hand-add an entry. Fix violations, then regenerate with +# scripts/ci/generate_application_database_sql_baseline.py to shrink the list. +# Widening it defeats the ratchet. +# +# Entries are keyed by sha256 of the ": " violation line -- content, +# never line numbers, so unrelated SQL edits cannot silently re-key an entry. +""" + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", type=Path, default=Path.cwd()) + parser.add_argument("--base-revision", required=True) + parser.add_argument("--head-revision", default="HEAD") + parser.add_argument( + "--ownership-manifest", + action="append", + type=Path, + default=[], + help="Typed ownership manifest; repeat for every authoritative source", + ) + parser.add_argument("--output", type=Path, default=_BASELINE_PATH) + parser.add_argument( + "--check", + action="store_true", + help="Exit non-zero if the committed baseline differs from this tree", + ) + return parser + + +def render_baseline(violations: tuple[str, ...], *, generated_at: str) -> str: + """Render the baseline document; sorted and content-keyed, so it is stable. + + Scalars are emitted with ``json.dumps``: violation messages routinely embed + single quotes around relation names, and Python ``repr`` escapes those with + backslashes, which YAML does not accept. JSON string syntax is a subset of + YAML's double-quoted style, so this round-trips. Getting this wrong is + quiet rather than loud -- an unparseable snapshot makes the loader fail + closed to empty, which grandfathers nothing and looks like "the baseline + did not take" instead of a render bug. + """ + lines = [ + _HEADER, + f"generated_at: {json.dumps(generated_at)}", + f"count: {len(violations)}", + ] + if not violations: + lines.append("violations: []") + return "\n".join(lines) + "\n" + lines.append("violations:") + for violation in sorted(violations): + path, _, message = violation.partition(": ") + lines.append(f" - key: {json.dumps(violation_key(violation))}") + lines.append(f" path: {json.dumps(path)}") + lines.append(f" violation: {json.dumps(message.strip())}") + return "\n".join(lines) + "\n" + + +def main() -> int: + args = _parser().parse_args() + repository = args.repository.resolve() + + # Regenerate against the RAW gate verdict: pass an explicitly absent baseline + # so the current snapshot cannot fold itself back in and freeze forever. + outcome = validate_changed_sql( + repository, + args.base_revision, + args.head_revision, + ownership_manifest_paths=tuple( + path if path.is_absolute() else repository / path + for path in args.ownership_manifest + ), + baseline_path=Path("/nonexistent-baseline-force-raw-verdict"), + ) + + existing = load_sql_baseline(args.output) + fresh_keys = {violation_key(violation) for violation in outcome.violations} + grew = sorted(fresh_keys - set(existing)) + + if args.check: + # Compare the KEY SET, not the rendered bytes. The repo's yamlfmt hook + # reflows this file on commit, so a byte comparison would report drift + # for pure formatting and train people to ignore this check. + committed_keys = set(existing) + if fresh_keys != committed_keys: + stale = sorted(committed_keys - fresh_keys) + print( + f"application_database_sql_baseline=DRIFT: {args.output.name} does not " + f"match this tree. {len(grew)} entr{'y' if len(grew) == 1 else 'ies'} " + f"missing from it, {len(stale)} no longer firing. Regenerate, and " + "confirm the change only SHRINKS the list." + ) + for key in grew[:5]: + print(f" + {key[:12]} (not baselined)") + for key in stale[:5]: + print(f" - {key[:12]} (stale)") + return 1 + print( + f"application_database_sql_baseline=OK ({len(outcome.violations)} entries)" + ) + return 0 + + if existing and grew: + print( + f"application_database_sql_baseline=REFUSED: regenerating would ADD " + f"{len(grew)} entr{'y' if len(grew) == 1 else 'ies'} not in the committed " + "baseline. The baseline is shrink-only -- fix the new violations instead." + ) + for key in grew[:10]: + print(f" + {key[:12]}") + return 1 + + args.output.write_text( + render_baseline( + outcome.violations, + generated_at=datetime.now(UTC).strftime("%Y-%m-%d"), + ), + encoding="utf-8", + ) + removed = len(set(existing) - fresh_keys) + print( + f"application_database_sql_baseline=WROTE {len(outcome.violations)} entries " + f"to {args.output} (removed {removed})" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/infra-node-allowlist.txt b/scripts/ci/infra-node-allowlist.txt index 62405a7e67..989e9e3b01 100644 --- a/scripts/ci/infra-node-allowlist.txt +++ b/scripts/ci/infra-node-allowlist.txt @@ -88,6 +88,10 @@ src/omnibase_infra/nodes/node_event_bus_wiring_effect/handlers/__init__.py # ap src/omnibase_infra/nodes/node_event_bus_wiring_effect/handlers/handler_event_bus_wiring.py # approved existing infra handler path; new business-domain handlers belong in omnimarket. src/omnibase_infra/nodes/node_event_forward_effect/handlers/__init__.py # approved existing infra handler path; new business-domain handlers belong in omnimarket. src/omnibase_infra/nodes/node_event_forward_effect/handlers/handler_event_forward.py # approved existing infra handler path; new business-domain handlers belong in omnimarket. +src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/__init__.py # infra-owned gateway attach/session control plane (OMN-15750); trust-boundary identity validation, not business-domain logic; sibling of node_bus_forwarder_effect. +src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_attach.py # infra-owned gateway attach/session control plane (OMN-15750); trust-boundary identity validation, not business-domain logic; sibling of node_bus_forwarder_effect. +src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_detach.py # infra-owned gateway attach/session control plane (OMN-15750); trust-boundary identity validation, not business-domain logic; sibling of node_bus_forwarder_effect. +src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_heartbeat.py # infra-owned gateway attach/session control plane (OMN-15750); trust-boundary identity validation, not business-domain logic; sibling of node_bus_forwarder_effect. src/omnibase_infra/nodes/node_github_pr_poller_effect/handlers/__init__.py # approved existing infra handler path; new business-domain handlers belong in omnimarket. src/omnibase_infra/nodes/node_github_pr_poller_effect/handlers/handler_github_api_poll.py # approved existing infra handler path; new business-domain handlers belong in omnimarket. src/omnibase_infra/nodes/node_gmail_archive_cleanup_effect/handlers/__init__.py # approved existing infra handler path; new business-domain handlers belong in omnimarket. diff --git a/scripts/ci/integration_skip_guard.yaml b/scripts/ci/integration_skip_guard.yaml index 600c99f741..d395a2a7d3 100644 --- a/scripts/ci/integration_skip_guard.yaml +++ b/scripts/ci/integration_skip_guard.yaml @@ -23,7 +23,8 @@ # ------------------------------- # The gate is keyed to the services the merge-gating guard job actually # PROVISIONS (see `required_services`). The `integration-guard` job in ci.yml -# provisions Postgres (postgres:16-alpine), applies all migrations, exports the +# provisions Postgres (postgres:16-alpine, pulled and started by explicit +# fail-closed steps per OMN-15249), applies all migrations, exports the # connection env, and runs a CURATED set of Postgres-only integration proofs # (see `curated_test_paths`). A skip only fails the gate when its reason names a # *provisioned* service's absence — i.e. Postgres is right there and the test @@ -51,7 +52,11 @@ require_executed_min: 1 # (grepped from tests/) so the gate stays correct if the curated set broadens. required_services: postgres: - provisioned_by: "ci.yml `integration-guard` job (postgres:16-alpine service, migrations applied, OMNIBASE_INFRA_DB_URL + POSTGRES_* exported)" + # OMN-15249: provisioning is explicit pull + `docker run` steps, NOT a + # GitHub-managed `services:` block — a services-block pull failure is not + # addressable from the workflow and let the verdict step fire past a + # container that never materialized (surfacing as a misattributed exit 127). + provisioned_by: "ci.yml `integration-guard` job (explicit fail-closed `docker pull`/`docker run` of postgres:16-alpine, migrations applied, OMNIBASE_INFRA_DB_URL + POSTGRES_* exported)" missing_skip_patterns: # test_postgres_repository_runtime_integration.py db_pool fixture - "PostgreSQL integration tests skipped" diff --git a/scripts/ci/parity_red_on_base_baseline.py b/scripts/ci/parity_red_on_base_baseline.py new file mode 100644 index 0000000000..82062ebcb4 --- /dev/null +++ b/scripts/ci/parity_red_on_base_baseline.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""FROZEN parity-RED-on-base debt baseline — OMN-15344 (child of OMN-14355). + +``PARITY_RED_ON_BASE_DEBT`` is the frozen set of nodes whose committed hand-flip proof +declares parity tests that are NOT red-on-their-own-assertion at the receipt's own +``base_ref``. Those tests run green on HEAD and prove nothing about the def-A -> def-B +behavior transfer, because they never discriminated it. + +Measured, not assumed. On 2026-07-28 assertion 7 of +``omnibase_core/scripts/ci/verify_flip_bundle.py`` (OMN-15340, merged omnibase_core#1519) +was EXECUTED against every committed ``*.handflip.json`` in this repo with the +newness/grandfathering rule bypassed. Every id below FAILED. +All 15 of this repo's committed hand-flip receipts are listed: none passed. +Across omnibase_infra + omnimarket that census read 21/22 receipts failing and 102/126 +declared parity ids not red-for-the-right-reason. + +This set is monotonically NON-INCREASING. It may only shrink, and the gate that reads it +(``verify_flip_bundle.py``, wired into this repo's CI in the same job as the +canonical-shape ratchet) enforces that: + +* A NEW hand-flip may NOT be added here. It must declare parity tests that are RED on + their own assertion at its ``base_ref``. Growth HARD-FAILS. +* An entry naming a hand-flip proof that does not exist HARD-FAILS, so this list cannot + rot into decoration and cannot be pre-seeded ahead of a flip. +* REMOVING an entry is a CLAIM that the node now passes, and the gate EXECUTES it: + assertion 7 is re-armed for that node and must pass. Removing an entry by deleting the + hand-flip proof instead HARD-FAILS too. + +``RED_EXCEPTION`` is inadmissible and stays that way. A base-tree exception is +indistinguishable from a test that is merely incompatible with the old code, so it is not +a discriminating claim — and it is the single largest slice of this census, which is +exactly why admitting it would erase most of this list without one test getting better. + +Retirement mechanism: burn down per node as each is next touched — rewrite its declared +parity tests so they assert the transition, prove them RED at the receipt's ``base_ref``, +then delete the entry in the same PR. +""" + +PARITY_RED_ON_BASE_DEBT: tuple[str, ...] = ( + "omnibase_infra.nodes.node_auth_gate_compute", + "omnibase_infra.nodes.node_broker_disk_watermark_compute", + "omnibase_infra.nodes.node_build_loop_projection_compute", + "omnibase_infra.nodes.node_checkpoint_validate_compute", + "omnibase_infra.nodes.node_impact_analyzer_compute", + "omnibase_infra.nodes.node_invariant_evaluate_compute", + "omnibase_infra.nodes.node_kafka_replay_compute", + "omnibase_infra.nodes.node_ledger_projection_compute", + "omnibase_infra.nodes.node_ledger_projection_compute.handlers", + "omnibase_infra.nodes.node_model_router_compute", + "omnibase_infra.nodes.node_pr_state_projection_compute", + "omnibase_infra.nodes.node_rsd_score_compute", + "omnibase_infra.nodes.node_runner_fleet_health_compute", + "omnibase_infra.nodes.node_runtime_source_attestor_effect", + "omnibase_infra.nodes.node_validation_ledger_projection_compute", +) diff --git a/scripts/ci/prove_application_database_acl.py b/scripts/ci/prove_application_database_acl.py new file mode 100644 index 0000000000..9da511ce6b --- /dev/null +++ b/scripts/ci/prove_application_database_acl.py @@ -0,0 +1,2398 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Rebuilt PostgreSQL 16 ACL/default-privilege and rollback proof.""" + +from __future__ import annotations + +import json +import os +import subprocess +from collections import Counter, defaultdict +from collections.abc import Callable, Iterable, Mapping, Sequence +from pathlib import Path +from typing import TypedDict, cast + +import psycopg2 +import psycopg2.extras +import yaml +from psycopg2 import sql + +from omnibase_core.enums.enum_database_grant_object_type import ( + EnumDatabaseGrantObjectType, +) +from omnibase_core.enums.enum_database_privilege import EnumDatabasePrivilege +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_core.models.core.model_deployment_topology import ModelDeploymentTopology +from omnibase_infra.validation.application_database_acl import ( + PUBLIC_PRINCIPAL, + build_application_database_acl_matrix, + render_application_database_acl_sql, +) +from omnibase_infra.validation.enums.enum_application_database_acl_authorization_scope import ( + EnumApplicationDatabaseAclAuthorizationScope, +) +from omnibase_infra.validation.enums.enum_application_database_acl_render_phase import ( + EnumApplicationDatabaseAclRenderPhase, +) +from omnibase_infra.validation.models.model_application_database_acl_matrix import ( + ModelApplicationDatabaseAclMatrix, +) +from omnibase_infra.validation.models.model_application_database_acl_object import ( + ModelApplicationDatabaseAclObject, +) +from omnibase_infra.validation.models.model_application_database_acl_policy import ( + ModelApplicationDatabaseAclPolicy, +) +from omnibase_infra.validation.models.model_application_database_acl_row import ( + ModelApplicationDatabaseAclRow, +) +from omnibase_infra.validation.models.model_application_database_acl_source import ( + ModelApplicationDatabaseAclSource, +) +from omnibase_infra.validation.models.model_application_database_default_acl_row import ( + ModelApplicationDatabaseDefaultAclRow, +) +from omnibase_infra.validation.models.model_application_database_observed_role_state import ( + ModelApplicationDatabaseObservedRoleState, +) +from omnibase_infra.validation.models.model_application_database_principal_inventory import ( + ModelApplicationDatabasePrincipalInventory, +) +from omnibase_infra.validation.models.model_application_database_role_membership import ( + ModelApplicationDatabaseRoleMembership, +) +from omnibase_infra.validation.models.model_application_database_role_state import ( + ModelApplicationDatabaseRoleState, +) +from omnibase_infra.validation.models.model_application_relation_evidence_inventory import ( + ModelApplicationRelationEvidenceInventory, +) + +ADMIN_DSN = os.environ["ADMIN_DSN"] +DATABASE = os.environ.get("PROOF_DATABASE", "omnidash_analytics") +DATABASE_HOST = os.environ.get("PROOF_DB_HOST", "postgres") +DATABASE_PORT = int(os.environ.get("PROOF_DB_PORT", "5432")) +ROLE_PASSWORD = "acl-proof-only" # pragma: allowlist secret +MANAGED_SCHEMAS = ("tenant", "omninode_internal", "platform_catalog") +MUTATED_SCHEMAS = (*MANAGED_SCHEMAS, "public") +LEGACY_DEFAULT_SCHEMA = "legacy_acl_sentinel" +OWNERS = ( + "owner_onex_tenant", + "owner_omninode_internal", + "owner_platform_catalog", +) +WORKLOADS = ( + "app_dashboard", + "omninode_runtime", + "onex_api", + "tenant_projection_writer", +) +SERVICE_DATABASE_ROLES = { + "keycloak": "keycloak_service", + "omnibase_infra": "omnibase_infra_service", + "omninode_cloud": "omninode_cloud_service", + "omniclaude": "omniclaude_service", + "omniintelligence": "omniintelligence_service", + "omnimemory": "omnimemory_service", + "umami": "umami_service", +} +EXPECTED_PRECHANGE = Path( + os.environ.get( + "EXPECTED_PRECHANGE", + "/app/proof/prechange-fixture-acl.json", + ) +) +OBSERVED_PRECHANGE = Path( + os.environ.get( + "OBSERVED_PRECHANGE", + "/output/observed-prechange-acl.json", + ) +) +FIXTURES = Path(os.environ.get("ACL_FIXTURES", "/app/proof/fixtures")) + + +class _AclSnapshot(TypedDict): + """Typed shape of the durable synthetic pre-change ACL artifact.""" + + schema_version: str + provenance: dict[str, object] + roles: list[dict[str, object]] + memberships: list[dict[str, object]] + database_owner: list[dict[str, object]] + database_acl: list[dict[str, object]] + schema_owners: list[dict[str, object]] + schema_acl: list[dict[str, object]] + object_owners: list[dict[str, object]] + object_acl: list[dict[str, object]] + column_acl: list[dict[str, object]] + default_acl: list[dict[str, object]] + default_acl_catalog_rows: list[dict[str, object]] + + +def _admin(dsn: str = ADMIN_DSN) -> psycopg2.extensions.connection: + connection = psycopg2.connect(dsn) + connection.autocommit = True + return connection + + +def _admin_dsn_for_database(database: str) -> str: + """Retarget the configured admin DSN without dropping authentication.""" + parameters = psycopg2.extensions.parse_dsn(ADMIN_DSN) + parameters["dbname"] = database + return cast("str", psycopg2.extensions.make_dsn(**parameters)) + + +def _dict_rows( + statement: str, + parameters: Sequence[object] = (), + *, + dsn: str = ADMIN_DSN, +) -> list[dict[str, object]]: + connection = _admin(dsn) + try: + with connection.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cursor: + cursor.execute(statement, parameters) + return [dict(row) for row in cursor.fetchall()] + finally: + connection.close() + + +def _sorted_rows(rows: Iterable[Mapping[str, object]]) -> list[dict[str, object]]: + materialized = [dict(row) for row in rows] + return sorted( + materialized, + key=lambda row: json.dumps(row, sort_keys=True, separators=(",", ":")), + ) + + +def _capture_legacy_default_acl( + *, + dsn: str = ADMIN_DSN, +) -> list[dict[str, object]]: + """Capture the unrelated-schema default ACL sentinel exactly.""" + return _sorted_rows( + _dict_rows( + """ + SELECT owner.rolname AS owner, namespace.nspname AS schema_name, + defaults.defaclobjtype AS object_type, + array_to_string( + ARRAY( + SELECT acl_item::text + FROM unnest(defaults.defaclacl) acl_item + ORDER BY acl_item::text + ), + ',' + ) AS raw_acl + FROM pg_default_acl defaults + JOIN pg_roles owner ON owner.oid = defaults.defaclrole + JOIN pg_namespace namespace ON namespace.oid = defaults.defaclnamespace + WHERE namespace.nspname = %s + """, + (LEGACY_DEFAULT_SCHEMA,), + dsn=dsn, + ) + ) + + +def _postgres_major() -> int: + row = _dict_rows("SHOW server_version_num")[0] + return int(str(row["server_version_num"])) // 10_000 + + +def _matrix_principals(matrix: ModelApplicationDatabaseAclMatrix) -> set[str]: + return { + principal + for principals in ( + *matrix.declared_principals.values(), + *matrix.observed_principals.values(), + *matrix.absent_principals.values(), + *matrix.observed_connect_principals.values(), + *matrix.absent_connect_principals.values(), + *matrix.allowed_connect_principals.values(), + ) + for principal in principals + } + + +def _owner_roles(matrix: ModelApplicationDatabaseAclMatrix) -> set[str]: + return ( + set(matrix.database_owners.values()) + | {obj.owner for obj in matrix.objects} + | {row.owner for row in matrix.default_privileges} + ) + + +def _acl_object_keyword(obj: ModelApplicationDatabaseAclObject) -> str: + """Return the PostgreSQL ACL keyword from typed object authority.""" + if obj.catalog_kind == "procedure": + return "PROCEDURE" + return obj.object_type.value + + +def _allowed_connect_roles(matrix: ModelApplicationDatabaseAclMatrix) -> set[str]: + return { + principal + for principals in matrix.allowed_connect_principals.values() + for principal in principals + } + + +def _controlled_roles(matrix: ModelApplicationDatabaseAclMatrix) -> set[str]: + return ( + _owner_roles(matrix) + | _matrix_principals(matrix) + | {membership.role for membership in matrix.allowed_memberships} + | {membership.member for membership in matrix.allowed_memberships} + ) + + +def _membership_member_roles(matrix: ModelApplicationDatabaseAclMatrix) -> set[str]: + """Roles whose parent edges are explicitly reconciled by the renderer.""" + return ( + _matrix_principals(matrix) + .union(_owner_roles(matrix)) + .difference(matrix.retained_administrative_principals) + ) + + +def _protected_parent_roles(matrix: ModelApplicationDatabaseAclMatrix) -> set[str]: + """Roles whose inheritance by any undeclared child breaks the allowlist.""" + return { + state.role for state in matrix.governed_role_states if state.manage_memberships + } + + +def _capture_snapshot( + matrix: ModelApplicationDatabaseAclMatrix, + *, + dsn: str = ADMIN_DSN, +) -> _AclSnapshot: + """Capture every fixture ACL input needed by rollback mechanics.""" + + def query( + statement: str, + parameters: Sequence[object] = (), + ) -> list[dict[str, object]]: + return _dict_rows(statement, parameters, dsn=dsn) + + owner_names = sorted(_owner_roles(matrix)) + controlled_roles = sorted(_controlled_roles(matrix)) + protected_parent_roles = sorted(_protected_parent_roles(matrix)) + membership_member_roles = sorted(_membership_member_roles(matrix)) + managed_schema_names = list(MANAGED_SCHEMAS) + mutated_schema_names = list(MUTATED_SCHEMAS) + roles = query( + """ + SELECT rolname, rolcanlogin, rolsuper, rolinherit, rolcreaterole, + rolcreatedb, rolreplication, rolbypassrls + FROM pg_roles + WHERE rolname = ANY(%s) + """, + (controlled_roles,), + ) + memberships = query( + """ + SELECT parent.rolname AS parent_role, member.rolname AS member_role, + grantor.rolname AS grantor, + membership.admin_option, membership.inherit_option, + membership.set_option + FROM pg_auth_members membership + JOIN pg_roles parent ON parent.oid = membership.roleid + JOIN pg_roles member ON member.oid = membership.member + JOIN pg_roles grantor ON grantor.oid = membership.grantor + WHERE member.rolname = ANY(%s) + OR parent.rolname = ANY(%s) + """, + (membership_member_roles, protected_parent_roles), + ) + database_owner = query( + """ + SELECT database.datname AS database_name, owner.rolname AS owner + FROM pg_database database + JOIN pg_roles owner ON owner.oid = database.datdba + WHERE database.datname = ANY(%s) + """, + (list(matrix.required_connect_databases),), + ) + database_acl = query( + """ + SELECT database.datname AS database_name, + COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + grantor.rolname AS grantor, acl.privilege_type, + acl.is_grantable + FROM pg_database database + CROSS JOIN LATERAL aclexplode( + COALESCE(database.datacl, acldefault('d', database.datdba)) + ) acl + LEFT JOIN pg_roles grantee ON grantee.oid = acl.grantee + JOIN pg_roles grantor ON grantor.oid = acl.grantor + WHERE database.datname = ANY(%s) + AND acl.grantee <> database.datdba + """, + (list(matrix.required_connect_databases),), + ) + schema_owners = query( + """ + SELECT namespace.nspname AS schema_name, owner.rolname AS owner + FROM pg_namespace namespace + JOIN pg_roles owner ON owner.oid = namespace.nspowner + WHERE namespace.nspname = ANY(%s) + """, + (mutated_schema_names,), + ) + schema_acl = query( + """ + SELECT namespace.nspname AS schema_name, + COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + grantor.rolname AS grantor, acl.privilege_type, + acl.is_grantable + FROM pg_namespace namespace + CROSS JOIN LATERAL aclexplode( + COALESCE(namespace.nspacl, acldefault('n', namespace.nspowner)) + ) acl + LEFT JOIN pg_roles grantee ON grantee.oid = acl.grantee + JOIN pg_roles grantor ON grantor.oid = acl.grantor + WHERE namespace.nspname = ANY(%s) + AND acl.grantee <> namespace.nspowner + """, + (mutated_schema_names,), + ) + relation_owners = query( + """ + SELECT namespace.nspname AS schema_name, relation.relname AS object_name, + CASE relation.relkind + WHEN 'r' THEN 'table' + WHEN 'p' THEN 'table' + WHEN 'v' THEN 'view' + WHEN 'm' THEN 'materialized_view' + WHEN 'S' THEN 'sequence' + END AS catalog_kind, + CASE relation.relkind + WHEN 'v' THEN 'VIEW' + WHEN 'm' THEN 'MATERIALIZED VIEW' + WHEN 'S' THEN 'SEQUENCE' + ELSE 'TABLE' + END AS owner_keyword, + CASE WHEN relation.relkind = 'S' THEN 'SEQUENCE' ELSE 'TABLE' END + AS object_type, + owner.rolname AS owner + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + JOIN pg_roles owner ON owner.oid = relation.relowner + WHERE namespace.nspname = ANY(%s) + AND relation.relkind IN ('r', 'p', 'v', 'm', 'S') + """, + (managed_schema_names,), + ) + relation_acl = query( + """ + SELECT namespace.nspname AS schema_name, relation.relname AS object_name, + CASE WHEN relation.relkind = 'S' THEN 'SEQUENCE' ELSE 'TABLE' END + AS object_type, + CASE relation.relkind + WHEN 'r' THEN 'table' + WHEN 'p' THEN 'table' + WHEN 'v' THEN 'view' + WHEN 'm' THEN 'materialized_view' + WHEN 'S' THEN 'sequence' + END AS catalog_kind, + COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + grantor.rolname AS grantor, acl.privilege_type, + acl.is_grantable + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + CROSS JOIN LATERAL aclexplode( + COALESCE( + relation.relacl, + acldefault( + (CASE WHEN relation.relkind = 'S' THEN 's' ELSE 'r' END)::"char", + relation.relowner + ) + ) + ) acl + LEFT JOIN pg_roles grantee ON grantee.oid = acl.grantee + JOIN pg_roles grantor ON grantor.oid = acl.grantor + WHERE namespace.nspname = ANY(%s) + AND relation.relkind IN ('r', 'p', 'v', 'm', 'S') + AND acl.grantee <> relation.relowner + """, + (managed_schema_names,), + ) + routine_owners = query( + """ + SELECT namespace.nspname AS schema_name, procedure.proname AS object_name, + '(' || pg_get_function_identity_arguments(procedure.oid) || ')' + AS function_signature, + CASE WHEN procedure.prokind = 'p' THEN 'procedure' ELSE 'function' END + AS catalog_kind, + CASE WHEN procedure.prokind = 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END + AS owner_keyword, + CASE WHEN procedure.prokind = 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END + AS object_type, + owner.rolname AS owner + FROM pg_proc procedure + JOIN pg_namespace namespace ON namespace.oid = procedure.pronamespace + JOIN pg_roles owner ON owner.oid = procedure.proowner + WHERE namespace.nspname = ANY(%s) AND procedure.prokind IN ('f', 'p') + """, + (managed_schema_names,), + ) + routine_acl = query( + """ + SELECT namespace.nspname AS schema_name, procedure.proname AS object_name, + '(' || pg_get_function_identity_arguments(procedure.oid) || ')' + AS function_signature, + CASE WHEN procedure.prokind = 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END + AS object_type, + CASE WHEN procedure.prokind = 'p' THEN 'procedure' ELSE 'function' END + AS catalog_kind, + COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + grantor.rolname AS grantor, acl.privilege_type, + acl.is_grantable + FROM pg_proc procedure + JOIN pg_namespace namespace ON namespace.oid = procedure.pronamespace + CROSS JOIN LATERAL aclexplode( + COALESCE(procedure.proacl, acldefault('f', procedure.proowner)) + ) acl + LEFT JOIN pg_roles grantee ON grantee.oid = acl.grantee + JOIN pg_roles grantor ON grantor.oid = acl.grantor + WHERE namespace.nspname = ANY(%s) AND procedure.prokind IN ('f', 'p') + AND acl.grantee <> procedure.proowner + """, + (managed_schema_names,), + ) + type_owners = query( + """ + SELECT namespace.nspname AS schema_name, type.typname AS object_name, + CASE type.typtype + WHEN 'b' THEN 'base_type' + WHEN 'r' THEN 'range_type' + WHEN 'm' THEN 'multirange_type' + ELSE 'type' + END AS catalog_kind, 'TYPE' AS owner_keyword, + 'TYPE' AS object_type, + owner.rolname AS owner + FROM pg_type type + JOIN pg_namespace namespace ON namespace.oid = type.typnamespace + JOIN pg_roles owner ON owner.oid = type.typowner + LEFT JOIN pg_class relation ON relation.oid = type.typrelid + WHERE namespace.nspname = ANY(%s) + AND type.typisdefined + AND ( + (type.typtype IN ('b', 'd', 'e', 'r', 'm') AND type.typelem = 0) + OR (type.typtype = 'c' AND relation.relkind = 'c') + ) + """, + (managed_schema_names,), + ) + type_acl = query( + """ + SELECT namespace.nspname AS schema_name, type.typname AS object_name, + 'TYPE' AS object_type, + CASE type.typtype + WHEN 'b' THEN 'base_type' + WHEN 'r' THEN 'range_type' + WHEN 'm' THEN 'multirange_type' + ELSE 'type' + END AS catalog_kind, + COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + grantor.rolname AS grantor, acl.privilege_type, + acl.is_grantable + FROM pg_type type + JOIN pg_namespace namespace ON namespace.oid = type.typnamespace + LEFT JOIN pg_class relation ON relation.oid = type.typrelid + CROSS JOIN LATERAL aclexplode( + COALESCE(type.typacl, acldefault('T', type.typowner)) + ) acl + LEFT JOIN pg_roles grantee ON grantee.oid = acl.grantee + JOIN pg_roles grantor ON grantor.oid = acl.grantor + WHERE namespace.nspname = ANY(%s) + AND type.typisdefined + AND ( + (type.typtype IN ('b', 'd', 'e', 'r', 'm') AND type.typelem = 0) + OR (type.typtype = 'c' AND relation.relkind = 'c') + ) + AND acl.grantee <> type.typowner + """, + (managed_schema_names,), + ) + column_acl = query( + """ + SELECT namespace.nspname AS schema_name, + relation.relname AS object_name, + CASE relation.relkind + WHEN 'r' THEN 'table' + WHEN 'p' THEN 'table' + WHEN 'v' THEN 'view' + WHEN 'm' THEN 'materialized_view' + END AS catalog_kind, + attribute.attname AS column_name, + COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + grantor.rolname AS grantor, acl.privilege_type, + acl.is_grantable + FROM pg_attribute attribute + JOIN pg_class relation ON relation.oid = attribute.attrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + CROSS JOIN LATERAL aclexplode(attribute.attacl) acl + LEFT JOIN pg_roles grantee ON grantee.oid = acl.grantee + JOIN pg_roles grantor ON grantor.oid = acl.grantor + WHERE namespace.nspname = ANY(%s) + AND relation.relkind IN ('r', 'p', 'v', 'm') + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + AND attribute.attacl IS NOT NULL + AND acl.grantee <> relation.relowner + """, + (managed_schema_names,), + ) + default_acl = query( + """ + SELECT owner.rolname AS owner, + namespace.nspname AS schema_name, + CASE defaults.defaclobjtype + WHEN 'r' THEN 'TABLE' + WHEN 'S' THEN 'SEQUENCE' + WHEN 'f' THEN 'FUNCTION' + WHEN 'T' THEN 'TYPE' + END AS object_type, + COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + grantor.rolname AS grantor, acl.privilege_type, + acl.is_grantable + FROM pg_default_acl defaults + JOIN pg_roles owner ON owner.oid = defaults.defaclrole + LEFT JOIN pg_namespace namespace ON namespace.oid = defaults.defaclnamespace + CROSS JOIN LATERAL aclexplode(defaults.defaclacl) acl + LEFT JOIN pg_roles grantee ON grantee.oid = acl.grantee + JOIN pg_roles grantor ON grantor.oid = acl.grantor + WHERE owner.rolname = ANY(%s) + AND (defaults.defaclnamespace = 0 OR namespace.nspname = ANY(%s)) + AND acl.grantee <> defaults.defaclrole + """, + (owner_names, managed_schema_names), + ) + default_acl_catalog_rows = query( + """ + SELECT owner.rolname AS owner, + namespace.nspname AS schema_name, + CASE defaults.defaclobjtype + WHEN 'r' THEN 'TABLE' + WHEN 'S' THEN 'SEQUENCE' + WHEN 'f' THEN 'FUNCTION' + WHEN 'T' THEN 'TYPE' + END AS object_type, + array_to_string( + ARRAY( + SELECT acl_item::text + FROM unnest(defaults.defaclacl) acl_item + ORDER BY acl_item::text + ), + ',' + ) AS raw_acl + FROM pg_default_acl defaults + JOIN pg_roles owner ON owner.oid = defaults.defaclrole + LEFT JOIN pg_namespace namespace ON namespace.oid = defaults.defaclnamespace + WHERE owner.rolname = ANY(%s) + AND (defaults.defaclnamespace = 0 OR namespace.nspname = ANY(%s)) + """, + (owner_names, managed_schema_names), + ) + return { + "schema_version": "3.0", + "provenance": { + "source": "sanitized_postgresql_16_fixture", + "live_database_read": False, + "dump_derived": False, + "authorization_scope": "synthetic_proof", + "postgres_major": _postgres_major(), + }, + "roles": _sorted_rows(roles), + "memberships": _sorted_rows(memberships), + "database_owner": _sorted_rows(database_owner), + "database_acl": _sorted_rows(database_acl), + "schema_owners": _sorted_rows(schema_owners), + "schema_acl": _sorted_rows(schema_acl), + "object_owners": _sorted_rows( + [ + *relation_owners, + *routine_owners, + *type_owners, + ] + ), + "object_acl": _sorted_rows([*relation_acl, *routine_acl, *type_acl]), + "column_acl": _sorted_rows(column_acl), + "default_acl": _sorted_rows(default_acl), + "default_acl_catalog_rows": _sorted_rows(default_acl_catalog_rows), + } + + +def _fixture_matrix() -> ModelApplicationDatabaseAclMatrix: + topology = ModelDeploymentTopology.from_yaml(FIXTURES / "topology.yaml") + inventory = ModelApplicationRelationEvidenceInventory.model_validate( + yaml.safe_load((FIXTURES / "inventory.yaml").read_text(encoding="utf-8")) + ) + principal_inventory = ModelApplicationDatabasePrincipalInventory.model_validate( + yaml.safe_load( + (FIXTURES / "principal-inventory-postgres16.yaml").read_text( + encoding="utf-8" + ) + ) + ) + external_principal_inventories = tuple( + ModelApplicationDatabasePrincipalInventory.model_validate(item) + for item in yaml.safe_load( + (FIXTURES / "principal-inventories-external.yaml").read_text( + encoding="utf-8" + ) + ) + ) + acl_policy = ModelApplicationDatabaseAclPolicy.model_validate( + yaml.safe_load( + (FIXTURES / "acl-policy-postgres16.yaml").read_text(encoding="utf-8") + ) + ) + external_inventory_sources = tuple( + ModelApplicationDatabaseAclSource( + source_key=f"synthetic_principal_inventory_{inventory.database_ref}", + repository="synthetic/docker-proof", + revision=f"{index:x}" * 40, + path="proof/fixtures/principal-inventories-external.yaml", + sha256=f"{index:x}" * 64, + purpose="principal_inventory", + ) + for index, inventory in enumerate(external_principal_inventories, start=3) + ) + sources = ( + ModelApplicationDatabaseAclSource( + source_key="synthetic_topology", + repository="synthetic/docker-proof", + revision="a" * 40, + path="proof/fixtures/topology.yaml", + sha256="b" * 64, + purpose="topology", + ), + ModelApplicationDatabaseAclSource( + source_key="synthetic_inventory", + repository="synthetic/docker-proof", + revision="c" * 40, + path="proof/fixtures/inventory.yaml", + sha256="d" * 64, + purpose="relation_inventory", + ), + ModelApplicationDatabaseAclSource( + source_key="synthetic_principal_inventory", + repository="synthetic/docker-proof", + revision="e" * 40, + path="proof/fixtures/principal-inventory-postgres16.yaml", + sha256="f" * 64, + purpose="principal_inventory", + ), + ModelApplicationDatabaseAclSource( + source_key="synthetic_acl_policy", + repository="synthetic/docker-proof", + revision="1" * 40, + path="proof/fixtures/acl-policy-postgres16.yaml", + sha256="2" * 64, + purpose="acl_policy", + ), + *external_inventory_sources, + ) + principal_inventories = { + "synthetic_principal_inventory": principal_inventory, + **{ + f"synthetic_principal_inventory_{inventory.database_ref}": inventory + for inventory in external_principal_inventories + }, + } + return build_application_database_acl_matrix( + topology=topology, + sources=sources, + relation_inventories={"synthetic_inventory": inventory}, + service_manifests={}, + principal_inventories=principal_inventories, + acl_policies={"synthetic_acl_policy": acl_policy}, + authorization_scope=EnumApplicationDatabaseAclAuthorizationScope.SYNTHETIC_PROOF, + required_connect_databases=tuple( + policy.physical_database for policy in acl_policy.connection_policies + ), + ) + + +def _expected_acl_set( + matrix: ModelApplicationDatabaseAclMatrix, +) -> set[tuple[object, ...]]: + objects = {obj.identity: obj for obj in matrix.objects} + expected: set[tuple[object, ...]] = { + ( + ( + "PROCEDURE" + if row.object_type is EnumDatabaseGrantObjectType.FUNCTION + and objects[ + ( + row.database_ref, + row.schema_ref or "", + row.object_type, + row.object_ref or "", + row.function_signature, + ) + ].catalog_kind + == "procedure" + else row.object_type.value + ), + row.physical_database, + row.schema_ref or "", + row.object_ref or "", + row.function_signature or "", + row.principal, + privilege.value, + False, + ) + for row in matrix.rows + for privilege in row.privileges + } + expected.update( + ( + "DATABASE", + physical_database, + "", + "", + "", + principal, + "CONNECT", + False, + ) + for physical_database, principals in matrix.allowed_connect_principals.items() + for principal in principals + ) + return expected + + +def _actual_acl_set( + snapshot: _AclSnapshot, + application_database: str, +) -> set[tuple[object, ...]]: + result: set[tuple[object, ...]] = set() + for row in snapshot["database_acl"]: + result.add( + ( + "DATABASE", + str(row["database_name"]), + "", + "", + "", + str(row["grantee"]), + str(row["privilege_type"]), + bool(row["is_grantable"]), + ) + ) + for row in snapshot["schema_acl"]: + result.add( + ( + "SCHEMA", + application_database, + str(row["schema_name"]), + "", + "", + str(row["grantee"]), + str(row["privilege_type"]), + bool(row["is_grantable"]), + ) + ) + for row in snapshot["object_acl"]: + result.add( + ( + str(row["object_type"]), + application_database, + str(row["schema_name"]), + str(row["object_name"]), + str(row.get("function_signature") or ""), + str(row["grantee"]), + str(row["privilege_type"]), + bool(row["is_grantable"]), + ) + ) + return result + + +def _expected_hardened_default_acl_catalog_rows( + matrix: ModelApplicationDatabaseAclMatrix, +) -> list[dict[str, object]]: + """Project the exact explicit rows needed to remove builtin PUBLIC defaults.""" + rows: list[dict[str, object]] = [] + for owner in sorted(_owner_roles(matrix)): + rows.extend( + ( + { + "owner": owner, + "schema_name": None, + "object_type": "FUNCTION", + "raw_acl": f"{owner}=X/{owner}", + }, + { + "owner": owner, + "schema_name": None, + "object_type": "TYPE", + "raw_acl": f"{owner}=U/{owner}", + }, + ) + ) + return _sorted_rows(rows) + + +def _catalog_violations( + snapshot: _AclSnapshot, + matrix: ModelApplicationDatabaseAclMatrix, +) -> set[str]: + violations: set[str] = set() + expected = _expected_acl_set(matrix) + application_databases = {row.physical_database for row in matrix.default_privileges} + assert len(application_databases) == 1, application_databases + actual = _actual_acl_set(snapshot, next(iter(application_databases))) + unexpected = actual - expected + missing = expected - actual + if missing: + violations.add("MISSING_DECLARED_PRIVILEGE") + if any(row[5] == PUBLIC_PRINCIPAL for row in unexpected): + violations.add("PUBLIC_PRIVILEGE") + if any(bool(row[7]) for row in actual): + violations.add("GRANT_OPTION") + if any( + row[0] == "DATABASE" and row[6] in {"CREATE", "TEMPORARY"} for row in unexpected + ): + violations.add("DATABASE_DDL_PRIVILEGE") + if snapshot["column_acl"]: + violations.add("COLUMN_PRIVILEGE") + + declared_principals = { + principal + for principals in matrix.declared_principals.values() + for principal in principals + } + if any( + row[5] not in declared_principals | {PUBLIC_PRINCIPAL} for row in unexpected + ): + violations.add("UNDECLARED_PRINCIPAL_PRIVILEGE") + + governed_roles = {state.role for state in matrix.governed_role_states} + owner_roles = _owner_roles(matrix) + governed_login_roles = { + state.role for state in matrix.governed_role_states if state.login + } + object_domains = { + ( + obj.object_type.value, + obj.schema_ref, + obj.object_ref, + obj.function_signature or "", + ): obj.domain + for obj in matrix.objects + } + database_ref_by_physical = { + row.physical_database: row.database_ref + for row in matrix.rows + if row.object_type is EnumDatabaseGrantObjectType.DATABASE + } + for row in unexpected: + object_type, _, schema_name, object_name, signature, grantee, _, _ = row + if object_type == "SCHEMA": + domain = matrix.schema_domains.get( + database_ref_by_physical.get(str(row[1]), ""), {} + ).get(str(schema_name)) + else: + normalized_object_type = str( + "FUNCTION" if object_type == "PROCEDURE" else object_type + ) + domain = object_domains.get( + ( + normalized_object_type, + str(schema_name), + str(object_name), + str(signature), + ) + ) + if domain is not None and domain not in matrix.principal_domains.get( + str(grantee), () + ): + violations.add("CROSS_DOMAIN_PRIVILEGE") + unexpected_objects = Counter( + (row[5], row[2], row[3], row[4]) + for row in unexpected + if row[0] in {"TABLE", "SEQUENCE", "FUNCTION", "PROCEDURE", "TYPE"} + and row[5] != PUBLIC_PRINCIPAL + ) + if len(unexpected_objects) >= 2 or any( + count > 1 for count in unexpected_objects.values() + ): + violations.add("BROAD_GRANT") + elif unexpected: + violations.add("UNEXPECTED_PRIVILEGE") + + if snapshot["default_acl"] or snapshot[ + "default_acl_catalog_rows" + ] != _expected_hardened_default_acl_catalog_rows(matrix): + violations.add("UNSAFE_DEFAULT_PRIVILEGE") + expected_owners = { + ( + obj.catalog_kind, + obj.schema_ref, + obj.object_ref, + obj.function_signature or "", + ): obj.owner + for obj in matrix.objects + } + actual_owners = { + ( + str(row["catalog_kind"]), + str(row["schema_name"]), + str(row["object_name"]), + str(row.get("function_signature") or ""), + ): str(row["owner"]) + for row in snapshot["object_owners"] + } + if actual_owners != expected_owners: + violations.add("OWNER_MISMATCH") + expected_schema_owners = { + (row.schema_ref, row.owner) for row in matrix.default_privileges + } + actual_schema_owners = { + (str(row["schema_name"]), str(row["owner"])) + for row in snapshot["schema_owners"] + if str(row["schema_name"]) in MANAGED_SCHEMAS + } + if expected_schema_owners != actual_schema_owners: + violations.add("OWNER_MISMATCH") + + expected_database_owners = dict(matrix.observed_connect_database_owners) + expected_database_owners.update( + { + row.physical_database: matrix.database_owners[row.database_ref] + for row in matrix.rows + if row.object_type is EnumDatabaseGrantObjectType.DATABASE + } + ) + actual_database_owners = { + str(row["database_name"]): str(row["owner"]) + for row in snapshot["database_owner"] + } + if actual_database_owners != expected_database_owners: + violations.add("OWNER_MISMATCH") + + runtime_owned = { + str(row["owner"]) + for row in ( + *snapshot["schema_owners"], + *snapshot["object_owners"], + ) + if str(row["owner"]) in declared_principals + } + application_database_owned = { + str(row["owner"]) + for row in snapshot["database_owner"] + if str(row["database_name"]) == DATABASE + and str(row["owner"]) in declared_principals + } + runtime_owned.update(application_database_owned) + if runtime_owned: + violations.add("RUNTIME_OWNERSHIP") + + for role in snapshot["roles"]: + role_name = str(role["rolname"]) + if role_name not in governed_roles: + continue + if ( + (role_name in governed_login_roles and not bool(role["rolcanlogin"])) + or (role_name in owner_roles and bool(role["rolcanlogin"])) + or bool(role["rolsuper"]) + or bool(role["rolinherit"]) + or bool(role["rolcreaterole"]) + or bool(role["rolcreatedb"]) + or bool(role["rolreplication"]) + or bool(role["rolbypassrls"]) + ): + violations.add("RUNTIME_ROLE_ATTRIBUTES") + + expected_memberships = { + ( + membership.role, + membership.member, + membership.admin_option, + membership.inherit_option, + membership.set_option, + ) + for membership in matrix.allowed_memberships + } + actual_memberships = { + ( + str(row["parent_role"]), + str(row["member_role"]), + bool(row["admin_option"]), + bool(row["inherit_option"]), + bool(row["set_option"]), + ) + for row in snapshot["memberships"] + } + if actual_memberships - expected_memberships: + violations.add("OWNER_MEMBERSHIP") + if expected_memberships - actual_memberships: + violations.add("MISSING_DECLARED_MEMBERSHIP") + return violations + + +def _execute_matrix( + matrix: ModelApplicationDatabaseAclMatrix, + *, + dsn: str = ADMIN_DSN, + phase: EnumApplicationDatabaseAclRenderPhase = ( + EnumApplicationDatabaseAclRenderPhase.FULL + ), +) -> None: + rendered = render_application_database_acl_sql( + matrix, + allow_synthetic_proof=True, + phase=phase, + ) + result = subprocess.run( + ["psql", "--no-psqlrc", "--dbname", dsn], + input=rendered, + text=True, + capture_output=True, + check=False, + ) + if result.returncode != 0: + raise AssertionError( + "psql ACL apply failed:\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + + +def _atomic_failure_proof( + matrix: ModelApplicationDatabaseAclMatrix, + snapshot: _AclSnapshot, + *, + dsn: str = ADMIN_DSN, + phase: EnumApplicationDatabaseAclRenderPhase = ( + EnumApplicationDatabaseAclRenderPhase.FULL + ), +) -> None: + """Prove the emitted psql script rolls back all prior statements on error.""" + rendered = render_application_database_acl_sql( + matrix, + allow_synthetic_proof=True, + phase=phase, + ) + injected = rendered.replace( + "COMMIT;\n", + "SELECT 1 / 0; -- injected atomicity control\nCOMMIT;\n", + ) + assert injected != rendered + result = subprocess.run( + ["psql", "--no-psqlrc", "--dbname", dsn], + input=injected, + text=True, + capture_output=True, + check=False, + ) + assert result.returncode != 0, result.stdout + assert _capture_snapshot(matrix, dsn=dsn) == snapshot, ( + "failed psql application leaked a partial ACL mutation" + ) + print("acl_phase=psql_atomic_failure red_control=ROLLBACK status=PASS") + + +def _identifier_list(values: Sequence[str]) -> sql.Composed: + return sql.SQL(", ").join(sql.Identifier(value) for value in values) + + +def _grant_rows( + cursor: psycopg2.extensions.cursor, + rows: Sequence[Mapping[str, object]], + *, + target: Callable[[Mapping[str, object]], sql.Composable], + prefix: Callable[[Mapping[str, object]], sql.Composable], +) -> None: + grouped: dict[tuple[str, str, str, bool], set[str]] = defaultdict(set) + row_by_key: dict[tuple[str, str, str, bool], Mapping[str, object]] = {} + for row in rows: + target_identity = { + key_name: value + for key_name, value in row.items() + if key_name not in {"privilege_type", "grantee", "grantor", "is_grantable"} + } + group_key = ( + json.dumps(target_identity, sort_keys=True), + str(row["grantee"]), + str(row["grantor"]), + bool(row["is_grantable"]), + ) + grouped[group_key].add(str(row["privilege_type"])) + row_by_key[group_key] = row + + remaining = set(grouped) + ordered: list[tuple[str, str, str, bool]] = [] + while remaining: + ready = sorted( + key + for key in remaining + if not any( + key[0] == prerequisite[0] + and key[2] == prerequisite[1] + and prerequisite[3] + and bool(grouped[key] & grouped[prerequisite]) + for prerequisite in remaining + if prerequisite != key + ) + ) + if not ready: + raise AssertionError("ACL grantor graph is cyclic or lacks a grant option") + ordered.extend(ready) + remaining.difference_update(ready) + + for key in ordered: + privileges = grouped[key] + row = row_by_key[key] + cursor.execute( + sql.SQL("SET ROLE {}").format(sql.Identifier(str(row["grantor"]))) + ) + cursor.execute( + sql.SQL("GRANT {} ON {} TO {}{}").format( + sql.SQL(", ").join(sql.SQL(item) for item in sorted(privileges)), + prefix(row) + target(row), + sql.SQL("PUBLIC") + if row["grantee"] == PUBLIC_PRINCIPAL + else sql.Identifier(str(row["grantee"])), + sql.SQL(" WITH GRANT OPTION") if row["is_grantable"] else sql.SQL(""), + ) + ) + cursor.execute("RESET ROLE") + + +def _restore_memberships( + cursor: psycopg2.extensions.cursor, + rows: Sequence[Mapping[str, object]], +) -> None: + """Restore exact PG16 membership grantors after their ADMIN paths exist.""" + remaining = {json.dumps(dict(row), sort_keys=True) for row in rows} + row_by_key = {json.dumps(dict(row), sort_keys=True): row for row in rows} + ordered: list[str] = [] + while remaining: + ready = sorted( + key + for key in remaining + if not any( + str(row_by_key[key]["parent_role"]) + == str(row_by_key[prerequisite]["parent_role"]) + and str(row_by_key[key]["grantor"]) + == str(row_by_key[prerequisite]["member_role"]) + and bool(row_by_key[prerequisite]["admin_option"]) + for prerequisite in remaining + if prerequisite != key + ) + ) + if not ready: + raise AssertionError( + "role membership grantor graph is cyclic or lacks ADMIN OPTION" + ) + ordered.extend(ready) + remaining.difference_update(ready) + + for key in ordered: + row = row_by_key[key] + cursor.execute( + sql.SQL("SET ROLE {}").format(sql.Identifier(str(row["grantor"]))) + ) + cursor.execute( + sql.SQL("GRANT {} TO {} WITH ADMIN {}, INHERIT {}, SET {}").format( + sql.Identifier(str(row["parent_role"])), + sql.Identifier(str(row["member_role"])), + sql.SQL("TRUE" if row["admin_option"] else "FALSE"), + sql.SQL("TRUE" if row["inherit_option"] else "FALSE"), + sql.SQL("TRUE" if row["set_option"] else "FALSE"), + ) + ) + cursor.execute("RESET ROLE") + + +def _object_target(row: Mapping[str, object]) -> sql.Composable: + qualified = sql.SQL("{}.{}").format( + sql.Identifier(str(row["schema_name"])), + sql.Identifier(str(row["object_name"])), + ) + if row["object_type"] in {"FUNCTION", "PROCEDURE"}: + return qualified + sql.SQL(str(row.get("function_signature", "()"))) + return qualified + + +def _object_prefix(row: Mapping[str, object]) -> sql.Composable: + return sql.SQL(str(row["object_type"]) + " ") + + +def _grant_column_rows( + cursor: psycopg2.extensions.cursor, + rows: Sequence[Mapping[str, object]], +) -> None: + """Restore exact per-column ACLs, including grantor and grant option.""" + grouped: dict[tuple[str, str, str, bool], set[str]] = defaultdict(set) + row_by_key: dict[tuple[str, str, str, bool], Mapping[str, object]] = {} + for row in rows: + target_identity = json.dumps( + { + "schema_name": row["schema_name"], + "object_name": row["object_name"], + "column_name": row["column_name"], + }, + sort_keys=True, + ) + key = ( + target_identity, + str(row["grantee"]), + str(row["grantor"]), + bool(row["is_grantable"]), + ) + grouped[key].add(str(row["privilege_type"])) + row_by_key[key] = row + remaining = set(grouped) + ordered: list[tuple[str, str, str, bool]] = [] + while remaining: + ready = sorted( + key + for key in remaining + if not any( + key[0] == prerequisite[0] + and key[2] == prerequisite[1] + and prerequisite[3] + and bool(grouped[key] & grouped[prerequisite]) + for prerequisite in remaining + if prerequisite != key + ) + ) + if not ready: + raise AssertionError( + "column ACL grantor graph is cyclic or lacks a grant option" + ) + ordered.extend(ready) + remaining.difference_update(ready) + + for key in ordered: + row = row_by_key[key] + grantee = ( + sql.SQL("PUBLIC") + if row["grantee"] == PUBLIC_PRINCIPAL + else sql.Identifier(str(row["grantee"])) + ) + cursor.execute( + sql.SQL("SET ROLE {}").format(sql.Identifier(str(row["grantor"]))) + ) + cursor.execute( + sql.SQL("GRANT {} ON TABLE {}.{} TO {}{}").format( + sql.SQL(", ").join( + sql.SQL("{} ({})").format( + sql.SQL(privilege), + sql.Identifier(str(row["column_name"])), + ) + for privilege in sorted(grouped[key]) + ), + sql.Identifier(str(row["schema_name"])), + sql.Identifier(str(row["object_name"])), + grantee, + sql.SQL(" WITH GRANT OPTION") if row["is_grantable"] else sql.SQL(""), + ) + ) + cursor.execute("RESET ROLE") + + +def _reset_target_default_acls( + cursor: psycopg2.extensions.cursor, + *, + owners: Sequence[str], + schemas: Sequence[str], +) -> None: + """Return governed global/schema defaults to PostgreSQL built-ins.""" + cursor.execute( + """ + SELECT owner.rolname AS owner, namespace.nspname AS schema_name, + CASE defaults.defaclobjtype + WHEN 'r' THEN 'TABLES' + WHEN 'S' THEN 'SEQUENCES' + WHEN 'f' THEN 'FUNCTIONS' + WHEN 'T' THEN 'TYPES' + END AS object_keyword, + COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + acl.privilege_type + FROM pg_default_acl defaults + JOIN pg_roles owner ON owner.oid = defaults.defaclrole + LEFT JOIN pg_namespace namespace ON namespace.oid = defaults.defaclnamespace + CROSS JOIN LATERAL aclexplode(defaults.defaclacl) acl + LEFT JOIN pg_roles grantee ON grantee.oid = acl.grantee + WHERE owner.rolname = ANY(%s) + AND (defaults.defaclnamespace = 0 OR namespace.nspname = ANY(%s)) + AND acl.grantee <> defaults.defaclrole + ORDER BY owner.rolname, namespace.nspname NULLS FIRST, + defaults.defaclobjtype, grantee.rolname NULLS FIRST, + acl.privilege_type + """, + (list(owners), list(schemas)), + ) + for owner, schema_name, object_keyword, grantee, privilege in cursor.fetchall(): + grantee_sql = ( + sql.SQL("PUBLIC") + if str(grantee) == PUBLIC_PRINCIPAL + else sql.Identifier(str(grantee)) + ) + scope = ( + sql.SQL(" IN SCHEMA {}").format(sql.Identifier(str(schema_name))) + if schema_name is not None + else sql.SQL("") + ) + cursor.execute( + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {}{} REVOKE {} ON {} FROM {} CASCADE" + ).format( + sql.Identifier(str(owner)), + scope, + sql.SQL(str(privilege)), + sql.SQL(str(object_keyword)), + grantee_sql, + ) + ) + + # Global function/type defaults include PUBLIC by definition. Table and + # sequence defaults do not; grant-then-revoke removes any explicit row that + # merely encodes the built-in state. + for owner in owners: + owner_identifier = sql.Identifier(owner) + cursor.execute( + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {} " + "GRANT EXECUTE ON FUNCTIONS TO PUBLIC" + ).format(owner_identifier) + ) + cursor.execute( + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {} GRANT USAGE ON TYPES TO PUBLIC" + ).format(owner_identifier) + ) + for object_keyword in ("TABLES", "SEQUENCES"): + cursor.execute( + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {} " + f"GRANT ALL PRIVILEGES ON {object_keyword} TO PUBLIC" + ).format(owner_identifier) + ) + cursor.execute( + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {} " + f"REVOKE ALL PRIVILEGES ON {object_keyword} FROM PUBLIC" + ).format(owner_identifier) + ) + + +def _restore_snapshot( + snapshot: _AclSnapshot, + matrix: ModelApplicationDatabaseAclMatrix, + *, + inject_failure: bool = False, + dsn: str = ADMIN_DSN, +) -> None: + """Atomically restore only from the durable pre-change catalog snapshot.""" + matrix_principals = _matrix_principals(matrix) + snapshot_grantees = { + str(row["grantee"]) + for rows in ( + snapshot["database_acl"], + snapshot["schema_acl"], + snapshot["object_acl"], + snapshot["column_acl"], + snapshot["default_acl"], + ) + for row in rows + if str(row["grantee"]) != PUBLIC_PRINCIPAL + } + membership_roles = { + str(row[key]) + for row in snapshot["memberships"] + for key in ("parent_role", "member_role", "grantor") + } + revocation_principals = sorted( + matrix_principals | snapshot_grantees | membership_roles + ) + database_revocation_principals = revocation_principals + membership_members = sorted(_membership_member_roles(matrix)) + protected_parents = sorted(_protected_parent_roles(matrix)) + snapshot_role_names = {str(row["rolname"]) for row in snapshot["roles"]} + renderer_created_roles = sorted( + ( + _owner_roles(matrix) + | { + principal + for principals in matrix.declared_principals.values() + for principal in principals + } + | _allowed_connect_roles(matrix) + ) + - snapshot_role_names + ) + snapshot_schema_names = { + str(row["schema_name"]) for row in snapshot["schema_owners"] + } + renderer_created_schemas = sorted(set(MANAGED_SCHEMAS) - snapshot_schema_names) + connection = psycopg2.connect(dsn) + try: + with connection.cursor() as cursor: + grantees = _identifier_list(revocation_principals) + database_grantees = _identifier_list(database_revocation_principals) + for database_name in matrix.required_connect_databases: + cursor.execute( + sql.SQL( + "REVOKE ALL PRIVILEGES ON DATABASE {} FROM PUBLIC, {} CASCADE" + ).format( + sql.Identifier(database_name), + database_grantees, + ) + ) + for schema_name in MUTATED_SCHEMAS: + cursor.execute( + sql.SQL( + "REVOKE ALL PRIVILEGES ON SCHEMA {} FROM PUBLIC, {} CASCADE" + ).format(sql.Identifier(schema_name), grantees) + ) + for obj in matrix.objects: + keyword = _acl_object_keyword(obj) + target = sql.SQL("{}.{}").format( + sql.Identifier(obj.schema_ref), sql.Identifier(obj.object_ref) + ) + if obj.object_type is EnumDatabaseGrantObjectType.FUNCTION: + target += sql.SQL(obj.function_signature or "()") + cursor.execute( + sql.SQL( + f"REVOKE ALL PRIVILEGES ON {keyword} {{}} FROM PUBLIC, {{}} CASCADE" + ).format(target, grantees) + ) + default_owners = sorted(_owner_roles(matrix)) + _reset_target_default_acls( + cursor, + owners=default_owners, + schemas=MANAGED_SCHEMAS, + ) + + if inject_failure: + cursor.execute("SELECT 1 / 0") + + for row in snapshot["roles"]: + attributes = [ + "LOGIN" if row["rolcanlogin"] else "NOLOGIN", + "SUPERUSER" if row["rolsuper"] else "NOSUPERUSER", + "INHERIT" if row["rolinherit"] else "NOINHERIT", + "CREATEROLE" if row["rolcreaterole"] else "NOCREATEROLE", + "CREATEDB" if row["rolcreatedb"] else "NOCREATEDB", + "REPLICATION" if row["rolreplication"] else "NOREPLICATION", + "BYPASSRLS" if row["rolbypassrls"] else "NOBYPASSRLS", + ] + cursor.execute( + sql.SQL("ALTER ROLE {} " + " ".join(attributes)).format( + sql.Identifier(str(row["rolname"])) + ) + ) + cursor.execute( + """ + SELECT parent.rolname, child.rolname, grantor.rolname + FROM pg_auth_members membership + JOIN pg_roles parent ON parent.oid = membership.roleid + JOIN pg_roles child ON child.oid = membership.member + JOIN pg_roles grantor ON grantor.oid = membership.grantor + WHERE child.rolname = ANY(%s) + OR parent.rolname = ANY(%s) + """, + (membership_members, protected_parents), + ) + current_memberships = cursor.fetchall() + for parent, member, grantor in current_memberships: + cursor.execute( + sql.SQL("REVOKE {} FROM {} GRANTED BY {} CASCADE").format( + sql.Identifier(str(parent)), + sql.Identifier(str(member)), + sql.Identifier(str(grantor)), + ) + ) + _restore_memberships(cursor, snapshot["memberships"]) + for row in snapshot["database_owner"]: + cursor.execute( + sql.SQL("ALTER DATABASE {} OWNER TO {}").format( + sql.Identifier(str(row["database_name"])), + sql.Identifier(str(row["owner"])), + ) + ) + for row in snapshot["schema_owners"]: + cursor.execute( + sql.SQL("ALTER SCHEMA {} OWNER TO {}").format( + sql.Identifier(str(row["schema_name"])), + sql.Identifier(str(row["owner"])), + ) + ) + owner_restore_order = { + "TABLE": 0, + "VIEW": 1, + "MATERIALIZED VIEW": 2, + "SEQUENCE": 3, + "TYPE": 4, + "FUNCTION": 5, + "PROCEDURE": 6, + } + for row in sorted( + snapshot["object_owners"], + key=lambda item: ( + owner_restore_order[str(item["owner_keyword"])], + str(item["schema_name"]), + str(item["object_name"]), + ), + ): + keyword = str(row["owner_keyword"]) + target = _object_target(row) + cursor.execute( + sql.SQL(f"ALTER {keyword} {{}} OWNER TO {{}}").format( + target, sql.Identifier(str(row["owner"])) + ) + ) + + _grant_rows( + cursor, + snapshot["database_acl"], + target=lambda row: sql.Identifier(str(row["database_name"])), + prefix=lambda row: sql.SQL("DATABASE "), + ) + _grant_rows( + cursor, + snapshot["schema_acl"], + target=lambda row: sql.Identifier(str(row["schema_name"])), + prefix=lambda row: sql.SQL("SCHEMA "), + ) + _grant_rows( + cursor, + snapshot["object_acl"], + target=_object_target, + prefix=_object_prefix, + ) + _grant_column_rows(cursor, snapshot["column_acl"]) + raw_default_rows = { + ( + str(row["owner"]), + row["schema_name"], + str(row["object_type"]), + ): str(row["raw_acl"]) + for row in snapshot["default_acl_catalog_rows"] + } + for owner in default_owners: + for object_type, privilege, marker in ( + ("FUNCTION", "EXECUTE", "=X"), + ("TYPE", "USAGE", "=U"), + ): + raw_acl = raw_default_rows.get((owner, None, object_type)) + public_builtin_present = raw_acl is not None and any( + item.startswith(marker) for item in raw_acl.split(",") + ) + if raw_acl is not None and not public_builtin_present: + cursor.execute( + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {} " + f"REVOKE {privilege} ON {object_type}S FROM PUBLIC" + ).format(sql.Identifier(owner)) + ) + default_rows = snapshot["default_acl"] + for row in default_rows: + if row["grantor"] != row["owner"]: + raise AssertionError( + "default ACL grantor must equal its owning role for exact restore" + ) + grantee = ( + sql.SQL("PUBLIC") + if row["grantee"] == PUBLIC_PRINCIPAL + else sql.Identifier(str(row["grantee"])) + ) + keyword = { + "TABLE": "TABLES", + "SEQUENCE": "SEQUENCES", + "FUNCTION": "FUNCTIONS", + "TYPE": "TYPES", + }[str(row["object_type"])] + default_schema_name = row["schema_name"] + scope = ( + sql.SQL(" IN SCHEMA {} ").format( + sql.Identifier(str(default_schema_name)) + ) + if default_schema_name is not None + else sql.SQL(" ") + ) + cursor.execute( + sql.SQL("SET ROLE {}").format(sql.Identifier(str(row["owner"]))) + ) + cursor.execute( + sql.SQL( + f"ALTER DEFAULT PRIVILEGES{{}}" + f"GRANT {row['privilege_type']} ON {keyword} TO {{}}{{}}" + ).format( + scope, + grantee, + sql.SQL(" WITH GRANT OPTION") + if row["is_grantable"] + else sql.SQL(""), + ) + ) + cursor.execute("RESET ROLE") + for schema_name in renderer_created_schemas: + cursor.execute( + sql.SQL("DROP SCHEMA {}").format(sql.Identifier(schema_name)) + ) + for role_name in renderer_created_roles: + # Return newly-created owners to PostgreSQL's built-in default ACL. + # DROP ROLE then remains a fail-closed emptiness/dependency check: + # it cannot cascade into a relation that appeared after apply. + cursor.execute( + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {} " + "GRANT EXECUTE ON FUNCTIONS TO PUBLIC" + ).format(sql.Identifier(role_name)) + ) + cursor.execute( + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {} " + "GRANT USAGE ON TYPES TO PUBLIC" + ).format(sql.Identifier(role_name)) + ) + cursor.execute( + sql.SQL("DROP ROLE {}").format(sql.Identifier(role_name)) + ) + connection.commit() + except Exception: + connection.rollback() + raise + finally: + connection.close() + + +def _role_connection( + role: str, + database: str = DATABASE, +) -> psycopg2.extensions.connection: + return psycopg2.connect( + host=DATABASE_HOST, + port=DATABASE_PORT, + dbname=database, + user=role, + password=ROLE_PASSWORD, + ) + + +def _expect_denied(connection: psycopg2.extensions.connection, statement: str) -> None: + try: + with connection.cursor() as cursor: + cursor.execute(statement) + except psycopg2.Error: + connection.rollback() + return + raise AssertionError(f"Expected PostgreSQL denial: {statement}") + + +def _expect_connection_denied(role: str, database: str) -> None: + try: + connection = _role_connection(role, database) + except psycopg2.OperationalError: + return + connection.close() + raise AssertionError(f"{role} unexpectedly connected to {database}") + + +def _legacy_default_behavior_proof() -> None: + """Prove the unrelated-schema default grant still affects future objects.""" + connection = psycopg2.connect(ADMIN_DSN) + try: + with connection.cursor() as cursor: + cursor.execute("SET ROLE owner_onex_tenant") + cursor.execute( + "CREATE TABLE legacy_acl_sentinel.default_acl_probe(id integer)" + ) + cursor.execute("RESET ROLE") + cursor.execute( + "SELECT has_table_privilege(%s, %s, 'SELECT')", + ( + "untrusted_login", + "legacy_acl_sentinel.default_acl_probe", + ), + ) + assert cursor.fetchone() == (True,) + finally: + connection.rollback() + connection.close() + + +def _builtin_default_rollback_proof() -> None: + """Prove rollback restores PostgreSQL's implicit PUBLIC routine/type ACLs.""" + connection = psycopg2.connect(ADMIN_DSN) + try: + with connection.cursor() as cursor: + cursor.execute("SET ROLE owner_onex_tenant") + cursor.execute( + "CREATE FUNCTION legacy_acl_sentinel.rollback_function() " + "RETURNS integer LANGUAGE sql AS $$SELECT 1$$" + ) + cursor.execute( + "CREATE TYPE legacy_acl_sentinel.rollback_type AS ENUM ('restored')" + ) + cursor.execute("RESET ROLE") + cursor.execute( + """ + SELECT has_function_privilege( + 'untrusted_login', + 'legacy_acl_sentinel.rollback_function()', + 'EXECUTE' + ), + has_type_privilege( + 'untrusted_login', + 'legacy_acl_sentinel.rollback_type', + 'USAGE' + ) + """ + ) + assert cursor.fetchone() == (True, True) + finally: + connection.rollback() + connection.close() + + +def _legacy_scaffold_behavior_proof() -> None: + """Prove P1 preserves a pre-existing login's CONNECT/read/write path.""" + connection = _role_connection("legacy_probe_login", "acl_scaffold_probe") + try: + with connection.cursor() as cursor: + cursor.execute("SELECT id FROM public.legacy_scaffold_data") + assert cursor.fetchall() == [(1,)] + cursor.execute("INSERT INTO public.legacy_scaffold_data VALUES (2)") + finally: + connection.rollback() + connection.close() + + +def _connection_isolation_proof(matrix: ModelApplicationDatabaseAclMatrix) -> None: + """Exercise positive and cross-database negative CONNECT cases.""" + assert set(matrix.required_connect_databases) == { + DATABASE, + *SERVICE_DATABASE_ROLES, + } + all_service_roles = set(SERVICE_DATABASE_ROLES.values()) + for database, allowed_role in SERVICE_DATABASE_ROLES.items(): + allowed = _role_connection(allowed_role, database) + allowed.close() + for denied_role in sorted( + set(WORKLOADS) | all_service_roles | {"shadow_login", "untrusted_login"} + ): + if denied_role != allowed_role: + _expect_connection_denied(denied_role, database) + for service_role in sorted(all_service_roles): + _expect_connection_denied(service_role, DATABASE) + print("acl_phase=eight_database_connect_isolation status=PASS") + + +def _behavioral_proof(matrix: ModelApplicationDatabaseAclMatrix) -> None: + tenant = _role_connection("tenant_projection_writer") + dashboard = _role_connection("app_dashboard") + api = _role_connection("onex_api") + runtime = _role_connection("omninode_runtime") + try: + with tenant.cursor() as cursor: + cursor.execute( + "INSERT INTO tenant.delegation_events (payload) VALUES ('matrix')" + ) + cursor.execute("CALL tenant.record_delegation('procedure')") + tenant.commit() + _expect_denied(tenant, "SELECT * FROM omninode_internal.runtime_state") + _expect_denied(tenant, "CREATE TABLE tenant.illegal(id integer)") + + with dashboard.cursor() as cursor: + cursor.execute("SELECT count(*) FROM tenant.delegation_events") + assert cursor.fetchone()[0] >= 1 + cursor.execute("SELECT display_name FROM tenant.tenant_account_names") + assert cursor.fetchone()[0] == "Synthetic" + cursor.execute("SELECT code FROM platform_catalog.plan_tier_snapshot") + assert cursor.fetchone()[0] == "beta" + _expect_denied( + dashboard, + "INSERT INTO tenant.delegation_events (payload) VALUES ('forbidden')", + ) + _expect_denied(dashboard, "SELECT * FROM omninode_internal.runtime_state") + + with api.cursor() as cursor: + cursor.execute( + "INSERT INTO tenant.tenant_accounts VALUES (2, 'API matrix')" + ) + cursor.execute("SELECT code FROM platform_catalog.plan_tiers") + assert cursor.fetchone()[0] == "beta" + cursor.execute("SELECT ROW(1, 'API')::tenant.account_ref") + cursor.execute("SELECT '[1,2)'::tenant.account_id_span") + cursor.execute("SELECT '{}'::tenant.account_id_span_set") + api.commit() + _expect_denied(api, "UPDATE platform_catalog.plan_tiers SET code='x'") + _expect_denied(api, "SELECT * FROM omninode_internal.runtime_state") + + with runtime.cursor() as cursor: + cursor.execute( + "INSERT INTO omninode_internal.runtime_state VALUES (2, 'blocked')" + ) + cursor.execute("SELECT 'runtime'::omninode_internal.runtime_code") + runtime.commit() + _expect_denied(runtime, "SELECT * FROM tenant.delegation_events") + _expect_denied(runtime, "SELECT * FROM platform_catalog.plan_tiers") + finally: + tenant.close() + dashboard.close() + api.close() + runtime.close() + + try: + untrusted = _role_connection("untrusted_login") + except psycopg2.OperationalError: + pass + else: + untrusted.close() + raise AssertionError("PUBLIC CONNECT was not revoked") + _connection_isolation_proof(matrix) + + +def _migration_membership_proof() -> None: + inherited = _dict_rows( + """ + SELECT has_table_privilege( + 'db_migrator', 'tenant.tenant_accounts', 'SELECT' + ) AS inherited + """ + )[0] + assert not bool(inherited["inherited"]), inherited + connection = _admin() + try: + with connection.cursor() as cursor: + cursor.execute("SET SESSION AUTHORIZATION db_migrator") + cursor.execute("SET ROLE owner_onex_tenant") + cursor.execute("SELECT count(*) FROM tenant.tenant_accounts") + assert cursor.fetchone()[0] >= 1 + cursor.execute("RESET SESSION AUTHORIZATION") + finally: + connection.close() + + +def _future_default_proof() -> None: + connection = _admin() + try: + with connection.cursor() as cursor: + cursor.execute("SET ROLE owner_onex_tenant") + cursor.execute("CREATE TABLE tenant.future_table(id integer)") + cursor.execute("CREATE SEQUENCE tenant.future_sequence") + cursor.execute( + "CREATE FUNCTION tenant.future_function() RETURNS integer " + "LANGUAGE sql AS $$SELECT 1$$" + ) + cursor.execute("CREATE TYPE tenant.future_type AS ENUM ('future')") + cursor.execute("RESET ROLE") + for role in (*WORKLOADS, "untrusted_login"): + checks = _dict_rows( + """ + SELECT has_table_privilege(%s, 'tenant.future_table', 'SELECT') AS table_ok, + has_sequence_privilege(%s, 'tenant.future_sequence', 'USAGE') AS sequence_ok, + has_function_privilege(%s, 'tenant.future_function()', 'EXECUTE') AS function_ok, + has_type_privilege(%s, 'tenant.future_type', 'USAGE') AS type_ok + """, + (role, role, role, role), + )[0] + assert not any(bool(value) for value in checks.values()), (role, checks) + with connection.cursor() as cursor: + cursor.execute("DROP TABLE tenant.future_table") + cursor.execute("DROP SEQUENCE tenant.future_sequence") + cursor.execute("DROP FUNCTION tenant.future_function()") + cursor.execute("DROP TYPE tenant.future_type") + finally: + connection.close() + + +def _extra_object_rejection_proof( + matrix: ModelApplicationDatabaseAclMatrix, +) -> None: + """Prove an undeclared runtime-owned object is a hard catalog violation.""" + connection = _admin() + try: + with connection.cursor() as cursor: + cursor.execute("CREATE TABLE tenant.undeclared_runtime_owned(id integer)") + cursor.execute( + "ALTER TABLE tenant.undeclared_runtime_owned OWNER TO app_dashboard" + ) + violations = _catalog_violations(_capture_snapshot(matrix), matrix) + assert {"OWNER_MISMATCH", "RUNTIME_OWNERSHIP"} <= violations, violations + with connection.cursor() as cursor: + cursor.execute("DROP TABLE tenant.undeclared_runtime_owned") + finally: + connection.close() + print("acl_phase=extra_object_red status=DETECTED") + + +def _unknown_acl_and_membership_reconciliation_proof( + matrix: ModelApplicationDatabaseAclMatrix, + green: _AclSnapshot, +) -> None: + """Prove live catalog sweeps remove grantees/children absent from evidence.""" + connection = _admin() + try: + with connection.cursor() as cursor: + cursor.execute("CREATE ROLE hostile_unknown") + cursor.execute( + "GRANT SELECT ON tenant.partitioned_events " + "TO rls_admin WITH GRANT OPTION" + ) + cursor.execute("GRANT USAGE ON SCHEMA tenant TO rls_admin") + cursor.execute("SET ROLE rls_admin") + cursor.execute( + "GRANT SELECT ON tenant.partitioned_events TO hostile_unknown" + ) + cursor.execute("RESET ROLE") + cursor.execute("REVOKE USAGE ON SCHEMA tenant FROM rls_admin") + cursor.execute("GRANT owner_onex_tenant TO hostile_unknown") + cursor.execute( + "ALTER DEFAULT PRIVILEGES FOR ROLE owner_onex_tenant " + "IN SCHEMA tenant GRANT SELECT ON TABLES TO hostile_unknown" + ) + _execute_matrix(matrix) + assert _capture_snapshot(matrix) == green + with connection.cursor() as cursor: + cursor.execute("DROP ROLE hostile_unknown") + finally: + connection.close() + print("acl_phase=unknown_catalog_rows_reconciled status=PASS") + + +def _scaffold_probe_matrix( + source_matrix: ModelApplicationDatabaseAclMatrix, +) -> ModelApplicationDatabaseAclMatrix: + """Build a typed scaffold whose desired roles and schemas are all absent.""" + database_ref = "probe" + physical_database = "acl_scaffold_probe" + principals_by_domain = { + EnumDatabaseSchemaDomain.TENANT: "probe_tenant_writer", + EnumDatabaseSchemaDomain.OMNINODE_INTERNAL: "probe_internal_writer", + EnumDatabaseSchemaDomain.PLATFORM_CATALOG: "probe_catalog_reader", + } + schema_domains = { + "tenant": EnumDatabaseSchemaDomain.TENANT, + "omninode_internal": EnumDatabaseSchemaDomain.OMNINODE_INTERNAL, + "platform_catalog": EnumDatabaseSchemaDomain.PLATFORM_CATALOG, + } + owners = { + "tenant": "probe_owner_tenant", + "omninode_internal": "probe_owner_internal", + "platform_catalog": "probe_owner_catalog", + } + declared = tuple(sorted(principals_by_domain.values())) + observed = ("probe_migrator",) + grantees = (PUBLIC_PRINCIPAL, *declared, *observed) + rows: list[ModelApplicationDatabaseAclRow] = [] + for principal in grantees: + rows.append( + ModelApplicationDatabaseAclRow( + principal=principal, + database_ref=database_ref, + physical_database=physical_database, + object_type=EnumDatabaseGrantObjectType.DATABASE, + privileges=(EnumDatabasePrivilege.CONNECT,) + if principal in declared + else (), + ) + ) + for schema_name, domain in schema_domains.items(): + rows.append( + ModelApplicationDatabaseAclRow( + principal=principal, + database_ref=database_ref, + physical_database=physical_database, + object_type=EnumDatabaseGrantObjectType.SCHEMA, + schema_ref=schema_name, + privileges=(EnumDatabasePrivilege.USAGE,) + if principals_by_domain.get(domain) == principal + else (), + ) + ) + defaults = tuple( + ModelApplicationDatabaseDefaultAclRow( + owner=owners[schema_name], + database_ref=database_ref, + physical_database=physical_database, + schema_ref=schema_name, + object_type=object_type, + grantee=grantee, + ) + for schema_name in sorted(schema_domains) + for object_type in ( + EnumDatabaseGrantObjectType.TABLE, + EnumDatabaseGrantObjectType.SEQUENCE, + EnumDatabaseGrantObjectType.FUNCTION, + EnumDatabaseGrantObjectType.TYPE, + ) + for grantee in grantees + ) + payload = { + "authorization_scope": ( + EnumApplicationDatabaseAclAuthorizationScope.SYNTHETIC_PROOF + ), + "scaffold_status": "READY", + "scaffold_blockers": (), + "status": "BLOCKED", + "blockers": ( + "scaffold proof deliberately withholds materialized object evidence", + ), + "sources": source_matrix.sources, + "declared_principals": {database_ref: declared}, + "observed_principals": {database_ref: observed}, + "absent_principals": {database_ref: declared}, + "observed_owner_roles": (), + "absent_owner_roles": tuple(sorted(owners.values())), + "observed_role_states": ( + ModelApplicationDatabaseObservedRoleState( + role="probe_migrator", + login=False, + superuser=False, + bypass_rls=False, + create_database=False, + create_role=False, + replication=False, + inherit=False, + ), + ), + "governed_role_states": ( + *( + ModelApplicationDatabaseRoleState( + role=owner, + role_kind="owner", + login=False, + ) + for owner in sorted(owners.values()) + ), + *( + ModelApplicationDatabaseRoleState( + role=principal, + role_kind="workload", + login=True, + ) + for principal in declared + ), + ModelApplicationDatabaseRoleState( + role="probe_migrator", + role_kind="migration", + login=False, + ), + ), + "retained_administrative_principals": (), + "database_owners": {database_ref: owners["platform_catalog"]}, + "required_connect_databases": (physical_database,), + "observed_connect_database_owners": {physical_database: "postgres"}, + "allowed_connect_principals": {physical_database: declared}, + "observed_connect_principals": {physical_database: observed}, + "absent_connect_principals": {physical_database: declared}, + "schema_domains": {database_ref: schema_domains}, + "observed_schema_owners": {database_ref: {}}, + "absent_schemas": {database_ref: tuple(sorted(schema_domains))}, + "principal_domains": { + principal: (domain,) for domain, principal in principals_by_domain.items() + }, + "allowed_memberships": tuple( + ModelApplicationDatabaseRoleMembership( + database_ref=database_ref, + role=owner, + member="probe_migrator", + ) + for owner in sorted(owners.values()) + ), + "observed_objects": (), + "objects": (), + "rows": tuple(rows), + "default_privileges": defaults, + "excluded_objects": (), + } + return ModelApplicationDatabaseAclMatrix.model_validate(payload) + + +def _scaffold_phase_proof(matrix: ModelApplicationDatabaseAclMatrix) -> None: + """Prove P1 can precede object materialization without weakening rollback.""" + probe = _scaffold_probe_matrix(matrix) + probe_dsn = _admin_dsn_for_database("acl_scaffold_probe") + assert probe.status == "BLOCKED" + assert probe.scaffold_status == "READY" + rendered = render_application_database_acl_sql( + probe, + allow_synthetic_proof=True, + phase=EnumApplicationDatabaseAclRenderPhase.SCAFFOLD, + ) + assert "-- Render phase: scaffold" in rendered + assert "ALTER TABLE " not in rendered + assert "ALTER VIEW " not in rendered + assert "ALTER MATERIALIZED VIEW " not in rendered + assert "ALTER FUNCTION " not in rendered + assert "ALTER PROCEDURE " not in rendered + assert "ALTER TYPE " not in rendered + assert "GRANT SELECT ON TABLE " not in rendered + + prechange = _capture_snapshot(probe, dsn=probe_dsn) + _legacy_scaffold_behavior_proof() + application_prechange = _capture_snapshot(matrix) + wrong_database = subprocess.run( + ["psql", "--no-psqlrc", "--dbname", ADMIN_DSN], + input=rendered, + text=True, + capture_output=True, + check=False, + ) + assert wrong_database.returncode != 0 + assert _capture_snapshot(probe, dsn=probe_dsn) == prechange + assert _capture_snapshot(matrix) == application_prechange + print("acl_phase=scaffold_wrong_database_guard status=PASS") + + collision_connection = _admin(probe_dsn) + try: + with collision_connection.cursor() as cursor: + cursor.execute( + "CREATE ROLE probe_catalog_reader LOGIN PASSWORD %s CREATEROLE", + (ROLE_PASSWORD,), + ) + hostile_role_snapshot = _capture_snapshot(probe, dsn=probe_dsn) + hostile_role_apply = subprocess.run( + ["psql", "--no-psqlrc", "--dbname", probe_dsn], + input=rendered, + text=True, + capture_output=True, + check=False, + ) + assert hostile_role_apply.returncode != 0 + assert _capture_snapshot(probe, dsn=probe_dsn) == hostile_role_snapshot, ( + "stale expected-absent role evidence leaked a mutation" + ) + with collision_connection.cursor() as cursor: + cursor.execute("DROP ROLE probe_catalog_reader") + assert _capture_snapshot(probe, dsn=probe_dsn) == prechange + + with collision_connection.cursor() as cursor: + cursor.execute("CREATE SCHEMA tenant") + cursor.execute("CREATE TABLE tenant.hostile_collision(id integer)") + hostile_schema_snapshot = _capture_snapshot(probe, dsn=probe_dsn) + hostile_schema_apply = subprocess.run( + ["psql", "--no-psqlrc", "--dbname", probe_dsn], + input=rendered, + text=True, + capture_output=True, + check=False, + ) + assert hostile_schema_apply.returncode != 0 + assert _capture_snapshot(probe, dsn=probe_dsn) == hostile_schema_snapshot, ( + "stale expected-absent schema evidence leaked a mutation" + ) + with collision_connection.cursor() as cursor: + cursor.execute("DROP SCHEMA tenant CASCADE") + assert _capture_snapshot(probe, dsn=probe_dsn) == prechange + finally: + collision_connection.close() + print("acl_phase=scaffold_stale_absence_collision status=REJECTED") + + _atomic_failure_proof( + probe, + prechange, + dsn=probe_dsn, + phase=EnumApplicationDatabaseAclRenderPhase.SCAFFOLD, + ) + for pass_number in (1, 2): + _execute_matrix( + probe, + dsn=probe_dsn, + phase=EnumApplicationDatabaseAclRenderPhase.SCAFFOLD, + ) + print(f"acl_phase=scaffold_apply pass={pass_number} status=PASS") + green = _capture_snapshot(probe, dsn=probe_dsn) + expected_additions = _expected_acl_set(probe) + actual_acl = _actual_acl_set(green, "acl_scaffold_probe") + assert expected_additions <= actual_acl, expected_additions - actual_acl + assert green["database_owner"] == prechange["database_owner"] + expected_schema_owners = { + (schema_name, owner) + for schema_name, owner in { + row.schema_ref: row.owner for row in probe.default_privileges + }.items() + } + assert expected_schema_owners <= { + (str(row["schema_name"]), str(row["owner"])) for row in green["schema_owners"] + } + for role in green["roles"]: + if str(role["rolname"]) == "probe_migrator": + continue + assert not any( + bool(role[field]) + for field in ( + "rolsuper", + "rolinherit", + "rolcreaterole", + "rolcreatedb", + "rolreplication", + "rolbypassrls", + ) + ), role + assert not green["object_owners"] + assert not green["object_acl"] + assert not green["column_acl"] + assert not green["default_acl"] + assert green[ + "default_acl_catalog_rows" + ] == _expected_hardened_default_acl_catalog_rows(probe) + _legacy_scaffold_behavior_proof() + + hostile_connection = _admin(probe_dsn) + try: + for create_statement, drop_statement in ( + ( + 'CREATE COLLATION tenant.hostile_collation FROM "C"', + "DROP COLLATION tenant.hostile_collation", + ), + ( + "CREATE TYPE tenant.hostile_enum AS ENUM ('collision')", + "DROP TYPE tenant.hostile_enum", + ), + ): + with hostile_connection.cursor() as cursor: + cursor.execute(create_statement) + hostile_snapshot = _capture_snapshot(probe, dsn=probe_dsn) + hostile_apply = subprocess.run( + ["psql", "--no-psqlrc", "--dbname", probe_dsn], + input=rendered, + text=True, + capture_output=True, + check=False, + ) + assert hostile_apply.returncode != 0 + assert "expected-absent schema collision for tenant" in hostile_apply.stderr + assert _capture_snapshot(probe, dsn=probe_dsn) == hostile_snapshot + with hostile_connection.cursor() as cursor: + cursor.execute(drop_statement) + assert _capture_snapshot(probe, dsn=probe_dsn) == green + finally: + hostile_connection.close() + + full_payload = probe.model_dump(mode="json") + full_payload.update({"status": "READY", "blockers": []}) + full_probe = ModelApplicationDatabaseAclMatrix.model_validate(full_payload) + for pass_number in (1, 2): + _execute_matrix(full_probe, dsn=probe_dsn) + print(f"acl_phase=scaffold_probe_full_apply pass={pass_number} status=PASS") + full_green = _capture_snapshot(full_probe, dsn=probe_dsn) + assert not _catalog_violations(full_green, full_probe), _catalog_violations( + full_green, + full_probe, + ) + _expect_connection_denied("legacy_probe_login", "acl_scaffold_probe") + _restore_snapshot(prechange, probe, dsn=probe_dsn) + assert _capture_snapshot(probe, dsn=probe_dsn) == prechange + _legacy_scaffold_behavior_proof() + print("acl_phase=scaffold_fresh_additive_round_trip status=PASS") + + +def main() -> None: + postgres_major = _postgres_major() + assert postgres_major == 16, postgres_major + matrix = _fixture_matrix() + assert matrix.status == "READY", matrix.blockers + assert matrix.scaffold_status == "BLOCKED" + assert any( + "additive scaffold intended roles are not already safe" in blocker + for blocker in matrix.scaffold_blockers + ) + + _scaffold_phase_proof(matrix) + _extra_object_rejection_proof(matrix) + legacy_default_acl = _capture_legacy_default_acl() + assert legacy_default_acl + _legacy_default_behavior_proof() + prechange = _capture_snapshot(matrix) + OBSERVED_PRECHANGE.write_text( + json.dumps(prechange, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + update = os.environ.get("UPDATE_PRECHANGE", "false").lower() == "true" + if update: + rollback_input = prechange + else: + rollback_input = cast( + "_AclSnapshot", + json.loads(EXPECTED_PRECHANGE.read_text(encoding="utf-8")), + ) + assert rollback_input == prechange, "durable pre-change ACL artifact drift" + + red = _catalog_violations(prechange, matrix) + required_red = { + "OWNER_MISMATCH", + "PUBLIC_PRIVILEGE", + "BROAD_GRANT", + "UNSAFE_DEFAULT_PRIVILEGE", + "CROSS_DOMAIN_PRIVILEGE", + "RUNTIME_ROLE_ATTRIBUTES", + "OWNER_MEMBERSHIP", + "UNDECLARED_PRINCIPAL_PRIVILEGE", + "MISSING_DECLARED_MEMBERSHIP", + "RUNTIME_OWNERSHIP", + "GRANT_OPTION", + "COLUMN_PRIVILEGE", + "DATABASE_DDL_PRIVILEGE", + } + assert required_red <= red, (required_red - red, red) + print(f"acl_phase=prechange red_controls={','.join(sorted(red))} status=DETECTED") + + _atomic_failure_proof(matrix, prechange) + for pass_number in (1, 2): + _execute_matrix(matrix) + print(f"acl_phase=apply pass={pass_number} status=PASS") + green = _capture_snapshot(matrix) + assert not _catalog_violations(green, matrix), _catalog_violations(green, matrix) + assert [row for row in green["roles"] if row["rolname"] == "rls_admin"] == [ + row for row in prechange["roles"] if row["rolname"] == "rls_admin" + ] + assert [ + row for row in green["database_owner"] if row["database_name"] != DATABASE + ] == [ + row for row in prechange["database_owner"] if row["database_name"] != DATABASE + ] + assert _capture_legacy_default_acl() == legacy_default_acl + _legacy_default_behavior_proof() + _unknown_acl_and_membership_reconciliation_proof(matrix, green) + try: + _restore_snapshot(rollback_input, matrix, inject_failure=True) + except psycopg2.errors.DivisionByZero: + pass + else: + raise AssertionError("injected rollback failure unexpectedly succeeded") + assert _capture_snapshot(matrix) == green, ( + "failed rollback leaked a partial catalog mutation" + ) + print("acl_phase=rollback_atomic_failure red_control=ROLLBACK status=PASS") + _behavioral_proof(matrix) + _migration_membership_proof() + _future_default_proof() + print("acl_phase=postgres16_readback status=PASS") + + _restore_snapshot(rollback_input, matrix) + restored = _capture_snapshot(matrix) + assert restored == prechange, "rollback did not reproduce durable pre-change ACL" + assert _capture_legacy_default_acl() == legacy_default_acl + _legacy_default_behavior_proof() + _builtin_default_rollback_proof() + print("acl_phase=rollback_round_trip status=PASS") + + for pass_number in (1, 2): + _execute_matrix(matrix) + print(f"acl_phase=reapply pass={pass_number} status=PASS") + final = _capture_snapshot(matrix) + assert not _catalog_violations(final, matrix), _catalog_violations(final, matrix) + assert _capture_legacy_default_acl() == legacy_default_acl + _legacy_default_behavior_proof() + print( + f"acl_status=PASS postgres_major={postgres_major} objects={len(matrix.objects)} " + f"rows={len(matrix.rows)} defaults={len(matrix.default_privileges)}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/prove_application_database_domain_enforcement.py b/scripts/ci/prove_application_database_domain_enforcement.py new file mode 100644 index 0000000000..c7704a413e --- /dev/null +++ b/scripts/ci/prove_application_database_domain_enforcement.py @@ -0,0 +1,2333 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Rebuilt PostgreSQL 16 proof for application-domain enforcement gates.""" + +from __future__ import annotations + +import os +import re +from collections.abc import Mapping, Sequence +from pathlib import Path +from uuid import UUID + +import psycopg2 +from psycopg2 import sql as pg_sql + +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_infra.topology.application_database import load_topology_profile +from omnibase_infra.validation.application_database_domain_enforcement import ( + application_database_function_definition_sha256, + lint_application_database_sql, + load_application_database_ownership_identities, + validate_application_database_catalog_census, + validate_application_database_pool_identities, + validate_application_database_relation_states, +) +from omnibase_infra.validation.application_database_source_tenant_authority import ( + resolve_application_database_authority_columns, +) +from omnibase_infra.validation.application_relation_ownership import ( + load_service_ownership_manifest, + validate_application_relation_ownership, +) +from omnibase_infra.validation.enums.enum_application_database_identity_root_operation import ( + EnumApplicationDatabaseIdentityRootOperation, +) +from omnibase_infra.validation.enums.enum_application_inventory_object_kind import ( + EnumApplicationInventoryObjectKind, +) +from omnibase_infra.validation.enums.enum_application_relation_kind import ( + EnumApplicationRelationKind, +) +from omnibase_infra.validation.models.model_application_database_catalog_identity import ( + ModelApplicationDatabaseCatalogIdentity, +) +from omnibase_infra.validation.models.model_application_database_column_state import ( + ModelApplicationDatabaseColumnState, +) +from omnibase_infra.validation.models.model_application_database_function_state import ( + ModelApplicationDatabaseFunctionState, +) +from omnibase_infra.validation.models.model_application_database_identity_root_control_state import ( + ModelApplicationDatabaseIdentityRootControlState, +) +from omnibase_infra.validation.models.model_application_database_policy_state import ( + ModelApplicationDatabasePolicyState, +) +from omnibase_infra.validation.models.model_application_database_pool_identity import ( + ModelApplicationDatabasePoolIdentity, +) +from omnibase_infra.validation.models.model_application_database_relation_state import ( + ModelApplicationDatabaseRelationState, +) +from omnibase_infra.validation.models.model_application_database_routine_dependency_state import ( + ModelApplicationDatabaseRoutineDependencyState, +) +from omnibase_infra.validation.models.model_application_database_tenant_isolation_evidence import ( + ModelApplicationDatabaseTenantIsolationEvidence, +) +from omnibase_infra.validation.models.model_application_relation_declaration import ( + ModelApplicationRelationDeclaration, +) +from omnibase_infra.validation.models.model_application_relation_inventory import ( + ModelApplicationRelationInventory, +) +from omnibase_infra.validation.models.model_database_object_evidence import ( + ModelDatabaseObjectEvidence, +) +from omnibase_infra.validation.models.model_live_application_relation import ( + ModelLiveApplicationRelation, +) +from omnibase_infra.validation.models.model_relation_evidence import ( + ModelRelationEvidence, +) + +_ADMIN_DSN = os.environ["ADMIN_DSN"] +_OWNERSHIP_MANIFEST = Path(os.environ["OWNERSHIP_MANIFEST"]) +_OWNERSHIP = load_service_ownership_manifest(_OWNERSHIP_MANIFEST) +_TOPOLOGY = load_topology_profile("local") +_DATABASE = _TOPOLOGY.databases["application"] +_TENANT_A = UUID("11111111-1111-1111-1111-111111111111") +_TENANT_B = UUID("22222222-2222-2222-2222-222222222222") +_EXPECTED_TENANT_ROWS = {_TENANT_A: 2, _TENANT_B: 1} +_POLICY_COMMANDS = { + "*": "ALL", + "r": "SELECT", + "a": "INSERT", + "w": "UPDATE", + "d": "DELETE", +} +_CATALOG_TO_RELATION_KIND = { + EnumApplicationInventoryObjectKind.TABLE: EnumApplicationRelationKind.TABLE, + EnumApplicationInventoryObjectKind.VIEW: EnumApplicationRelationKind.VIEW, + EnumApplicationInventoryObjectKind.MATERIALIZED_VIEW: ( + EnumApplicationRelationKind.MATERIALIZED_VIEW + ), + EnumApplicationInventoryObjectKind.FOREIGN_TABLE: ( + EnumApplicationRelationKind.FOREIGN_TABLE + ), + EnumApplicationInventoryObjectKind.FUNCTION: EnumApplicationRelationKind.FUNCTION, +} + + +def _pool_dsn(binding_ref: str) -> str: + """Resolve proof plumbing from a topology-derived binding name.""" + return os.environ[f"POOL_DSN_{binding_ref.upper()}"] + + +def _connect(dsn: str) -> psycopg2.extensions.connection: + return psycopg2.connect(dsn) + + +def _relation_evidence( + declaration: ModelApplicationRelationDeclaration, +) -> ModelRelationEvidence: + """Resolve one manifest classification for a relation identity.""" + matches = tuple( + evidence + for evidence in _OWNERSHIP.relation_evidence + if evidence.database_ref == declaration.database_ref + and evidence.schema == declaration.schema + and evidence.name == declaration.name + and evidence.kind is declaration.kind + and evidence.function_signature == declaration.function_signature + ) + if len(matches) != 1: + raise AssertionError( + f"{declaration.identity!r} requires exactly one typed relation " + f"classification; observed {len(matches)}" + ) + return matches[0] + + +def _database_object_evidence( + declaration: ModelApplicationRelationDeclaration, +) -> ModelDatabaseObjectEvidence: + """Resolve one independently authored database-object audit declaration.""" + matches = tuple( + evidence + for evidence in _OWNERSHIP.database_objects + if evidence.database_ref == declaration.database_ref + and evidence.schema == declaration.schema + and evidence.name == declaration.name + and evidence.kind.value == declaration.kind.value + and evidence.function_signature == declaration.function_signature + ) + if len(matches) != 1: + raise AssertionError( + f"{declaration.identity!r} requires exactly one typed database-object " + f"audit; observed {len(matches)}" + ) + return matches[0] + + +def _column_states( + connection: psycopg2.extensions.connection, + schema: str, + name: str, +) -> tuple[ModelApplicationDatabaseColumnState, ...]: + """Read exact columns for tables, views, materialized views, and foreign tables.""" + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT attribute.attname, + format_type(attribute.atttypid, attribute.atttypmod), + NOT attribute.attnotnull, + CASE WHEN attribute.attgenerated = '' + THEN pg_get_expr(default_row.adbin, default_row.adrelid) + END, + CASE WHEN attribute.attgenerated <> '' + THEN pg_get_expr(default_row.adbin, default_row.adrelid) + END + FROM pg_attribute attribute + JOIN pg_class relation ON relation.oid = attribute.attrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + LEFT JOIN pg_attrdef default_row + ON default_row.adrelid = attribute.attrelid + AND default_row.adnum = attribute.attnum + WHERE namespace.nspname = %s + AND relation.relname = %s + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + ORDER BY attribute.attnum + """, + (schema, name), + ) + return tuple( + ModelApplicationDatabaseColumnState( + name=row[0], + data_type=row[1], + nullable=row[2], + default_expression=row[3], + generated_expression=row[4], + ) + for row in cursor.fetchall() + ) + + +def _prove_identity_root_enumeration( + *, + schema: str, + name: str, + role: str, +) -> bool: + relation = pg_sql.SQL("{}.{}").format( + pg_sql.Identifier(schema), + pg_sql.Identifier(name), + ) + admin = _connect(_ADMIN_DSN) + try: + with admin.cursor() as cursor: + cursor.execute( + pg_sql.SQL("SET LOCAL ROLE {}").format(pg_sql.Identifier(role)) + ) + cursor.execute(pg_sql.SQL("SELECT count(*) FROM {}").format(relation)) + control_count = int(cursor.fetchone()[0]) + admin.rollback() + finally: + admin.close() + + runtime = _connect(_pool_dsn("onex_api")) + try: + with runtime.cursor() as cursor: + cursor.execute("SET LOCAL app.tenant_id = %s", (str(_TENANT_A),)) + cursor.execute(pg_sql.SQL("SELECT count(*) FROM {}").format(relation)) + tenant_count = int(cursor.fetchone()[0]) + runtime.rollback() + finally: + runtime.close() + + unset_runtime = _connect(_pool_dsn("onex_api")) + try: + with unset_runtime.cursor() as cursor: + cursor.execute(pg_sql.SQL("SELECT count(*) FROM {}").format(relation)) + unset_count = int(cursor.fetchone()[0]) + unset_runtime.rollback() + finally: + unset_runtime.close() + return control_count == 2 and tenant_count == 1 and unset_count == 0 + + +def _prove_identity_root_creation( + *, + schema: str, + name: str, + role: str, +) -> bool: + relation = pg_sql.SQL("{}.{}").format( + pg_sql.Identifier(schema), + pg_sql.Identifier(name), + ) + new_tenant = UUID("33333333-3333-3333-3333-333333333333") + admin = _connect(_ADMIN_DSN) + try: + with admin.cursor() as cursor: + cursor.execute( + pg_sql.SQL("SET LOCAL ROLE {}").format(pg_sql.Identifier(role)) + ) + cursor.execute( + pg_sql.SQL("INSERT INTO {} (id, tenant_name) VALUES (%s, %s)").format( + relation + ), + (str(new_tenant), "control-created"), + ) + control_inserted = True + admin.rollback() + except psycopg2.Error: + admin.rollback() + control_inserted = False + finally: + admin.close() + + runtime = _connect(_pool_dsn("onex_api")) + try: + try: + with runtime.cursor() as cursor: + cursor.execute("SET LOCAL app.tenant_id = %s", (str(_TENANT_A),)) + cursor.execute( + pg_sql.SQL( + "INSERT INTO {} (id, tenant_name) VALUES (%s, %s)" + ).format(relation), + (str(new_tenant), "runtime-cross-tenant"), + ) + except psycopg2.Error as exc: + runtime.rollback() + runtime_denied = str(exc.pgcode) == "42501" + else: + runtime.rollback() + runtime_denied = False + finally: + runtime.close() + return control_inserted and runtime_denied + + +def _runtime_identity_root_membership_principals( + connection: psycopg2.extensions.connection, + role: str, +) -> tuple[str, ...]: + """Return runtime roles with a zero-hop, direct, or transitive path to role.""" + runtime_principals = sorted( + binding.principal for binding in _DATABASE.bindings.values() + ) + with connection.cursor() as cursor: + cursor.execute( + """ + WITH RECURSIVE runtime_role_path AS ( + SELECT runtime_role.oid AS root_member, + runtime_role.oid AS reachable_role, + ARRAY[runtime_role.oid]::oid[] AS path + FROM pg_roles AS runtime_role + WHERE runtime_role.rolname = ANY(%s) + + UNION ALL + + SELECT prior.root_member, + membership.roleid AS reachable_role, + prior.path || membership.roleid + FROM runtime_role_path AS prior + JOIN pg_auth_members AS membership + ON membership.member = prior.reachable_role + WHERE NOT membership.roleid = ANY(prior.path) + ) + SELECT DISTINCT runtime_role.rolname + FROM runtime_role_path AS reachable + JOIN pg_roles AS runtime_role + ON runtime_role.oid = reachable.root_member + JOIN pg_roles AS reached_role + ON reached_role.oid = reachable.reachable_role + WHERE reached_role.rolname = %s + ORDER BY runtime_role.rolname + """, + (runtime_principals, role), + ) + return tuple(str(row[0]) for row in cursor.fetchall()) + + +def _runtime_identity_root_set_role_denials(role: str) -> tuple[str, ...]: + """Attempt SET ROLE from every topology pool and retain only 42501 denials.""" + denied_principals: list[str] = [] + for binding_ref in sorted(_DATABASE.bindings): + runtime = _connect(_pool_dsn(binding_ref)) + current_user: str | None = None + try: + try: + with runtime.cursor() as cursor: + cursor.execute("SELECT current_user") + current_user = str(cursor.fetchone()[0]) + cursor.execute( + pg_sql.SQL("SET LOCAL ROLE {}").format(pg_sql.Identifier(role)) + ) + except psycopg2.Error as exc: + runtime.rollback() + if str(exc.pgcode) != "42501": + raise AssertionError( + f"pool {binding_ref!r} SET ROLE proof failed with unexpected " + f"SQLSTATE {exc.pgcode!r}" + ) from exc + if current_user is None: + raise AssertionError( + f"pool {binding_ref!r} did not expose current_user" + ) from exc + denied_principals.append(current_user) + else: + runtime.rollback() + finally: + runtime.close() + return tuple(sorted(denied_principals)) + + +def _identity_root_control_state( + connection: psycopg2.extensions.connection, + classification: ModelRelationEvidence, +) -> ModelApplicationDatabaseIdentityRootControlState | None: + if classification.identity_root_contract is None: + return None + role = classification.identity_root_control_role + if role is None or classification.schema is None: + raise AssertionError("identity-root classification lacks control authority") + qualified_relation = f"{classification.schema}.{classification.name}" + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT role.rolcanlogin, role.rolsuper, role.rolbypassrls, + has_table_privilege(role.rolname, %s, 'INSERT'), + has_table_privilege(role.rolname, %s, 'SELECT') + FROM pg_roles role + WHERE role.rolname = %s + """, + (qualified_relation, qualified_relation, role), + ) + rows = cursor.fetchall() + if len(rows) != 1: + raise AssertionError( + f"identity-root control role {role!r} must exist exactly once" + ) + can_login, superuser, bypass_rls, can_insert, can_select = rows[0] + observed: list[EnumApplicationDatabaseIdentityRootOperation] = [] + proof_ids: list[str] = [] + if can_insert and _prove_identity_root_creation( + schema=classification.schema, + name=classification.name, + role=role, + ): + observed.append(EnumApplicationDatabaseIdentityRootOperation.TENANT_CREATION) + proof_ids.append("postgres16:identity-root-control-create-and-runtime-deny") + if can_select and _prove_identity_root_enumeration( + schema=classification.schema, + name=classification.name, + role=role, + ): + observed.append( + EnumApplicationDatabaseIdentityRootOperation.CROSS_TENANT_ENUMERATION + ) + proof_ids.append("postgres16:identity-root-control-enumerate-and-runtime-scope") + return ModelApplicationDatabaseIdentityRootControlState( + role=role, + role_can_login=can_login, + role_superuser=superuser, + role_bypass_rls=bypass_rls, + runtime_membership_principals=( + _runtime_identity_root_membership_principals(connection, role) + ), + runtime_set_role_denied_principals=( + _runtime_identity_root_set_role_denials(role) + ), + declared_operations=classification.identity_root_control_operations, + observed_operations=tuple(observed), + behavioral_proof_ids=tuple(proof_ids), + ) + + +def _catalog_routine_dependency_states( + cursor: psycopg2.extensions.cursor, + target_relation_oid: int, +) -> tuple[ModelApplicationDatabaseRoutineDependencyState, ...]: + """Load immutable routine definitions and exact catalog dependency edges.""" + cursor.execute( + """ + SELECT routine.oid, + namespace.nspname, + routine.proname, + language.lanname, + routine.prosrc, + routine.proargtypes::oid[], + ARRAY( + SELECT NULLIF(argument.argument_name, '') + FROM unnest( + COALESCE( + routine.proargnames, + array_fill( + NULL::text, + ARRAY[ + cardinality( + COALESCE( + routine.proallargtypes, + routine.proargtypes::oid[] + ) + ) + ] + ) + ), + COALESCE( + routine.proargmodes, + array_fill('i'::"char", ARRAY[routine.pronargs]) + ) + ) AS argument(argument_name, argument_mode) + WHERE argument.argument_mode IN ( + 'i'::"char", + 'b'::"char", + 'v'::"char" + ) + ), + routine.prorettype IN ( + 'pg_catalog.trigger'::regtype, + 'pg_catalog.event_trigger'::regtype + ), + ARRAY( + SELECT DISTINCT dependency.refobjid + FROM pg_depend dependency + WHERE dependency.classid = 'pg_proc'::regclass + AND dependency.objid = routine.oid + AND dependency.refclassid = 'pg_proc'::regclass + ORDER BY dependency.refobjid + ), + ARRAY( + SELECT DISTINCT attribute.attname + FROM pg_depend dependency + JOIN pg_attribute attribute + ON attribute.attrelid = dependency.refobjid + AND attribute.attnum = dependency.refobjsubid + WHERE dependency.classid = 'pg_proc'::regclass + AND dependency.objid = routine.oid + AND dependency.refclassid = 'pg_class'::regclass + AND dependency.refobjid = %s + AND dependency.refobjsubid > 0 + ORDER BY attribute.attname + ), + EXISTS ( + SELECT 1 + FROM pg_depend dependency + WHERE dependency.classid = 'pg_proc'::regclass + AND dependency.objid = routine.oid + AND dependency.refclassid = 'pg_class'::regclass + AND dependency.refobjid = %s + AND dependency.refobjsubid = 0 + ) + FROM pg_proc routine + JOIN pg_namespace namespace ON namespace.oid = routine.pronamespace + JOIN pg_language language ON language.oid = routine.prolang + ORDER BY routine.oid + """, + (target_relation_oid, target_relation_oid), + ) + return tuple( + ModelApplicationDatabaseRoutineDependencyState( + object_id=row[0], + namespace=row[1], + name=row[2], + language=row[3], + source_body=row[4], + argument_type_ids=tuple(row[5]), + argument_names=tuple(row[6]), + returns_trigger=row[7], + referenced_routine_ids=tuple(row[8]), + referenced_target_columns=tuple(row[9]), + references_target_whole_row=row[10], + ) + for row in cursor.fetchall() + ) + + +def _catalog_surface_authority_columns( + cursor: psycopg2.extensions.cursor, + *, + roots: Sequence[tuple[str, int]], + target_relation_oid: int, + target_composite_type_id: int, + columns: Sequence[ModelApplicationDatabaseColumnState], + routines: Sequence[ModelApplicationDatabaseRoutineDependencyState], +) -> tuple[str, ...]: + """Resolve a surface through pg_depend and reachable routine definitions.""" + if not roots: + return () + roots_by_class: dict[str, list[int]] = {} + for class_name, object_id in roots: + roots_by_class.setdefault(class_name, []).append(object_id) + + direct_columns: set[str] = set() + direct_whole_row_reference = False + routine_ids: set[int] = set() + for class_name, object_ids in roots_by_class.items(): + cursor.execute( + """ + WITH RECURSIVE dependency_walk( + refclassid, + refobjid, + refobjsubid, + visited + ) AS ( + SELECT dependency.refclassid, + dependency.refobjid, + dependency.refobjsubid, + ARRAY[ + dependency.refclassid::text || ':' || + dependency.refobjid::text || ':' || + dependency.refobjsubid::text + ] + FROM pg_depend dependency + WHERE dependency.classid = %s::regclass + AND dependency.objid = ANY(%s::oid[]) + UNION ALL + SELECT dependency.refclassid, + dependency.refobjid, + dependency.refobjsubid, + dependency_walk.visited || ( + dependency.refclassid::text || ':' || + dependency.refobjid::text || ':' || + dependency.refobjsubid::text + ) + FROM dependency_walk + JOIN pg_depend dependency + ON dependency.classid = dependency_walk.refclassid + AND dependency.objid = dependency_walk.refobjid + WHERE dependency_walk.refclassid <> 'pg_class'::regclass + AND NOT ( + dependency.refclassid::text || ':' || + dependency.refobjid::text || ':' || + dependency.refobjsubid::text + ) = ANY(dependency_walk.visited) + ) + SELECT DISTINCT dependency_walk.refclassid = 'pg_proc'::regclass, + dependency_walk.refobjid, + attribute.attname, + dependency_walk.refclassid = 'pg_class'::regclass + AND dependency_walk.refobjid = %s + AND dependency_walk.refobjsubid = 0 + AS references_target_whole_row + FROM dependency_walk + LEFT JOIN pg_attribute attribute + ON dependency_walk.refclassid = 'pg_class'::regclass + AND dependency_walk.refobjid = attribute.attrelid + AND dependency_walk.refobjsubid = attribute.attnum + AND dependency_walk.refobjid = %s + AND dependency_walk.refobjsubid > 0 + WHERE dependency_walk.refclassid = 'pg_proc'::regclass + OR attribute.attname IS NOT NULL + OR ( + dependency_walk.refclassid = 'pg_class'::regclass + AND dependency_walk.refobjid = %s + AND dependency_walk.refobjsubid = 0 + ) + """, + ( + class_name, + list(dict.fromkeys(object_ids)), + target_relation_oid, + target_relation_oid, + target_relation_oid, + ), + ) + for ( + is_routine, + object_id, + column_name, + references_target_whole_row, + ) in cursor.fetchall(): + if is_routine: + routine_ids.add(int(object_id)) + elif references_target_whole_row: + direct_whole_row_reference = True + elif column_name is not None: + direct_columns.add(str(column_name)) + + return resolve_application_database_authority_columns( + target_columns=tuple(column.name for column in columns), + target_composite_type_id=target_composite_type_id, + direct_referenced_columns=tuple(direct_columns), + direct_whole_row_reference=direct_whole_row_reference, + root_routine_ids=tuple(routine_ids), + routines=routines, + governed_schemas=tuple(_DATABASE.schemas), + ) + + +def _table_state( + connection: psycopg2.extensions.connection, + declaration: ModelApplicationRelationDeclaration, +) -> ModelApplicationDatabaseRelationState: + schema = declaration.schema + name = declaration.name + classification = _relation_evidence(declaration) + columns = _column_states(connection, schema, name) + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT relation.oid, relation.reltype + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = %s AND relation.relname = %s + """, + (schema, name), + ) + relation_rows = cursor.fetchall() + if len(relation_rows) != 1: + raise AssertionError( + f"{declaration.identity!r} requires exactly one catalog relation; " + f"observed {len(relation_rows)}" + ) + target_relation_oid, target_composite_type_id = relation_rows[0] + cursor.execute( + """ + SELECT attribute.attname + FROM pg_index index_row + CROSS JOIN LATERAL unnest(index_row.indkey) + WITH ORDINALITY AS key_column(attnum, position) + JOIN pg_attribute attribute + ON attribute.attrelid = index_row.indrelid + AND attribute.attnum = key_column.attnum + JOIN pg_class relation ON relation.oid = index_row.indrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE index_row.indisprimary + AND key_column.position <= index_row.indnkeyatts + AND namespace.nspname = %s + AND relation.relname = %s + ORDER BY key_column.position + """, + (schema, name), + ) + primary_key_columns = tuple(row[0] for row in cursor.fetchall()) + cursor.execute( + """ + SELECT index_row.indexrelid, + index_relation.relname, + ARRAY( + SELECT attribute.attname + FROM unnest(index_row.indkey) + WITH ORDINALITY AS key_column(attnum, position) + JOIN pg_attribute attribute + ON attribute.attrelid = index_row.indrelid + AND attribute.attnum = key_column.attnum + WHERE key_column.position <= index_row.indnkeyatts + ORDER BY key_column.position + ), + pg_get_expr(index_row.indexprs, index_row.indrelid), + pg_get_expr(index_row.indpred, index_row.indrelid) + FROM pg_index index_row + JOIN pg_class relation ON relation.oid = index_row.indrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + JOIN pg_class index_relation ON index_relation.oid = index_row.indexrelid + WHERE index_row.indisunique + AND namespace.nspname = %s + AND relation.relname = %s + ORDER BY index_relation.relname + """, + (schema, name), + ) + unique_index_rows = tuple( + (row[0], tuple(row[2]), row[3], row[4]) for row in cursor.fetchall() + ) + cursor.execute( + """ + SELECT constraint_row.conname, + array_agg(attribute.attname ORDER BY key_column.position) + FROM pg_constraint constraint_row + JOIN pg_class relation ON relation.oid = constraint_row.conrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + CROSS JOIN LATERAL unnest(constraint_row.conkey) + WITH ORDINALITY AS key_column(attnum, position) + JOIN pg_attribute attribute + ON attribute.attrelid = constraint_row.conrelid + AND attribute.attnum = key_column.attnum + WHERE constraint_row.contype = 'f' + AND namespace.nspname = %s + AND relation.relname = %s + GROUP BY constraint_row.conname + ORDER BY constraint_row.conname + """, + (schema, name), + ) + foreign_key_column_sets = tuple(tuple(row[1]) for row in cursor.fetchall()) + cursor.execute( + """ + SELECT attribute.attname + FROM pg_partitioned_table partitioned + JOIN pg_class relation ON relation.oid = partitioned.partrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + CROSS JOIN LATERAL unnest(partitioned.partattrs) + WITH ORDINALITY AS key_column(attnum, position) + JOIN pg_attribute attribute + ON attribute.attrelid = partitioned.partrelid + AND attribute.attnum = key_column.attnum + WHERE namespace.nspname = %s + AND relation.relname = %s + ORDER BY key_column.position + """, + (schema, name), + ) + direct_partition_key_columns = tuple(row[0] for row in cursor.fetchall()) + cursor.execute( + """ + SELECT pg_get_expr(partitioned.partexprs, partitioned.partrelid) + FROM pg_partitioned_table partitioned + JOIN pg_class relation ON relation.oid = partitioned.partrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = %s + AND relation.relname = %s + """, + (schema, name), + ) + partition_expressions = tuple( + row[0] for row in cursor.fetchall() if row[0] is not None + ) + cursor.execute( + """ + SELECT relation.relrowsecurity, relation.relforcerowsecurity + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = %s AND relation.relname = %s + """, + (schema, name), + ) + rls_enabled, rls_forced = cursor.fetchone() + cursor.execute( + """ + SELECT policy.polname, policy.polpermissive, policy.polcmd, + ARRAY( + SELECT CASE + WHEN policy_role.role_oid = 0 THEN 'PUBLIC' + ELSE role.rolname + END + FROM unnest(policy.polroles) WITH ORDINALITY + AS policy_role(role_oid, position) + LEFT JOIN pg_roles role ON role.oid = policy_role.role_oid + ORDER BY policy_role.position + ), + pg_get_expr(policy.polqual, policy.polrelid), + pg_get_expr(policy.polwithcheck, policy.polrelid) + FROM pg_policy policy + JOIN pg_class relation ON relation.oid = policy.polrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = %s AND relation.relname = %s + ORDER BY policy.polname + """, + (schema, name), + ) + policies = tuple( + ModelApplicationDatabasePolicyState( + name=row[0], + permissive=row[1], + command=_POLICY_COMMANDS[row[2]], + roles=tuple(row[3]), + using_expression=row[4], + with_check_expression=row[5], + ) + for row in cursor.fetchall() + ) + cursor.execute( + """ + SELECT policy.oid + FROM pg_policy policy + JOIN pg_class relation ON relation.oid = policy.polrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = %s AND relation.relname = %s + ORDER BY policy.oid + """, + (schema, name), + ) + policy_roots = tuple(("pg_policy", row[0]) for row in cursor.fetchall()) + cursor.execute( + """ + SELECT CASE constraint_row.contype + WHEN 'c' THEN pg_get_expr( + constraint_row.conbin, + constraint_row.conrelid + ) + ELSE pg_get_constraintdef(constraint_row.oid, true) + END + FROM pg_constraint constraint_row + JOIN pg_class relation ON relation.oid = constraint_row.conrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE constraint_row.contype IN ('c', 'x') + AND namespace.nspname = %s + AND relation.relname = %s + UNION ALL + SELECT pg_get_triggerdef(trigger_row.oid, true) + FROM pg_trigger trigger_row + JOIN pg_class relation ON relation.oid = trigger_row.tgrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE NOT trigger_row.tgisinternal + AND namespace.nspname = %s + AND relation.relname = %s + UNION ALL + SELECT pg_get_functiondef(trigger_row.tgfoid) + FROM pg_trigger trigger_row + JOIN pg_class relation ON relation.oid = trigger_row.tgrelid + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE NOT trigger_row.tgisinternal + AND namespace.nspname = %s + AND relation.relname = %s + """, + (schema, name, schema, name, schema, name), + ) + write_eligibility_expressions = tuple( + row[0] for row in cursor.fetchall() if row[0] is not None + ) + cursor.execute( + """ + WITH RECURSIVE dependent_views(rewrite_oid, relation_oid) AS ( + SELECT DISTINCT rewrite_row.oid, dependent_relation.oid + FROM pg_depend dependency + JOIN pg_rewrite rewrite_row ON rewrite_row.oid = dependency.objid + JOIN pg_class dependent_relation + ON dependent_relation.oid = rewrite_row.ev_class + WHERE dependency.classid = 'pg_rewrite'::regclass + AND dependency.refclassid = 'pg_class'::regclass + AND dependency.refobjid = %s + AND dependent_relation.relkind IN ('v', 'm') + AND dependent_relation.oid <> %s + UNION + SELECT DISTINCT rewrite_row.oid, dependent_relation.oid + FROM dependent_views source_view + JOIN pg_depend dependency + ON dependency.refclassid = 'pg_class'::regclass + AND dependency.refobjid = source_view.relation_oid + JOIN pg_rewrite rewrite_row ON rewrite_row.oid = dependency.objid + JOIN pg_class dependent_relation + ON dependent_relation.oid = rewrite_row.ev_class + WHERE dependency.classid = 'pg_rewrite'::regclass + AND dependent_relation.relkind IN ('v', 'm') + AND dependent_relation.oid <> source_view.relation_oid + ) + SELECT rewrite_oid, pg_get_viewdef(relation_oid, true) + FROM dependent_views + ORDER BY relation_oid + """, + (target_relation_oid, target_relation_oid), + ) + dependent_view_rows = tuple(cursor.fetchall()) + dependent_view_expressions = tuple(row[1] for row in dependent_view_rows) + cursor.execute( + """ + SELECT 'pg_constraint', constraint_row.oid + FROM pg_constraint constraint_row + WHERE constraint_row.conrelid = %s + AND constraint_row.contype IN ('c', 'x') + UNION ALL + SELECT 'pg_trigger', trigger_row.oid + FROM pg_trigger trigger_row + WHERE trigger_row.tgrelid = %s + AND NOT trigger_row.tgisinternal + """, + (target_relation_oid, target_relation_oid), + ) + write_eligibility_roots = tuple( + (str(row[0]), int(row[1])) for row in cursor.fetchall() + ) + cursor.execute( + """ + SELECT attribute.attname, default_row.oid + FROM pg_attribute attribute + JOIN pg_attrdef default_row + ON default_row.adrelid = attribute.attrelid + AND default_row.adnum = attribute.attnum + WHERE attribute.attrelid = %s + AND attribute.attgenerated <> '' + ORDER BY attribute.attnum + """, + (target_relation_oid,), + ) + generated_expression_roots = tuple(cursor.fetchall()) + has_source_tenant = any(column.name == "source_tenant_id" for column in columns) + if has_source_tenant != ( + classification.source_tenant_provenance_contract is not None + ): + raise AssertionError( + f"{declaration.identity!r} source_tenant_id and its typed provenance " + "classification must be declared together" + ) + semantic_fields = { + "deduplication_key_columns": classification.deduplication_key_columns, + "authorization_dependency_columns": ( + classification.authorization_dependency_columns + ), + "write_eligibility_dependency_columns": ( + classification.write_eligibility_dependency_columns + ), + } + missing_semantics = [ + name for name, value in semantic_fields.items() if value is None + ] + if missing_semantics: + raise AssertionError( + f"{declaration.identity!r} lacks explicit typed semantic fields: " + f"{missing_semantics!r}" + ) + policy_expressions = tuple( + expression + for policy in policies + for expression in (policy.using_expression, policy.with_check_expression) + if expression is not None + ) + + def referenced_columns(expressions: Sequence[str]) -> tuple[str, ...]: + return tuple( + column.name + for column in columns + if any( + re.search( + rf"(? tuple[str, ...]: + """Make semantic authority through generated aliases explicit.""" + expanded: list[str] = [] + pending = list(column_names) + while pending: + column_name = pending.pop(0) + if column_name in expanded: + continue + expanded.append(column_name) + pending.extend(generated_dependencies.get(column_name, ())) + return tuple(expanded) + + unique_index_column_sets = tuple( + expand_generated_dependencies(combined) + for index_oid, direct_columns, expression, predicate in unique_index_rows + if ( + combined := tuple( + dict.fromkeys( + ( + *direct_columns, + *referenced_columns((expression,) if expression else ()), + *referenced_columns((predicate,) if predicate else ()), + *catalog_unique_dependencies.get(index_oid, ()), + ) + ) + ) + ) + ) + partition_key_columns = expand_generated_dependencies( + tuple( + dict.fromkeys( + ( + *direct_partition_key_columns, + *referenced_columns(partition_expressions), + *catalog_partition_dependencies, + ) + ) + ) + ) + deduplication_columns = classification.deduplication_key_columns + authorization_columns = classification.authorization_dependency_columns + write_eligibility_columns = classification.write_eligibility_dependency_columns + assert deduplication_columns is not None + assert authorization_columns is not None + assert write_eligibility_columns is not None + return ModelApplicationDatabaseRelationState( + declaration=declaration, + columns=columns, + primary_key_columns=expand_generated_dependencies(primary_key_columns), + unique_index_column_sets=unique_index_column_sets, + foreign_key_column_sets=tuple( + expand_generated_dependencies(column_set) + for column_set in foreign_key_column_sets + ), + partition_key_columns=partition_key_columns, + deduplication_key_columns=expand_generated_dependencies(deduplication_columns), + authorization_dependency_columns=expand_generated_dependencies( + tuple( + dict.fromkeys( + ( + *authorization_columns, + *referenced_columns( + (*policy_expressions, *dependent_view_expressions) + ), + *catalog_authorization_dependencies, + ) + ) + ) + ), + write_eligibility_dependency_columns=expand_generated_dependencies( + tuple( + dict.fromkeys( + ( + *write_eligibility_columns, + *referenced_columns(write_eligibility_expressions), + *catalog_write_dependencies, + ) + ) + ) + ), + rls_enabled=rls_enabled, + rls_forced=rls_forced, + policies=policies, + tenant_identity_column=classification.tenant_identity_column, + identity_root_contract=classification.identity_root_contract, + identity_root_control_state=_identity_root_control_state( + connection, + classification, + ), + canonical_policy_name=classification.canonical_policy_name, + source_tenant_provenance_contract=( + classification.source_tenant_provenance_contract + ), + ) + + +def _surface_count(query: pg_sql.Composable, tenant_id: UUID | None) -> int: + connection = _connect(_pool_dsn("app_dashboard")) + try: + connection.autocommit = False + with connection.cursor() as cursor: + if tenant_id is not None: + cursor.execute("SET LOCAL app.tenant_id = %s", (str(tenant_id),)) + cursor.execute(query) + count = int(cursor.fetchone()[0]) + connection.rollback() + return count + finally: + connection.close() + + +def _malformed_context_denied(query: pg_sql.Composable) -> bool: + connection = _connect(_pool_dsn("app_dashboard")) + try: + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.execute("SET LOCAL app.tenant_id = 'not-a-uuid'") + cursor.execute(query) + except psycopg2.Error as exc: + connection.rollback() + return str(exc.pgcode) == "22P02" + connection.rollback() + return False + finally: + connection.close() + + +def _behavioral_evidence( + query: pg_sql.Composable, +) -> ModelApplicationDatabaseTenantIsolationEvidence: + observed = { + tenant_id: _surface_count(query, tenant_id) + for tenant_id in _EXPECTED_TENANT_ROWS + } + return ModelApplicationDatabaseTenantIsolationEvidence( + expected_rows_by_tenant=_EXPECTED_TENANT_ROWS, + observed_rows_by_tenant=observed, + unset_context_rows=_surface_count(query, None), + malformed_context_denied=_malformed_context_denied(query), + ) + + +def _view_state( + connection: psycopg2.extensions.connection, + declaration: ModelApplicationRelationDeclaration, + evidence: ModelApplicationDatabaseTenantIsolationEvidence | None, +) -> ModelApplicationDatabaseRelationState: + _relation_evidence(declaration) + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT COALESCE(relation.reloptions, ARRAY[]::text[]) + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = %s AND relation.relname = %s + """, + (declaration.schema, declaration.name), + ) + options = set(cursor.fetchone()[0]) + return ModelApplicationDatabaseRelationState( + declaration=declaration, + columns=_column_states(connection, declaration.schema, declaration.name), + security_invoker="security_invoker=true" in options, + view_tenant_isolation_evidence=evidence, + ) + + +def _function_state( + connection: psycopg2.extensions.connection, + declaration: ModelApplicationRelationDeclaration, + evidence: ModelApplicationDatabaseTenantIsolationEvidence | None, +) -> ModelApplicationDatabaseRelationState: + authority = _database_object_evidence(declaration) + signature = declaration.function_signature + if signature is None: + raise AssertionError( + f"{declaration.identity!r} lacks an exact routine signature" + ) + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT owner.rolname, routine.prosecdef, + COALESCE(routine.proconfig, ARRAY[]::text[]), + EXISTS ( + SELECT 1 + FROM aclexplode( + COALESCE(routine.proacl, acldefault('f', routine.proowner)) + ) acl + WHERE acl.grantee = 0 AND acl.privilege_type = 'EXECUTE' + ) AS public_execute, + language.lanname, + routine.prosrc, + routine.prosqlbody::text, + routine.proleakproof, + routine.provolatile, + routine.proparallel, + routine.prokind, + routine.proisstrict, + routine.proretset, + pg_get_function_result(routine.oid) + FROM pg_proc routine + JOIN pg_namespace namespace ON namespace.oid = routine.pronamespace + JOIN pg_roles owner ON owner.oid = routine.proowner + JOIN pg_language language ON language.oid = routine.prolang + WHERE namespace.nspname = %s + AND routine.proname = %s + AND '(' || pg_get_function_identity_arguments(routine.oid) || ')' = %s + """, + (declaration.schema, declaration.name, signature), + ) + rows = cursor.fetchall() + if len(rows) != 1: + raise AssertionError( + f"{declaration.identity!r} requires exactly one live routine; " + f"observed {len(rows)}" + ) + ( + owner, + security_definer, + config, + public_execute, + language, + source_body, + parsed_sql_body, + leakproof, + volatility, + parallel, + kind, + strict, + returns_set, + result_type, + ) = rows[0] + search_path: tuple[str, ...] = () + for setting in config: + if setting.startswith("search_path="): + search_path = tuple( + part.strip() for part in setting.removeprefix("search_path=").split(",") + ) + definition_sha256 = application_database_function_definition_sha256( + schema=declaration.schema, + name=declaration.name, + signature=signature, + language=language, + source_body=source_body, + parsed_sql_body=parsed_sql_body, + security_definer=security_definer, + leakproof=leakproof, + volatility=volatility, + parallel=parallel, + config=config, + kind=kind, + strict=strict, + returns_set=returns_set, + result_type=result_type, + ) + return ModelApplicationDatabaseRelationState( + declaration=declaration, + function_state=ModelApplicationDatabaseFunctionState( + owner=owner, + security_definer=security_definer, + search_path=search_path, + public_execute=public_execute, + audit_id=authority.audit_id, + definition_sha256=definition_sha256, + audited_definition_sha256=authority.definition_sha256, + tenant_isolation_evidence=evidence, + ), + ) + + +def _catalog_identities( + connection: psycopg2.extensions.connection, +) -> tuple[ModelApplicationDatabaseCatalogIdentity, ...]: + """Read every class, routine, type, and extension in application schemas.""" + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT schema_name, object_name, object_kind, function_signature + FROM ( + SELECT namespace.nspname AS schema_name, + relation.relname AS object_name, + CASE relation.relkind + WHEN 'r' THEN 'table' + WHEN 'p' THEN 'table' + WHEN 'v' THEN 'view' + WHEN 'm' THEN 'materialized_view' + WHEN 'S' THEN 'sequence' + WHEN 'f' THEN 'foreign_table' + END AS object_kind, + NULL::text AS function_signature + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE relation.relkind IN ('r', 'p', 'v', 'm', 'S', 'f') + AND namespace.nspname !~ '^pg_' + AND namespace.nspname <> 'information_schema' + + UNION ALL + + SELECT namespace.nspname, + routine.proname, + CASE routine.prokind + WHEN 'p' THEN 'procedure' + WHEN 'a' THEN 'aggregate' + WHEN 'w' THEN 'window_function' + ELSE 'function' + END, + '(' || pg_get_function_identity_arguments(routine.oid) || ')' + FROM pg_proc routine + JOIN pg_namespace namespace ON namespace.oid = routine.pronamespace + WHERE routine.prokind IN ('f', 'p', 'a', 'w') + AND namespace.nspname !~ '^pg_' + AND namespace.nspname <> 'information_schema' + + UNION ALL + + SELECT namespace.nspname, + type_row.typname, + CASE type_row.typtype + WHEN 'b' THEN 'base_type' + WHEN 'r' THEN 'range_type' + WHEN 'm' THEN 'multirange_type' + ELSE 'type' + END, + NULL::text + FROM pg_type type_row + JOIN pg_namespace namespace ON namespace.oid = type_row.typnamespace + LEFT JOIN pg_class relation ON relation.oid = type_row.typrelid + WHERE ( + type_row.typtype IN ('d', 'e', 'r', 'm') + OR (type_row.typtype = 'c' AND relation.relkind = 'c') + OR ( + type_row.typtype = 'b' + AND type_row.typelem = 0 + AND type_row.typrelid = 0 + ) + ) + AND namespace.nspname !~ '^pg_' + AND namespace.nspname <> 'information_schema' + + UNION ALL + + SELECT namespace.nspname, + extension.extname, + 'extension', + NULL::text + FROM pg_extension extension + JOIN pg_namespace namespace ON namespace.oid = extension.extnamespace + WHERE namespace.nspname !~ '^pg_' + AND namespace.nspname <> 'information_schema' + ) catalog + ORDER BY schema_name, object_name, object_kind, function_signature + """ + ) + return tuple( + ModelApplicationDatabaseCatalogIdentity( + schema=row[0], + name=row[1], + kind=EnumApplicationInventoryObjectKind(row[2]), + function_signature=row[3], + ) + for row in cursor.fetchall() + ) + + +def _ownership_inventory( + observed: Sequence[ModelApplicationDatabaseCatalogIdentity], +) -> ModelApplicationRelationInventory: + """Project the exhaustive census into the predecessor's ownership validator.""" + live_relations: list[ModelLiveApplicationRelation] = [] + excluded_objects: list[str] = [] + for identity in observed: + relation_kind = _CATALOG_TO_RELATION_KIND.get(identity.kind) + if relation_kind is None: + excluded_objects.append( + f"{identity.schema}.{identity.name}:{identity.kind.value}" + ) + continue + live_relations.append( + ModelLiveApplicationRelation( + name=identity.name, + database_ref="application", + schema=identity.schema, + kind=relation_kind, + domain=_TOPOLOGY.schema_domain("application", identity.schema), + function_signature=identity.function_signature, + ) + ) + return ModelApplicationRelationInventory( + schema_version="1.0", + relations=tuple(live_relations), + completion_status="complete", + source_relation_count=len(observed), + excluded_database_objects=tuple(excluded_objects), + ) + + +def _authoritative_declarations( + observed: Sequence[ModelApplicationDatabaseCatalogIdentity], +) -> tuple[ModelApplicationRelationDeclaration, ...]: + """Resolve exactly one owner per live relation from the checked manifest.""" + report = validate_application_relation_ownership( + topology=_TOPOLOGY, + node_contract_paths=(), + service_manifest_paths=(_OWNERSHIP_MANIFEST,), + inventory=_ownership_inventory(observed), + ) + if not report.is_valid: + messages = tuple( + f"{violation.code.value}: {violation.message}" + for violation in report.violations + ) + raise AssertionError(f"authoritative ownership proof failed: {messages}") + owners = tuple( + declaration + for declaration in report.declarations + if declaration.owner_declaration is not None + ) + owner_identities = [owner.identity for owner in owners] + if len(owner_identities) != len(set(owner_identities)): + raise AssertionError("ownership report did not resolve unique owner identities") + if len(owners) != len(_ownership_inventory(observed).relations): + raise AssertionError( + "ownership report owner count does not match live relation projection" + ) + return tuple( + sorted( + owners, + key=lambda owner: tuple( + "" if item is None else str(item) for item in owner.identity + ), + ) + ) + + +def _tenant_surface_query( + declaration: ModelApplicationRelationDeclaration, +) -> pg_sql.Composable: + if declaration.kind is EnumApplicationRelationKind.VIEW: + return pg_sql.SQL("SELECT count(*) FROM {}.{}").format( + pg_sql.Identifier(declaration.schema), + pg_sql.Identifier(declaration.name), + ) + if declaration.kind is EnumApplicationRelationKind.FUNCTION: + return pg_sql.SQL("SELECT {}.{}()").format( + pg_sql.Identifier(declaration.schema), + pg_sql.Identifier(declaration.name), + ) + raise AssertionError( + f"tenant behavioral surface is unsupported for {declaration.kind.value}" + ) + + +def _relation_state( + connection: psycopg2.extensions.connection, + declaration: ModelApplicationRelationDeclaration, +) -> ModelApplicationDatabaseRelationState: + """Join one manifest-authoritative declaration to live catalog evidence.""" + if declaration.kind is EnumApplicationRelationKind.TABLE: + return _table_state(connection, declaration) + evidence = ( + _behavioral_evidence(_tenant_surface_query(declaration)) + if declaration.domain is EnumDatabaseSchemaDomain.TENANT + and declaration.kind + in {EnumApplicationRelationKind.VIEW, EnumApplicationRelationKind.FUNCTION} + else None + ) + if declaration.kind is EnumApplicationRelationKind.VIEW: + return _view_state(connection, declaration, evidence) + if declaration.kind is EnumApplicationRelationKind.FUNCTION: + return _function_state(connection, declaration, evidence) + if declaration.kind in { + EnumApplicationRelationKind.MATERIALIZED_VIEW, + EnumApplicationRelationKind.FOREIGN_TABLE, + }: + _relation_evidence(declaration) + return ModelApplicationDatabaseRelationState( + declaration=declaration, + columns=_column_states(connection, declaration.schema, declaration.name), + ) + raise AssertionError(f"unsupported application relation kind: {declaration.kind}") + + +def _pool_identities() -> tuple[ModelApplicationDatabasePoolIdentity, ...]: + identities: list[ModelApplicationDatabasePoolIdentity] = [] + for binding_ref in _DATABASE.bindings: + connection = _connect(_pool_dsn(binding_ref)) + try: + with connection.cursor() as cursor: + cursor.execute("SELECT current_database(), current_user") + current_database, current_user = cursor.fetchone() + finally: + connection.close() + identities.append( + ModelApplicationDatabasePoolIdentity( + pool=binding_ref, + current_database=current_database, + current_user=current_user, + ) + ) + return tuple(identities) + + +def _assert_red(control: str, violations: Sequence[str], expected: str) -> None: + text = "\n".join(violations) + if expected not in text: + raise AssertionError( + f"seeded control {control!r} did not discriminate {expected!r}: {text}" + ) + print(f"domain_control={control} status=PASS") + + +def _run_relation_controls( + identity_root: ModelApplicationDatabaseRelationState, + tenant: ModelApplicationDatabaseRelationState, + internal: ModelApplicationDatabaseRelationState, + view: ModelApplicationDatabaseRelationState, + function: ModelApplicationDatabaseRelationState, +) -> int: + tenant_column = next( + column for column in tenant.columns if column.name == "tenant_id" + ) + canonical = tenant.policies[0] + observed_function = function.function_state + if observed_function is None: + raise AssertionError("green function state is missing") + root_control = identity_root.identity_root_control_state + if root_control is None: + raise AssertionError("green identity-root control state is missing") + controls: tuple[tuple[str, ModelApplicationDatabaseRelationState, str], ...] = ( + ( + "identity-root-runtime-login", + identity_root.model_copy( + update={ + "identity_root_control_state": root_control.model_copy( + update={"role_can_login": True} + ) + } + ), + "NOLOGIN", + ), + ( + "identity-root-unproven-enumeration", + identity_root.model_copy( + update={ + "identity_root_control_state": root_control.model_copy( + update={ + "observed_operations": ( + EnumApplicationDatabaseIdentityRootOperation.TENANT_CREATION, + ), + "behavioral_proof_ids": ( + "postgres16:identity-root-control-create-and-runtime-deny", + ), + } + ) + } + ), + "differ from the declared operation set", + ), + ( + "tenant-text-key", + tenant.model_copy( + update={ + "columns": tuple( + column.model_copy(update={"data_type": "text"}) + if column.name == "tenant_id" + else column + for column in tenant.columns + ) + } + ), + "UUID", + ), + ( + "tenant-nullable", + tenant.model_copy( + update={ + "columns": tuple( + tenant_column.model_copy(update={"nullable": True}) + if column.name == "tenant_id" + else column + for column in tenant.columns + ) + } + ), + "NOT NULL", + ), + ( + "tenant-default", + tenant.model_copy( + update={ + "columns": tuple( + tenant_column.model_copy(update={"default_expression": "0"}) + if column.name == "tenant_id" + else column + for column in tenant.columns + ) + } + ), + "default", + ), + ( + "missing-enable-rls", + tenant.model_copy(update={"rls_enabled": False}), + "ENABLE ROW LEVEL SECURITY", + ), + ( + "missing-force-rls", + tenant.model_copy(update={"rls_forced": False}), + "FORCE ROW LEVEL SECURITY", + ), + ( + "using-drift", + tenant.model_copy( + update={ + "policies": ( + canonical.model_copy(update={"using_expression": "true"}), + ) + } + ), + "USING", + ), + ( + "with-check-drift", + tenant.model_copy( + update={ + "policies": ( + canonical.model_copy(update={"with_check_expression": "true"}), + ) + } + ), + "WITH CHECK", + ), + ( + "uncontracted-identity-root", + tenant.model_copy(update={"tenant_identity_column": "event_id"}), + "identity-root contract", + ), + ( + "owner-security-view", + view.model_copy(update={"security_invoker": False}), + "security_invoker", + ), + ( + "unproven-security-view", + view.model_copy(update={"view_tenant_isolation_evidence": None}), + "behavioral evidence", + ), + ( + "internal-tenant-id", + internal.model_copy( + update={ + "columns": ( + *internal.columns, + ModelApplicationDatabaseColumnState( + name="tenant_id", + data_type="uuid", + nullable=False, + ), + ) + } + ), + "tenant_id", + ), + ( + "uncontracted-source-tenant", + internal.model_copy(update={"source_tenant_provenance_contract": None}), + "provenance contract", + ), + ( + "source-tenant-uniqueness-authority", + internal.model_copy( + update={ + "unique_index_column_sets": ( + *internal.unique_index_column_sets, + ("source_tenant_id",), + ) + } + ), + "drive uniqueness", + ), + ( + "source-tenant-foreign-key-authority", + internal.model_copy( + update={"foreign_key_column_sets": (("source_tenant_id",),)} + ), + "drive foreign key", + ), + ( + "source-tenant-partition-authority", + internal.model_copy( + update={"partition_key_columns": ("source_tenant_id",)} + ), + "drive partition", + ), + ( + "source-tenant-deduplication-authority", + internal.model_copy( + update={ + "deduplication_key_columns": ( + *internal.deduplication_key_columns, + "source_tenant_id", + ) + } + ), + "drive deduplication", + ), + ( + "source-tenant-authorization-authority", + internal.model_copy( + update={"authorization_dependency_columns": ("source_tenant_id",)} + ), + "drive authorization", + ), + ( + "source-tenant-write-eligibility-authority", + internal.model_copy( + update={"write_eligibility_dependency_columns": ("source_tenant_id",)} + ), + "drive write eligibility", + ), + ( + "unsafe-security-definer", + function.model_copy( + update={ + "function_state": observed_function.model_copy( + update={ + "owner": "app_dashboard", + "search_path": ("public", "pg_temp"), + "public_execute": True, + "audit_id": None, + } + ) + } + ), + "PUBLIC EXECUTE", + ), + ( + "unproven-security-definer", + function.model_copy( + update={ + "function_state": observed_function.model_copy( + update={"tenant_isolation_evidence": None} + ) + } + ), + "behavioral evidence", + ), + ( + "security-definer-definition-audit-drift", + function.model_copy( + update={ + "function_state": observed_function.model_copy( + update={"definition_sha256": "0" * 64} + ) + } + ), + "does not match the audited definition hash", + ), + ) + for control, state, expected in controls: + _assert_red( + control, + validate_application_database_relation_states((state,), _TOPOLOGY), + expected, + ) + return len(controls) + + +def _run_catalog_controls( + admin: psycopg2.extensions.connection, + states: tuple[ModelApplicationDatabaseRelationState, ...], + observed: tuple[ModelApplicationDatabaseCatalogIdentity, ...], + authority: tuple[ModelApplicationDatabaseCatalogIdentity, ...], +) -> int: + _assert_red( + "incomplete-catalog-census", + validate_application_database_catalog_census( + states, + observed[1:], + _TOPOLOGY, + authoritative_identities=authority, + ), + "missing", + ) + _assert_red( + "empty-authoritative-relation-set", + validate_application_database_catalog_census( + (), + observed, + _TOPOLOGY, + authoritative_identities=(), + ), + "cannot be empty", + ) + admin.commit() + with admin.cursor() as cursor: + cursor.execute("CREATE UNLOGGED TABLE public.rogue_shadow (id uuid)") + leaked_catalog = _catalog_identities(admin) + _assert_red( + "public-catalog-leak", + validate_application_database_catalog_census( + states, + leaked_catalog, + _TOPOLOGY, + authoritative_identities=authority, + ), + "undeclared", + ) + admin.rollback() + return 3 + + +def _run_live_source_tenant_controls( + admin: psycopg2.extensions.connection, + declaration: ModelApplicationRelationDeclaration, +) -> int: + """Prove catalog extraction sees hidden source-tenant authority surfaces.""" + admin.commit() + controls: tuple[tuple[str, str, str | tuple[str, ...]], ...] = ( + ( + "source-tenant-check-constraint", + """ + ALTER TABLE omninode_internal.runtime_state + ADD CONSTRAINT runtime_state_source_write_guard + CHECK (source_tenant_id IS NULL) + """, + "drive write eligibility", + ), + ( + "source-tenant-partial-unique-predicate", + """ + CREATE UNIQUE INDEX runtime_state_partial_source_guard + ON omninode_internal.runtime_state (payload) + WHERE source_tenant_id IS NOT NULL + """, + "drive uniqueness", + ), + ( + "source-tenant-generated-unique-alias", + """ + ALTER TABLE omninode_internal.runtime_state + ADD COLUMN source_tenant_copy uuid + GENERATED ALWAYS AS (source_tenant_id) STORED; + CREATE UNIQUE INDEX runtime_state_generated_source_guard + ON omninode_internal.runtime_state (source_tenant_copy) + """, + "drive uniqueness", + ), + ( + "source-tenant-transitive-whole-row-helper", + """ + CREATE FUNCTION omninode_internal.source_tenant_key( + omninode_internal.runtime_state + ) RETURNS uuid LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT $1.source_tenant_id + $$; + CREATE FUNCTION omninode_internal.nested_source_tenant_key( + omninode_internal.runtime_state + ) RETURNS uuid LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT omninode_internal.source_tenant_key + /* comments are PostgreSQL whitespace */ ($1) + $$; + CREATE UNIQUE INDEX runtime_state_transitive_source_guard + ON omninode_internal.runtime_state ( + omninode_internal.nested_source_tenant_key(runtime_state.*) + ); + CREATE FUNCTION omninode_internal.nested_source_tenant_guard() + RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + PERFORM omninode_internal.nested_source_tenant_key(NEW); + RETURN NEW; + END + $$; + CREATE TRIGGER runtime_state_transitive_source_guard + BEFORE INSERT OR UPDATE ON omninode_internal.runtime_state + FOR EACH ROW EXECUTE FUNCTION + omninode_internal.nested_source_tenant_guard(); + CREATE VIEW omninode_internal.runtime_state_transitive_authority AS + SELECT state_id, + omninode_internal.nested_source_tenant_key(runtime_state.*) + AS source_key + FROM omninode_internal.runtime_state + """, + ("drive uniqueness", "drive write eligibility", "drive authorization"), + ), + ( + "source-tenant-named-whole-row-helper", + """ + CREATE FUNCTION omninode_internal.named_runtime_state_digest( + row_value omninode_internal.runtime_state + ) RETURNS text LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT pg_catalog.md5(row_value::text) + $$; + CREATE UNIQUE INDEX runtime_state_named_whole_row_guard + ON omninode_internal.runtime_state ( + omninode_internal.named_runtime_state_digest(runtime_state.*) + ); + CREATE FUNCTION omninode_internal.named_whole_row_guard() + RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + PERFORM omninode_internal.named_runtime_state_digest(NEW); + RETURN NEW; + END + $$; + CREATE TRIGGER runtime_state_named_whole_row_guard + BEFORE INSERT OR UPDATE ON omninode_internal.runtime_state + FOR EACH ROW EXECUTE FUNCTION + omninode_internal.named_whole_row_guard(); + CREATE VIEW omninode_internal.runtime_state_named_whole_row_authority AS + SELECT state_id, + omninode_internal.named_runtime_state_digest(runtime_state.*) + AS row_digest + FROM omninode_internal.runtime_state + """, + ("drive uniqueness", "drive write eligibility", "drive authorization"), + ), + ( + "source-tenant-trigger-body", + """ + CREATE FUNCTION omninode_internal.reject_source_tenant() + RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.source_tenant_id IS NOT NULL THEN + RAISE EXCEPTION 'source tenant cannot select writes'; + END IF; + RETURN NEW; + END + $$; + CREATE TRIGGER runtime_state_source_guard + BEFORE INSERT OR UPDATE ON omninode_internal.runtime_state + FOR EACH ROW EXECUTE FUNCTION omninode_internal.reject_source_tenant() + """, + "drive write eligibility", + ), + ( + "source-tenant-dependent-view", + """ + CREATE VIEW omninode_internal.runtime_state_authorized AS + SELECT state_id, source_tenant_id, payload + FROM omninode_internal.runtime_state + WHERE source_tenant_id = current_setting('app.tenant_id', true)::uuid + """, + "drive authorization", + ), + ) + for control, statement, expected in controls: + with admin.cursor() as cursor: + cursor.execute(statement) + state = _table_state(admin, declaration) + violations = validate_application_database_relation_states((state,), _TOPOLOGY) + if isinstance(expected, tuple): + missing = tuple( + fragment + for fragment in expected + if not any(fragment in violation for violation in violations) + ) + if missing: + raise AssertionError( + f"seeded control {control!r} did not discriminate {missing!r}: " + + "; ".join(violations) + ) + print(f"domain_control={control} status=PASS") + else: + _assert_red(control, violations, expected) + admin.rollback() + return len(controls) + + +def _run_live_function_definition_control( + admin: psycopg2.extensions.connection, + declaration: ModelApplicationRelationDeclaration, + evidence: ModelApplicationDatabaseTenantIsolationEvidence | None, +) -> int: + """Prove non-body routine metadata drift changes the audited fingerprint.""" + admin.commit() + with admin.cursor() as cursor: + cursor.execute("ALTER FUNCTION tenant.safe_report() STABLE") + state = _function_state(admin, declaration, evidence) + _assert_red( + "security-definer-volatility-drift", + validate_application_database_relation_states((state,), _TOPOLOGY), + "does not match the audited definition hash", + ) + admin.rollback() + return 1 + + +def _run_live_policy_role_control( + admin: psycopg2.extensions.connection, + tenant: ModelApplicationDatabaseRelationState, +) -> int: + """Prove exact pg_policy.polroles drift is observed and rejected.""" + canonical_policy = tenant.canonical_policy_name + if canonical_policy is None: + raise AssertionError("green tenant state is missing its canonical policy name") + declaration = tenant.declaration + admin.commit() + with admin.cursor() as cursor: + cursor.execute( + pg_sql.SQL("ALTER POLICY {} ON {}.{} TO {}").format( + pg_sql.Identifier(canonical_policy), + pg_sql.Identifier(declaration.schema), + pg_sql.Identifier(declaration.name), + pg_sql.Identifier("omninode_runtime"), + ) + ) + state = _table_state(admin, declaration) + _assert_red( + "canonical-policy-unrelated-role", + validate_application_database_relation_states((state,), _TOPOLOGY), + "role scope", + ) + admin.rollback() + return 1 + + +def _run_live_identity_root_role_controls( + admin: psycopg2.extensions.connection, + identity_root: ModelApplicationDatabaseRelationState, +) -> int: + """Seed real role reachability and prove both catalog and SET ROLE controls.""" + root_control = identity_root.identity_root_control_state + if root_control is None: + raise AssertionError("green identity-root control state is missing") + runtime_principal = _DATABASE.bindings["onex_api"].principal + bridge_role = "seeded_identity_root_bridge" + + with admin.cursor() as cursor: + cursor.execute("SAVEPOINT identity_root_membership_control") + try: + cursor.execute( + pg_sql.SQL("CREATE ROLE {} NOLOGIN").format( + pg_sql.Identifier(bridge_role) + ) + ) + cursor.execute( + pg_sql.SQL("GRANT {} TO {}").format( + pg_sql.Identifier(root_control.role), + pg_sql.Identifier(bridge_role), + ) + ) + cursor.execute( + pg_sql.SQL("GRANT {} TO {}").format( + pg_sql.Identifier(bridge_role), + pg_sql.Identifier(runtime_principal), + ) + ) + membership_principals = _runtime_identity_root_membership_principals( + admin, + root_control.role, + ) + finally: + cursor.execute("ROLLBACK TO SAVEPOINT identity_root_membership_control") + cursor.execute("RELEASE SAVEPOINT identity_root_membership_control") + _assert_red( + "identity-root-runtime-membership", + validate_application_database_relation_states( + ( + identity_root.model_copy( + update={ + "identity_root_control_state": root_control.model_copy( + update={ + "runtime_membership_principals": (membership_principals) + } + ) + } + ), + ), + _TOPOLOGY, + ), + "membership path", + ) + admin.commit() + + try: + with admin.cursor() as cursor: + cursor.execute( + pg_sql.SQL("GRANT {} TO {}").format( + pg_sql.Identifier(root_control.role), + pg_sql.Identifier(runtime_principal), + ) + ) + admin.commit() + denied_principals = _runtime_identity_root_set_role_denials(root_control.role) + finally: + admin.rollback() + with admin.cursor() as cursor: + cursor.execute( + pg_sql.SQL("REVOKE {} FROM {}").format( + pg_sql.Identifier(root_control.role), + pg_sql.Identifier(runtime_principal), + ) + ) + admin.commit() + restored_memberships = _runtime_identity_root_membership_principals( + admin, + root_control.role, + ) + restored_denials = _runtime_identity_root_set_role_denials(root_control.role) + expected_runtime_principals = tuple( + sorted(binding.principal for binding in _DATABASE.bindings.values()) + ) + if restored_memberships or restored_denials != expected_runtime_principals: + raise AssertionError( + "identity-root role-control RED proof did not restore the green " + "runtime authority state" + ) + _assert_red( + "identity-root-runtime-set-role", + validate_application_database_relation_states( + ( + identity_root.model_copy( + update={ + "identity_root_control_state": root_control.model_copy( + update={ + "runtime_set_role_denied_principals": ( + denied_principals + ) + } + ) + } + ), + ), + _TOPOLOGY, + ), + "SET ROLE denial", + ) + return 2 + + +def _run_cross_gate_controls( + identities: tuple[ModelApplicationDatabasePoolIdentity, ...], +) -> int: + sql_controls: Mapping[str, tuple[str, str]] = { + "public-table": ("CREATE TABLE public.events (id uuid);", "public"), + "unqualified-table": ( + "CREATE TABLE events (id uuid);", + "schema-qualified", + ), + "unknown-schema": ( + "CREATE TABLE mystery.events (id uuid);", + "unknown topology schema", + ), + "unqualified-read": ("SELECT * FROM events;", "schema-qualified"), + "unqualified-merge": ( + "MERGE INTO events USING tenant.incoming ON false WHEN NOT MATCHED THEN DO NOTHING;", + "schema-qualified", + ), + "unqualified-grant": ( + "GRANT SELECT ON TABLE events TO app_dashboard;", + "schema-qualified", + ), + "unqualified-foreign-key": ( + "CREATE TABLE tenant.child (id uuid REFERENCES parent(id));", + "schema-qualified", + ), + } + for control, (statement, expected) in sql_controls.items(): + _assert_red( + control, + lint_application_database_sql(statement, _TOPOLOGY), + expected, + ) + + wrong_database = identities[0].model_copy( + update={"current_database": "omninode_cloud"} + ) + _assert_red( + "old-application-database", + validate_application_database_pool_identities( + (wrong_database, *identities[1:]), _TOPOLOGY + ), + "one physical database", + ) + duplicate_user = identities[1].model_copy( + update={"current_user": identities[0].current_user} + ) + _assert_red( + "duplicate-pool-user", + validate_application_database_pool_identities( + (identities[0], duplicate_user, *identities[2:]), _TOPOLOGY + ), + "distinct", + ) + return len(sql_controls) + 2 + + +def main() -> None: + admin = _connect(_ADMIN_DSN) + try: + observed_catalog = _catalog_identities(admin) + catalog_authority = load_application_database_ownership_identities( + (_OWNERSHIP_MANIFEST,) + ) + declarations = _authoritative_declarations(observed_catalog) + relation_states = tuple( + _relation_state(admin, declaration) for declaration in declarations + ) + states_by_name = { + (state.declaration.schema, state.declaration.name): state + for state in relation_states + } + tenant = states_by_name[("tenant", "events")] + identity_root = states_by_name[("tenant", "tenants")] + internal = states_by_name[("omninode_internal", "runtime_state")] + view = states_by_name[("tenant", "events_view")] + function = states_by_name[("tenant", "safe_report")] + + relation_violations = validate_application_database_catalog_census( + relation_states, + observed_catalog, + _TOPOLOGY, + authoritative_identities=catalog_authority, + ) + if relation_violations: + raise AssertionError( + f"green exact relation catalog failed: {relation_violations}" + ) + control_count = _run_catalog_controls( + admin, + relation_states, + observed_catalog, + catalog_authority, + ) + control_count += _run_live_source_tenant_controls( + admin, + internal.declaration, + ) + function_state = function.function_state + if function_state is None: + raise AssertionError("green function state is missing") + control_count += _run_live_function_definition_control( + admin, + function.declaration, + function_state.tenant_isolation_evidence, + ) + control_count += _run_live_policy_role_control(admin, tenant) + control_count += _run_live_identity_root_role_controls( + admin, + identity_root, + ) + finally: + admin.close() + + identities = _pool_identities() + pool_violations = validate_application_database_pool_identities( + identities, _TOPOLOGY + ) + if pool_violations: + raise AssertionError(f"green pool identity proof failed: {pool_violations}") + if lint_application_database_sql( + "CREATE TABLE tenant.seeded_green (id uuid PRIMARY KEY);", _TOPOLOGY + ): + raise AssertionError("qualified green migration SQL was rejected") + + control_count += _run_relation_controls( + identity_root, + tenant, + internal, + view, + function, + ) + control_count += _run_cross_gate_controls(identities) + print( + "application_domain_enforcement_status=PASS " + f"postgres_major=16 relations={len(relation_states)} " + f"catalog_objects={len(observed_catalog)} pools={len(identities)} " + f"red_controls={control_count}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/resolve_modern_bash.sh b/scripts/ci/resolve_modern_bash.sh new file mode 100755 index 0000000000..bba812223e --- /dev/null +++ b/scripts/ci/resolve_modern_bash.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# resolve_modern_bash.sh -- OMN-15617: explicit, PATH-order-independent +# resolution of a bash interpreter new enough for `declare -A`. +# +# Root cause: on stickybeatz-studio (.200, the rule-11a default gate host), +# non-interactive ssh sessions resolve `bash` / `env bash` to the system +# 3.2.57 shell (no associative arrays) even though a modern bash 5.x sits at +# /opt/homebrew/bin/bash -- it simply is not first on PATH for that session +# class. Callers that spawn `bash ` via bare "bash" +# silently run the wrong interpreter and fail with a bash syntax error deep +# inside the script, not a resolvable "wrong bash" diagnostic. +# +# This script performs the resolution ONCE, explicitly, independent of PATH +# order, and is the single source of truth callers (the pre-push canary and +# the runner-monitor.sh pytest harness) both use -- so they can never drift. +# +# MUST remain runnable under bash 3.2 itself (no `declare -A`, no `[[ ]]` +# reliance beyond what 3.2 supports) -- interpreter resolution cannot +# presuppose the thing it is resolving. +# +# Output contract: on success, prints the resolved interpreter's absolute +# path to stdout and exits 0. On failure, prints nothing to stdout, prints a +# pointed ERROR + REMEDIATION message to stderr, and exits 1. Never a silent +# fallback and never a quiet skip -- an unresolvable modern bash is a hard +# failure the caller must surface. +# +# Env overrides: +# OMNIBASE_INFRA_BASH_BIN explicit interpreter path to try FIRST. +# OMNIBASE_INFRA_MIN_BASH_MAJOR minimum BASH_VERSINFO[0] required (default 5). + +set -euo pipefail + +MIN_MAJOR="${OMNIBASE_INFRA_MIN_BASH_MAJOR:-5}" + +# Prints the candidate's BASH_VERSINFO[0] on stdout, or nothing + nonzero exit +# if the candidate is not an executable bash at all. +bash_major_version() { + local bin="$1" + [ -x "$bin" ] || return 1 + "$bin" -c 'printf "%s" "${BASH_VERSINFO[0]:-}"' 2> /dev/null +} + +seen="" +try_candidate() { + local bin="$1" major + case " ${seen} " in + *" ${bin} "*) return 1 ;; + esac + seen="${seen} ${bin}" + major="$(bash_major_version "$bin")" || return 1 + [ -n "$major" ] || return 1 + [ "$major" -ge "$MIN_MAJOR" ] || return 1 + printf '%s\n' "$bin" + return 0 +} + +# Ordered candidate list: explicit override, the two brew prefixes (Apple +# Silicon + Intel), then every "bash" found walking $PATH -- so a modern bash +# installed somewhere non-standard is still picked up without PATH surgery. +# +# Built as an array (not a space-joined string) so an interpreter path or +# PATH entry containing a space is never word-split into non-existent +# fragments -- indexed arrays are bash-3.2-safe (only `declare -A` requires +# bash>=4, which this script must not presuppose). A dropped candidate would +# be a silent-wrong-answer mode in the exact script whose purpose is to +# eliminate silent wrong interpreter resolution. +CANDIDATES=() +if [ -n "${OMNIBASE_INFRA_BASH_BIN:-}" ]; then + CANDIDATES+=("${OMNIBASE_INFRA_BASH_BIN}") +fi +CANDIDATES+=("/opt/homebrew/bin/bash" "/usr/local/bin/bash") + +old_ifs="$IFS" +IFS=':' +for _dir in $PATH; do + [ -n "$_dir" ] || continue + CANDIDATES+=("${_dir}/bash") +done +IFS="$old_ifs" + +for _candidate in "${CANDIDATES[@]}"; do + if try_candidate "$_candidate"; then + exit 0 + fi +done + +{ + printf 'ERROR: no bash interpreter >= %s found (declare -A requires bash>=4; OMN-15617 requires >=%s).\n' "$MIN_MAJOR" "$MIN_MAJOR" + printf 'Checked: OMNIBASE_INFRA_BASH_BIN, /opt/homebrew/bin/bash, /usr/local/bin/bash, and every "bash" on PATH.\n' + printf 'REMEDIATION: install a modern bash (e.g. `brew install bash`) and/or set OMNIBASE_INFRA_BASH_BIN to its absolute path.\n' +} >&2 +exit 1 diff --git a/scripts/ci/runner_broker_dispatch_wedge_rerun.sh b/scripts/ci/runner_broker_dispatch_wedge_rerun.sh new file mode 100755 index 0000000000..3af6a518c1 --- /dev/null +++ b/scripts/ci/runner_broker_dispatch_wedge_rerun.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# runner_broker_dispatch_wedge_rerun.sh — OMN-15776 +# +# Targeted, signature-keyed auto-rerun for the proven GitHub Actions +# broker-dispatch/reconnect race (2026-08-09 ``omn15776-wedge`` ledger +# finding). +# +# MECHANISM (proven, not speculative — see the ticket comment history and +# docs/runbooks/runner-fleet-broker-dispatch-wedge.md): +# GitHub's Actions broker dispatches a job to a self-hosted runner within +# 2-7s of that SAME runner finishing its previous (unrelated) job, exactly +# while the runner's Runner.Listener is mid-reconnect on its broker +# long-poll (every job completion triggers a retry/backoff storm on that +# connection, observed 5-12s exponential backoff). The new dispatch lands +# in that reconnect gap and is never delivered to the runner's active +# message loop — no Runner.Worker process is ever spawned locally (so +# local _diag logs show ZERO "Running job: " entry — this is NOT a +# crashed step 1) — while GitHub's server side records the assignment, +# sets started_at, and independently times the orphaned assignment out at +# a FIXED ~10m0-1s, unrelated to any declared workflow timeout-minutes or +# any local watchdog threshold. +# +# CORRECTION (2026-08-10, omn15776-wedge-verify): the paragraph originally +# here asserted "the Listener in this failure class is actively chattering +# through broker retries, never silent" and used that to conclude no local +# fix applies. That claim is RETRACTED — it was never verified against the +# local listener log and is false. Live evidence (same incident's captured +# job + the runner's own _diag/Runner_*.log for that window): after the +# reconnect-backoff line is written, the listener goes SILENT — no further +# _diag lines until the OMN-14564 hung-listener watchdog +# (docker/runners/entrypoint.sh) independently fires on that same silence at +# LISTENER_HEARTBEAT_MAX_AGE_SECONDS (3600s) + up to +# (LISTENER_HEARTBEAT_MISSES-1)*LISTENER_SUPERVISE_INTERVAL, i.e. ~3600s + +# ~2-3min. That watchdog firing is CAUSALLY DOWNSTREAM of the same silence +# this script's rerun logic targets — it is real, and it does eventually +# recycle the wedged listener — but at ~53 minutes after GitHub's fixed +# ~600s server-side orphan timeout has already burned the job as a false +# red. Each occurrence of this signature costs ~62 minutes of runner +# capacity (600s orphaned-job wait the CI consumer observes, plus the ~53min +# until the watchdog recycles the listener) before the runner is usable +# again, without this script's targeted rerun closing the gap for the CI +# consumer immediately. +# +# WHY THIS SCRIPT STILL EXISTS (revised 2026-08-10, omn15776-wedge-verify — +# rescoped, see below): the OMN-14564 watchdog is real and does eventually +# recover the runner, but it is not a fix for the ~600s orphan-timeout +# false-red a CI consumer sees, and it is far too slow (~53min later) to +# prevent it. Nothing in the GitHub Actions client/broker protocol path is +# controllable from this repo, so the false-red itself cannot be prevented +# locally ON THE SILENCE-THRESHOLD SURFACE — i.e. a fix that waits for and +# measures listener silence duration (OMN-14564's surface) cannot, by +# construction, resolve faster than a silence-duration threshold allows. +# That scoping is deliberate: it does NOT claim no local fix of any kind can +# ever beat the ~600s orphan timeout. A separate, non-threshold local +# detector — matching the "Acknowledging runner request " / +# no-"Running job" structural signature directly, instead of inferring the +# wedge from elapsed silence — could recycle the listener far sooner; that +# surface is tracked as its own follow-up, OMN-15806, not addressed here. +# The remediation for the wedged LISTENER via the silence-threshold surface +# (getting the runner back into service) is the OMN-14564 watchdog in +# entrypoint.sh, whose detection threshold is addressed separately (see +# OMN-14564 threshold PR, same ticket family). The remediation for the CI +# consumer's ~600s orphan false-red (fast recovery of the specific orphaned +# job, independent of either local watchdog) lives here via signature-keyed +# rerun. +# +# STRUCTURAL SIGNATURE (matched via the Jobs API, never log-text grepping — +# there is no log content to grep, because no Worker ever ran): +# 1. job.runner_name is set (self-hosted; GitHub-hosted jobs are excluded — +# they are never serviced by this fleet and must never be touched here) +# 2. job.conclusion in {failure, cancelled} +# 3. job.steps is an empty array (no Worker spawned — distinguishes this +# from a genuine content failure, which always has at least one step) +# 4. duration = completed_at - started_at is within +# [WEDGE_MIN_DURATION_SECONDS, WEDGE_MAX_DURATION_SECONDS] — a tight +# band around the observed fixed ~10m0-1s GitHub-side orphan timeout. +# This is deliberately narrow: a job that happens to run steps=[] for +# some other reason and complete at, say, 9 or 11 minutes is NOT this +# signature and must not be silently relaunched. +# +# A job matching all four is a REPLAY of a dispatch that never reached local +# execution — reissuing it is not "retry on red" (which would launder a +# genuine content failure); it targets exactly the drop this mechanism +# describes, and only that. +# +# Usage: +# ./scripts/ci/runner_broker_dispatch_wedge_rerun.sh +# ./scripts/ci/runner_broker_dispatch_wedge_rerun.sh --dry-run +# ./scripts/ci/runner_broker_dispatch_wedge_rerun.sh --repos omnibase_infra,omnibase_core +# ./scripts/ci/runner_broker_dispatch_wedge_rerun.sh --state-dir /path/to/state +# +# Env (mirrors scripts/ci/runner_fleet_canary.sh's testable curl+env-override +# convention, NOT the gh CLI, so this script can be driven against a local +# HTTP stub in tests without touching the real GitHub API): +# GITHUB_API_URL API base (default https://api.github.com) +# GITHUB_ORG org name (default OmniNode-ai) +# REPOS_CSV comma-separated repo list to scan +# RUNNER_GITHUB_TOKEN bearer token (falls back to CROSS_REPO_PAT) +# LOOKBACK_HOURS how far back to scan runs (default 6) +# MAX_RUNS_PER_REPO runs listed per repo per scan (default 30) +# WEDGE_MIN_DURATION_SECONDS / WEDGE_MAX_DURATION_SECONDS +# signature duration band (default 595 / 605 — a +# tight collar around the proven fixed ~600-601s) +# MAX_RERUNS circuit breaker on reruns issued per invocation +# +# Exit codes: 0 = scan completed (reruns issued or no candidates), 1 = error + +set -euo pipefail + +GITHUB_API_URL="${GITHUB_API_URL:-https://api.github.com}" +GITHUB_ORG="${GITHUB_ORG:-OmniNode-ai}" +REPOS_CSV="${REPOS_CSV:-omnibase_infra,omnibase_core,omniclaude,omnimarket,onex_change_control}" +LOOKBACK_HOURS="${LOOKBACK_HOURS:-6}" +MAX_RUNS_PER_REPO="${MAX_RUNS_PER_REPO:-30}" +WEDGE_MIN_DURATION_SECONDS="${WEDGE_MIN_DURATION_SECONDS:-595}" +WEDGE_MAX_DURATION_SECONDS="${WEDGE_MAX_DURATION_SECONDS:-605}" +MAX_RERUNS="${MAX_RERUNS:-10}" +STATE_DIR="${ONEX_STATE_DIR:-.onex_state}/runner-broker-dispatch-wedge-rerun" +DRY_RUN=false + +TOKEN="${RUNNER_GITHUB_TOKEN:-${CROSS_REPO_PAT:-}}" + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=true; shift ;; + --repos) + REPOS_CSV="${2:?--repos requires a value}"; shift 2 ;; + --state-dir) + STATE_DIR="${2:?--state-dir requires a path}"; shift 2 ;; + --help|-h) + grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) + echo "Unknown argument: $1" >&2; exit 1 ;; + esac +done + +if [[ -z "${TOKEN}" ]]; then + echo "ERROR: no API token — set RUNNER_GITHUB_TOKEN or CROSS_REPO_PAT" >&2 + exit 1 +fi + +mkdir -p "${STATE_DIR}" +SESSION_RERUN_LOG="${STATE_DIR}/session-reruns.json" +LOG_FILE="${STATE_DIR}/scan.log" + +log() { + local ts + ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "[${ts}] [broker-dispatch-wedge-rerun] $*" | tee -a "${LOG_FILE}" >&2 +} + +api_get() { + local path="$1" + curl -fsS \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "${GITHUB_API_URL}${path}" 2>>"${LOG_FILE}" || echo "" +} + +api_post_rerun() { + local repo="$1" + local job_id="$2" + curl -fsS -X POST \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "${GITHUB_API_URL}/repos/${GITHUB_ORG}/${repo}/actions/jobs/${job_id}/rerun" \ + >>"${LOG_FILE}" 2>&1 +} + +already_rerun() { + local job_id="$1" + [[ -f "${SESSION_RERUN_LOG}" ]] || return 1 + jq -e --arg id "${job_id}" '.rerun_ids | index($id) != null' \ + "${SESSION_RERUN_LOG}" >/dev/null 2>&1 +} + +record_rerun() { + local job_id="$1" repo="$2" + local ts + ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + if [[ -f "${SESSION_RERUN_LOG}" ]]; then + tmp="$(mktemp)" + jq --arg id "${job_id}" --arg repo "${repo}" --arg ts "${ts}" \ + '.rerun_ids += [$id] | .entries += [{job_id: $id, repo: $repo, rerun_at: $ts}]' \ + "${SESSION_RERUN_LOG}" > "${tmp}" && mv "${tmp}" "${SESSION_RERUN_LOG}" + else + jq -n --arg id "${job_id}" --arg repo "${repo}" --arg ts "${ts}" \ + '{rerun_ids: [$id], entries: [{job_id: $id, repo: $repo, rerun_at: $ts}]}' \ + > "${SESSION_RERUN_LOG}" + fi +} + +cutoff_iso="$(python3 -c " +from datetime import datetime, timezone, timedelta +print((datetime.now(timezone.utc) - timedelta(hours=${LOOKBACK_HOURS})).strftime('%Y-%m-%dT%H:%M:%SZ')) +")" + +log "Starting broker-dispatch-wedge scan (OMN-15776)" +log " repos: ${REPOS_CSV} | lookback: ${LOOKBACK_HOURS}h | dry_run: ${DRY_RUN}" +log " signature band: [${WEDGE_MIN_DURATION_SECONDS}s, ${WEDGE_MAX_DURATION_SECONDS}s], steps=[], runner_name set, conclusion in {failure,cancelled}" + +rerun_count=0 +candidate_count=0 +scanned_count=0 + +for repo in $(echo "${REPOS_CSV}" | tr ',' '\n'); do + repo="$(echo "${repo}" | xargs)" + [[ -z "${repo}" ]] && continue + + log "Scanning ${GITHUB_ORG}/${repo} (last ${LOOKBACK_HOURS}h)..." + + runs_json="$(api_get "/repos/${GITHUB_ORG}/${repo}/actions/runs?status=completed&per_page=${MAX_RUNS_PER_REPO}")" + if [[ -z "${runs_json}" ]]; then + log " WARN: could not list runs for ${repo} — skipping" + continue + fi + + run_ids="$(echo "${runs_json}" | jq -r --arg cutoff "${cutoff_iso}" ' + .workflow_runs[]? + | select(.created_at >= $cutoff) + | .id + ' 2>/dev/null || true)" + + while IFS= read -r run_id; do + [[ -z "${run_id}" ]] && continue + + jobs_json="$(api_get "/repos/${GITHUB_ORG}/${repo}/actions/runs/${run_id}/jobs")" + [[ -z "${jobs_json}" ]] && continue + + while IFS= read -r job_entry; do + [[ -z "${job_entry}" ]] && continue + scanned_count=$((scanned_count + 1)) + + job_id="$(echo "${job_entry}" | jq -r '.id')" + job_name="$(echo "${job_entry}" | jq -r '.name')" + runner_name="$(echo "${job_entry}" | jq -r '.runner_name // empty')" + conclusion="$(echo "${job_entry}" | jq -r '.conclusion // empty')" + step_count="$(echo "${job_entry}" | jq -r '(.steps // []) | length')" + started_at="$(echo "${job_entry}" | jq -r '.started_at // empty')" + completed_at="$(echo "${job_entry}" | jq -r '.completed_at // empty')" + + # 1. self-hosted only. + [[ -z "${runner_name}" ]] && continue + # 2. conclusion class. + if [[ "${conclusion}" != "failure" && "${conclusion}" != "cancelled" ]]; then + continue + fi + # 3. zero steps — no Worker ever spawned. + [[ "${step_count}" != "0" ]] && continue + # started_at/completed_at required to compute duration. + [[ -z "${started_at}" || -z "${completed_at}" ]] && continue + + duration="$(python3 -c " +from datetime import datetime +fmt = '%Y-%m-%dT%H:%M:%SZ' +a = datetime.strptime('${started_at}', fmt) +b = datetime.strptime('${completed_at}', fmt) +print(int((b - a).total_seconds())) +" 2>/dev/null || echo "")" + [[ -z "${duration}" ]] && continue + + # 4. tight band around the proven fixed ~10m0-1s server-side timeout. + if (( duration < WEDGE_MIN_DURATION_SECONDS || duration > WEDGE_MAX_DURATION_SECONDS )); then + continue + fi + + log " CANDIDATE job ${job_id} (${job_name}, runner=${runner_name}, conclusion=${conclusion}, duration=${duration}s, steps=0) — broker-dispatch-wedge signature (OMN-15776)" + candidate_count=$((candidate_count + 1)) + echo "CANDIDATE job_id=${job_id} repo=${repo} run_id=${run_id} name=${job_name}" + + if already_rerun "${job_id}"; then + log " SKIP job ${job_id}: already rerun this session" + continue + fi + + if (( rerun_count >= MAX_RERUNS )); then + log " SKIP job ${job_id}: max reruns (${MAX_RERUNS}) reached for this invocation" + continue + fi + + if [[ "${DRY_RUN}" == "true" ]]; then + log " [DRY-RUN] Would rerun job ${job_id} (${repo})" + else + log " Issuing targeted rerun for job ${job_id} (${repo})" + if api_post_rerun "${repo}" "${job_id}"; then + log " RERUN issued: job ${job_id}" + record_rerun "${job_id}" "${repo}" + rerun_count=$((rerun_count + 1)) + else + log " WARN: rerun request failed for job ${job_id}" + fi + fi + done < <(echo "${jobs_json}" | jq -c '.jobs[]?') + done <<< "${run_ids}" +done + +log "Scan complete: scanned=${scanned_count} candidates=${candidate_count} reruns_issued=${rerun_count} dry_run=${DRY_RUN}" diff --git a/scripts/ci/runner_fleet_canary.sh b/scripts/ci/runner_fleet_canary.sh index 945e47babf..32f154ebfd 100755 --- a/scripts/ci/runner_fleet_canary.sh +++ b/scripts/ci/runner_fleet_canary.sh @@ -1,12 +1,11 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT -# runner_fleet_canary.sh — scheduled fleet-status canary (OMN-13915) +# runner_fleet_canary.sh — scheduled fleet-status canary (OMN-13915, OMN-16030) # -# Compares the GitHub org self-hosted runner registry (the AUTHORITATIVE view -# of whether runners are serving jobs) against the expected fleet size declared -# in config/runner_fleet.yaml, and FAILS LOUDLY when the offline count crosses -# a threshold — BEFORE queued CI runs pile up. +# Compares the GitHub org self-hosted runner registry against the expected fleet +# size declared in config/runner_fleet.yaml, and FAILS LOUDLY on the signals that +# actually prove the fleet stopped serving jobs — BEFORE queued CI runs pile up. # # Why this exists: on 2026-07-03 the org API showed 37/48 runners offline while # every runner container on .201 reported "Up (healthy)". Docker-side checks @@ -14,6 +13,49 @@ # host; this canary runs on GitHub-hosted compute so it stays alive when the # fleet — or the whole .201 host — is dead. # +# OMN-16030 — WHY `status == "offline"` ALONE IS NOT A LIVENESS SIGNAL. +# This script previously treated the org REST `status` field as "the +# AUTHORITATIVE view of whether runners are serving jobs" and failed whenever +# more than RUNNER_CANARY_MAX_OFFLINE runners reported offline. On 2026-08-14 +# that assumption was measured and falsified against the 72-runner fleet: +# +# * Runners reporting `offline` were concurrently executing jobs. runner-51 +# completed a job at 10:24:22Z and runner-67 at 10:25:20Z while both were +# labelled offline; runner-30 held an in-progress job while labelled +# offline; `ps` inside runner-23 showed Runner.Worker running `uv sync` +# while the registry reported it offline+busy. +# * Over a 2h40m window the 13 persistently-offline-labelled runners served +# 153 jobs (mean 11.8/runner) versus 14.7/runner for online-labelled ones — +# ~80% of nominal throughput, not zero. +# * `missing` was 0 in every sample all day and RestartCount was 0 on all 72 +# containers: nothing ever actually de-registered or crashed. +# * The offline count correlates POSITIVELY with concurrent job count +# (Pearson r=+0.55, n=7) — it reads worst exactly when the fleet is +# busiest, which is the opposite of a liveness signal. +# +# Mechanism: these runners use the Actions V2 broker flow (`useV2Flow: true`, +# serverUrlV2 = broker.actions.githubusercontent.com). The org REST `status` +# field reflects broker-session bookkeeping that goes stale under load; it is +# not a heartbeat. A transient stale read is not a dead listener. +# +# Cost of getting this wrong: a persistently-red canary is indistinguishable +# from a real outage, so it trains operators to ignore it AND it halts landing +# sweeps on a false alarm (observed 2026-08-14: two sweeps halted, and a +# proposed "recovery" would have force-recreated healthy runners, killing +# in-flight jobs and wiping the warm tool cache). +# +# What this canary fails on now — signals that cannot be produced by a stale +# read, in descending order of certainty: +# 1. missing > 0 — a runner lost its REGISTRATION. Unambiguous. +# 2. offline fraction >= RUNNER_CANARY_MASS_OFFLINE_PCT — mass listener death +# (the 2026-07-03 mode was 77%). Load-induced staleness has never exceeded +# ~22% in measurement, so this band separates the two cleanly. +# A `busy` runner is counted ALIVE regardless of `status`: it is demonstrably +# executing a job, which is the thing the canary exists to protect. +# Anything between the advisory threshold and the mass-outage threshold is a +# WARNING (visible in the step summary + Slack) on a GREEN run — loud enough to +# investigate, not loud enough to block landing. +# # Enforcement surface: .github/workflows/runner-fleet-canary.yml runs this on a # 15-minute schedule on ubuntu-latest. A threshold breach fails the workflow # run (red X + owner notification). This is not an opt-in script. @@ -23,7 +65,10 @@ # GET /orgs/{org}/actions/runners (classic PAT: admin:org read; # fine-grained: org "Self-hosted runners" read). # Optional env: -# RUNNER_CANARY_MAX_OFFLINE max offline runners tolerated (default 5) +# RUNNER_CANARY_MAX_OFFLINE offline count above which the run WARNS (default 5). +# Advisory only — see OMN-16030 note above. +# RUNNER_CANARY_MASS_OFFLINE_PCT percent of the fleet reporting offline-and-not-busy +# at which the run FAILS (default 50). # RUNNER_FLEET_CONFIG_PATH path to runner_fleet.yaml (default config/runner_fleet.yaml) # GITHUB_API_URL API base (set by Actions; default https://api.github.com) # GITHUB_STEP_SUMMARY if set, a markdown summary is appended @@ -33,6 +78,7 @@ set -euo pipefail RUNNER_FLEET_CONFIG_PATH="${RUNNER_FLEET_CONFIG_PATH:-config/runner_fleet.yaml}" RUNNER_CANARY_MAX_OFFLINE="${RUNNER_CANARY_MAX_OFFLINE:-5}" +RUNNER_CANARY_MASS_OFFLINE_PCT="${RUNNER_CANARY_MASS_OFFLINE_PCT:-50}" GITHUB_API_URL="${GITHUB_API_URL:-https://api.github.com}" log() { echo "[fleet-canary] $*"; } @@ -107,28 +153,45 @@ online_count=$(jq '[ .[] | select(.status == "online") ] | length' <<< "${fleet} offline_count=$(jq '[ .[] | select(.status != "online") ] | length' <<< "${fleet}") missing_count=$(( EXPECTED_RUNNERS - total_registered )) [[ "${missing_count}" -lt 0 ]] && missing_count=0 -# A runner that dropped its registration entirely is offline in every way that -# matters — count it against the same threshold. -effective_offline=$(( offline_count + missing_count )) + +# OMN-16030: a runner reporting offline while `busy` is demonstrably executing a +# job — the registry read is stale, the listener is not dead. Only offline AND +# not-busy runners are candidates for "actually unreachable". +offline_idle_count=$(jq '[ .[] | select(.status != "online") | select(.busy != true) ] | length' <<< "${fleet}") +offline_busy_count=$(( offline_count - offline_idle_count )) + +# Mass-outage fraction is computed against offline-and-not-busy plus lost +# registrations — the two states a stale read cannot manufacture. +unreachable=$(( offline_idle_count + missing_count )) +mass_threshold=$(( EXPECTED_RUNNERS * RUNNER_CANARY_MASS_OFFLINE_PCT / 100 )) offline_names=$(jq -r '[ .[] | select(.status != "online") | .name ] | join(", ")' <<< "${fleet}") -log "expected=${EXPECTED_RUNNERS} registered=${total_registered} online=${online_count} offline=${offline_count} missing=${missing_count} threshold=${RUNNER_CANARY_MAX_OFFLINE}" +log "expected=${EXPECTED_RUNNERS} registered=${total_registered} online=${online_count} offline=${offline_count} offline_but_busy=${offline_busy_count} offline_idle=${offline_idle_count} missing=${missing_count} unreachable=${unreachable} warn_threshold=${RUNNER_CANARY_MAX_OFFLINE} fail_threshold=${mass_threshold}" summary() { cat < A runner reporting \`offline\` is NOT proof it is dead. These runners use the +> Actions V2 broker flow, whose REST \`status\` goes stale under load — measured +> 2026-08-14, offline-labelled runners served ~80% of nominal job throughput +> (OMN-16030). Only lost registrations and mass offline-idle fail this gate. EOF } @@ -137,19 +200,39 @@ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then fi slack_alert() { + local severity="${1}" + local detail="${2}" [[ -n "${SLACK_BOT_TOKEN:-}" && -n "${SLACK_CHANNEL_ID:-}" ]] || return 0 curl -s -X POST https://slack.com/api/chat.postMessage \ -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ -H "Content-Type: application/json" \ -d "$(jq -n \ --arg channel "${SLACK_CHANNEL_ID}" \ - --arg text "*[RUNNER FLEET CANARY]* ${effective_offline}/${EXPECTED_RUNNERS} runners offline-or-missing (online=${online_count}, threshold=${RUNNER_CANARY_MAX_OFFLINE}). Offline: ${offline_names:-none}. Docker 'Up (healthy)' is NOT sufficient evidence — see OMN-13915 runbook." \ + --arg text "*[RUNNER FLEET CANARY — ${severity}]* ${detail} (expected=${EXPECTED_RUNNERS} online=${online_count} offline=${offline_count} offline_but_busy=${offline_busy_count} offline_idle=${offline_idle_count} missing=${missing_count}). Offline: ${offline_names:-none}. See docs/runbooks/runner-fleet-listener-liveness.md" \ '{channel: $channel, text: $text}')" > /dev/null 2>&1 || true } -if [[ "${effective_offline}" -gt "${RUNNER_CANARY_MAX_OFFLINE}" ]]; then - slack_alert - fail "${effective_offline}/${EXPECTED_RUNNERS} runners offline-or-missing (> ${RUNNER_CANARY_MAX_OFFLINE}). Offline: ${offline_names:-}. The fleet is degrading silently — do NOT trust Docker 'Up (healthy)'. See docs/runbooks/runner-fleet-listener-liveness.md" +# --- FAIL 1: lost registrations. A stale status read cannot remove a runner +# from the registry, so this is unambiguous evidence of real fleet loss. +if [[ "${missing_count}" -gt 0 ]]; then + slack_alert "FAIL" "${missing_count} runner registration(s) LOST (registered=${total_registered}/${EXPECTED_RUNNERS})" + fail "${missing_count} runner registration(s) missing (registered=${total_registered}, expected=${EXPECTED_RUNNERS}). A runner dropped its registration entirely — this is real fleet loss, not a stale status read. See docs/runbooks/runner-fleet-listener-liveness.md" +fi + +# --- FAIL 2: mass listener death (the 2026-07-03 mode, 77% of fleet). +if [[ "${unreachable}" -ge "${mass_threshold}" ]]; then + slack_alert "FAIL" "${unreachable}/${EXPECTED_RUNNERS} runners offline-and-idle (>= ${mass_threshold})" + fail "${unreachable}/${EXPECTED_RUNNERS} runners offline-and-not-busy (>= ${mass_threshold} = ${RUNNER_CANARY_MASS_OFFLINE_PCT}% of fleet). At this scale it is no longer explainable as broker-status staleness — treat as mass listener death. Offline: ${offline_names:-none}. Do NOT trust Docker 'Up (healthy)'. See docs/runbooks/runner-fleet-listener-liveness.md" +fi + +# --- WARN: elevated but within the band that measurement attributes to V2 +# broker-status staleness. Visible, not blocking. See OMN-16030. +if [[ "${unreachable}" -gt "${RUNNER_CANARY_MAX_OFFLINE}" ]]; then + slack_alert "WARN" "${unreachable}/${EXPECTED_RUNNERS} runners offline-and-idle (> ${RUNNER_CANARY_MAX_OFFLINE}, below fail threshold ${mass_threshold})" + log "WARN: ${unreachable}/${EXPECTED_RUNNERS} offline-and-idle — above the advisory threshold (${RUNNER_CANARY_MAX_OFFLINE}) but below the mass-outage threshold (${mass_threshold})." + log "WARN: this band is attributed to V2 broker-status staleness (OMN-16030). Confirm with actual job throughput before any restart — offline-labelled runners are usually still serving jobs." + log "OK: fleet serving; not failing on a status-staleness signal." + exit 0 fi log "OK: fleet within threshold." diff --git a/scripts/ci/test_selection_adjacency.yaml b/scripts/ci/test_selection_adjacency.yaml index 9f871f19cb..5918c65a38 100644 --- a/scripts/ci/test_selection_adjacency.yaml +++ b/scripts/ci/test_selection_adjacency.yaml @@ -108,6 +108,8 @@ adjacency: reverse_deps: [] tools: reverse_deps: [event_bus, topics] + topology: + reverse_deps: [] # --- leaf / near-leaf modules --- deep_dive: {reverse_deps: []} cli: {reverse_deps: []} diff --git a/scripts/ci/test_selection_models.py b/scripts/ci/test_selection_models.py index 257eeb9465..83513f3103 100644 --- a/scripts/ci/test_selection_models.py +++ b/scripts/ci/test_selection_models.py @@ -18,11 +18,29 @@ class EnumFullSuiteReason(StrEnum): MERGE_GROUP = "merge_group" SCHEDULED = "scheduled" FEATURE_FLAG_OFF = "feature_flag_off" + # OMN-15245: a changed test path that cannot be narrowed below `tests/` + # itself (a test module living directly in the tests/ root). Emitting + # "tests/" as a smart selection would run the whole suite on the smart + # step's split count; escalate to the real full suite instead. + CHANGED_TEST_UNNARROWABLE = "changed_test_unnarrowable" +# A selectable pytest target: a directory under the root-collected `tests/` +# tree, OR a collocated `tests/` directory anywhere in the repo. +# +# OMN-15410 added the second alternative. The original `tests/`-only pattern +# encoded an assumption that stopped being true when pyproject `testpaths` +# grew to include four collocated roots (scripts/ci/tests/, scripts/tests/, +# scripts/runtime_build/tests/, and the agent_actions root): the selector could +# not emit them, so a narrowed run could never reach them and constructing the +# selection raised a pattern_mismatch ValidationError. The constraint stays +# tight — the final path component must still be `tests`, so the selector can +# never emit an arbitrary source directory to pytest. TestPath = Annotated[ str, - StringConstraints(pattern=r"^tests(/[A-Za-z0-9_./-]+)?/$|^tests/$"), + StringConstraints( + pattern=r"^tests(/[A-Za-z0-9_./-]+)?/$|^[A-Za-z0-9_-]+(/[A-Za-z0-9_-]+)*/tests/$" + ), ] ModuleName = Annotated[ str, diff --git a/scripts/ci/verify_pypi_pin_resolvability.py b/scripts/ci/verify_pypi_pin_resolvability.py index 648bb1a57e..fdd714dd7d 100644 --- a/scripts/ci/verify_pypi_pin_resolvability.py +++ b/scripts/ci/verify_pypi_pin_resolvability.py @@ -37,22 +37,81 @@ python3 scripts/ci/verify_pypi_pin_resolvability.py dist/ -Exit codes: ``0`` all declared pins resolve; ``1`` a pin failed to resolve (or -dist/ did not contain exactly one wheel); ``2`` bad invocation. +Exit codes: ``0`` all declared pins resolve; ``1`` a pin failed to resolve, the +check exceeded its wall-clock budget, or dist/ did not contain exactly one +wheel; ``2`` bad invocation. A timeout and an unresolvable pin both exit ``1`` +but print distinct reports -- see :data:`_INSTALL_TIMEOUT_SECONDS` and +:class:`PinResolveTimeoutError` (OMN-16047); do not read a timeout as evidence +that a pin is broken. """ from __future__ import annotations +import os import shutil import subprocess # nosec B404 - invokes `uv venv`/`uv pip install` with a fixed, non-shell argv import sys import tempfile from pathlib import Path -#: Wall-clock ceiling for the scratch-venv creation + install. Generous -#: because it has to hit the real PyPI index for every transitive dependency -#: with no local cache warm-up. -_INSTALL_TIMEOUT_SECONDS = 300 +#: Env var overriding :data:`_INSTALL_TIMEOUT_SECONDS`, so a slower fleet can be +#: accommodated from the workflow without editing this file. +_INSTALL_TIMEOUT_ENV_VAR = "PYPI_PIN_RESOLVE_TIMEOUT_SECONDS" + +#: Wall-clock ceiling for the ``uv pip install`` of the built wheel. +#: +#: This is a *throughput* budget, not a resolution budget: ``--no-cache`` forces a +#: cold download + unpack of the whole transitive closure on every run. For +#: ``omnibase_infra`` that closure is **135 packages / 242 MB** (measured +#: 2026-08-14), and the ``self-hosted``/``omnibase-ci`` fleet needs 110-402s just +#: for a *cached* ``uv sync`` that downloads nothing at all. The original 300s +#: ceiling was therefore unreachable here and hard-blocked every release -- +#: v0.38.4 timed out at exactly 300s twice in a row before publish was ever +#: attempted, stranding PyPI at 0.36.1. See OMN-16047. +#: +#: Keep this comfortably under the release job's ``timeout-minutes`` so a timeout +#: surfaces as this script's diagnostic failure rather than an opaque job kill. +_INSTALL_TIMEOUT_SECONDS = int(os.environ.get(_INSTALL_TIMEOUT_ENV_VAR, "1800")) + +#: Creating the scratch venv is purely local work (no index access), so it gets a +#: much smaller budget of its own. Sharing the install budget would let a hung +#: ``uv venv`` silently consume the entire allowance before the install starts. +_VENV_TIMEOUT_SECONDS = 120 + + +class PinResolveTimeoutError(Exception): + """A subprocess in the pin-resolvability check exceeded its wall-clock budget. + + Raised instead of letting :class:`subprocess.TimeoutExpired` escape, because a + bare traceback (a) discards whatever ``uv`` had already written and (b) reads + identically to the unresolvable-pin failure this gate exists to detect. A + timeout says nothing about whether the pins resolve -- it says the fleet was + too slow -- and the report must not conflate the two. + """ + + def __init__( + self, + step: str, + budget_seconds: int, + cause: subprocess.TimeoutExpired, + prior_output: str, + ) -> None: + self.step = step + self.budget_seconds = budget_seconds + self.prior_output = prior_output + self.partial_output = _decode_stream(cause.stdout) + _decode_stream( + cause.stderr + ) + super().__init__(f"{step} exceeded {budget_seconds}s") + + +def _decode_stream(stream: str | bytes | None) -> str: + """Best-effort text for a ``TimeoutExpired`` stdout/stderr capture.""" + if stream is None: + return "" + if isinstance(stream, bytes): + return stream.decode("utf-8", errors="replace") + return stream def _resolve_uv() -> str: @@ -98,14 +157,20 @@ def verify_pin_resolvability(wheel_path: Path) -> tuple[bool, str]: tmp_path = Path(tmp) venv_dir = tmp_path / "venv" - create = subprocess.run( # nosec B603 - fixed argv, no shell, fully-qualified uv path - [uv_bin, "venv", str(venv_dir)], - cwd=tmp_path, - capture_output=True, - text=True, - timeout=_INSTALL_TIMEOUT_SECONDS, - check=False, - ) + try: + create = subprocess.run( # nosec B603 - fixed argv, no shell, fully-qualified uv path + [uv_bin, "venv", str(venv_dir)], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=_VENV_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise PinResolveTimeoutError( + "uv venv", _VENV_TIMEOUT_SECONDS, exc, "" + ) from exc + if create.returncode != 0: return False, create.stdout + create.stderr @@ -113,22 +178,31 @@ def verify_pin_resolvability(wheel_path: Path) -> tuple[bool, str]: scratch_wheel.write_bytes(wheel_path.read_bytes()) venv_python = venv_dir / "bin" / "python" - proc = subprocess.run( # nosec B603 - fixed argv, no shell, fully-qualified uv path - [ - uv_bin, - "pip", - "install", - "--python", - str(venv_python), - "--no-cache", - str(scratch_wheel), - ], - cwd=tmp_path, - capture_output=True, - text=True, - timeout=_INSTALL_TIMEOUT_SECONDS, - check=False, - ) + try: + proc = subprocess.run( # nosec B603 - fixed argv, no shell, fully-qualified uv path + [ + uv_bin, + "pip", + "install", + "--python", + str(venv_python), + "--no-cache", + str(scratch_wheel), + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=_INSTALL_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise PinResolveTimeoutError( + "uv pip install", + _INSTALL_TIMEOUT_SECONDS, + exc, + create.stdout + create.stderr, + ) from exc + return ( proc.returncode == 0, create.stdout + create.stderr + proc.stdout + proc.stderr, @@ -151,7 +225,26 @@ def main(argv: list[str]) -> int: "resolve from the real PyPI index (no [tool.uv.sources] overrides " "reachable)..." ) - ok, log = verify_pin_resolvability(wheel) + try: + ok, log = verify_pin_resolvability(wheel) + except PinResolveTimeoutError as timeout: + print( + f"ERROR: `{timeout.step}` exceeded its {timeout.budget_seconds}s budget " + f"while checking {wheel.name}." + ) + print( + "This is a THROUGHPUT failure, not evidence that the declared pins are " + "unresolvable: the check installs the wheel's entire transitive closure " + "with --no-cache, so it re-downloads every dependency on every run. If " + "the CI fleet is saturated or its egress is degraded, raise the budget " + f"via the {_INSTALL_TIMEOUT_ENV_VAR} environment variable (and the " + "release job's timeout-minutes with it) rather than assuming a bad pin. " + "See OMN-16047." + ) + print(f"---- partial `{timeout.step}` output before the timeout ----") + print(timeout.prior_output + timeout.partial_output) + return 1 + if not ok: print( f"ERROR: {wheel.name}'s declared dependency pins do not resolve " diff --git a/scripts/ci_check_runtime_env_passthrough.py b/scripts/ci_check_runtime_env_passthrough.py index 5a6d3255b1..92e3c11cad 100644 --- a/scripts/ci_check_runtime_env_passthrough.py +++ b/scripts/ci_check_runtime_env_passthrough.py @@ -45,6 +45,7 @@ "OMNIBASE_INFRA_DB_URL", "ONEX_CONTRACTS_DIR", "POSTGRES_PASSWORD", + "DEPLOY_AGENT_HMAC_SECRET", "USE_EVENT_ROUTING", } ) diff --git a/scripts/create_kafka_topics.py b/scripts/create_kafka_topics.py index e6d736a850..300edb067c 100644 --- a/scripts/create_kafka_topics.py +++ b/scripts/create_kafka_topics.py @@ -10,7 +10,7 @@ # and creates any missing topics on the Kafka broker. Idempotent and safe # for repeated runs. # -# Ticket: OMN-2965 +# Ticket: OMN-2965, OMN-15395 # # Usage: # # Dry-run: print plan, no broker connection @@ -20,38 +20,62 @@ # uv run python scripts/create_kafka_topics.py \ # --bootstrap-servers localhost:19092 # -# # Override defaults +# # Override the fallback used ONLY for topics whose contract declares none # uv run python scripts/create_kafka_topics.py \ # --bootstrap-servers localhost:19092 \ # --partitions 3 \ -# --replication-factor 1 \ # --contracts-root src/omnibase_infra/nodes/ # # Exit Codes: # 0 Success (always in --dry-run; or all topics ensured in non-dry-run) -# 1 Broker or create failure +# 1 Broker or create failure, or a contract that violates the environment's +# replication policy (RF1 against managed staging) # 2 Missing --bootstrap-servers in non-dry-run mode # # Algorithm: -# 1. Extract topics via ContractTopicExtractor (no broker connection) -# 2. list_topics() from broker -# 3. Diff: determine which topics are missing -# 4. create_topics() for missing topics -# 5. list_topics() again (source of truth — do NOT branch on create_topics return) -# 6. Report final created count based on list_topics() diff +# 1. Extract topics + per-topic topic_config via ContractTopicExtractor +# 2. list_topics() from broker — this also carries the live broker count +# 3. Bind the replication policy to that MEASURED broker count +# 4. Diff: determine which topics are missing +# 5. Resolve EVERY missing topic's replication factor through the policy, +# fail-closed, BEFORE the first create_topics() +# 6. create_topics() for missing topics, each at its own resolved spec +# 7. list_topics() again (source of truth — do NOT branch on create_topics return) +# 8. Report final created count based on list_topics() diff # # Design decisions: # - confluent-kafka (sync): CLI tool — no async event loop needed. # - list_topics() is the source of truth, not create_topics() return value. # - Repo-root is discovered via Path(__file__).resolve(), not CWD. # - --dry-run never attempts a broker connection, even if --bootstrap-servers given. +# +# OMN-15395 (D2) — why this script shares the runtime's policy seam: +# This is the SECOND live CreateTopics path in the repository, and it is the +# one docs/operations/README.md tells operators to run and the one +# compare_environments.py names in its fix_hint for cloud/local topic parity. +# It previously created every topic with a flat `--replication-factor` whose +# default was 1, discarding each contract's declared +# `topic_config.replication_factor` entirely — an operator following the +# documented runbook against MSK would recreate the exact +# AWS_KAFKA_HIGH_RISK_CONFIG_RF_EQUALS_ONE condition OMN-15395 exists to +# eliminate, with the runtime provisioner's fail-closed gate never consulted. +# Replication is now resolved by the SAME ModelTopicProvisioningPolicy, with +# the SAME measured capacity ceiling and the SAME fail-closed batch check +# before any CreateTopics is issued. There is no flat replication default and +# no --replication-factor flag: durability is declared in the contract. from __future__ import annotations import argparse import importlib.metadata import sys +from collections.abc import Sequence from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from omnibase_infra.tools.contract_topic_extractor import ModelContractTopicEntry + from omnibase_infra.topics.model_topic_spec import ModelTopicSpec # --------------------------------------------------------------------------- # Repo-root discovery @@ -153,6 +177,11 @@ def _build_parser() -> argparse.ArgumentParser: uv run python scripts/create_kafka_topics.py \\ --bootstrap-servers localhost:19092 \\ --contracts-root src/omnibase_infra/nodes/ + +Replication factor is NOT a CLI flag: it comes from each topic's owning +contract (topic_config.replication_factor) and is resolved through the same +ModelTopicProvisioningPolicy the runtime provisioner uses — managed (MSK) +clusters reject RF1 fail-closed before any topic is created (OMN-15395). """, ) parser.add_argument( @@ -167,16 +196,15 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument( "--partitions", type=int, - default=1, - metavar="N", - help="Number of partitions for new topics (default: 1).", - ) - parser.add_argument( - "--replication-factor", - type=int, - default=1, + default=None, metavar="N", - help="Replication factor for new topics (default: 1).", + help=( + "Partition count for topics whose contract declares no " + "topic_config.partitions. A contract-declared value always wins. " + "Defaults to the runtime provisioner's own default; the lane cap " + "ONEX_TOPIC_PROVISIONER_MAX_PARTITIONS applies either way, so this " + "script and the runtime never disagree about a topic's shape." + ), ) parser.add_argument( "--dry-run", @@ -277,17 +305,65 @@ def _run_dry( return 0 +def _build_specs( + entries: Sequence[ModelContractTopicEntry], + *, + partitions_fallback: int | None, +) -> list[ModelTopicSpec]: + """Build one contract-driven ``ModelTopicSpec`` per unique topic. + + The topic's OWN contract supplies partitions / replication_factor / + kafka_config (the OMN-13238 ``topic_config`` seam). ``replication_factor`` + stays ``None`` when the contract declared nothing — "undeclared", which the + policy resolves or refuses. It is never coerced to a flat 1 here. + """ + from omnibase_infra.topics.model_topic_spec import ( + DEFAULT_EVENT_TOPIC_PARTITIONS, + ModelTopicSpec, + ) + + default_partitions = ( + partitions_fallback + if partitions_fallback is not None + else DEFAULT_EVENT_TOPIC_PARTITIONS + ) + merged: dict[str, ModelContractTopicEntry] = {} + for entry in entries: + existing = merged.get(entry.topic) + merged[entry.topic] = ( + entry if existing is None else existing.merge_sources(entry) + ) + return [ + ModelTopicSpec( + suffix=entry.topic, + provisioning_priority=entry.provisioning_priority, + partitions=( + entry.partitions if entry.partitions is not None else default_partitions + ), + replication_factor=entry.replication_factor, + kafka_config=( + dict(entry.kafka_config) if entry.kafka_config is not None else None + ), + ) + for _, entry in sorted(merged.items()) + ] + + def _run_live( - topics: list[str], + specs: Sequence[ModelTopicSpec], bootstrap_servers: str, - partitions: int, - replication_factor: int, contracts_root: Path, ) -> int: """ Connect to broker, diff existing topics, create missing ones. - Returns 0 on success, 1 on broker or creation failure. + Each topic is created at its OWN contract-declared spec, with the + replication factor resolved through ``ModelTopicProvisioningPolicy`` against + a MEASURED broker count. Every missing topic is resolved before the first + ``create_topics`` call, so an RF1 contract against managed staging aborts + the run with nothing created (OMN-15395 D2). + + Returns 0 on success, 1 on broker, policy, or creation failure. """ try: from confluent_kafka.admin import ( # type: ignore[attr-defined] @@ -302,18 +378,49 @@ def _run_live( ) return 1 + from omnibase_infra.errors import TopicReplicationPolicyError + from omnibase_infra.event_bus.service_topic_manager import ( + topic_partition_cap_from_env, + ) + from omnibase_infra.topics.broker_capacity_probe import ( + bind_policy_to_broker_count, + broker_count_from_cluster_metadata, + is_invalid_replication_factor_error, + ) + from omnibase_infra.topics.model_topic_provisioning_policy import ( + ModelTopicProvisioningPolicy, + resolve_specs_for_creation, + ) + + partition_cap = topic_partition_cap_from_env() + admin: AdminClient | None = None try: print(f"Connecting to broker: {bootstrap_servers}") admin = AdminClient({"bootstrap.servers": bootstrap_servers}) - # Step 1: List existing topics (source of truth — before) + # Step 1: List existing topics (source of truth — before). The same + # response carries the live broker list, so capacity is MEASURED off + # the metadata request the diff already needs — no extra round trip and + # no inference from the auth mechanism. print("Listing existing topics...") cluster_metadata = admin.list_topics(timeout=10) existing_topics: set[str] = set(cluster_metadata.topics.keys()) + policy = bind_policy_to_broker_count( + ModelTopicProvisioningPolicy.from_env(), + broker_count_from_cluster_metadata(cluster_metadata), + ) + print( + f"Replication policy: profile={policy.profile.value} " + f"floor={policy.minimum_replication_factor} " + f"measured_brokers={policy.broker_count} " + f"ceiling={policy.capacity_replication_factor}" + ) + # Step 2: Diff — missing topics only - topic_set = set(topics) + spec_by_name = {spec.suffix: spec for spec in specs} + topic_set = set(spec_by_name) missing = sorted(topic_set - existing_topics) if not missing: @@ -324,19 +431,36 @@ def _run_live( for t in missing: print(f" + {t}") - # Step 3: Create missing topics + # Step 3: Resolve EVERY missing topic through the policy BEFORE the + # first CreateTopics. Fail-closed and batch-scoped — one offending + # contract aborts the whole run with zero topics created. + try: + resolved_specs = resolve_specs_for_creation( + policy, [spec_by_name[name] for name in missing] + ) + except TopicReplicationPolicyError as policy_exc: + print(f"ERROR: {policy_exc}", file=sys.stderr) + return 1 + + # Step 4: Create missing topics, each at its own contract-declared spec. new_topics = [ NewTopic( - t, - num_partitions=partitions, - replication_factor=replication_factor, + spec.suffix, + num_partitions=( + spec.partitions + if partition_cap is None + else min(spec.partitions, partition_cap) + ), + replication_factor=spec.replication_factor, + config=dict(spec.kafka_config) if spec.kafka_config else {}, ) - for t in missing + for spec in resolved_specs ] futures = admin.create_topics(new_topics) # Collect create results (best-effort: log per-topic errors) create_errors: list[str] = [] + unhostable: list[str] = [] for topic_name, future in futures.items(): topic_exc = future.exception() if topic_exc is not None: @@ -350,14 +474,31 @@ def _run_live( ): # Harmless — topic was created concurrently continue + if is_invalid_replication_factor_error(topic_exc): + # (D5) A replica count the broker cannot host is a + # durability failure, not a best-effort miss. + unhostable.append(f" {topic_name}: {topic_exc}") + continue create_errors.append(f" {topic_name}: {topic_exc}") + if unhostable: + print( + f"ERROR: {len(unhostable)} topic(s) were REFUSED by the broker " + "with INVALID_REPLICATION_FACTOR — the declared replication " + "factor exceeds what this cluster can host and no measured " + f"capacity ceiling reduced it (measured_brokers=" + f"{policy.broker_count}). These topics do NOT exist:", + file=sys.stderr, + ) + for err in unhostable: + print(err, file=sys.stderr) + if create_errors: print("WARNING: Some topics failed to create:", file=sys.stderr) for err in create_errors: print(err, file=sys.stderr) - # Step 4: list_topics() is the source of truth — re-check after create + # Step 5: list_topics() is the source of truth — re-check after create cluster_metadata_after = admin.list_topics(timeout=10) existing_after: set[str] = set(cluster_metadata_after.topics.keys()) # Topics from our set that now exist but didn't before (newly_present is the truth) @@ -471,10 +612,8 @@ def main() -> int: return _run_dry(topics, args.bootstrap_servers, contracts_root) return _run_live( - topics, + _build_specs(entries, partitions_fallback=args.partitions), bootstrap_servers=args.bootstrap_servers, - partitions=args.partitions, - replication_factor=args.replication_factor, contracts_root=contracts_root, ) @@ -545,10 +684,8 @@ def main() -> int: ) return _run_live( - unique_topics, + _build_specs(all_entries, partitions_fallback=args.partitions), bootstrap_servers=args.bootstrap_servers, - partitions=args.partitions, - replication_factor=args.replication_factor, contracts_root=packages[0][1], # display first package path ) diff --git a/scripts/deploy-agent/deploy_agent/executor.py b/scripts/deploy-agent/deploy_agent/executor.py index 0cb8993905..a5156a1448 100644 --- a/scripts/deploy-agent/deploy_agent/executor.py +++ b/scripts/deploy-agent/deploy_agent/executor.py @@ -75,6 +75,10 @@ ) _BUILD_SOURCE_ALLOWED = ", ".join(source.value for source in BuildSource) +_BUILD_PROVENANCE_BY_SOURCE: dict[BuildSource, tuple[str, str]] = { + BuildSource.WORKSPACE: ("stability-candidate", "true"), + BuildSource.RELEASE: ("clean-main", "false"), +} # Promotion-lineage guard (OMN-12626, R1). Loaded from scripts/ by file path # because scripts/ is not an importable package. The guard refuses to build a @@ -168,6 +172,18 @@ class ModelLaneConfig(BaseModel): _STABILITY_OVERLAY = f"{REPO_DIR}/docker/docker-compose.stability-test.yml" _PROD_OVERLAY = f"{REPO_DIR}/docker/docker-compose.prod.yml" +# OMN-15379: the dev/lab lane's own overlay. Its only content is +# ``ONEX_MIGRATION_LANE=dev`` on forward-migration -- the lane indicator that +# releases the node_projection_registration trio (0000/0001/0002, CREATE + +# heartbeat + ENABLE/FORCE ROW LEVEL SECURITY) from the operator fence, per +# operator ruling 15 which makes the lab the FORCE proving ground. It is a +# SEPARATE file, not a line in the base compose, so that no non-dev lane can +# inherit it: every lane overlay merges the base, and stability-test's +# forward-migration override inherits the base ``environment:`` block wholesale. +# Unset indicator = FULL fence, so this list is fail-closed on omission. +# Must stay matched with ``resolve_compose_file_args`` in +# ``scripts/deploy-runtime.sh``. +_DEV_LANE_OVERLAY = f"{REPO_DIR}/docker/docker-compose.dev-lane.yml" # OMN-15181 round 3 (Finding 9): maps each prod runtime service to the compose # env var that repoints its `image:` field (docker-compose.prod.yml). This is @@ -181,7 +197,7 @@ class ModelLaneConfig(BaseModel): _LANE_CONFIGS: dict[EnumRuntimeLane, ModelLaneConfig] = { EnumRuntimeLane.DEV: ModelLaneConfig( lane=EnumRuntimeLane.DEV, - compose_files=(COMPOSE_FILE,), + compose_files=(COMPOSE_FILE, _DEV_LANE_OVERLAY), compose_project=COMPOSE_PROJECT, postgres_container="omnibase-infra-postgres", runtime_health_targets=RUNTIME_HEALTH_TARGETS, @@ -362,12 +378,18 @@ def _build_source_build_args( if selected == BuildSource.WORKSPACE and not omni_home: raise RuntimeError("BUILD_SOURCE=workspace requires OMNI_HOME before build") + promotion_class, non_main_lineage = _BUILD_PROVENANCE_BY_SOURCE[selected] + return [ "--build-arg", f"BUILD_SOURCE={selected.value}", "--build-arg", f"EXPECTED_BUILD_SOURCE={expected.value}", "--build-arg", + f"PROMOTION_CLASS={promotion_class}", + "--build-arg", + f"NON_MAIN_LINEAGE={non_main_lineage}", + "--build-arg", f"OMNI_HOME={omni_home}", ] diff --git a/scripts/deploy-agent/pyproject.toml b/scripts/deploy-agent/pyproject.toml index af77e7ef66..8d07019d19 100644 --- a/scripts/deploy-agent/pyproject.toml +++ b/scripts/deploy-agent/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.12" dependencies = [ "kafka-python>=2.0.2", "pydantic>=2.0", - "aiohttp>=3.14.1", + "aiohttp>=3.14.3", ] [project.optional-dependencies] diff --git a/scripts/deploy-agent/tests/unit/test_executor_build_source.py b/scripts/deploy-agent/tests/unit/test_executor_build_source.py index eaca3c868c..2cefb2d330 100644 --- a/scripts/deploy-agent/tests/unit/test_executor_build_source.py +++ b/scripts/deploy-agent/tests/unit/test_executor_build_source.py @@ -18,7 +18,7 @@ PhaseStatus, Scope, ) -from deploy_agent.executor import DeployExecutor +from deploy_agent.executor import DeployExecutor, _build_source_build_args from pydantic import ValidationError REPO_ROOT = Path(__file__).resolve().parents[4] @@ -83,6 +83,8 @@ def fake_run(cmd: list[str], timeout: int, **kwargs) -> subprocess.CompletedProc assert "--build-arg" in build_cmd assert "BUILD_SOURCE=workspace" in build_cmd assert "EXPECTED_BUILD_SOURCE=workspace" in build_cmd + assert "PROMOTION_CLASS=stability-candidate" in build_cmd + assert "NON_MAIN_LINEAGE=true" in build_cmd assert "OMNI_HOME=/data/omninode/omni_home" in build_cmd # OMN-12965: the workspace build must stamp the full OCI image-identity quad @@ -98,6 +100,27 @@ def fake_run(cmd: list[str], timeout: int, **kwargs) -> subprocess.CompletedProc assert any(a.startswith("BUILD_DATE=") for a in build_cmd) +@pytest.mark.parametrize( + ("build_source", "promotion_class", "non_main_lineage"), + [ + (BuildSource.WORKSPACE, "stability-candidate", "true"), + (BuildSource.RELEASE, "clean-main", "false"), + ], +) +def test_deploy_agent_build_provenance_matches_canonical_deploy_contract( + build_source: BuildSource, + promotion_class: str, + non_main_lineage: str, +) -> None: + """Agent and deploy-runtime.sh must stamp the same promotion metadata.""" + env = {"OMNI_HOME": "/data/omninode/omni_home"} + + build_args = _build_source_build_args(build_source, env=env) + + assert f"PROMOTION_CLASS={promotion_class}" in build_args + assert f"NON_MAIN_LINEAGE={non_main_lineage}" in build_args + + def test_unknown_build_source_fails_before_compose_build( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/scripts/deploy-agent/tests/unit/test_executor_lane_selection.py b/scripts/deploy-agent/tests/unit/test_executor_lane_selection.py index e0e3c8f6c8..c0af57ab4b 100644 --- a/scripts/deploy-agent/tests/unit/test_executor_lane_selection.py +++ b/scripts/deploy-agent/tests/unit/test_executor_lane_selection.py @@ -7,7 +7,12 @@ file(s), compose project, and runtime health targets by ``runtime_lane`` so the agent can deploy ``stability-test`` (18085/18086, project ``omnibase-infra-stability-test``) and ``prod`` (28085/28086, project -``omnibase-infra-prod``). The dev lane is unchanged. +``omnibase-infra-prod``). + +OMN-15379 later gave the dev/lab lane its own overlay too +(``docker-compose.dev-lane.yml``) — the migration-fence lane indicator, per +operator ruling 15 making the lab the FORCE proving ground. Dev is no longer +single-file; it layers exactly like stability-test and prod do. """ from __future__ import annotations @@ -48,7 +53,10 @@ def test_dev_lane_matches_legacy_module_constants(self) -> None: cfg = lane_config_for(EnumRuntimeLane.DEV) assert cfg.compose_project == COMPOSE_PROJECT == "omnibase-infra" assert cfg.postgres_container == "omnibase-infra-postgres" - assert cfg.compose_files == (COMPOSE_FILE,) + # OMN-15379: dev/lab now layers its own migration-fence overlay on top + # of the base infra compose file (previously single-file). + assert cfg.compose_files[0] == COMPOSE_FILE + assert any("docker-compose.dev-lane.yml" in f for f in cfg.compose_files) assert cfg.runtime_health_targets == RUNTIME_HEALTH_TARGETS assert cfg.runtime_health_targets == ( ("omninode-runtime", 8085), @@ -115,8 +123,9 @@ def fake_run( assert cmd[3] == COMPOSE_FILE assert "-p" in cmd assert cmd[cmd.index("-p") + 1] == "omnibase-infra" - # dev does not layer an overlay - assert cmd.count("-f") == 1 + # OMN-15379: dev/lab layers its own migration-fence overlay. + assert cmd.count("-f") == 2, f"dev lane must layer its overlay: {cmd}" + assert any("docker-compose.dev-lane.yml" in tok for tok in cmd) def test_stability_lane_compose_up_layers_overlay_and_project(self) -> None: executor = DeployExecutor() diff --git a/scripts/deploy-agent/tests/unit/test_executor_prod_image_slot_omn15181.py b/scripts/deploy-agent/tests/unit/test_executor_prod_image_slot_omn15181.py index e16d73d65e..81f45c539a 100644 --- a/scripts/deploy-agent/tests/unit/test_executor_prod_image_slot_omn15181.py +++ b/scripts/deploy-agent/tests/unit/test_executor_prod_image_slot_omn15181.py @@ -83,6 +83,7 @@ _DUMMY_BOOTSTRAP_ENV: dict[str, str] = dict.fromkeys( ( "GITHUB_TOKEN", + "DEPLOY_AGENT_HMAC_SECRET", "LINEAR_API_KEY", "LLM_CODER_FAST_URL", "POSTGRES_PASSWORD", @@ -103,6 +104,15 @@ "INFISICAL_ENCRYPTION_KEY", "INFISICAL_AUTH_SECRET", "PROD_REDPANDA_ADVERTISE_HOST", + # OMN-15378: docker-compose.infra.yml (merged as the base file for + # EVERY lane, including prod) hard-requires DEV_REDPANDA_ADVERTISE_HOST + # with no default (OMN-15173's deliberate no-silent-default design). + # The original 2026-07-26 enumeration missed it because the author's + # shell already had it set via a sourced ~/.omnibase/.env -- these + # tests never ran anywhere else (this uncollected-tests-root ticket) + # until now, so a CI runner's clean shell was the first environment to + # expose the gap. + "DEV_REDPANDA_ADVERTISE_HOST", ), "dummy-placeholder-value", ) diff --git a/scripts/deploy-agent/tests/unit/test_runtime_worker_census.py b/scripts/deploy-agent/tests/unit/test_runtime_worker_census.py index 5db099627a..5f70d36191 100644 --- a/scripts/deploy-agent/tests/unit/test_runtime_worker_census.py +++ b/scripts/deploy-agent/tests/unit/test_runtime_worker_census.py @@ -4,9 +4,11 @@ The stability-test lane's required state includes a running ``runtime-worker`` container (4-container census: main, effects, worker, projection-api). The base -``docker-compose.infra.yml`` defaults the worker to ``replicas: 0`` -(``${WORKER_REPLICAS:-0}``), so a plain compose ``up``/recreate that does not -set replicas silently drops the worker — zero errors, zero signal. +``docker-compose.infra.yml`` used to default the worker to ``replicas: 0`` via a +bare ``${WORKER_REPLICAS:-0}``, so a plain compose ``up``/recreate that did not +set replicas silently dropped the worker — zero errors, zero signal. OMN-14968 +made that base line lane-prefixed and fail-closed (``${DEV_WORKER_REPLICAS:?}``) +as well; the census ratchets below stay the runtime-side backstop. Two ratchets guard against that: @@ -47,13 +49,26 @@ def test_runtime_worker_in_runtime_scope_census() -> None: assert "runtime-worker" in services_for_scope(Scope.FULL) -def test_stability_override_pins_worker_replicas_to_literal_one() -> None: - """The stability override pins worker ``replicas: 1`` as a literal. +def test_stability_override_pins_worker_replicas_fail_closed() -> None: + """The stability override pins worker replicas fail-closed on the policy value. - Env indirection (``${STABILITY_TEST_WORKER_REPLICAS:-1}``) is a silent-drop - surface: an exported ``STABILITY_TEST_WORKER_REPLICAS=0`` or a future edit - removing the ``:-1`` fallback would scale the worker to 0 with no signal. - The pin must be the literal integer ``1``. + A SOFT env indirection (``${STABILITY_TEST_WORKER_REPLICAS:-1}``) is the + silent-drop surface: an exported ``STABILITY_TEST_WORKER_REPLICAS=0``, or a + future edit removing the ``:-1`` fallback, scales the worker to 0 with no + signal. + + OMN-12988 first closed that with a LITERAL ``1``. OMN-12990 then deliberately + replaced the literal with the ledgered, fail-fast + ``${STABILITY_TEST_WORKER_REPLICAS:?...}`` form so the value comes from + ``contracts/services/runtime_policy.contract.yaml`` and a recreate that omits + the policy env ABORTS instead of dropping the worker. This test kept asserting + the superseded literal and has been RED since 2026-06-23 — invisible because + ``scripts/deploy-agent/tests/`` is a separate pytest root that the repo suite + does not collect. Corrected under OMN-14968, which applied the same fail-closed + form to the base compose (dev lane) as ``${DEV_WORKER_REPLICAS:?...}``. + + Accepted: the literal ``1``, or a ``:?`` fail-fast reference to the ledgered + lane value. Rejected: any ``:-`` soft default, and a literal ``0``. """ # The stability override uses docker compose custom tags (!override, # !!merge) that yaml.safe_load cannot parse, so assert against the raw text: @@ -80,8 +95,13 @@ def test_stability_override_pins_worker_replicas_to_literal_one() -> None: ) value = replicas_line.group(1) - assert value == "1", ( - "stability-test runtime-worker must pin deploy.replicas to the literal " - f"integer 1 (no env-interpolation default that could silently scale to " - f"0), got {value!r}" + assert value == "1" or value.startswith("${STABILITY_TEST_WORKER_REPLICAS:?"), ( + "stability-test runtime-worker must pin deploy.replicas either to the " + "literal integer 1 or to the ledgered policy value with the ':?' " + "fail-fast form (OMN-12990); a soft ':-' default can silently scale the " + f"worker to 0, got {value!r}" + ) + assert ":-" not in value, ( + "a soft ':-' default re-opens the silent-drop hole: a recreate that omits " + f"the policy env would scale the worker without any signal, got {value!r}" ) diff --git a/scripts/deploy-agent/uv.lock b/scripts/deploy-agent/uv.lock index 2821b734f8..9cc017cfee 100644 --- a/scripts/deploy-agent/uv.lock +++ b/scripts/deploy-agent/uv.lock @@ -13,7 +13,7 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.14.1" +version = "3.14.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, @@ -25,90 +25,90 @@ dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.13'" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/21/151624b51cd92553d95424daf4bf19f19ce9be9002d19253e7e7ce67197b/aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480", size = 757402, upload-time = "2026-06-07T21:06:40.311Z" }, - { url = "https://files.pythonhosted.org/packages/c2/82/280619e0bd7bf2454987e19282616e84762255dd9c8468f62382e8c191f1/aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d", size = 512310, upload-time = "2026-06-07T21:06:42.207Z" }, - { url = "https://files.pythonhosted.org/packages/55/b2/2aac325583aaa1353045f96dffa586d8a34e8322e14a7ba49cffeb103ab4/aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2", size = 512448, upload-time = "2026-06-07T21:06:43.813Z" }, - { url = "https://files.pythonhosted.org/packages/8a/72/a60607cb849faa8af8a356c9329ea2eb6f395d49e82cc82ccba1fd8deb8f/aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2", size = 1766854, upload-time = "2026-06-07T21:06:45.391Z" }, - { url = "https://files.pythonhosted.org/packages/b5/d3/d9fe1c9ec7557ab4d0d82bebaa728c6418f0b93295ec2f4ab015f7710cc7/aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a", size = 1740884, upload-time = "2026-06-07T21:06:47.413Z" }, - { url = "https://files.pythonhosted.org/packages/c1/dc/f2cecfaf9337ba3e63f181500814ff502aa3d00d9c7ec93a9d23d10a27b2/aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264", size = 1810034, upload-time = "2026-06-07T21:06:50.165Z" }, - { url = "https://files.pythonhosted.org/packages/66/d7/2ff65c5e65c0d7476daf7e15c032e0805e36811185b9623e3238ad6c763e/aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842", size = 1904054, upload-time = "2026-06-07T21:06:52.035Z" }, - { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" }, - { url = "https://files.pythonhosted.org/packages/4d/aa/bf04cb4d865fc6101c2229a294ad744973b72e513fdc5a6b791e6983d72a/aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95", size = 1591795, upload-time = "2026-06-07T21:06:55.911Z" }, - { url = "https://files.pythonhosted.org/packages/dc/b4/4dac0038960427ba832f6609dfb4ea5437d7fd80c72001b9e48f834f428b/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199", size = 1728397, upload-time = "2026-06-07T21:06:57.777Z" }, - { url = "https://files.pythonhosted.org/packages/2b/f9/7cd4e8ad7aa3b75f17d56bb5498dd604a93d4e6eece822ba0568c413fff0/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817", size = 1766504, upload-time = "2026-06-07T21:07:00.009Z" }, - { url = "https://files.pythonhosted.org/packages/f9/df/fc01d9fcad0f73fed3f3d361f1f94f975947b50dff82919f6dc2bf4316cc/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a", size = 1777806, upload-time = "2026-06-07T21:07:02.064Z" }, - { url = "https://files.pythonhosted.org/packages/41/09/47e2d090bddcc8fb4ccb4c314aadc32d7c5d9bb55f50f6ad1c92fc15d501/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4", size = 1580707, upload-time = "2026-06-07T21:07:03.942Z" }, - { url = "https://files.pythonhosted.org/packages/3d/36/f1a4ce904ae0b6930cfe9afc96d0896f7ec1a620c400405d63783bb95a9c/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087", size = 1798121, upload-time = "2026-06-07T21:07:05.987Z" }, - { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/a0c0a8f327a9c52095cdd8e312391b00d3ed64ab6c72bb5c33d8ec251cf7/aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4", size = 452771, upload-time = "2026-06-07T21:07:09.669Z" }, - { url = "https://files.pythonhosted.org/packages/df/d9/ea367c75f16ac9c6cdc8febb25e8318fa21a2b1bc8d6514d4b2d890bface/aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271", size = 479873, upload-time = "2026-06-07T21:07:11.538Z" }, - { url = "https://files.pythonhosted.org/packages/03/64/8d96784a7851156db8a4c6c3f6f91042fdf39fb15a4cc38c8b3c14833c45/aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847", size = 448073, upload-time = "2026-06-07T21:07:13.637Z" }, - { url = "https://files.pythonhosted.org/packages/bc/97/bd137012dd97e1649162b099135a80e1fd59aaa807b2430fc448d1029aff/aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178", size = 506882, upload-time = "2026-06-07T21:07:15.501Z" }, - { url = "https://files.pythonhosted.org/packages/ef/79/e5cc690e9d922a66887ceeaca53a8ffd5a7b0be3816142b7abc433742d89/aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf", size = 515270, upload-time = "2026-06-07T21:07:17.53Z" }, - { url = "https://files.pythonhosted.org/packages/fe/22/a73ccbf9dbd6e26dda0b24d5fd5db7da92ee3383a79f47677ffb834c5c5b/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd", size = 485841, upload-time = "2026-06-07T21:07:19.555Z" }, - { url = "https://files.pythonhosted.org/packages/3b/b9/57ed8eaf596321c2ad747bd480fb1700dbd7177c60dfc9e4c187f629662e/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe", size = 492088, upload-time = "2026-06-07T21:07:21.581Z" }, - { url = "https://files.pythonhosted.org/packages/78/c0/5ebe5270a7c140d7c6f79dcb018640225f14d406c149e4eec04a7d82fe71/aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4", size = 501564, upload-time = "2026-06-07T21:07:23.388Z" }, - { url = "https://files.pythonhosted.org/packages/75/7f/8cdaa24fc7983865e0915153b96a9ac5bcdd3548d64c5a27d17cecccad2d/aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876", size = 751998, upload-time = "2026-06-07T21:07:25.046Z" }, - { url = "https://files.pythonhosted.org/packages/b2/f4/c4227aacfacc5cb0cc2d119b65301d177912a6842cd64e120c47af76064f/aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da", size = 510918, upload-time = "2026-06-07T21:07:27.28Z" }, - { url = "https://files.pythonhosted.org/packages/ab/01/a2d5f96cd4e74424864d30bc0a7e44d0a12dacdcfa91b5b2d1bd3dca6bf3/aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6", size = 508657, upload-time = "2026-06-07T21:07:29.252Z" }, - { url = "https://files.pythonhosted.org/packages/e8/ed/3c0fb5c500fdd8e7ebc10d1889c04384fffa1a9163eac1356088ca9da1b1/aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96", size = 1757907, upload-time = "2026-06-07T21:07:31.03Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ab/d4c924d9bd5be3050c226612413ce68cb54c70d2c31b661bfc8d9a5b6a70/aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f", size = 1737565, upload-time = "2026-06-07T21:07:33.031Z" }, - { url = "https://files.pythonhosted.org/packages/19/2a/37326821ff779084020cdc33224d20b19f42f4183a500ff92022a739eda7/aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296", size = 1799018, upload-time = "2026-06-07T21:07:35.003Z" }, - { url = "https://files.pythonhosted.org/packages/b3/4f/6e947ba73e4ce09070761c05ed3a8ceb7c21f5e46798671d8b2aac0e4626/aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa", size = 1894416, upload-time = "2026-06-07T21:07:36.956Z" }, - { url = "https://files.pythonhosted.org/packages/9d/6e/dbf1d0625dc711fb2851f4f3c3055c39ed58bae92082d8c627dbe6013736/aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451", size = 1783881, upload-time = "2026-06-07T21:07:39.063Z" }, - { url = "https://files.pythonhosted.org/packages/44/c2/5e25098a67268ed369483ae7d1a58bd0a13d03aab860d2a0e4a6eb25b046/aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c", size = 1587572, upload-time = "2026-06-07T21:07:41.058Z" }, - { url = "https://files.pythonhosted.org/packages/2a/bd/cf9cee17e140f942a3de73e658a543aa8fbf35a5fc67a9d2538d52d77f0b/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca", size = 1722137, upload-time = "2026-06-07T21:07:43.014Z" }, - { url = "https://files.pythonhosted.org/packages/89/6d/5684f8c59045c96f81a18cefbc1fbbd79d25b88f1c622f2a5c5c08fcb632/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09", size = 1755953, upload-time = "2026-06-07T21:07:45.933Z" }, - { url = "https://files.pythonhosted.org/packages/a8/40/35caf3170f8359760740a7d9aa0fff2e344bef98e1d1186f5a0f6dec17e6/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397", size = 1766479, upload-time = "2026-06-07T21:07:48.047Z" }, - { url = "https://files.pythonhosted.org/packages/6d/a1/b0c61e7a137f0d81de49a82023a6df73c3c16d6fefb0f8e4a93d21639002/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080", size = 1580077, upload-time = "2026-06-07T21:07:50.069Z" }, - { url = "https://files.pythonhosted.org/packages/0b/41/194ea4623693009fcefebef7aef63c141754f153e9cd0d39d3b9e36c175c/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345", size = 1791688, upload-time = "2026-06-07T21:07:52.106Z" }, - { url = "https://files.pythonhosted.org/packages/ba/45/4de841f005cfe1fd63e2a2fe011262c515e2a62aa6994b15947e7d717ac9/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588", size = 1761094, upload-time = "2026-06-07T21:07:54.113Z" }, - { url = "https://files.pythonhosted.org/packages/e4/ae/dbce10533d3896d544d5053939ed75b7dc31a1b0973d959b1b5ae21028d6/aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780", size = 452662, upload-time = "2026-06-07T21:07:56.06Z" }, - { url = "https://files.pythonhosted.org/packages/7b/d9/0bf1a19362c32f06229da5e7ddfcec91f93474d6307f7a2d3135e9c674dc/aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a", size = 479748, upload-time = "2026-06-07T21:07:58.319Z" }, - { url = "https://files.pythonhosted.org/packages/22/0a/62e7232dc9484fbec112ceb32efb6a624cc7994ec6e2b019286f17c4e8f2/aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8", size = 447723, upload-time = "2026-06-07T21:08:00.154Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a1/5fafa04e1ca91ddb47608699d60649c1c6db3cf41c99e78fc4056f9513db/aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15", size = 508531, upload-time = "2026-06-07T21:08:02.093Z" }, - { url = "https://files.pythonhosted.org/packages/fa/2e/bfa02f699d87ffc86d5959270b28f1cb410add3ccaced8ed2e0b8a5238fc/aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8", size = 514718, upload-time = "2026-06-07T21:08:04.476Z" }, - { url = "https://files.pythonhosted.org/packages/85/a5/9594ad6289eebbc97d167c44213d557807f90e59115caad24de21ad2c3b1/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3", size = 487918, upload-time = "2026-06-07T21:08:06.377Z" }, - { url = "https://files.pythonhosted.org/packages/b4/61/16a32c36c3c49edec122a3dc811f2057df2f94d3b14aa107c8017d981618/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba", size = 494014, upload-time = "2026-06-07T21:08:08.263Z" }, - { url = "https://files.pythonhosted.org/packages/9b/89/3ebcf96ed99c05bec9c434aaac6963fd3cbab4a786ae739908a144d9ce44/aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397", size = 502398, upload-time = "2026-06-07T21:08:10.244Z" }, - { url = "https://files.pythonhosted.org/packages/fd/3d/b74870a0c2d40c355928cd5b96c7a11fa821b8a40fc41365e64479b151fb/aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448", size = 758018, upload-time = "2026-06-07T21:08:12.447Z" }, - { url = "https://files.pythonhosted.org/packages/d3/66/f42f5c984d99e49c6cff5f26f590750f2e2f7ef1fcfb99966ab5be1b632e/aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004", size = 512462, upload-time = "2026-06-07T21:08:14.624Z" }, - { url = "https://files.pythonhosted.org/packages/e9/a7/248e1aebe0c7810b0271e021a0f2a5eb6e78a051885b3c9df49f42a5802d/aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983", size = 512824, upload-time = "2026-06-07T21:08:16.572Z" }, - { url = "https://files.pythonhosted.org/packages/26/97/2aa0e5ba0727dc3bd5aaebb7ccbc510f7dfb7fb961ec87497cd496635ab1/aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe", size = 1749898, upload-time = "2026-06-07T21:08:18.635Z" }, - { url = "https://files.pythonhosted.org/packages/00/8d/e97f6c96c891d457c8479d92a514ba194d0412f981d72c70341ee18488ed/aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333", size = 1710114, upload-time = "2026-06-07T21:08:20.892Z" }, - { url = "https://files.pythonhosted.org/packages/6f/e6/aa8d7e863048c8fceb5cd6ce74017311cec3ead07847387e12265fb4444e/aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0", size = 1802541, upload-time = "2026-06-07T21:08:23.044Z" }, - { url = "https://files.pythonhosted.org/packages/83/a8/72193137de57fda4ebfae4563182d082c8856e3b6e9871d0b46f028fb369/aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a", size = 1875776, upload-time = "2026-06-07T21:08:25.288Z" }, - { url = "https://files.pythonhosted.org/packages/a0/18/938441025db6769a3464596b2410af3afde0b21eb2f204c6f766f68af4bd/aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602", size = 1760329, upload-time = "2026-06-07T21:08:27.363Z" }, - { url = "https://files.pythonhosted.org/packages/60/29/bf2496b4065e76e09fe48015aaffe5ce161d8f089b06ac6982070f653076/aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca", size = 1587293, upload-time = "2026-06-07T21:08:29.805Z" }, - { url = "https://files.pythonhosted.org/packages/49/a2/2136674d52123b1354bd05dd5753c318db47dc0c927cc70b27bab3755456/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35", size = 1714756, upload-time = "2026-06-07T21:08:32.094Z" }, - { url = "https://files.pythonhosted.org/packages/a7/b9/e5fd2e6f915503081c0f9b1e8540947037929c70c191da2e4d54b31a21a1/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844", size = 1721052, upload-time = "2026-06-07T21:08:34.167Z" }, - { url = "https://files.pythonhosted.org/packages/63/5a/2833e324a2263e104e31e2e91bc5bbee81bc499afd32203faee048a883f0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496", size = 1766888, upload-time = "2026-06-07T21:08:36.95Z" }, - { url = "https://files.pythonhosted.org/packages/57/fa/dea6511870913162f3b2e8c42a7614eb203a4540b8c2da43e0bfb0548f3c/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5", size = 1581679, upload-time = "2026-06-07T21:08:39.292Z" }, - { url = "https://files.pythonhosted.org/packages/14/bd/3cf0d55e71784b33534e9710a67d382d900598b4787fbce6cc7317f8c42a/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95", size = 1782021, upload-time = "2026-06-07T21:08:41.407Z" }, - { url = "https://files.pythonhosted.org/packages/c1/af/14bb5843eccbe234f4dfb78ab73e549d99727247e62ae5d62cbd22eaf5b0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444", size = 1742574, upload-time = "2026-06-07T21:08:43.795Z" }, - { url = "https://files.pythonhosted.org/packages/f2/1e/fbeb7af9210a67ac0f9c9bec0f8f4568497924e33137a3d5b48e1cf85f3f/aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0", size = 457773, upload-time = "2026-06-07T21:08:46.168Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2b/13e8d741a9ec5db7d900c060554cf8352ab85e44e2a4469ebb9d377bda17/aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719", size = 485001, upload-time = "2026-06-07T21:08:48.401Z" }, - { url = "https://files.pythonhosted.org/packages/df/30/491acfa2c4d6c3ff59c49a14fc1b50be3241e25bbb0c84c09e2da4d11395/aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec", size = 453809, upload-time = "2026-06-07T21:08:50.7Z" }, - { url = "https://files.pythonhosted.org/packages/34/e3/19dbe1a1f4cc6230eb9e314de7fe68053b0992f9302b27d12141a0b5db53/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2", size = 793320, upload-time = "2026-06-07T21:08:52.775Z" }, - { url = "https://files.pythonhosted.org/packages/7f/20/1b7182219ba1b108430d6e4dc53d25ae02dcfcf5a045b33af4e8c5167527/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340", size = 529077, upload-time = "2026-06-07T21:08:55Z" }, - { url = "https://files.pythonhosted.org/packages/b9/c8/14ce60ec31a2e5f5274bb17d383a6f7a3aabca31ac04eee05585bbadab16/aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d", size = 532476, upload-time = "2026-06-07T21:08:57.176Z" }, - { url = "https://files.pythonhosted.org/packages/7e/02/9ac85e081e53da2e061b02fa7758fe0a12d17b8ce2d1f5e6c7cb76730328/aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94", size = 1922347, upload-time = "2026-06-07T21:08:59.563Z" }, - { url = "https://files.pythonhosted.org/packages/c0/3e/d3ba07a0ab38b5389e10bec4362d21e10a4f667cba2d79ba30837b3a5059/aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc", size = 1786465, upload-time = "2026-06-07T21:09:01.909Z" }, - { url = "https://files.pythonhosted.org/packages/0b/cb/e2ee978a00cfb2df829704a69528b18154eba5939f45bc1efa8f33aee4c5/aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251", size = 1909423, upload-time = "2026-06-07T21:09:04.357Z" }, - { url = "https://files.pythonhosted.org/packages/73/5d/1430334858b1022b58ae50399a918f0bd6fe8fa7fa183598d657ff61e040/aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1", size = 2001906, upload-time = "2026-06-07T21:09:06.722Z" }, - { url = "https://files.pythonhosted.org/packages/66/4e/560c7472d3d198a23aa5c8b19a5115bf6a9b77b7d3e4bb363da320430ad2/aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3", size = 1877095, upload-time = "2026-06-07T21:09:09.011Z" }, - { url = "https://files.pythonhosted.org/packages/0d/f1/4745806578d447db4a784a8591e2dae3afdfc2bcb96f8f81271b13df6543/aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c", size = 1676222, upload-time = "2026-06-07T21:09:11.461Z" }, - { url = "https://files.pythonhosted.org/packages/6a/c9/48255813cca749a229ef0ab476004ec623728ad79a9c0840616f6c076325/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203", size = 1842922, upload-time = "2026-06-07T21:09:14.118Z" }, - { url = "https://files.pythonhosted.org/packages/3d/c0/bbd054e2bee909f529523a5af3891052606af5143c09f5f183ec3b234676/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1", size = 1825035, upload-time = "2026-06-07T21:09:16.447Z" }, - { url = "https://files.pythonhosted.org/packages/a8/ae/90395d4376deceb74e09ec26b6adf7d2015a6f8802d6d84446af860fef04/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665", size = 1849512, upload-time = "2026-06-07T21:09:18.742Z" }, - { url = "https://files.pythonhosted.org/packages/93/bd/fb25f3049957553d4ce0ba6ae480aa2f592a6985497fca590837d16c1be0/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1", size = 1668571, upload-time = "2026-06-07T21:09:21.458Z" }, - { url = "https://files.pythonhosted.org/packages/3f/22/7f73303d64dd567ff3addca90b556690ed1233a47b8f55d242fb90af3681/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d", size = 1881159, upload-time = "2026-06-07T21:09:23.813Z" }, - { url = "https://files.pythonhosted.org/packages/44/be/0474c5a8b5640e1e4aa1923430a91f4151be82e511373fe764189b89aef5/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c", size = 1841409, upload-time = "2026-06-07T21:09:26.207Z" }, - { url = "https://files.pythonhosted.org/packages/7b/3c/bb4a7cba26956cb3da4553cc2056cf67be5b5ff6e6d8fa4fbdff73bfb7ae/aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365", size = 494166, upload-time = "2026-06-07T21:09:28.505Z" }, - { url = "https://files.pythonhosted.org/packages/8a/84/ec80c2c1f66a952555a9f86df6b33af65108a6febfa0471b69013a12f807/aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9", size = 530255, upload-time = "2026-06-07T21:09:30.843Z" }, - { url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" }, + { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" }, + { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" }, + { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" }, + { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" }, + { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" }, + { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" }, + { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" }, + { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" }, + { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" }, + { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" }, + { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" }, + { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" }, + { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" }, + { url = "https://files.pythonhosted.org/packages/57/be/5afd201cc0ab139029aadb75392efe85a293403d9dd3a3226161c21ce00c/aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86", size = 506269, upload-time = "2026-07-23T01:54:49.075Z" }, + { url = "https://files.pythonhosted.org/packages/22/09/dec8189d62b45ade009f6792a2264b942a90cb88aeaf181239933cd72c3c/aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627", size = 515166, upload-time = "2026-07-23T01:54:51.894Z" }, + { url = "https://files.pythonhosted.org/packages/28/24/2854869d29ed8a8b19d74f9ec6629515f7e04d02dd329d9d179201e58e47/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82", size = 486263, upload-time = "2026-07-23T01:54:54.223Z" }, + { url = "https://files.pythonhosted.org/packages/d4/dd/57187c8be2a35aea65eaee3bd2c3dcbbcf0204f5106c89637e3610380cd1/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c", size = 492299, upload-time = "2026-07-23T01:54:56.236Z" }, + { url = "https://files.pythonhosted.org/packages/b9/11/06ae6ed8f0d414edf4068861e233d8fe23ee699bfd4b3ceb8663db948a62/aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f", size = 502235, upload-time = "2026-07-23T01:54:58.377Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a3/559639c34a345d2cf7c52dff6838119f2eaf29eb508227b5b83f573af813/aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80", size = 750883, upload-time = "2026-07-23T01:55:00.65Z" }, + { url = "https://files.pythonhosted.org/packages/91/cd/41e131f13afd1e7b0172a9d9eda085ef90eb8439f41f0d279db81ed3ae60/aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0", size = 508473, upload-time = "2026-07-23T01:55:02.945Z" }, + { url = "https://files.pythonhosted.org/packages/bc/6b/e7f13410d391c6e55b4c007a8de024355389d7d459e3d64c42b2d33617e5/aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf", size = 509190, upload-time = "2026-07-23T01:55:05.173Z" }, + { url = "https://files.pythonhosted.org/packages/97/21/6464573e53d69672cc1eada3e5c5cb2d2efa82701e8305a0f2047a576967/aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd", size = 1761478, upload-time = "2026-07-23T01:55:07.383Z" }, + { url = "https://files.pythonhosted.org/packages/1a/81/d217043a4c17fbce360905e3b2bdd20139ebc9a2de836d035d179c4da006/aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807", size = 1735092, upload-time = "2026-07-23T01:55:09.803Z" }, + { url = "https://files.pythonhosted.org/packages/a1/66/e13a02d0eeb1a9a502402a977abb4e4abff9fe4051c26f80558c57a7c975/aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8", size = 1800546, upload-time = "2026-07-23T01:55:12.012Z" }, + { url = "https://files.pythonhosted.org/packages/26/5e/57d42fca1d18cb5acc1cad945d017fabc5d6ae71d8a08ad66be8dc3ee544/aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24", size = 1895250, upload-time = "2026-07-23T01:55:14.357Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1c/7da8d08e74d56f00070822f9638ff3f1c563f8ad87d1efa996c87bfc8644/aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5", size = 1789289, upload-time = "2026-07-23T01:55:16.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/0f/cf16bcf56896981c1a0319f5d5db9337994b5165730c48a8fa07e9b34be6/aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4", size = 1586706, upload-time = "2026-07-23T01:55:18.913Z" }, + { url = "https://files.pythonhosted.org/packages/fe/6f/76eac12a7f2480e1e304f842efdb07db33256b0d9165b866b6ef0806c202/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9", size = 1724652, upload-time = "2026-07-23T01:55:21.296Z" }, + { url = "https://files.pythonhosted.org/packages/39/b6/19c8c592baeeb94b75f966547d40c02ac7590902306ec5863d5c027cf506/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1", size = 1756239, upload-time = "2026-07-23T01:55:23.705Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c9/4e9383150296f97f873b680c4de8fb2cd88608fb9f48c79edcb111611abc/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371", size = 1769161, upload-time = "2026-07-23T01:55:26.082Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1e/147bdc6cc5de5f3ab011be8bf5d6e786633249f22c20bae06f85e45f5387/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde", size = 1578759, upload-time = "2026-07-23T01:55:28.846Z" }, + { url = "https://files.pythonhosted.org/packages/fd/31/78388a9d6040ece2e11df62ea229a822cf5e52d238374b220ae9975b2623/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e", size = 1792025, upload-time = "2026-07-23T01:55:31.457Z" }, + { url = "https://files.pythonhosted.org/packages/03/51/a3d29fdf2c25d796746af8ad6fe56a45d6256c38b0a8a2ed752e1160b3a2/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71", size = 1768477, upload-time = "2026-07-23T01:55:33.87Z" }, + { url = "https://files.pythonhosted.org/packages/29/a6/442e18b5afeade534d877a2dc3c3e392aff8d49787890b0cf84790410267/aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0", size = 451069, upload-time = "2026-07-23T01:55:36.121Z" }, + { url = "https://files.pythonhosted.org/packages/9d/69/3d876ac02659f271cf7f6769f14a8e3de5b6e888ed8b5a7e998086a4cec8/aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883", size = 476518, upload-time = "2026-07-23T01:55:38.303Z" }, + { url = "https://files.pythonhosted.org/packages/b2/0e/50d6e6471cd31edce8b282bdec59375a3a69124d8a989a0b1313355cae52/aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2", size = 447676, upload-time = "2026-07-23T01:55:40.451Z" }, + { url = "https://files.pythonhosted.org/packages/c8/20/887fdcf832326571b370ffc347b3e70abe101096f3720126aac161b1d872/aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062", size = 509067, upload-time = "2026-07-23T01:55:42.618Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a3/92cec936f78cc4bf0fa5554ebe593b73459d94e3c62303e1902a4cccb6f7/aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6", size = 514774, upload-time = "2026-07-23T01:55:44.937Z" }, + { url = "https://files.pythonhosted.org/packages/29/ba/2a0c38df3fc557620b6a5acd98364af050053b6285b4dc7ee74100c63c18/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919", size = 488134, upload-time = "2026-07-23T01:55:47.135Z" }, + { url = "https://files.pythonhosted.org/packages/48/d6/d51b7d4bf309af3693940d8ffd2b9ed0b682434ef85959b7c9c137f60cf8/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7", size = 494201, upload-time = "2026-07-23T01:55:49.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5a/8f624384e5f1efabb5229b94157eb966b021e97bdb188c62860c2ae243c2/aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0", size = 502766, upload-time = "2026-07-23T01:55:51.656Z" }, + { url = "https://files.pythonhosted.org/packages/a6/26/4ff0164370deec18fb19254ee4ab10b7a73304ac0c860b13f5f84663759b/aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924", size = 756557, upload-time = "2026-07-23T01:55:53.964Z" }, + { url = "https://files.pythonhosted.org/packages/97/a3/7056b86dc0d9ec709ea9777eae3b0161428f943372f8b98c01c11593b682/aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646", size = 510168, upload-time = "2026-07-23T01:55:56.22Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/0357a015892fd68058bf2d39d3fd1958e459b997a7db30aaa6aaa434ae96/aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b", size = 512957, upload-time = "2026-07-23T01:55:58.437Z" }, + { url = "https://files.pythonhosted.org/packages/47/d1/8aba53f15ccb2238405f5e9d30e2a8ca44f93878c26e7165ade00d374b1c/aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30", size = 1750149, upload-time = "2026-07-23T01:56:00.856Z" }, + { url = "https://files.pythonhosted.org/packages/49/bd/40c3fee327529284375c6701cbb0fa4600cc2e8432af1378f897e2ef7d3a/aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9", size = 1707685, upload-time = "2026-07-23T01:56:03.371Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a3/ca0cc6724cca8114b05694abd916060758c79894c3aa5b012cdadc1bc28e/aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f", size = 1803911, upload-time = "2026-07-23T01:56:05.817Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/85b099c299c3ffd38ad9b3e43694c8a346934e4a30c88c4fd5a841234f77/aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d", size = 1876929, upload-time = "2026-07-23T01:56:08.413Z" }, + { url = "https://files.pythonhosted.org/packages/d5/b7/1da684a04175473fa4cddbf9a2f572e79514c3fd27a74597f43057d4f3da/aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147", size = 1761112, upload-time = "2026-07-23T01:56:10.918Z" }, + { url = "https://files.pythonhosted.org/packages/d1/16/bc4b55e3e5cb175fd69c53c90d60d2f47797cb343da5106e23863dc4dba4/aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c", size = 1583500, upload-time = "2026-07-23T01:56:13.613Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e8/13a9d957a1ee40837f46aa30f0f4c657e673ad86a2e6362a9f9be20d26d9/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a", size = 1713940, upload-time = "2026-07-23T01:56:15.969Z" }, + { url = "https://files.pythonhosted.org/packages/38/05/d33c680c1bcf1c7e130f9cbfc1fc02fe8bb0c4af2a94a53dd5fb56131e5c/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0", size = 1724413, upload-time = "2026-07-23T01:56:18.591Z" }, + { url = "https://files.pythonhosted.org/packages/85/1d/af798d306f7a74b6a632dbcabcf62a4c91391b7582d2a8c6d7712e2cc54e/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661", size = 1770748, upload-time = "2026-07-23T01:56:21.074Z" }, + { url = "https://files.pythonhosted.org/packages/a8/92/ad720d472556a995049206867765e9410969684f86ee09423ff9969044c1/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22", size = 1577564, upload-time = "2026-07-23T01:56:23.475Z" }, + { url = "https://files.pythonhosted.org/packages/60/ad/0ed7586cbef7a884e23a752fa2bb987a122e6a5dd50dab109258d0a95193/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41", size = 1782080, upload-time = "2026-07-23T01:56:25.994Z" }, + { url = "https://files.pythonhosted.org/packages/97/ea/dbaed0d73e8a69aad653b045dab451c67c2454bb731a37b45a86593e9422/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf", size = 1745813, upload-time = "2026-07-23T01:56:28.604Z" }, + { url = "https://files.pythonhosted.org/packages/81/1b/6893d4bc57e434fc93a6c9217c637d967a0b651d989f6e3265179375754a/aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da", size = 455872, upload-time = "2026-07-23T01:56:31.031Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8b/c7baa1ba1eda4db6989baefe5de6d99834921b84ebd7918624febcb9f290/aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100", size = 481030, upload-time = "2026-07-23T01:56:33.365Z" }, + { url = "https://files.pythonhosted.org/packages/22/8c/c29d067df825a2df88ca432db848aa2fe8199598359cc06c12b09320cac9/aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc", size = 453669, upload-time = "2026-07-23T01:56:35.731Z" }, + { url = "https://files.pythonhosted.org/packages/6a/a4/9c033beb355d39b6147980597ec9645e4729243f686ee4dc73945de72030/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b", size = 791403, upload-time = "2026-07-23T01:56:37.972Z" }, + { url = "https://files.pythonhosted.org/packages/80/ca/87c32a0a7704583cfc49660bd817889bae5b830bf53b5dcb4e92145ac2da/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0", size = 526413, upload-time = "2026-07-23T01:56:40.523Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d8/8ec0e471248c500acdce2be3f46db8fb62b5eb60efef072529cc85ee1d26/aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e", size = 532135, upload-time = "2026-07-23T01:56:42.876Z" }, + { url = "https://files.pythonhosted.org/packages/fe/45/f8919fd936e8b79fcd9bda7b6d8e62613462a713f4f17987fd7c34399142/aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716", size = 1922742, upload-time = "2026-07-23T01:56:45.528Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ec/9ca76b28a27525b0cc53e20842e0228b022f301ce1f436b7d814b4aaf2df/aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f", size = 1787371, upload-time = "2026-07-23T01:56:48.045Z" }, + { url = "https://files.pythonhosted.org/packages/b1/04/6acdbf17315f7b55f1937e3387acb89a3cddeb4995689553d064af8e92ab/aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553", size = 1912623, upload-time = "2026-07-23T01:56:50.605Z" }, + { url = "https://files.pythonhosted.org/packages/86/e6/438b0c79ca6f45eb9fd9817dd4c01a91919a38c0de5ee9e05e2b4dc0ece7/aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100", size = 2005515, upload-time = "2026-07-23T01:56:53.153Z" }, + { url = "https://files.pythonhosted.org/packages/bb/6b/62cbd6577758699525f5c712d1ddef57d9875fbab0ae8d5f5a202fd598f8/aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85", size = 1879906, upload-time = "2026-07-23T01:56:55.818Z" }, + { url = "https://files.pythonhosted.org/packages/00/95/18bcbf830a21dc3aae24d8f6b6feaf3db1d2090242d00a7868db2ffb0b67/aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33", size = 1675849, upload-time = "2026-07-23T01:56:58.861Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/47f4968659c5e23606c3790c80fc624e691c153d036148449ee84d31b287/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f", size = 1843496, upload-time = "2026-07-23T01:57:01.591Z" }, + { url = "https://files.pythonhosted.org/packages/64/af/38c33c4dd82fddcb4e56c4653b6f1072a8edbc6b7fa15809f14932c41e2d/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0", size = 1827746, upload-time = "2026-07-23T01:57:05.131Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9d/0537cda4885ac8f5b7053d164dd06312f4c483a4edcb8ee5b8aaf2a989bf/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098", size = 1853810, upload-time = "2026-07-23T01:57:08.043Z" }, + { url = "https://files.pythonhosted.org/packages/19/fe/26f9c5e6458385aa86497836b0dea6fb2f027827d63f37c7856cce9286ee/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25", size = 1668895, upload-time = "2026-07-23T01:57:10.837Z" }, + { url = "https://files.pythonhosted.org/packages/ec/4c/618b1db9b9ba079b8875d2cdf78e7c4a3bf72903bd5850fee7dd9544600a/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9", size = 1883833, upload-time = "2026-07-23T01:57:13.672Z" }, + { url = "https://files.pythonhosted.org/packages/94/c6/bd959bd1e4771f9fd944e9e436224c48c77b018b73b519b5aad346335bcc/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb", size = 1844251, upload-time = "2026-07-23T01:57:16.593Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/08d41839658bdd44a0ed2480f3891705ecb487ce28c0dde62c9040c997e0/aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963", size = 474180, upload-time = "2026-07-23T01:57:19.306Z" }, + { url = "https://files.pythonhosted.org/packages/99/5d/3cd6ef0a2b2851f7ab913b5b079334781bd50ff56a323e4454063377a080/aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b", size = 500528, upload-time = "2026-07-23T01:57:21.762Z" }, + { url = "https://files.pythonhosted.org/packages/a4/37/cfd1ed540a4d318da025590d96b728e63713c09e9377950fc655dadeb856/aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7", size = 469280, upload-time = "2026-07-23T01:57:24.241Z" }, ] [[package]] @@ -171,7 +171,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "aiohttp", specifier = ">=3.14.1" }, + { name = "aiohttp", specifier = ">=3.14.3" }, { name = "kafka-python", specifier = ">=2.0.2" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, diff --git a/scripts/deploy-gateway.sh b/scripts/deploy-gateway.sh new file mode 100755 index 0000000000..f3a20cf0dd --- /dev/null +++ b/scripts/deploy-gateway.sh @@ -0,0 +1,812 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# deploy-gateway.sh -- Sanctioned deploy path for the .201 omninode-gateway lane +# (OMN-15521). +# +# Before this script, the compose project `omninode-gateway` (the standalone +# operator-edge forwarder at /opt/omninode/gateway, systemd unit +# onex-gateway-forwarder) had no repo-resident deploy path at all -- it was +# stood up by hand-copying docker/docker-compose.gateway.yml and +# docker/gateway/beta-gateway-canary.yaml into a root-owned directory and +# building/running compose there directly. That left the lane invisible to +# deploy-runtime.sh (whose -p omnibase-infra scope never touches it), stamped +# with no org.opencontainers.image.revision / com.omninode.build_source +# labels, and with no recorded rollback target. +# +# This script builds the gateway-forwarder image FROM THIS REPO CHECKOUT (the +# same src/omnibase_infra/ tree deploy-runtime.sh's dev lane builds from -- pull +# to the merged-dev tip first), stamps the same OCI provenance labels every +# omnibase-infra runtime container carries, pins the running container to the +# resulting image DIGEST (never a moving :latest tag -- the existing systemd +# unit's ExecStartPre already asserts GATEWAY_IMAGE is a real sha256 digest), +# and records a rollback target the same way the omnibase-infra lane does via +# registry.json. +# +# Scope: this is the .201 gateway lane ONLY (compose project +# `omninode-gateway`). It does not touch the omnibase-infra runtime lane, does +# not run migrations, and does not restart any RUNTIME_SERVICES container. +# +# Usage: +# ./scripts/deploy-gateway.sh # Dry-run preview (default) +# ./scripts/deploy-gateway.sh --execute # Build + deploy + reload +# ./scripts/deploy-gateway.sh --print-compose-cmd # Show the exact build command +# ./scripts/deploy-gateway.sh --help # Full usage +# +# Runbook: docs/runbooks/gateway-lane-deploy.md + +set -euo pipefail + +SCRIPT_NAME="$(basename "$0")" +readonly SCRIPT_NAME +readonly SCRIPT_VERSION="1.0.0" + +readonly COMPOSE_PROJECT="omninode-gateway" +readonly SERVICE_NAME="gateway-forwarder" +readonly CONTAINER_NAME="omninode-gateway-forwarder" +readonly SYSTEMD_UNIT="onex-gateway-forwarder" + +# Host paths the running lane reads from. Overridable so tests can point them +# at a scratch directory instead of the real root-owned locations. +GATEWAY_HOST_DIR="${GATEWAY_HOST_DIR:-/opt/omninode/gateway}" +GATEWAY_ENV_FILE="${GATEWAY_ENV_FILE:-/etc/omninode/gateway/gateway.env}" +GATEWAY_REGISTRY_DIR="${GATEWAY_REGISTRY_DIR:-${HOME}/.omnibase/gateway}" +readonly GATEWAY_REGISTRY_FILE="${GATEWAY_REGISTRY_DIR}/registry.json" + +# Build-time image tag. The running lane is pinned to a resolved digest (see +# resolve_image_digest), never to this moving tag -- it exists only so +# `docker compose build` has something to name the image it produces. +readonly BUILD_IMAGE_TAG="docker-gateway-forwarder:build" + +# ============================================================================= +# Logging +# ============================================================================= + +log_info() { printf '[deploy-gateway] %s\n' "$*"; } +log_warn() { printf '[deploy-gateway] WARNING: %s\n' "$*" >&2; } +log_error() { printf '[deploy-gateway] ERROR: %s\n' "$*" >&2; } +log_step() { printf '\n[deploy-gateway] === %s ===\n' "$*"; } +log_cmd() { printf '[deploy-gateway] > %s\n' "$*"; } + +# ============================================================================= +# Usage +# ============================================================================= + +usage() { + cat <). + 6. Sync docker/docker-compose.gateway.yml and + docker/gateway/beta-gateway-canary.yaml from this checkout into + ${GATEWAY_HOST_DIR} (root-owned, mode 0444 -- same posture the files + already have), replacing the hand-copied originals. + 7. Rewrite ${GATEWAY_ENV_FILE}'s GATEWAY_IMAGE= line to the new digest, + preserving every other key untouched. + 8. Record the previous digest + this deploy's identity in + ${GATEWAY_REGISTRY_FILE} (rollback target). rollback_command is null + when no previous image was retained (first deploy, or the previous + image had already been pruned) -- never a command built from an empty + digest. + 9. 'systemctl reload ${SYSTEMD_UNIT}' (force-recreates the container on + the new digest; requires sudo) unless --skip-reload. + 10. Verify: the container is actually running the digest just built (not + just that labels are non-empty -- a reload that silently fails to + recreate the container is caught here instead of reporting success), + image labels are non-empty, and the two OMN-12912 files are present. + +ROLLBACK + ${GATEWAY_REGISTRY_FILE}'s "rollback_command" field carries the exact + restore command, already pre-filled with "previous_digest" -- the image + this script retagged as ${ROLLBACK_IMAGE_TAG} before building, so it + stays resolvable even after a later 'docker image prune'. Read it out of + the registry and run it verbatim; do NOT reconstruct it by hand: + jq -r .rollback_command ${GATEWAY_REGISTRY_FILE} + bash -c "\$(jq -r .rollback_command ${GATEWAY_REGISTRY_FILE})" + + "rollback_command" is null when there is no rollback target -- the first + deploy ever run against this lane, or a deploy whose previous running + image had already been pruned before it could be retagged. There is + nothing to roll back to in that case: deploy forward instead. Do not + hand-build a substitution from "previous_digest" -- a JSON null printed + through 'jq -r' renders as the literal string "null", which substitutes + straight into ${GATEWAY_ENV_FILE}'s GATEWAY_IMAGE= line and corrupts it; + the systemd unit's ExecStartPre digest-format assertion then refuses to + start on the next restart/reboot. + + (mirrors the omnibase-infra lane's manual rollback-via-registry.json + pattern -- deploy-runtime.sh has no automated --rollback flag either.) + Full procedure: docs/runbooks/gateway-lane-deploy.md + +EXAMPLES + # Preview what would be built and deployed + ${SCRIPT_NAME} + + # Print the exact build command + ${SCRIPT_NAME} --print-compose-cmd + + # Build + deploy + reload + ${SCRIPT_NAME} --execute + + # Verify after deploy + docker inspect ${CONTAINER_NAME} \\ + --format='rev={{index .Config.Labels "org.opencontainers.image.revision"}} src={{index .Config.Labels "com.omninode.build_source"}}' + docker exec ${CONTAINER_NAME} ls /app/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/ + docker exec ${CONTAINER_NAME} ls /app/src/omnibase_infra/idempotency/ + diff ${GATEWAY_HOST_DIR}/docker-compose.gateway.yml docker/docker-compose.gateway.yml +EOF + exit 0 +} + +# ============================================================================= +# Argument parsing +# ============================================================================= + +MODE="dry-run" +PRINT_COMPOSE_CMD=false +SKIP_RELOAD=false + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --execute) + MODE="execute" + shift + ;; + --print-compose-cmd) + PRINT_COMPOSE_CMD=true + shift + ;; + --skip-reload) + SKIP_RELOAD=true + shift + ;; + --help | -h) + usage + ;; + *) + log_error "Unknown argument: $1" + log_error "Run '${SCRIPT_NAME} --help' for usage." + exit 64 + ;; + esac + done +} + +# ============================================================================= +# Identity +# ============================================================================= + +resolve_repo_root() { + local dir + dir="$(cd "$(dirname "$0")" && pwd)" + while [[ "${dir}" != "/" ]]; do + if [[ -f "${dir}/pyproject.toml" ]]; then + echo "${dir}" + return 0 + fi + dir="$(dirname "${dir}")" + done + log_error "Cannot find repository root (no pyproject.toml found above script)." + exit 1 +} + +validate_repo_structure() { + local repo_root="$1" + local missing=() + [[ -f "${repo_root}/docker/docker-compose.gateway.yml" ]] || missing+=("docker/docker-compose.gateway.yml") + [[ -f "${repo_root}/docker/gateway/beta-gateway-canary.yaml" ]] || missing+=("docker/gateway/beta-gateway-canary.yaml") + [[ -f "${repo_root}/docker/Dockerfile.runtime" ]] || missing+=("docker/Dockerfile.runtime") + [[ -d "${repo_root}/src/omnibase_infra/nodes/node_bus_forwarder_effect/services" ]] || missing+=("src/omnibase_infra/nodes/node_bus_forwarder_effect/services/") + [[ -d "${repo_root}/src/omnibase_infra/idempotency" ]] || missing+=("src/omnibase_infra/idempotency/") + if [[ ${#missing[@]} -gt 0 ]]; then + log_error "Repository structure validation failed. Missing:" + for item in "${missing[@]}"; do + log_error " - ${item}" + done + exit 1 + fi +} + +read_version() { + local repo_root="$1" + local version + version="$(awk ' + /^\[project\]/ { in_section=1; next } + /^\[/ { in_section=0 } + in_section && /^version[[:space:]]*=/ { + gsub(/.*=[[:space:]]*"/, ""); + gsub(/".*/, ""); + print; + exit + } + ' "${repo_root}/pyproject.toml")" + if [[ -z "${version}" ]]; then + log_error "Could not read version from pyproject.toml [project] section" + exit 1 + fi + echo "${version}" +} + +read_git_sha() { + local repo_root="$1" + local sha + sha="$(git -C "${repo_root}" rev-parse --short=12 HEAD 2>/dev/null || true)" + if [[ -z "${sha}" ]]; then + log_error "Could not determine git SHA. Is this a git repository?" + exit 1 + fi + echo "${sha}" +} + +read_repo_ref_or_main() { + # Same helper as deploy-runtime.sh's read_repo_ref_or_main (OMN-15521 + # remediation): resolve a full git SHA for a sibling workspace repo when + # available, falling back to "main" (or the repo's own default via the + # Dockerfile ARG default) when OMNI_HOME or the sibling clone is absent. + local repo_path="$1" fallback="$2" + local sha + sha="$(git -C "${repo_path}" rev-parse HEAD 2>/dev/null || true)" + if [[ -n "${sha}" ]]; then + echo "${sha}" + else + echo "${fallback}" + fi +} + +check_git_dirty() { + local repo_root="$1" + local status_output + status_output="$(git -C "${repo_root}" status --porcelain 2>/dev/null || true)" + if [[ -n "${status_output}" ]]; then + log_warn "Working tree has uncommitted changes." + log_warn "The deployed SHA will not match the actual file contents." + fi +} + +# ============================================================================= +# Build +# ============================================================================= + +# resolve_build_args REPO_ROOT GIT_SHA VERSION -- prints the --build-arg argv +# entries (one per line) so both build_image() and print_compose_commands() +# stay in lockstep instead of maintaining two copies of the same list. +resolve_build_args() { + local repo_root="$1" git_sha="$2" version="$3" + local build_date + build_date="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + local build_source="${BUILD_SOURCE:-release}" + local expected_build_source="${EXPECTED_BUILD_SOURCE:-${build_source}}" + local promotion_class="clean-main" + local non_main_lineage="false" + if [[ "${build_source}" == "workspace" ]]; then + promotion_class="stability-candidate" + non_main_lineage="true" + fi + # OMN-15521 remediation: these three sibling-ref build-args are NOT + # optional extras -- deploy-runtime.sh's build_images() passes them + # unconditionally on every build_source (scripts/deploy-runtime.sh + # resolve+pass at build_images()). Omitting them silently falls back to + # the Dockerfile's hardcoded ARG defaults (OMNIBASE_COMPAT_REF=v0.5.5, + # ONEX_CHANGE_CONTROL_REF=v0.5.3, OMNIMARKET_REF=dev), which is exactly + # how the gateway image drifted from the omnibase-infra runtime image's + # onex-change-control pin on the same box (0.5.3 vs 0.5.1). + local omni_home="${OMNI_HOME:-}" + local compat_ref="main" + local omnimarket_ref="dev" + local occ_ref="main" + if [[ -n "${omni_home}" ]]; then + compat_ref="$(read_repo_ref_or_main "${omni_home}/omnibase_compat" "main")" + omnimarket_ref="$(read_repo_ref_or_main "${omni_home}/omnimarket" "dev")" + occ_ref="$(read_repo_ref_or_main "${omni_home}/onex_change_control" "main")" + fi + cat </dev/null || true)" + if [[ -z "${digest}" ]]; then + log_error "Could not resolve digest for built image ${image_tag}" + exit 1 + fi + echo "${digest}" +} + +# ============================================================================= +# Workspace staging (BUILD_SOURCE=workspace) -- OMN-15521 remediation. +# +# resolve_build_args() already honours BUILD_SOURCE=workspace for the label +# values (promotion_class/non_main_lineage), but a prior version of this +# script never actually populated workspace/sibling-repos/ -- unlike +# scripts/deploy-runtime.sh's build_images(), which always calls +# stage_workspace_if_needed() first. docker/Dockerfile.runtime unconditionally +# COPYs workspace/sibling-repos/, so an unstaged workspace build silently used +# the committed placeholder (or whatever stale staging happened to be sitting +# in the checkout) while still stamping workspace-provenance labels. This is +# the same helper deploy-runtime.sh uses -- same underlying +# scripts/runtime_build/stage_workspace.sh and +# scripts/runtime_build/check_sibling_lock_pins.py, invoked the same way, not +# reimplemented -- adapted only to this script's own build-source resolution +# (no COLD_FULL_BRINGUP concept here). +# ============================================================================= + +stage_workspace_if_needed() { + # Populate workspace/sibling-repos/ from the operator-selected OMNI_HOME so + # Dockerfile.runtime can install exact local sibling repo contents. + local repo_root="$1" + local build_source omni_home stage_script + build_source="${BUILD_SOURCE:-release}" + if [[ "${build_source}" != "workspace" ]]; then + return 0 + fi + + omni_home="${OMNI_HOME:-}" + if [[ -z "${omni_home}" ]]; then + log_error "BUILD_SOURCE=workspace requires OMNI_HOME before staging or build." + exit 64 + fi + + stage_script="${repo_root}/scripts/runtime_build/stage_workspace.sh" + if [[ ! -f "${stage_script}" ]]; then + log_error "Workspace staging script not found: ${stage_script}" + log_error "Cannot proceed with BUILD_SOURCE=workspace." + exit 1 + fi + + log_step "Stage Workspace Sibling Repos" + log_cmd "OMNI_HOME=${omni_home} bash ${stage_script}" + (cd "${repo_root}" && OMNI_HOME="${omni_home}" bash "${stage_script}") + + check_sibling_lock_pins "${repo_root}" "${omni_home}" +} + +check_sibling_lock_pins() { + # Fail-fast preflight (OMN-12987, same guard deploy-runtime.sh runs): every + # vendored sibling's version/SHA must match the consuming repo's + # (omnimarket) uv.lock pin. A stale vendored sibling produced the + # 2026-06-11 stability crash; this guard refuses to build against one. + local repo_root="$1" + local omni_home="$2" + local guard="${repo_root}/scripts/runtime_build/check_sibling_lock_pins.py" + if [[ ! -f "${guard}" ]]; then + log_error "Sibling lock-pin preflight not found: ${guard}" + log_error "Cannot verify vendored siblings match the consuming lock. Aborting." + exit 1 + fi + + log_step "Sibling Lock-Pin Preflight (OMN-12987)" + mkdir -p "${repo_root}/workspace/sibling-repos" + local provenance_out="${repo_root}/workspace/sibling-repos/.sibling-lock-pins.json" + local python_bin + if [[ -x "${repo_root}/.venv/bin/python" ]]; then + python_bin="${repo_root}/.venv/bin/python" + elif command -v uv &>/dev/null; then + python_bin="uv-run" + elif command -v python3 &>/dev/null; then + python_bin="python3" + else + log_error "No Python interpreter available to run the sibling lock-pin preflight." + exit 1 + fi + + local lock_path="${omni_home}/omnimarket/uv.lock" + local guard_args=( + --lock "${lock_path}" + --repo "omnibase-infra=${omni_home}/omnibase_infra" + --repo "omnibase-core=${omni_home}/omnibase_core" + --repo "omnibase-spi=${omni_home}/omnibase_spi" + --repo "omnibase-compat=${omni_home}/omnibase_compat" + --repo "onex-change-control=${omni_home}/onex_change_control" + --output "${provenance_out}" + --build-source workspace + ) + if [[ "${ALLOW_SIBLING_PIN_DRIFT:-0}" == "1" ]]; then + guard_args+=(--allow-drift) + log_warn "ALLOW_SIBLING_PIN_DRIFT=1 -- passing --allow-drift to sibling lock-pin preflight (OMN-12977)" + fi + + log_cmd "OMNI_HOME=${omni_home} ${guard} ${guard_args[*]}" + if [[ "${python_bin}" == "uv-run" ]]; then + if ! OMNI_HOME="${omni_home}" uv run --project "${repo_root}" python "${guard}" \ + "${guard_args[@]}"; then + log_error "Sibling lock-pin preflight FAILED. Refusing to build a stale image." + exit 1 + fi + else + if ! OMNI_HOME="${omni_home}" "${python_bin}" "${guard}" \ + "${guard_args[@]}"; then + log_error "Sibling lock-pin preflight FAILED. Refusing to build a stale image." + exit 1 + fi + fi + log_info "Sibling lock-pin preflight passed: all vendored siblings match the lock." +} + +# ============================================================================= +# Host-file sync (AC4) -- eliminates the 2026-07-29 hand-copy as the source of +# truth. Every deploy re-syncs from this checkout so the host copy can never +# silently drift from a merged commit again. +# ============================================================================= + +sync_host_files() { + local repo_root="$1" host_dir="$2" + log_step "Sync host files -> ${host_dir}" + sudo install -d -m 0755 -o root -g root "${host_dir}" + sudo install -d -m 0755 -o root -g root "${host_dir}/gateway" + sudo install -m 0444 -o root -g root \ + "${repo_root}/docker/docker-compose.gateway.yml" \ + "${host_dir}/docker-compose.gateway.yml" + sudo install -m 0444 -o root -g root \ + "${repo_root}/docker/gateway/beta-gateway-canary.yaml" \ + "${host_dir}/gateway/beta-gateway-canary.yaml" + log_info "Synced docker-compose.gateway.yml + gateway/beta-gateway-canary.yaml" +} + +verify_host_files_match() { + local repo_root="$1" host_dir="$2" + if ! diff -q "${host_dir}/docker-compose.gateway.yml" "${repo_root}/docker/docker-compose.gateway.yml" >/dev/null 2>&1; then + log_error "AC4 FAILED: ${host_dir}/docker-compose.gateway.yml differs from the repo copy after sync." + return 1 + fi + log_info "AC4 OK: ${host_dir}/docker-compose.gateway.yml matches the repo copy." + return 0 +} + +# ============================================================================= +# Rollback target retention (AC6) -- OMN-15521 remediation. +# +# The rollback target must be derived from what the CONTAINER is actually +# running, never from gateway.env's GATEWAY_IMAGE= line: that line can go +# stale relative to the running container (manual edits, a previous deploy +# that wrote the file but was killed before reload, etc.), which produces a +# recorded "rollback target" that was never the last-known-good image. +# +# It must also be RETAGGED under a durable name before build_image() moves +# BUILD_IMAGE_TAG onto the freshly built image -- otherwise the previous +# image becomes untagged/dangling the instant the build succeeds and is +# eligible for collection by any routine `docker image prune`, so the +# recorded digest resolves to nothing by the time anyone needs it. +# ============================================================================= + +readonly ROLLBACK_IMAGE_TAG="docker-gateway-forwarder:previous" + +resolve_running_container_image() { + # Prints the image id (sha256:<64 hex>) CONTAINER_NAME is currently + # running, or empty if the container does not exist yet (first deploy). + # Read straight off the live container's own state -- this is the true + # last-known-good and cannot go stale the way gateway.env's line can. + docker inspect "${CONTAINER_NAME}" --format '{{.Image}}' 2>/dev/null || true +} + +retain_previous_image() { + # Retag the previous running image under ROLLBACK_IMAGE_TAG so it + # survives the build moving BUILD_IMAGE_TAG onto the new image. Returns + # non-zero (and the caller must then treat the rollback target as + # unavailable) if the image no longer resolves locally -- `docker tag` + # against a missing source image fails, which is exactly the existence + # check a bare env-file digest never got. + local previous_digest="$1" + if [[ -z "${previous_digest}" ]]; then + log_info "No previous running container image to retain (first deploy)." + return 1 + fi + if docker tag "${previous_digest}" "${ROLLBACK_IMAGE_TAG}" 2>/dev/null; then + log_info "Retained previous image ${previous_digest} as ${ROLLBACK_IMAGE_TAG} (rollback target, survives prune)." + return 0 + fi + log_warn "Previous running image ${previous_digest} could not be retagged (already pruned / not local); no rollback target will be recorded for this deploy." + return 1 +} + +# ============================================================================= +# gateway.env digest pin +# ============================================================================= + +update_gateway_env_digest() { + local env_file="$1" new_digest="$2" + log_step "Pin GATEWAY_IMAGE -> ${new_digest}" + local tmp_file + tmp_file="$(mktemp)" + awk -v new="GATEWAY_IMAGE=${new_digest}" ' + BEGIN { done=0 } + /^GATEWAY_IMAGE=/ { print new; done=1; next } + { print } + END { if (!done) print new } + ' "${env_file}" >"${tmp_file}" + sudo install -m 0444 -o root -g root "${tmp_file}" "${env_file}" + rm -f "${tmp_file}" + log_info "gateway.env now pins GATEWAY_IMAGE=${new_digest}" +} + +# ============================================================================= +# Registry (AC6 -- rollback target, same convention as ~/.omnibase/infra/registry.json) +# ============================================================================= + +write_registry() { + local version="$1" git_sha="$2" new_digest="$3" previous_digest="$4" repo_root="$5" + log_step "Write Registry" + mkdir -p "${GATEWAY_REGISTRY_DIR}" + local deployed_at + deployed_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + local tmp_file="${GATEWAY_REGISTRY_FILE}.tmp" + jq -n \ + --arg active_version "${version}" \ + --arg git_sha "${git_sha}" \ + --arg active_digest "${new_digest}" \ + --arg previous_digest "${previous_digest}" \ + --arg source_repo "${repo_root}" \ + --arg deployed_at "${deployed_at}" \ + --arg compose_project "${COMPOSE_PROJECT}" \ + --arg gateway_env_file "${GATEWAY_ENV_FILE}" \ + --arg host_compose_file "${GATEWAY_HOST_DIR}/docker-compose.gateway.yml" \ + '{ + active_version: $active_version, + git_sha: $git_sha, + active_digest: $active_digest, + previous_digest: ($previous_digest | select(. != "") // null), + source_repo: $source_repo, + deployed_at: $deployed_at, + compose_project: $compose_project, + gateway_env_file: $gateway_env_file, + host_compose_file: $host_compose_file, + rollback_command: ( + if ($previous_digest != "") then + ("sudo sed -i \"s|^GATEWAY_IMAGE=.*|GATEWAY_IMAGE=" + $previous_digest + "|\" " + $gateway_env_file + " && sudo systemctl reload onex-gateway-forwarder") + else + null + end + ) + }' >"${tmp_file}" + mv "${tmp_file}" "${GATEWAY_REGISTRY_FILE}" + log_info "Registry written: ${GATEWAY_REGISTRY_FILE}" + log_info " active_digest: ${new_digest}" + log_info " previous_digest: ${previous_digest:-}" +} + +# ============================================================================= +# Reload +# ============================================================================= + +reload_service() { + log_step "Reload ${SYSTEMD_UNIT}" + log_cmd "sudo systemctl reload ${SYSTEMD_UNIT}" + sudo systemctl reload "${SYSTEMD_UNIT}" +} + +# ============================================================================= +# Verify (AC2 + AC3 probes) +# ============================================================================= + +verify_deployment() { + # verify_deployment NEW_DIGEST -- OMN-15521 remediation: the first check + # must be that the container is actually running the digest this + # invocation just built. Without this, a `systemctl reload` that + # silently fails to recreate the container (e.g. a transient compose + # error swallowed by the unit) leaves the OLD container running, which + # still passes every other check here (non-empty labels, both files + # present) -- the deploy reports success while nothing actually changed. + local new_digest="$1" + log_step "Verify" + local running_image + running_image="$(resolve_running_container_image)" + if [[ "${running_image}" != "${new_digest}" ]]; then + log_error "AC-VERIFY FAILED: ${CONTAINER_NAME} is running ${running_image:-}, expected the newly built ${new_digest}. The reload did not take effect." + return 1 + fi + log_info "AC-VERIFY OK: ${CONTAINER_NAME} is running the newly deployed digest ${new_digest}." + + local rev src + rev="$(docker inspect "${CONTAINER_NAME}" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' 2>/dev/null || true)" + src="$(docker inspect "${CONTAINER_NAME}" --format '{{index .Config.Labels "com.omninode.build_source"}}' 2>/dev/null || true)" + if [[ -z "${rev}" ]]; then + log_error "AC2 FAILED: org.opencontainers.image.revision is empty on ${CONTAINER_NAME}" + return 1 + fi + log_info "AC2 OK: org.opencontainers.image.revision=${rev} com.omninode.build_source=${src}" + + if docker exec "${CONTAINER_NAME}" test -f /app/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_delivery.py \ + && docker exec "${CONTAINER_NAME}" test -f /app/src/omnibase_infra/idempotency/store_sqlite.py; then + log_info "AC3 OK: service_gateway_delivery.py + store_sqlite.py present in the running container." + else + log_error "AC3 FAILED: one or both OMN-12912 files are absent from the running container." + return 1 + fi + return 0 +} + +# ============================================================================= +# Main +# ============================================================================= + +main() { + parse_args "$@" + + local repo_root version git_sha + repo_root="$(resolve_repo_root)" + validate_repo_structure "${repo_root}" + version="$(read_version "${repo_root}")" + git_sha="$(read_git_sha "${repo_root}")" + check_git_dirty "${repo_root}" + + if [[ "${PRINT_COMPOSE_CMD}" == true ]]; then + print_compose_commands "${repo_root}" "${git_sha}" "${version}" + exit 0 + fi + + log_step "Identity" + log_info "repo_root: ${repo_root}" + log_info "version: ${version}" + log_info "git_sha: ${git_sha}" + log_info "compose_project: ${COMPOSE_PROJECT}" + + if [[ "${MODE}" != "execute" ]]; then + log_step "Dry Run (default) -- no mutation performed" + print_compose_commands "${repo_root}" "${git_sha}" "${version}" + log_info "Would sync ${repo_root}/docker/docker-compose.gateway.yml -> ${GATEWAY_HOST_DIR}/docker-compose.gateway.yml" + log_info "Would sync ${repo_root}/docker/gateway/beta-gateway-canary.yaml -> ${GATEWAY_HOST_DIR}/gateway/beta-gateway-canary.yaml" + log_info "Would pin GATEWAY_IMAGE in ${GATEWAY_ENV_FILE} to the resolved build digest" + log_info "Would write ${GATEWAY_REGISTRY_FILE}" + if [[ "${SKIP_RELOAD}" != true ]]; then + log_info "Would run: sudo systemctl reload ${SYSTEMD_UNIT}" + fi + log_info "Re-run with --execute to perform this deploy." + exit 0 + fi + + if [[ ! -f "${GATEWAY_ENV_FILE}" ]]; then + log_error "GATEWAY_ENV_FILE not found: ${GATEWAY_ENV_FILE}" + log_error "This file supplies the AWS/TPM/UID variables the compose file requires." + exit 64 + fi + + local previous_digest + previous_digest="$(resolve_running_container_image)" + if ! retain_previous_image "${previous_digest}"; then + previous_digest="" + fi + log_info "Previous running image (rollback target): ${previous_digest:-}" + + set -a + # shellcheck source=/dev/null + source "${GATEWAY_ENV_FILE}" + set +a + + stage_workspace_if_needed "${repo_root}" + + build_image "${repo_root}" "${git_sha}" "${version}" + local new_digest + new_digest="$(resolve_image_digest "${BUILD_IMAGE_TAG}")" + log_info "Built image digest: ${new_digest}" + + sync_host_files "${repo_root}" "${GATEWAY_HOST_DIR}" + verify_host_files_match "${repo_root}" "${GATEWAY_HOST_DIR}" + + update_gateway_env_digest "${GATEWAY_ENV_FILE}" "${new_digest}" + write_registry "${version}" "${git_sha}" "${new_digest}" "${previous_digest}" "${repo_root}" + + if [[ "${SKIP_RELOAD}" == true ]]; then + log_warn "--skip-reload set: gateway.env is updated but the running container still has the OLD digest until you reload." + exit 0 + fi + + reload_service + verify_deployment "${new_digest}" + log_step "Done" + log_info "omninode-gateway deployed: version=${version} git_sha=${git_sha} digest=${new_digest}" +} + +main "$@" diff --git a/scripts/deploy-runtime.sh b/scripts/deploy-runtime.sh index c3464d49a3..e7fce22719 100755 --- a/scripts/deploy-runtime.sh +++ b/scripts/deploy-runtime.sh @@ -101,6 +101,12 @@ readonly DEPLOY_ROOT="${HOME}/.omnibase/infra" readonly REGISTRY_FILE="${DEPLOY_ROOT}/registry.json" readonly LOCK_DIR="${DEPLOY_ROOT}/.deploy.lock" +# OMN-15218: env-var NAMES the lane-deploy attribution preflight reads. Held as +# names (not values) so the guard's error text and the Python preflight can never +# drift into naming two different knobs. +readonly ONEX_DEPLOY_REASON_VAR="ONEX_DEPLOY_REASON" +readonly ONEX_DEPLOY_GRANT_ACK_VAR="ONEX_DEPLOY_GRANT_ACK" + # Maximum number of deployed versions to retain. Older deployments are pruned # after each successful deployment. The currently active deployment (tracked in # registry.json) is never removed regardless of age. @@ -211,6 +217,14 @@ readonly HEALTH_CHECK_INTERVAL=4 MODE="dry-run" # dry-run | execute FORCE=false RESTART=false +# OMN-15218: the raw argv this invocation was called with, captured before +# parse_args consumes it, so the attribution record names the exact command that +# mutated the lane instead of a reconstruction. +DEPLOY_INVOCATION_ARGS=() +# OMN-15218: the attribution record emitted by the lane-deploy preflight, folded +# into registry.json by write_registry() so "who/what/why" is readable from the +# same file that already answers "what version is deployed". +LANE_ATTRIBUTION_RECORD_JSON="" # Set after rsync to enable automatic cleanup of orphaned deployment directories # on failure. If this is non-empty and the deployment directory is NOT the active # deployment in registry.json, the trap handler will remove it. @@ -253,6 +267,20 @@ MIGRATION_TREE_SNAPSHOT_DIR="" # Set to true only when ALL deployment phases complete successfully. # Used by cleanup_on_exit to determine if the --force backup can be safely removed. DEPLOYMENT_COMPLETE=false +# OMN-15352: path to a file recording each RUNTIME_BUILD_SERVICES image's +# pre-build `:latest` id (or empty if none existed), taken by +# snapshot_latest_image_tags() right before build_images() runs. On a failed +# deploy, cleanup_on_exit() restores every snapshotted tag (or removes an +# unverified tag that had no prior state) so a later `docker compose up -d` +# without --build can never silently resolve an untested image (F3). Empty +# until the snapshot is taken; the snapshot file is removed on exit. +LATEST_TAG_SNAPSHOT_FILE="" +# OMN-15352: compose project name, mirrored into a global right after +# resolve_compose_project() resolves it in main(). cleanup_on_exit() is an +# EXIT-trap handler with no arguments, so it cannot receive compose_project as +# a parameter -- it reads this global to resolve the same image names +# snapshot_latest_image_tags() recorded them under. +DEPLOY_COMPOSE_PROJECT="" # ============================================================================= # Logging @@ -328,10 +356,28 @@ OPTIONS origin/main. Also honored via ONEX_DEPLOY_LANE=prod. --help Show this help message and exit. +ATTRIBUTION + GRANT INTERLOCK (OMN-15218) + ONEX_DEPLOY_REASON REQUIRED for the governed lanes (stability-test, prod, + judge). A real justification, ideally naming a ticket; + placeholders are rejected. Recorded durably with the + actor (user/uid/host/ssh peer/parent command) and the + invoking command line. + ONEX_DEPLOY_TICKET Optional explicit OMN-#### (otherwise parsed from the + reason). + ONEX_DEPLOY_GRANT_ACK Comma-separated grant ids. A stability-test deploy is + REFUSED while unconsumed, unexpired prod-promotion + grants exist at onex_change_control@main; proceeding + requires naming EVERY live grant id here (or the token + 'unreadable-grant-state' when grant state cannot be + resolved — which also fails closed). The + acknowledgement is written into the record. + DEPLOYMENT ROOT ~/.omnibase/infra/ +-- .deploy.lock/ mkdir-based concurrency guard +-- registry.json tracks active deployment + +-- deploy-log.jsonl append-only lane-deploy attribution log + +-- deploy-attribution/ per-run attribution records +-- deployed/ +-- {version}/ build directory +-- pyproject.toml @@ -496,6 +542,26 @@ resolve_compose_project() { # scripts/deploy-agent/deploy_agent/executor.py (_LANE_CONFIGS): stability-test # layers docker-compose.stability-test.yml, prod layers docker-compose.prod.yml, # judge layers docker-compose.judge.yml. The dev project gets no overlay. +resolve_lane_name() { + # Echo the LANE name derived from a compose project (OMN-15218). + # omnibase-infra -> dev + # omnibase-infra-stability-test -> stability-test + # omnibase-infra-prod -> prod + # omnibase-infra-judge -> judge + # Single derivation shared by the hot-patch preflight and the lane-deploy + # attribution guard, so one deploy can never be recorded under two different + # lane names. Unknown suffixes echo through unchanged; the callers that must + # fail closed on an unknown lane (resolve_lane_overlay_filename) do their own + # allowlist check. + local compose_project="$1" + local lane="${compose_project#omnibase-infra}" + lane="${lane#-}" + if [[ -z "${lane}" ]]; then + lane="dev" + fi + echo "${lane}" +} + resolve_lane_overlay_filename() { # Echo the overlay compose FILENAME (relative to docker/) for a compose # project, or nothing for the bare dev project. Fails closed: an unknown @@ -541,17 +607,28 @@ resolve_compose_file_args() { # local -a compose_args # resolve_compose_file_args compose_args "${deploy_target}" "${compose_project}" # docker compose -p "${compose_project}" "${compose_args[@]}" ... - local -n _out_args="$1" + local _out_args_name="$1" local deploy_target="$2" local compose_project="$3" local docker_dir="${deploy_target}/docker" - _out_args=("-f" "${docker_dir}/docker-compose.infra.yml") + eval "${_out_args_name}=(-f $(printf '%q' "${docker_dir}/docker-compose.infra.yml"))" local overlay_filename overlay_filename="$(resolve_lane_overlay_filename "${compose_project}")" if [[ -n "${overlay_filename}" ]]; then - _out_args+=("-f" "${docker_dir}/${overlay_filename}") + eval "${_out_args_name}+=( -f $(printf '%q' "${docker_dir}/${overlay_filename}") )" + else + # Dev/lab lane (bare omnibase-infra project). OMN-15379: layer the + # dev-lane overlay, whose ONLY content is ONEX_MIGRATION_LANE=dev for + # forward-migration — the lane indicator that releases the + # node_projection_registration trio (operator ruling 15, lab lane is the + # FORCE proving ground). It is a separate file precisely so no non-dev + # lane can inherit it from the base: see the header of + # docker/docker-compose.dev-lane.yml. Unset indicator = FULL fence, so + # omitting this file degrades safely (the lane comes up without + # node_service_registry) rather than dangerously. + eval "${_out_args_name}+=( -f $(printf '%q' "${docker_dir}/docker-compose.dev-lane.yml") )" fi } @@ -1043,6 +1120,85 @@ guard_prod_promotion_lineage() { log_info "Prod promotion-lineage guard passed: source clean + promoted." } +guard_lane_deploy_attribution() { + # Lane-deploy ATTRIBUTION + live-grant INTERLOCK preflight (OMN-15218). + # + # Runs BEFORE any mutation (and in dry-run, so an operator sees the refusal + # during preview rather than after a build starts). Delegates every rule to + # scripts/preflight_lane_deploy_attribution.py — one tested source of truth + # for: mandatory ONEX_DEPLOY_REASON on governed lanes (stability-test / prod + # / judge), durable actor+command+ticket capture, and the refuse-by-default + # interlock when live, unconsumed prod-promotion grants at + # onex_change_control@main pin the stability lane's proof. + # + # The preflight prints its human summary on stderr and the JSON attribution + # record on stdout; the record is captured here and folded into registry.json + # by write_registry(). + local repo_root="$1" + local compose_project="$2" + + local lane + lane="$(resolve_lane_name "${compose_project}")" + + log_step "Lane Deploy Attribution + Grant Interlock (OMN-15218)" + + local preflight="${repo_root}/scripts/preflight_lane_deploy_attribution.py" + if [[ ! -f "${preflight}" ]]; then + log_error "Lane-deploy attribution preflight not found: ${preflight}" + log_error "Refusing to deploy lane '${lane}' with no attribution mechanism." + log_error " Two unattributed stability rebuilds in two days (2026-07-26, 2026-07-27)" + log_error " are exactly what this preflight exists to make impossible (OMN-15218)." + exit 1 + fi + + local python_bin="" + if [[ -x "${repo_root}/.venv/bin/python" ]]; then + python_bin="${repo_root}/.venv/bin/python" + elif command -v uv &>/dev/null; then + python_bin="uv-run" + elif command -v python3 &>/dev/null; then + python_bin="python3" + else + log_error "No Python interpreter available to run the lane-deploy attribution preflight." + exit 1 + fi + + local preflight_args=( + --lane "${lane}" + --compose-project "${compose_project}" + --source "deploy-runtime.sh" + --invoking-command "${SCRIPT_NAME} ${DEPLOY_INVOCATION_ARGS[*]}" + ) + # Dry-run evaluates and reports but writes no durable record — nothing was + # deployed, so nothing is attributed; the verdict is still shown. + if [[ "${MODE}" != "execute" ]]; then + preflight_args+=(--check-only) + fi + + log_cmd "${preflight} ${preflight_args[*]}" + + local preflight_exit=0 + if [[ "${python_bin}" == "uv-run" ]]; then + LANE_ATTRIBUTION_RECORD_JSON="$(uv run --project "${repo_root}" python "${preflight}" \ + "${preflight_args[@]}")" || preflight_exit=$? + else + LANE_ATTRIBUTION_RECORD_JSON="$("${python_bin}" "${preflight}" \ + "${preflight_args[@]}")" || preflight_exit=$? + fi + + if [[ "${preflight_exit}" -ne 0 ]]; then + log_error "Lane-deploy attribution preflight REFUSED this deploy (exit ${preflight_exit})." + log_error " Lane: ${lane} (${compose_project})" + log_error " Set ${ONEX_DEPLOY_REASON_VAR} to a real justification, and — when live" + log_error " prod-promotion grants pin this lane — acknowledge each grant id via" + log_error " ${ONEX_DEPLOY_GRANT_ACK_VAR}. Both are recorded in the attribution record." + log_error " Never route around this by calling docker compose directly (OMN-15218)." + exit 1 + fi + + log_info "Lane-deploy attribution recorded; grant interlock clear." +} + guard_hotpatch_ledger() { # Hot-patch ledger rebuild preflight (OMN-13014, retro B-1). # @@ -1075,11 +1231,8 @@ guard_hotpatch_ledger() { # Lane = compose project suffix (omnibase-infra-stability-test -> stability-test); # the bare dev project (omnibase-infra) maps to lane 'dev'. - local lane="${compose_project#omnibase-infra}" - lane="${lane#-}" - if [[ -z "${lane}" ]]; then - lane="dev" - fi + local lane + lane="$(resolve_lane_name "${compose_project}")" # Workspace builds vendor sibling repos from OMNI_HOME clones; the gate # resolves each ledger row's repo build ref (clone HEAD unless overridden @@ -1293,8 +1446,12 @@ cleanup_on_exit() { # backup's stale snapshot (which dropped a forward migration in # the 2026-06-19 stability redeploy). restore_migration_tree_after_revert "${original_dir}" - log_warn "NOTE: registry.json may contain stale metadata (git_sha, deployed_at)" - log_warn "from the failed deployment. Verify or re-deploy to restore consistency." + # OMN-15352: registry.json is no longer stale here. write_registry() + # is now commit-on-success -- it only runs after every phase that + # can fail has passed, so on any non-success exit (including this + # restore branch) it never ran this invocation, and registry.json + # still holds whatever it held before this deploy started. + log_info "registry.json is unaffected by this restore (written only on full deploy success, OMN-15352)." fi else # Full deployment succeeded -- backup is stale, clean it up. @@ -1304,6 +1461,19 @@ cleanup_on_exit() { FORCE_BACKUP_DIR="" fi + # OMN-15352 F3: restore every RUNTIME_BUILD_SERVICES `:latest` tag to its + # pre-build state on any non-success exit. This is independent of whether a + # --force backup exists -- a build/restart/verify/readback failure can leave + # `:latest` pointed at an unverified image even on a first-ever (non-force) + # deploy, so it is not covered by the FORCE_BACKUP_DIR branch above. + if [[ "${DEPLOYMENT_COMPLETE}" != "true" ]]; then + restore_latest_image_tags + fi + if [[ -n "${LATEST_TAG_SNAPSHOT_FILE}" && -f "${LATEST_TAG_SNAPSHOT_FILE}" ]]; then + rm -f "${LATEST_TAG_SNAPSHOT_FILE}" 2>/dev/null || true + fi + LATEST_TAG_SNAPSHOT_FILE="" + # OMN-13364: remove the migration-tree snapshot taken after sync_files. if [[ -n "${MIGRATION_TREE_SNAPSHOT_DIR}" && -d "${MIGRATION_TREE_SNAPSHOT_DIR}" ]]; then rm -rf "${MIGRATION_TREE_SNAPSHOT_DIR}" 2>/dev/null || true @@ -1634,19 +1804,19 @@ resolve_core_contracts_dir() { # `local resolved=""` here produced an empty resolved-dir output AND # incorrectly returned success on the fail-closed case once the caller's # own variable was also named "resolved". - local -n _out_probed="$1" - local -n _out_resolved="$2" - _out_resolved="" + local _out_probed_name="$1" + local _out_resolved_name="$2" + eval "${_out_resolved_name}=''" local _resolve_ccd_dir="" if [[ -n "${OMNI_HOME:-}" ]]; then local fs_candidate="${OMNI_HOME}/omnibase_core/src/omnibase_core/contracts/runtime_data" - _out_probed+=("${fs_candidate}") + eval "${_out_probed_name}+=( $(printf '%q' "${fs_candidate}") )" if [[ -d "${fs_candidate}" ]]; then _resolve_ccd_dir="${fs_candidate}" fi else - _out_probed+=("") + eval "${_out_probed_name}+=( $(printf '%q' "") )" fi if [[ -z "${_resolve_ccd_dir}" ]]; then @@ -1664,15 +1834,15 @@ if spec and spec.origin: print(runtime_data) " 2>/dev/null || true)" if [[ -n "${py_candidate}" ]]; then - _out_probed+=("${py_candidate} (python find_spec('omnibase_core') resolution)") + eval "${_out_probed_name}+=( $(printf '%q' "${py_candidate} (python find_spec('omnibase_core') resolution)") )" _resolve_ccd_dir="${py_candidate}" else - _out_probed+=("") + eval "${_out_probed_name}+=( $(printf '%q' "") )" fi fi if [[ -n "${_resolve_ccd_dir}" ]]; then - _out_resolved="${_resolve_ccd_dir}" + eval "${_out_resolved_name}=$(printf '%q' "${_resolve_ccd_dir}")" return 0 fi return 1 @@ -1765,6 +1935,24 @@ sync_files() { exit 1 fi + # 3c. Config (repo-tracked deploy-time config baked into the image) + # OMN-15696: Dockerfile.runtime COPYs config/runner_fleet.yaml (OMN-15676), + # but sync_files() never rsynced config/ into the deployed build context, so + # any --force redeploy or cold bring-up that recreates deployed// + # fails the image build with "failed to calculate checksum of ref + # ...:/config/runner_fleet.yaml: not found" -- the same COPY-without-matching- + # rsync class OMN-12987 fixed for workspace/. Sync the whole directory (not + # just runner_fleet.yaml) so a future config/ COPY addition doesn't reopen + # the same gap. + if [[ -d "${repo_root}/config/" ]]; then + log_info "Syncing config/..." + log_cmd "rsync -a --delete config/ -> deployed" + rsync -a --delete \ + "${repo_root}/config/" "${deploy_target}/config/" + else + log_info "No config/ directory present, skipping config sync." + fi + # 4. Docker files -- with preserve allowlist # .env, .env.local, certs/, overrides/ survive --delete # Excludes use a leading '/' to anchor them to the transfer root (docker/), @@ -2013,6 +2201,19 @@ write_registry() { old_umask="$(umask)" umask 077 + # OMN-15218: carry the attribution record (actor, reason, ticket, invoking + # command, grant-interlock verdict + any acknowledgement) into registry.json. + # Before this, registry.json answered "what is deployed" but nothing on the + # box answered "who deployed it and why" — the exact gap that made two + # stability rebuilds unattributable. Defensive: if the record is missing or + # not valid JSON, write `null` rather than corrupting the registry (the + # preflight already hard-failed the deploy if it could not produce one). + local attribution_json="null" + if [[ -n "${LANE_ATTRIBUTION_RECORD_JSON}" ]] \ + && jq -e . >/dev/null 2>&1 <<<"${LANE_ATTRIBUTION_RECORD_JSON}"; then + attribution_json="${LANE_ATTRIBUTION_RECORD_JSON}" + fi + jq -n \ --arg active_version "${version}" \ --arg git_sha "${git_sha}" \ @@ -2021,6 +2222,7 @@ write_registry() { --arg deployed_at "${deployed_at}" \ --arg compose_project "${compose_project}" \ --arg profile "${COMPOSE_PROFILE}" \ + --argjson attribution "${attribution_json}" \ '{ active_version: $active_version, git_sha: $git_sha, @@ -2028,7 +2230,8 @@ write_registry() { source_repo: $source_repo, deployed_at: $deployed_at, compose_project: $compose_project, - profile: $profile + profile: $profile, + attribution: $attribution }' > "${tmp_file}" # Restore original umask before continuing @@ -2042,6 +2245,77 @@ write_registry() { log_info " git_sha: ${git_sha}" log_info " deployed_at: ${deployed_at}" log_info " compose_project: ${compose_project}" + if [[ "${attribution_json}" != "null" ]]; then + log_info " actor: $(jq -r '.actor.identity // "unknown"' <<<"${attribution_json}")" + log_info " reason: $(jq -r '.reason // ""' <<<"${attribution_json}")" + log_info " grant verdict: $(jq -r '.grant_guard.verdict // "unknown"' <<<"${attribution_json}")" + fi +} + +# ============================================================================= +# Image tag snapshot -- protect `:latest` from a failed build (OMN-15352 F3) +# ============================================================================= + +snapshot_latest_image_tags() { + # Record the pre-build `:latest` image id for every RUNTIME_BUILD_SERVICES + # member, so a failed deploy can restore each tag to what it resolved to + # before this invocation (or remove a tag that had no prior state). Runs + # unconditionally right before build_images(): `docker compose build` only + # (re)tags `:latest` on a SUCCESSFUL build, so whatever we capture here is + # always a correct pre-image of the tag regardless of how this run ends. + local compose_project="$1" + + local snapshot_file + snapshot_file="$(mktemp "${DEPLOY_ROOT}/.latest-tag-snapshot.XXXXXX" 2>/dev/null || true)" + if [[ -z "${snapshot_file}" ]]; then + log_warn "Could not create :latest tag snapshot file; :latest rollback protection is disabled for this run." + return 0 + fi + + local service image_name prior_id + for service in "${RUNTIME_BUILD_SERVICES[@]}"; do + image_name="${compose_project}-${service}" + prior_id="$(docker image inspect "${image_name}:latest" --format '{{.Id}}' 2>/dev/null || true)" + printf '%s\t%s\n' "${service}" "${prior_id}" >>"${snapshot_file}" + done + + LATEST_TAG_SNAPSHOT_FILE="${snapshot_file}" + log_info "Snapshotted pre-build :latest image ids for ${#RUNTIME_BUILD_SERVICES[@]} service(s) (rollback safety)." +} + +restore_latest_image_tags() { + # Restore every RUNTIME_BUILD_SERVICES `:latest` tag to its pre-build state + # (OMN-15352 F3). Called only from cleanup_on_exit() on a non-success exit. + # A service that had no prior `:latest` image (recorded as an empty id by + # snapshot_latest_image_tags()) has its now-unverified tag removed instead + # of being left resolvable by a later `docker compose up -d` without + # --build. + if [[ -z "${LATEST_TAG_SNAPSHOT_FILE}" || ! -f "${LATEST_TAG_SNAPSHOT_FILE}" ]]; then + return 0 + fi + if [[ -z "${DEPLOY_COMPOSE_PROJECT}" ]]; then + log_warn "DEPLOY_COMPOSE_PROJECT is unset; cannot restore :latest image tags." + return 0 + fi + + local service prior_id image_name + while IFS=$'\t' read -r service prior_id; do + [[ -n "${service}" ]] || continue + image_name="${DEPLOY_COMPOSE_PROJECT}-${service}" + if [[ -n "${prior_id}" ]]; then + if docker tag "${prior_id}" "${image_name}:latest" 2>/dev/null; then + log_warn "Restored ${image_name}:latest to its pre-build image ${prior_id}." + else + log_error "Failed to restore ${image_name}:latest to pre-build image ${prior_id}." + log_error "Manual recovery: docker tag ${prior_id} ${image_name}:latest" + fi + else + # No prior :latest existed for this service -- remove the tag this + # failed run may have created rather than leave it pointing at an + # image that was never proven to deploy. + docker rmi "${image_name}:latest" 2>/dev/null || true + fi + done <"${LATEST_TAG_SNAPSHOT_FILE}" } # ============================================================================= @@ -2588,10 +2862,24 @@ readback_deployed_ref() { # and exits non-zero on any mismatch. It is NOT an optional flag -- it runs # unconditionally after every restart / cold bring-up. Without it, # "deployed" / "live-readback" proof classes are unfalsifiable. + # + # OMN-15348: verifies EXACTLY the services this run actually rebuilt/ + # recreated -- RUNTIME_BUILD_SERVICES (which already resolves to the + # RUNTIME_BUILD_SERVICES_OVERRIDE subset when OMN-14873 scoping is in + # play, else the full RUNTIME_SERVICES set). Prior to this fix the + # readback was hardcoded to the single omninode-runtime container + # regardless of scope: a scoped rebuild of e.g. runtime-effects left + # omninode-runtime's stale label untouched, RT-6 read THAT container, + # false-FAILed, and auto-triggered restore-previous-deployment on a + # deploy that never touched omninode-runtime at all. Looping the + # verified set over RUNTIME_BUILD_SERVICES means an out-of-scope + # container's stale label is never probed, so it can neither fail the + # deploy nor trigger the restore. local git_sha="$1" local version="$2" local compose_project="$3" local repo_root="$4" + local deploy_target="$5" log_step "Deploy Readback (RT-6, fail-closed) [OMN-14469]" @@ -2603,8 +2891,12 @@ readback_deployed_ref() { exit 1 fi - local runtime_container_name - runtime_container_name="$(resolve_lane_runtime_container_name "${compose_project}")" + # Compose file args to resolve non-runtime service containers below by + # `docker compose ps -q ` -- robust to lanes/services (e.g. + # runtime-worker on the dev lane) that have no fixed container_name and + # get a compose-assigned one. + local -a compose_args + resolve_compose_file_args compose_args "${deploy_target}" "${compose_project}" # The readback script lives next to this script (sync_files does NOT copy # scripts/ into the deploy target), so resolve it from the repo root. @@ -2627,30 +2919,61 @@ readback_deployed_ref() { exit 1 fi - # Always assert the running container's revision == the intended git SHA. - # Also assert the runtime package version inside the container matches the - # built version (always true on every lane); operators can declare extra - # sibling versions to assert via READBACK_EXPECTED_VERSIONS. + # Always assert each in-scope container's revision == the intended git SHA. + # Also assert the runtime package version inside the primary + # omninode-runtime container matches the built version (always true on + # every lane, and only meaningful for that container's image); operators + # can declare extra sibling versions to assert via + # READBACK_EXPECTED_VERSIONS. local expected_versions="omnibase-infra=${version}" if [[ -n "${READBACK_EXPECTED_VERSIONS:-}" ]]; then expected_versions="${expected_versions},${READBACK_EXPECTED_VERSIONS}" fi - local readback_args=( - --container "${runtime_container_name}" - --expected-revision "${git_sha}" - --versions "${expected_versions}" - ) + log_info "Verifying in-scope service(s): ${RUNTIME_BUILD_SERVICES[*]}" - log_cmd "${python_bin} ${readback} ${readback_args[*]}" - if ! "${python_bin}" "${readback}" "${readback_args[@]}"; then - log_error "Deploy readback FAILED (RT-6): the running container is NOT the intended ref ${git_sha}." - log_error "Deployed code != intended ref (stale / mis-targeted image, or version drift)." - log_error "Refusing to certify this deploy. Rebuild + recreate the lane's runtime and re-run." - exit 1 - fi + local service + for service in "${RUNTIME_BUILD_SERVICES[@]}"; do + local container_name="" + if [[ "${service}" == "omninode-runtime" ]]; then + # Keep the pre-existing, individually-tested resolver for the + # primary runtime container (lane-prefixed container_name). + container_name="$(resolve_lane_runtime_container_name "${compose_project}")" + else + # Every other RUNTIME_SERVICES member either has no fixed + # container_name (e.g. dev-lane runtime-worker, compose-assigned) + # or a lane-prefix convention that differs per service + # (projection-api -> omnimarket-*, intelligence-api -> + # omnibase-*). Resolve live via the compose service key instead + # of hardcoding a second name map (OMN-13826-class lesson). + container_name="$(docker compose -p "${compose_project}" "${compose_args[@]}" ps -q "${service}" 2>/dev/null || true)" + if [[ -z "${container_name}" ]]; then + log_error "Deploy readback FAILED (RT-6): could not resolve a running container for in-scope service '${service}'." + log_error "Refusing to certify this deploy. Rebuild + recreate the lane's runtime and re-run." + exit 1 + fi + fi + + local -a readback_args=( + --container "${container_name}" + --expected-revision "${git_sha}" + ) + if [[ "${service}" == "omninode-runtime" ]]; then + readback_args+=(--versions "${expected_versions}") + fi + + log_cmd "${python_bin} ${readback} ${readback_args[*]}" + if ! "${python_bin}" "${readback}" "${readback_args[@]}"; then + log_error "Deploy readback FAILED (RT-6): service '${service}' (container ${container_name}) is NOT the intended ref ${git_sha}." + log_error "Deployed code != intended ref (stale / mis-targeted image, or version drift)." + log_error "Refusing to certify this deploy. Rebuild + recreate the lane's runtime and re-run." + exit 1 + fi - log_info "Deploy readback passed: ${runtime_container_name} revision == ${git_sha}, runtime version == ${version} (RT-6)." + log_info "Deploy readback passed: ${service} (${container_name}) revision == ${git_sha} (RT-6)." + done + + log_info "Deploy readback passed for all ${#RUNTIME_BUILD_SERVICES[@]} in-scope service(s) (RT-6)." } # ============================================================================= @@ -2805,6 +3128,9 @@ show_summary() { main() { # Orchestrate the full deployment workflow from validation through verification. + # OMN-15218: capture raw argv before parse_args consumes it so the attribution + # record carries the literal command that touched the lane. + DEPLOY_INVOCATION_ARGS=("$@") parse_args "$@" # Phase 1: Validate prerequisites @@ -2856,15 +3182,20 @@ main() { # during preview, not after a build starts (OMN-12626, R1). guard_prod_promotion_lineage "${repo_root}" - # Prod lane: hard-fail on dirty/non-promoted source before any build/deploy. - # Runs in both dry-run and execute modes so operators see the rejection - # during preview, not after a build starts (OMN-12626, R1). - guard_prod_promotion_lineage "${repo_root}" - # Compute paths local deploy_target="${DEPLOY_ROOT}/deployed/${version}" local compose_project compose_project="$(resolve_compose_project)" + # OMN-15352: mirror into the global cleanup_on_exit() (a no-argument EXIT + # trap handler) reads to resolve :latest image names on a failed deploy. + DEPLOY_COMPOSE_PROJECT="${compose_project}" + + # Lane-deploy attribution + live-grant interlock (OMN-15218). FIRST gate that + # runs once the target lane is known and BEFORE anything is built, recreated, + # or restarted: an unattributed deploy must not get as far as touching an + # image, and a stability refresh must not silently erode the stability-proven + # premise of a live prod-promotion grant. + guard_lane_deploy_attribution "${repo_root}" "${compose_project}" # Hot-patch ledger preflight: refuse to rebuild over live in-container # hot-patches whose source PRs are not merged into the build ref. @@ -2923,9 +3254,11 @@ main() { # deployed migrations to the backup's stale, pre-build snapshot. snapshot_migration_tree "${deploy_target}" - # Mark deployment directory for cleanup on failure. If registry write or - # build fails after rsync, cleanup_on_exit() will remove this orphaned - # directory (unless registry.json already points to it). + # Mark deployment directory for cleanup on failure. If the build or any + # later phase fails, cleanup_on_exit() will remove this orphaned directory + # (unless registry.json already points to it). OMN-15352: stays armed for + # the whole deploy now that the registry write is commit-on-success -- there + # is no longer an early point at which disarming it would be safe. DEPLOY_DIR_TO_CLEANUP="${deploy_target}" # Phase 7: Env setup -- REMOVED (F65 / OMN-6910) @@ -2934,11 +3267,16 @@ main() { # Phase 8: Sanity check sanity_check "${deploy_target}" "${compose_project}" - # Phase 9: Registry - write_registry "${version}" "${git_sha}" "${deploy_target}" "${repo_root}" "${compose_project}" + # Phase 9: Registry write is DEFERRED to commit-on-success, after Phase 12 + # (OMN-15352). Everything that can actually fail -- build, migration + # preflight, restart, readback -- runs first; registry.json is written only + # once none of it failed, so a failed deploy never leaves the registry + # asserting a version that was never running. See the write_registry() call + # near the deployment-complete marker below. - # Registry now points to this deployment -- disable partial cleanup - DEPLOY_DIR_TO_CLEANUP="" + # Snapshot the pre-build `:latest` image id for every service this build + # will retag, so a failed deploy can restore it (OMN-15352 F3). + snapshot_latest_image_tags "${compose_project}" # Phase 10: Build build_images "${deploy_target}" "${compose_project}" "${git_sha}" @@ -2996,9 +3334,19 @@ main() { # only when this invocation actually started containers (there is nothing # to read back otherwise). A stale / mis-targeted running container is # rejected here instead of passing with only verify_deployment's warning. - readback_deployed_ref "${git_sha}" "${version}" "${compose_project}" "${repo_root}" + readback_deployed_ref "${git_sha}" "${version}" "${compose_project}" "${repo_root}" "${deploy_target}" fi + # Phase 9 (commit-on-success, OMN-15352): every phase that can fail -- + # build, migration preflight, restart, readback -- has now passed. Write + # the registry only now, closing the write-ahead window that let a failed + # deploy leave registry.json asserting a version that was never actually + # running. + write_registry "${version}" "${git_sha}" "${deploy_target}" "${repo_root}" "${compose_project}" + + # Registry now points to this deployment -- disable partial cleanup. + DEPLOY_DIR_TO_CLEANUP="" + # All phases completed successfully. Mark deployment as complete so that # cleanup_on_exit knows the backup can be safely removed rather than restored. DEPLOYMENT_COMPLETE=true diff --git a/scripts/disk-gc.sh b/scripts/disk-gc.sh index 79b5f81493..75f76fa316 100755 --- a/scripts/disk-gc.sh +++ b/scripts/disk-gc.sh @@ -102,6 +102,16 @@ fi MIN_AGE_DAYS="$(echo "$PLAN_JSON" | python3 -c 'import json,sys;print(json.load(sys.stdin)["min_age_days"])')" IMAGE_IDS="$(echo "$PLAN_JSON" | python3 -c 'import json,sys;[print(i) for i in json.load(sys.stdin)["remove_image_ids"]]')" CONTAINER_IDS="$(echo "$PLAN_JSON" | python3 -c 'import json,sys;[print(c) for c in json.load(sys.stdin)["remove_container_ids"]]')" +# OMN-15804: idref1,ref2,... — one line per removal-candidate image id, refs +# comma-joined (empty string when the id is dangling / has no repo:tag to untag). +IMAGE_REFS_TSV="$(echo "$PLAN_JSON" | python3 -c ' +import json, sys +plan = json.load(sys.stdin) +refs = plan.get("remove_image_refs", {}) +for iid in plan["remove_image_ids"]: + joined = ",".join(refs.get(iid, [])) + print(f"{iid}\t{joined}") +')" log "Plan: $(echo "$IMAGE_IDS" | grep -c . || true) image(s), $(echo "$CONTAINER_IDS" | grep -c . || true) stopped container(s), builder cache > ${MIN_AGE_DAYS}d" @@ -124,13 +134,75 @@ if [[ -n "$CONTAINER_IDS" ]]; then done <<< "$CONTAINER_IDS" fi -if [[ -n "$IMAGE_IDS" ]]; then - while IFS= read -r iid; do +if [[ -n "$IMAGE_REFS_TSV" ]]; then + while IFS=$'\t' read -r iid refs_csv; do [[ -z "$iid" ]] && continue - if docker rmi "$iid" >>"$LOG_FILE" 2>&1; then log "removed image $iid"; else log "kept/failed image $iid (likely still referenced)"; fi - done <<< "$IMAGE_IDS" + + # OMN-15804 fresh in-use re-check: the plan's protect_running decision was + # made against a docker-inventory SNAPSHOT taken before the builder-cache + # prune / stopped-container removal above ran. Re-derive liveness right + # before deletion via docker's own `ancestor` filter, which resolves an + # image id or repo:tag to every container (running OR stopped) built from + # it — closing the snapshot-staleness + short-vs-full-id gaps a static + # substring match against a pre-captured `docker ps` list cannot catch. + if [[ -n "$(docker ps -a --filter "ancestor=${iid}" --format '{{.ID}}' 2>/dev/null)" ]]; then + log "kept image $iid (fresh in-use re-check: referenced by a container)" + continue + fi + + # OMN-15804: untag every repo:tag ref before the final by-id remove. + # `docker rmi ` alone fails "must be forced - referenced in multiple + # repositories" whenever more than one repo:tag points at the same id — + # this was why the native timer identified ~101 candidates and removed + # ZERO across 3+ cycles. Untagging each ref first (never -f) drops the + # tag; the final `docker rmi ` (or the last untag itself) frees the + # underlying image once no ref remains. + untag_failed=false + if [[ -n "$refs_csv" ]]; then + IFS=',' read -ra refs_arr <<< "$refs_csv" + for ref in "${refs_arr[@]}"; do + [[ -z "$ref" ]] && continue + if docker rmi "$ref" >>"$LOG_FILE" 2>&1; then + log "untagged $ref" + else + log "FAILED to untag $ref" + untag_failed=true + fi + done + fi + + if [[ "$untag_failed" == true ]]; then + log "kept/failed image $iid (one or more tag refs failed to untag; not force-removing)" + continue + fi + + # Dangling images (no refs) or an id whose underlying layers survived + # untagging (should not happen once every ref above succeeded, but the + # `docker image ls` id may still resolve if this id was ALSO a parent + # layer of another image) still need this final by-id remove. + if docker image inspect "$iid" >/dev/null 2>&1; then + if docker rmi "$iid" >>"$LOG_FILE" 2>&1; then + log "removed image $iid" + else + log "kept/failed image $iid (likely still referenced)" + fi + else + log "removed image $iid (freed by untag)" + fi + done <<< "$IMAGE_REFS_TSV" fi log "Done. df after:" -df -h /data 2>/dev/null | tee -a "$LOG_FILE" >&2 || df -h / | tee -a "$LOG_FILE" >&2 +DF_OUT="$(df -h /data 2>/dev/null || df -h /)" +echo "$DF_OUT" | tee -a "$LOG_FILE" >&2 + +# OMN-15804: surface the watermark threshold (shared with disk-watermark-check.sh, +# no new alerting path — this is a log-line-only warning) directly in the GC +# summary so a breach is visible without cross-referencing a second log file. +DF_USED_PCT="$(echo "$DF_OUT" | awk 'NR==2 {gsub(/%/,"",$5); print $5}')" +WATERMARK_WARN_PCT=85 +if [[ "$DF_USED_PCT" =~ ^[0-9]+$ ]] && [[ "$DF_USED_PCT" -ge "$WATERMARK_WARN_PCT" ]]; then + log "WARNING: disk usage ${DF_USED_PCT}% >= watermark ${WATERMARK_WARN_PCT}% — see disk-watermark-check.sh" +fi + exit 0 diff --git a/scripts/disk_gc_plan.py b/scripts/disk_gc_plan.py index 1239356216..e75c4e72aa 100644 --- a/scripts/disk_gc_plan.py +++ b/scripts/disk_gc_plan.py @@ -22,6 +22,19 @@ { "min_age_days": int, "remove_image_ids": [str, ...], + "remove_image_refs": {image_id: [repo:tag, ...]}, # OMN-15804: every tag + # reference an id carries, + # so the executor can + # untag-then-remove a + # multi-tag image instead + # of `docker rmi `, + # which docker refuses + # ("referenced in + # multiple repositories") + # once >1 repo:tag point + # at the same id. Empty + # list = dangling (no tag + # to untag), remove by id. "remove_container_ids": [str, ...], "kept_reasons": {image_id: reason, ...} # for dry-run transparency } @@ -33,6 +46,13 @@ - never remove anything younger than min_age_days - retain the newest superseded_image_keep_generations of each kept repo +In-use matching (OMN-15804): `inuse_refs` entries can be a repo:tag string OR an +image ID, and image IDs can appear TRUNCATED (12 hex chars, as printed by +`docker ps --format '{{.Image}}'` when a container was started directly by ID) +or FULL (`sha256:<64hex>`, as printed by `docker image ls --no-trunc`). Exact +string equality misses the truncated-vs-full case, so matching also compares +the normalized 12-hex-char short form of both sides. + PR-state reaping invariants (OMN-13225, tested in test_disk_gc_pr_state.py): - ghcr CI tags pr- / sha-* are DISPOSABLE and bypassed the age/generation window ONLY when their associated PR is merged or closed (via pr_state_lookup) @@ -100,6 +120,36 @@ def _repo_protected(repo: str, keep_repos: list[str]) -> bool: return any(sub in repo for sub in keep_repos) +_HEX_ID_RE = re.compile(r"^[0-9a-f]{12,64}$") + + +def _short_id(value: str) -> str | None: + """Return the 12-hex-char short form of an image ID, or None if `value` + is not a hex image ID at all (OMN-15804). + + `docker image ls --no-trunc` prints `sha256:<64hex>`; `docker ps --format + '{{.Image}}'` prints the bare 12-hex short form when a container was + started directly by image ID (not by repo:tag). Only hex-ID-shaped + strings are normalized — a repo:tag ref (e.g. "myrepo:v1") never matches + this shape, so it can never false-positive-collide with a short id. + """ + v = value.strip() + if v.startswith("sha256:"): + v = v[len("sha256:") :] + if not _HEX_ID_RE.match(v): + return None + return v[:12] + + +def _is_in_use(image_id: str, ref: str, inuse_refs: set[str]) -> bool: + if image_id in inuse_refs or ref in inuse_refs: + return True + short = _short_id(image_id) + if short is None: + return False + return any(short == _short_id(entry) for entry in inuse_refs) + + def _is_disposable_ci_tag(tag: str) -> bool: """Return True if the tag looks like a disposable CI artifact (pr-N or sha-*).""" return bool(_PR_TAG_RE.match(tag) or _SHA_TAG_RE.match(tag)) @@ -186,6 +236,12 @@ def build_plan( remove_image_ids: list[str] = [] kept_reasons: dict[str, str] = {} + # Every repo:tag ref seen for a given image id, across ALL rows (not just + # removal candidates) — OMN-15804: the executor needs the full tag set to + # untag-then-remove a multi-tag image (`docker rmi ` alone fails with + # "referenced in multiple repositories" once >1 tag points at one id). + refs_by_id: dict[str, list[str]] = {} + # Group superseded candidates per repo so we can keep the N newest. superseded_by_repo: dict[str, list[tuple[float, str]]] = {} @@ -197,12 +253,19 @@ def build_plan( age = _parse_created_at(created, now) ref = f"{repo}:{tag}" if repo and tag and repo != "" else image_id + if repo and tag and repo != "" and tag != "": + tag_list = refs_by_id.setdefault(image_id, []) + if ref not in tag_list: + tag_list.append(ref) + else: + refs_by_id.setdefault(image_id, []) + # --- Hard safety guards (always win, regardless of PR state) --- if tag in keep_tags and tag and tag != "": kept_reasons[image_id] = f"tag '{tag}' in keep_image_tags" continue - if protect_running and (ref in inuse_refs or image_id in inuse_refs): + if protect_running and _is_in_use(image_id, ref, inuse_refs): kept_reasons[image_id] = "referenced by a container (protect_running)" continue @@ -300,9 +363,14 @@ def build_plan( safe_remove.append(image_id) remove_image_ids = safe_remove + remove_image_refs: dict[str, list[str]] = { + image_id: refs_by_id.get(image_id, []) for image_id in remove_image_ids + } + return { "min_age_days": min_age_days, "remove_image_ids": remove_image_ids, + "remove_image_refs": remove_image_refs, "remove_container_ids": remove_container_ids, "kept_reasons": kept_reasons, } diff --git a/scripts/enforcement_parity_manifest.yaml b/scripts/enforcement_parity_manifest.yaml index cc3631ad49..d15ec8ad69 100644 --- a/scripts/enforcement_parity_manifest.yaml +++ b/scripts/enforcement_parity_manifest.yaml @@ -82,6 +82,31 @@ repos: coverage: direct rule: >- OMN-14556: node-migration-sync.yml is its OWN workflow file (separate run_id from ci.yml), so ci_summary_gate.py's poll of actions/runs/${RUN_ID}/jobs structurally cannot observe it -- a needs_child declaration would be a false COVERED. Confirmed live on PR #2288 (2026-07-13): node-migration-sync=FAIL, CI Summary=PASS, different run_ids. Absent from required_status_checks today (MISSING) -- this entry makes the report-only ratchet flag it until the branch-protection PUT lands. + # OMN-15378 AC3 -- RESOLVED, so the former `deploy-agent-tests` direct-gate + # entry is deliberately GONE rather than left as a permanent MISSING finding. + # The gap it recorded was real: deploy-agent-tests.yml was its own + # separately-triggered workflow (own run_id), so ci_summary_gate.py's poll of + # actions/runs/${RUN_ID}/jobs structurally could not observe it and a RED run + # left "CI Summary" -- the sole required context here -- green. The fix was NOT + # the branch-protection PUT this entry anticipated: that workflow is path-filtered + # (scripts/deploy-agent/**), and a required context that does not report on every + # PR shape wedges merges indefinitely. Instead ci.yml now CALLS the workflow from + # an unconditional `deploy-agent-tests` job, so it runs inside ci.yml's own run as + # the check-run "Deploy Agent Tests (OMN-15378) / deploy-agent-tests", registered + # in scripts/ci/ci_summary_gate.py STRICT_GATE_JOBS -- absent/red/skipped all fail + # the required context. Pinned by tests/ci/test_ci_summary_gate.py + # (test_deploy_agent_tests_gate_is_strict_and_fails_closed + + # test_every_gate_name_resolves_to_a_ci_yml_job). + # + # RESIDUAL (flagged, not silently dropped): this manifest has no coverage mode + # for "strict-gated by the CI Summary poller". `direct` is false (no own + # context) and `needs_child` is false (ci-summary is a NO-`needs` poller, so a + # needs-closure check reports NEEDS_CLOSURE for every poller-strict gate). Until + # a `poller_strict_gate` mode exists -- verifying aggregator-required AND + # membership in STRICT_GATE_JOBS AND job-present-in-ci.yml -- the ratchet cannot + # audit this class, and the tests above are the enforcement of record. The same + # limitation applies to the 17 other STRICT_GATE_JOBS entries, none of which this + # manifest declares either. # --- The two repos the audit CONFIRMED are missing the gates (M1-M4) --- omnimarket: dev: diff --git a/scripts/gateway_restart_safety_proof.sh b/scripts/gateway_restart_safety_proof.sh new file mode 100755 index 0000000000..35db6f4d67 --- /dev/null +++ b/scripts/gateway_restart_safety_proof.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# gateway_restart_safety_proof.sh -- OMN-15521 remediation: a real, +# executable restart-durability smoke proof for the OMN-12912 gateway +# idempotency store, run against the ALREADY-DEPLOYED omninode-gateway +# lane on `.201`. +# +# What this is NOT: the full cross-broker at-least-once/exactly-once +# redelivery proof (an in-flight message deliberately killed mid-delivery, +# then confirmed not to duplicate or drop on the far side). That needs a +# synthetic in-flight cloud MSK message and is OMN-12912's own test suite's +# job -- see PR #2556's own scope-boundary note: "Docker Compose/runtime +# proof is delegated to hosted CI because neither the development Mac nor +# .200 has a Docker CLI", i.e. that proof has never run against a REAL +# deployed container. +# +# What this IS: a concrete, mechanical proof that the durable SQLite +# idempotency store (WAL + synchronous=FULL, per OMN-12912's PR +# description) actually survives a real container restart on the real box, +# and that the container comes back genuinely healthy rather than +# false-green -- the restart-safety half of OMN-12912's own scope ("make +# delivery-loop or heartbeat failure remove readiness ... instead of +# leaving a false-green gateway"). +# +# The receipt this prints is meant to be pasted into an OMN-12912 comment +# (per OMN-15521's own AC5 wording: "that receipt lands on OMN-12912, not +# this ticket") -- this script does not file it anywhere itself. +# +# Usage: +# ./scripts/gateway_restart_safety_proof.sh # run the proof +# ./scripts/gateway_restart_safety_proof.sh --help # usage + +set -euo pipefail + +SCRIPT_NAME="$(basename "$0")" +readonly SCRIPT_NAME + +CONTAINER_NAME="${GATEWAY_RESTART_PROOF_CONTAINER:-omninode-gateway-forwarder}" +SYSTEMD_UNIT="${GATEWAY_RESTART_PROOF_UNIT:-onex-gateway-forwarder}" +SQLITE_PATH="${GATEWAY_RESTART_PROOF_SQLITE_PATH:-/app/data/gateway/delivery.sqlite3}" +HEALTHY_TIMEOUT_SECONDS="${GATEWAY_RESTART_PROOF_HEALTHY_TIMEOUT_SECONDS:-120}" + +log_info() { printf '[gateway-restart-proof] %s\n' "$*"; } +log_warn() { printf '[gateway-restart-proof] WARNING: %s\n' "$*" >&2; } +log_error() { printf '[gateway-restart-proof] ERROR: %s\n' "$*" >&2; } +log_step() { printf '\n[gateway-restart-proof] === %s ===\n' "$*"; } + +usage() { + cat <\t" read from the running + # container's idempotency store via the stdlib sqlite3 module (no + # sqlite3 CLI binary is installed in the runtime image -- see + # docker/Dockerfile.runtime's minimal apt-get install list). Read-only; + # "0\t0" if the store does not exist yet (container never processed + # anything). Callers MUST call require_reachable() first -- this function + # no longer distinguishes "genuinely empty store" from "container + # unreachable" itself (both used to collapse to "0\t0", which made the + # after/dev/null || printf '0\t0\n' +} + +require_reachable() { + # Fail loudly (not a silent 0\t0) if the container cannot be reached via + # `docker exec` at all -- distinct from a genuinely empty idempotency + # store, which is a legitimate (if weak) reading. Without this, an + # unreachable container after the restart made before_count/after_count + # both resolve to 0 via snapshot()'s own exception handler, the + # after/dev/null; then + log_error "AC5-PROOF FAILED: ${CONTAINER_NAME} is not reachable via 'docker exec' (${label}) -- cannot prove restart safety against a container that cannot be inspected." + exit 1 + fi +} + +container_identity() { + # "|" -- proves a restart actually RECREATED the + # container, not merely that a health probe happened to return "healthy" + # against a stale, never-touched container (the exact false-green a fake + # `sudo systemctl reload` that exits 0 and does nothing produces: the + # health check and row-count check both pass trivially because nothing + # changed). Empty if the container does not exist / is not inspectable. + docker inspect "${CONTAINER_NAME}" --format '{{.Id}}|{{.State.StartedAt}}' 2>/dev/null || true +} + +wait_healthy() { + local timeout="$1" waited=0 + while (( waited < timeout )); do + if [[ "$(docker inspect "${CONTAINER_NAME}" --format '{{.State.Health.Status}}' 2>/dev/null || true)" == "healthy" ]]; then + return 0 + fi + sleep 3 + waited=$((waited + 3)) + done + return 1 +} + +main() { + log_step "Restart-safety proof: ${CONTAINER_NAME}" + log_info "Idempotency store: ${SQLITE_PATH}" + + local before_identity + before_identity="$(container_identity)" + if [[ -z "${before_identity}" ]]; then + log_error "AC5-PROOF FAILED: ${CONTAINER_NAME} is not inspectable before the restart -- cannot prove restart safety against a container that isn't running." + exit 1 + fi + + require_reachable "before restart" + + local before before_count before_max + before="$(snapshot)" + before_count="${before%%$'\t'*}" + before_max="${before#*$'\t'}" + log_info "Before restart: rows=${before_count} max_processed_at=${before_max:-}" + + if [[ "${before_count}" == "0" ]]; then + log_warn "No idempotency records exist yet -- this run will only show an EMPTY store surviving a restart, which is trivially true and is not evidence of durability. Re-run once real traffic has flowed." + fi + + log_step "Reload ${SYSTEMD_UNIT}" + log_info "Same mechanism scripts/deploy-gateway.sh's --execute uses to recreate the container." + sudo systemctl reload "${SYSTEMD_UNIT}" + + if ! wait_healthy "${HEALTHY_TIMEOUT_SECONDS}"; then + log_error "AC5-PROOF FAILED: ${CONTAINER_NAME} did not report Docker-healthy within ${HEALTHY_TIMEOUT_SECONDS}s of restart." + exit 1 + fi + log_info "Container reports healthy after restart." + + local after_identity + after_identity="$(container_identity)" + if [[ -z "${after_identity}" ]]; then + log_error "AC5-PROOF FAILED: ${CONTAINER_NAME} is not inspectable after the restart." + exit 1 + fi + if [[ "${after_identity}" == "${before_identity}" ]]; then + log_error "AC5-PROOF FAILED: ${CONTAINER_NAME} was NOT actually recreated by 'systemctl reload ${SYSTEMD_UNIT}' (container identity unchanged: ${before_identity}). A reload that exits 0 and reports healthy without recreating the container is a false-green, not a real restart -- 'healthy' alone (checked above) is not sufficient proof." + exit 1 + fi + log_info "Container identity changed across restart (${before_identity} -> ${after_identity}): a real restart occurred, not a stale no-op." + + require_reachable "after restart" + + local after after_count after_max + after="$(snapshot)" + after_count="${after%%$'\t'*}" + after_max="${after#*$'\t'}" + log_info "After restart: rows=${after_count} max_processed_at=${after_max:-}" + + if (( after_count < before_count )); then + log_error "AC5-PROOF FAILED: idempotency record count dropped from ${before_count} to ${after_count} across restart -- the durable marker store did NOT survive." + exit 1 + fi + + log_step "Done" + log_info "AC5-PROOF OK: idempotency store survived restart (rows ${before_count} -> ${after_count}, no durable-marker loss); container was genuinely recreated and returned healthy (not false-green)." + log_info "File this receipt on OMN-12912 (not OMN-15521, per that ticket's own AC5 filing instruction) -- this script does not post it anywhere itself." +} + +main "$@" diff --git a/scripts/generate_application_database_acl.py b/scripts/generate_application_database_acl.py new file mode 100644 index 0000000000..57906f221a --- /dev/null +++ b/scripts/generate_application_database_acl.py @@ -0,0 +1,279 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Generate the one-database ACL matrix from immutable Git blobs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +from pathlib import Path +from typing import Literal + +import yaml +from pydantic import BaseModel, ConfigDict + +from omnibase_core.models.core.model_deployment_topology import ModelDeploymentTopology +from omnibase_infra.validation.application_database_acl import ( + build_application_database_acl_matrix, + render_application_database_acl_sql, +) +from omnibase_infra.validation.enums.enum_application_database_acl_authorization_scope import ( + EnumApplicationDatabaseAclAuthorizationScope, +) +from omnibase_infra.validation.enums.enum_application_database_acl_render_phase import ( + EnumApplicationDatabaseAclRenderPhase, +) +from omnibase_infra.validation.models.model_application_database_acl_policy import ( + ModelApplicationDatabaseAclPolicy, +) +from omnibase_infra.validation.models.model_application_database_acl_source import ( + ModelApplicationDatabaseAclSource, +) +from omnibase_infra.validation.models.model_application_database_activity_result_evidence import ( + ModelApplicationDatabaseActivityResultEvidence, +) +from omnibase_infra.validation.models.model_application_database_catalog_result_evidence import ( + ModelApplicationDatabaseCatalogResultEvidence, +) +from omnibase_infra.validation.models.model_application_database_principal_inventory import ( + ModelApplicationDatabasePrincipalInventory, +) +from omnibase_infra.validation.models.model_application_relation_evidence_inventory import ( + ModelApplicationRelationEvidenceInventory, +) +from omnibase_infra.validation.models.model_migration_ownership_manifest import ( + ModelMigrationOwnershipManifest, +) + + +class _SourceLock(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: Literal["1.0"] + required_connect_databases: tuple[str, ...] + sources: tuple[ModelApplicationDatabaseAclSource, ...] + + +class _IndentedSafeDumper(yaml.SafeDumper): + """Match the repository YAML formatter's nested-sequence indentation.""" + + def increase_indent( + self, + flow: bool = False, + indentless: bool = False, + ) -> None: + return super().increase_indent(flow=flow, indentless=False) + + +def _parse_repository_roots(values: list[str]) -> dict[str, Path]: + result: dict[str, Path] = {} + for value in values: + repository, separator, raw_path = value.partition("=") + if not separator or not repository or not raw_path: + raise ValueError("--repository-root must be REPOSITORY=/absolute/git/clone") + root = Path(raw_path).resolve() + if not root.is_dir(): + raise ValueError(f"Repository root does not exist: {root}") + result[repository] = root + return result + + +def _git_blob(root: Path, source: ModelApplicationDatabaseAclSource) -> bytes: + result = subprocess.run( + ["git", "-C", str(root), "show", f"{source.revision}:{source.path}"], + check=False, + capture_output=True, + ) + if result.returncode != 0: + detail = result.stderr.decode("utf-8", errors="replace").strip() + raise ValueError( + f"Cannot read {source.source_key} from {source.repository}@" + f"{source.revision}: {detail}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != source.sha256: + raise ValueError( + f"Source-lock digest mismatch for {source.source_key}: " + f"expected {source.sha256}, got {digest}" + ) + return result.stdout + + +def _mapping(blob: bytes, source_id: str) -> object: + try: + if blob.lstrip().startswith((b"{", b"[")): + return json.loads(blob) + return yaml.safe_load(blob) + except (json.JSONDecodeError, yaml.YAMLError) as exc: + raise ValueError(f"Cannot parse locked source {source_id}: {exc}") from exc + + +def _write_yaml(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + yaml.dump( + value, + Dumper=_IndentedSafeDumper, + default_flow_style=False, + sort_keys=True, + width=1_000_000, + ), + encoding="utf-8", + ) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--source-lock", + type=Path, + default=Path("docker/application-acl-proof/source-lock.yaml"), + ) + parser.add_argument( + "--repository-root", + action="append", + default=[], + metavar="REPOSITORY=PATH", + help="Local Git clone containing every locked revision (repeatable)", + ) + parser.add_argument("--matrix-output", type=Path, required=True) + parser.add_argument("--sql-output", type=Path) + parser.add_argument( + "--render-phase", + type=EnumApplicationDatabaseAclRenderPhase, + choices=tuple(EnumApplicationDatabaseAclRenderPhase), + default=EnumApplicationDatabaseAclRenderPhase.FULL, + help="Render the additive scaffold before full object materialization", + ) + parser.add_argument( + "--allow-blocked-matrix", + action="store_true", + help="Write candidate evidence but never SQL when dependencies are incomplete", + ) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + lock = _SourceLock.model_validate( + yaml.safe_load(args.source_lock.read_text(encoding="utf-8")) + ) + roots = _parse_repository_roots(args.repository_root) + missing_roots = sorted( + {source.repository for source in lock.sources} - roots.keys() + ) + if missing_roots: + raise ValueError(f"Missing --repository-root values: {missing_roots!r}") + + blobs = { + source.source_key: _git_blob(roots[source.repository], source) + for source in lock.sources + } + topology_sources = [ + source for source in lock.sources if source.purpose == "topology" + ] + if len(topology_sources) != 1: + raise ValueError("Source lock must contain exactly one topology source") + topology_source = topology_sources[0] + topology = ModelDeploymentTopology.model_validate( + _mapping(blobs[topology_source.source_key], topology_source.source_key) + ) + inventories = { + source.source_key: ModelApplicationRelationEvidenceInventory.model_validate( + _mapping(blobs[source.source_key], source.source_key) + ) + for source in lock.sources + if source.purpose == "relation_inventory" + } + manifests = { + source.source_key: ModelMigrationOwnershipManifest.model_validate( + _mapping(blobs[source.source_key], source.source_key) + ) + for source in lock.sources + if source.purpose == "service_ownership" + } + principal_inventories = { + source.source_key: ModelApplicationDatabasePrincipalInventory.model_validate( + _mapping(blobs[source.source_key], source.source_key) + ) + for source in lock.sources + if source.purpose == "principal_inventory" + } + catalog_results = { + source.source_key: ModelApplicationDatabaseCatalogResultEvidence.model_validate( + _mapping(blobs[source.source_key], source.source_key) + ) + for source in lock.sources + if source.purpose == "catalog_result_evidence" + } + activity_results = { + source.source_key: ModelApplicationDatabaseActivityResultEvidence.model_validate( + _mapping(blobs[source.source_key], source.source_key) + ) + for source in lock.sources + if source.purpose == "activity_result_evidence" + } + acl_policies = { + source.source_key: ModelApplicationDatabaseAclPolicy.model_validate( + _mapping(blobs[source.source_key], source.source_key) + ) + for source in lock.sources + if source.purpose == "acl_policy" + } + matrix = build_application_database_acl_matrix( + topology=topology, + sources=lock.sources, + relation_inventories=inventories, + service_manifests=manifests, + principal_inventories=principal_inventories, + acl_policies=acl_policies, + authorization_scope=EnumApplicationDatabaseAclAuthorizationScope.DEPLOYMENT, + required_connect_databases=lock.required_connect_databases, + catalog_results=catalog_results, + activity_results=activity_results, + ) + _write_yaml(args.matrix_output, matrix.model_dump(mode="json")) + print( + f"matrix_status={matrix.status} scaffold_status={matrix.scaffold_status} " + f"objects={len(matrix.objects)} " + f"rows={len(matrix.rows)} defaults={len(matrix.default_privileges)} " + f"blockers={len(matrix.blockers)} " + f"scaffold_blockers={len(matrix.scaffold_blockers)}" + ) + phase_blocked = ( + matrix.status == "BLOCKED" + if args.render_phase is EnumApplicationDatabaseAclRenderPhase.FULL + else matrix.scaffold_status == "BLOCKED" + ) + phase_blockers = ( + matrix.blockers + if args.render_phase is EnumApplicationDatabaseAclRenderPhase.FULL + else matrix.scaffold_blockers + ) + if phase_blocked: + for blocker in phase_blockers: + print(f"blocker={blocker}") + if args.sql_output is not None and args.sql_output.exists(): + raise ValueError( + "Refusing to overwrite an existing SQL output from a blocked phase" + ) + return 0 if args.allow_blocked_matrix else 2 + if args.sql_output is not None: + args.sql_output.parent.mkdir(parents=True, exist_ok=True) + args.sql_output.write_text( + render_application_database_acl_sql(matrix, phase=args.render_phase), + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, ValueError) as exc: + print(f"application ACL generation failed: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/scripts/generate_application_database_table_grants.py b/scripts/generate_application_database_table_grants.py new file mode 100644 index 0000000000..b818f9fcc3 --- /dev/null +++ b/scripts/generate_application_database_table_grants.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Generate and drift-check application-database ``TABLE`` grants (OMN-15656). + +The typed topology instances under ``src/omnibase_infra/topology/instances`` are +the platform authority for which principal may touch which relation. Their +``object_type: TABLE`` grants are a **projection of node contract** +``db_io.db_tables`` **declarations** — this script is the only sanctioned way to +write them. Hand-listing is what ADR-0027 exists to remove. + +Modes +----- +``--write`` + Regenerate the TABLE grants in every instance from the contracts. +``--check`` + Fail when the checked-in grants differ from the derivation, in either + direction: a contract-declared table with no grant, or a granted table no + contract declares. +``--prove`` + Resolve every contract declaration through the real + ``_resolve_projection_database_target`` against the real shipped + ``load_topology_profile(profile)`` for every supported profile, and report + ``PASS``/``FAIL`` per profile. This is the end-to-end assertion that the + grants actually satisfy the OMN-15418 validator. + +``--check`` and ``--prove`` compose; CI runs both. + +After ``--write`` the rendered catalogs must be regenerated with +``scripts/render_application_database_topology.py`` — the topology unit tests +fail closed if they drift. +""" + +from __future__ import annotations + +import argparse +import sys +from collections.abc import Sequence +from pathlib import Path + +import yaml + +from omnibase_core.enums.enum_database_grant_object_type import ( + EnumDatabaseGrantObjectType, +) +from omnibase_core.models.core import ModelDeploymentTopology +from omnibase_infra.topology import load_topology_profile +from omnibase_infra.topology.application_database import ( + SUPPORTED_TOPOLOGY_PROFILES, + TOPOLOGY_PROFILE_INSTANCE_MAP, +) +from omnibase_infra.topology.table_grant_derivation import ( + STATE_IO_TABLE_DECLARATIONS, + ContractTableDeclaration, + TopologyTableGrants, + derive_topology_table_grants, + load_contract_declarations, +) + +_REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +_INSTANCE_ROOT = _REPOSITORY_ROOT / "src" / "omnibase_infra" / "topology" / "instances" +DEFAULT_CONTRACTS_ROOT = ( + _REPOSITORY_ROOT + / ".proof-dependencies" + / "omnimarket" + / "src" + / "omnimarket" + / "nodes" +) + +# Every instance that must carry the derived grants. The three files are +# byte-identical in their principal blocks by construction; this script is the +# mechanism that keeps them so, because the topology schema has no include or +# inheritance surface. +INSTANCE_NAMES = tuple(sorted(set(TOPOLOGY_PROFILE_INSTANCE_MAP.values()))) + + +class _FlowList(list[object]): + """Sequence rendered inline to preserve the hand-authored grant style.""" + + +class _InstanceDumper(yaml.SafeDumper): + """Dumper that reproduces the checked-in instance formatting exactly.""" + + def increase_indent(self, flow: bool = False, indentless: bool = False) -> None: + return super().increase_indent(flow, False) + + +def _represent_flow_list(dumper: yaml.SafeDumper, data: _FlowList) -> yaml.Node: + return dumper.represent_sequence("tag:yaml.org,2002:seq", data, flow_style=True) + + +def _represent_str(dumper: yaml.SafeDumper, data: str) -> yaml.Node: + """Double-quote scalars that would otherwise stop resolving as strings. + + Keeps ``schema_version: "2.0"`` intact instead of round-tripping it into + the single-quoted form, so a regeneration diff shows only real changes. + """ + style: str | None = None + try: + if not isinstance(yaml.safe_load(data), str): + style = '"' + except yaml.YAMLError: + style = '"' + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style) + + +_InstanceDumper.add_representer(_FlowList, _represent_flow_list) +_InstanceDumper.add_representer(str, _represent_str) + + +def _mark_flow(node: object) -> object: + """Render ``privileges`` inline and everything else in block style.""" + if isinstance(node, dict): + return { + key: _FlowList(value) if key == "privileges" else _mark_flow(value) + for key, value in node.items() + } + if isinstance(node, list): + return [_mark_flow(item) for item in node] + return node + + +def _grant_to_document(grant: object) -> dict[str, object]: + payload = grant.model_dump(mode="json", exclude_defaults=False) # type: ignore[attr-defined] + document: dict[str, object] = {"object_type": payload["object_type"]} + if payload.get("objects"): + document["objects"] = list(payload["objects"]) + document["privileges"] = list(payload["privileges"]) + if payload.get("schema") is not None: + document["schema"] = payload["schema"] + return document + + +def _render_instance(path: Path, derived: TopologyTableGrants) -> str: + """Return the instance text with derived TABLE grants substituted in. + + Every logical database in the instance is rendered, not just + ``application``: the omniintelligence service database (OMN-15655 AC-2) + carries its own principal, and a renderer scoped to one database would + leave that principal permanently grant-less while ``--check`` reported + green. + """ + original = path.read_text(encoding="utf-8") + document = yaml.safe_load(original) + for database_ref, database in document["databases"].items(): + database_grants = derived.per_database.get(database_ref) + for principal_name, principal in database["principals"].items(): + # Drop every existing TABLE grant: this script owns that subset + # entirely, so a stale entry must not survive a regeneration. + retained = [ + grant + for grant in principal.get("grants", []) + if grant.get("object_type") != EnumDatabaseGrantObjectType.TABLE.value + ] + generated = ( + [] + if database_grants is None + else [ + _grant_to_document(grant) + for grant in database_grants.grants.get(principal_name, ()) + ] + ) + principal["grants"] = retained + generated + header = "".join( + line for line in original.splitlines(keepends=True) if line.startswith("#") + ) + body = yaml.dump( + _mark_flow(document), + Dumper=_InstanceDumper, + sort_keys=True, + default_flow_style=False, + width=4096, + allow_unicode=True, + ) + return header + body + + +def _derivation_for_instance( + instance_name: str, declarations: Sequence[ContractTableDeclaration] +) -> TopologyTableGrants: + topology = ModelDeploymentTopology.from_yaml( + _INSTANCE_ROOT / f"{instance_name}.yaml" + ) + return derive_topology_table_grants(topology, declarations) + + +def _run_write(declarations: Sequence[ContractTableDeclaration]) -> int: + changed = [] + for instance_name in INSTANCE_NAMES: + path = _INSTANCE_ROOT / f"{instance_name}.yaml" + rendered = _render_instance( + path, _derivation_for_instance(instance_name, declarations) + ) + if rendered != path.read_text(encoding="utf-8"): + path.write_text(rendered, encoding="utf-8") + changed.append(instance_name) + print(f"instances updated: {', '.join(changed) if changed else '(none)'}") + print( + "reminder: regenerate the rendered catalogs with " + "scripts/render_application_database_topology.py" + ) + return 0 + + +def _run_check(declarations: Sequence[ContractTableDeclaration]) -> int: + failures: list[str] = [] + for instance_name in INSTANCE_NAMES: + path = _INSTANCE_ROOT / f"{instance_name}.yaml" + expected = _render_instance( + path, _derivation_for_instance(instance_name, declarations) + ) + if expected != path.read_text(encoding="utf-8"): + failures.append( + f"{path.relative_to(_REPOSITORY_ROOT)} drifted from the contract " + "derivation; regenerate with " + "scripts/generate_application_database_table_grants.py --write" + ) + for failure in failures: + print(f"::error::{failure}", file=sys.stderr) + if not failures: + print(f"grant derivation check: {len(INSTANCE_NAMES)} instance(s) in sync") + return 1 if failures else 0 + + +def _run_prove(declarations: Sequence[ContractTableDeclaration]) -> int: + # Imported lazily so --write/--check do not depend on the wiring module's + # import cost, and so the private resolver import stays localised. + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _resolve_projection_database_target, + ) + + by_node: dict[str, list[ContractTableDeclaration]] = {} + for declaration in declarations: + by_node.setdefault(declaration.node, []).append(declaration) + + overall_failures = 0 + for profile in sorted(SUPPORTED_TOPOLOGY_PROFILES): + topology = load_topology_profile(profile) + derived = derive_topology_table_grants(topology, declarations) + residual_keys = {residual.key for residual in derived.unmappable} + passed = 0 + failed: list[str] = [] + allowed: list[str] = [] + for node, node_declarations in sorted(by_node.items()): + tables = tuple(item.table for item in node_declarations) + node_residuals = [ + (table.database_ref, table.schema, table.name) + for table in tables + if (table.database_ref, table.schema, table.name) in residual_keys + ] + try: + _resolve_projection_database_target(tables, topology) + except ValueError as exc: + if node_residuals: + allowed.append(f"{node}: {exc}") + else: + failed.append(f"{node}: {exc}") + else: + passed += 1 + overall_failures += len(failed) + print( + f"profile={profile:<15} PASS={passed:>3} " + f"FAIL={len(failed):>3} RESIDUAL={len(allowed):>3}" + ) + for message in failed: + print(f" ::error::{message}", file=sys.stderr) + if overall_failures: + print( + f"::error::{overall_failures} contract/profile resolution failure(s)", + file=sys.stderr, + ) + return 1 if overall_failures else 0 + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--contracts-root", + type=Path, + default=DEFAULT_CONTRACTS_ROOT, + help="Directory containing node contract.yaml files (cross-repo checkout).", + ) + parser.add_argument( + "--write", action="store_true", help="Rewrite instance TABLE grants." + ) + parser.add_argument( + "--check", action="store_true", help="Fail on grant/contract drift." + ) + parser.add_argument( + "--prove", + action="store_true", + help="Resolve every contract against every shipped profile.", + ) + args = parser.parse_args(argv) + if not (args.write or args.check or args.prove): + parser.error("one of --write, --check, or --prove is required") + if args.write and args.check: + parser.error("--write and --check are mutually exclusive") + + declarations = ( + load_contract_declarations(args.contracts_root) + STATE_IO_TABLE_DECLARATIONS + ) + print( + f"loaded {len(declarations) - len(STATE_IO_TABLE_DECLARATIONS)} " + f"db_io.db_tables declaration(s) from {args.contracts_root} plus " + f"{len(STATE_IO_TABLE_DECLARATIONS)} state_io declaration(s)" + ) + + exit_code = 0 + if args.write: + exit_code |= _run_write(declarations) + if args.check: + exit_code |= _run_check(declarations) + if args.prove: + exit_code |= _run_prove(declarations) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/git-hooks/canonical-clone/commit-msg b/scripts/git-hooks/canonical-clone/commit-msg new file mode 120000 index 0000000000..eb1aa510bc --- /dev/null +++ b/scripts/git-hooks/canonical-clone/commit-msg @@ -0,0 +1 @@ +../canonical_clone_guard.sh \ No newline at end of file diff --git a/scripts/git-hooks/canonical-clone/pre-commit b/scripts/git-hooks/canonical-clone/pre-commit new file mode 120000 index 0000000000..eb1aa510bc --- /dev/null +++ b/scripts/git-hooks/canonical-clone/pre-commit @@ -0,0 +1 @@ +../canonical_clone_guard.sh \ No newline at end of file diff --git a/scripts/git-hooks/canonical-clone/pre-merge-commit b/scripts/git-hooks/canonical-clone/pre-merge-commit new file mode 120000 index 0000000000..eb1aa510bc --- /dev/null +++ b/scripts/git-hooks/canonical-clone/pre-merge-commit @@ -0,0 +1 @@ +../canonical_clone_guard.sh \ No newline at end of file diff --git a/scripts/git-hooks/canonical-clone/pre-push b/scripts/git-hooks/canonical-clone/pre-push new file mode 120000 index 0000000000..eb1aa510bc --- /dev/null +++ b/scripts/git-hooks/canonical-clone/pre-push @@ -0,0 +1 @@ +../canonical_clone_guard.sh \ No newline at end of file diff --git a/scripts/git-hooks/canonical_clone_guard.sh b/scripts/git-hooks/canonical_clone_guard.sh new file mode 100755 index 0000000000..f39aced85b --- /dev/null +++ b/scripts/git-hooks/canonical_clone_guard.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# Canonical-clone worktree-discipline guard, chained into the real hook chain +# (OMN-7018 guard + OMN-15071 chaining fix). +# +# Installed on a host by pointing `core.hooksPath` at the sibling +# `canonical-clone/` directory, whose per-hook-type symlinks all resolve to this +# script. `core.hooksPath` REPLACES git's hook lookup outright -- git never falls +# back to `$GIT_COMMON_DIR/hooks/` -- so anything this script does not explicitly +# invoke simply does not run. +# +# OMN-15071: the pre-chaining revision of this guard `exit 0`-ed for every +# worktree path, which meant that on `.200` -- the host root CLAUDE.md rule 11a +# makes the DEFAULT target for pushes and gate runs -- EVERY `git commit` in a +# worktree ran ZERO hooks and reported success. Silently: the guard printed +# nothing, so a clean commit was indistinguishable from a commit that had passed +# every gate. Concrete false negative (2026-07-30): a pattern-ratchet violation +# was correctly rejected on the Mac (no `core.hooksPath` override there) and +# committed clean on `.200`; it was caught by ordering luck, not by a gate. +# +# Behaviour: +# 1. Commits in a registry canonical clone are refused (the OMN-7018 rule: +# canonical clones are pull/index mirrors, work happens in a worktree). +# 2. Everywhere the guard permits the operation, it CHAINS to the real hook of +# the same type instead of returning success on its own -- the installed +# pre-commit-framework hook when one is present, otherwise `pre-commit +# hook-impl` directly when the repo carries a `.pre-commit-config.yaml`. +# 3. If the repo has a pre-commit config but no runnable pre-commit at all, +# the hook FAILS CLOSED rather than reporting a vacuous success. +# +# Escape hatch (pre-existing, unchanged): `ALLOW_CANONICAL_CLONE_COMMIT=1` +# suppresses the canonical-clone refusal only. It does not skip the chain. + +set -euo pipefail + +hook_name="$(basename "$0")" + +# --- path helpers ----------------------------------------------------------- + +resolve_path() { + # Portable realpath: macOS `readlink` has no -f. Resolves symlinks in the + # final component, which is all this script needs (the hook symlinks). + local target="$1" + local dir base link hops=0 + while [[ -L "$target" ]]; do + hops=$((hops + 1)) + if [[ "$hops" -gt 32 ]]; then + printf 'ERROR: symlink loop resolving %s\n' "$1" >&2 + exit 1 + fi + link="$(readlink "$target")" + case "$link" in + /*) target="$link" ;; + *) target="$(dirname "$target")/$link" ;; + esac + done + dir="$(cd "$(dirname "$target")" && pwd -P)" + base="$(basename "$target")" + printf '%s/%s\n' "$dir" "$base" +} + +absolutize() { + # git may hand back a relative --git-common-dir / --git-dir. + local p="$1" + case "$p" in + /*) printf '%s\n' "$p" ;; + *) printf '%s\n' "$(cd "$p" && pwd -P)" ;; + esac +} + +is_under() { + # is_under -- strict descendant, not equal. + local candidate="$1" ancestor="$2" + [[ -n "$ancestor" ]] || return 1 + [[ "$candidate" == "$ancestor"/* ]] +} + +# --- repository facts ------------------------------------------------------- + +top_level="$(git rev-parse --show-toplevel 2>/dev/null || true)" +if [[ -z "$top_level" ]]; then + # Not inside a work tree (bare repo, or git internals) -- nothing to guard. + exit 0 +fi +# Compare physical paths throughout: git_common_dir below is resolved with +# `pwd -P`, so a symlinked registry root would otherwise defeat the +# canonical-clone refusal by making the two sides incomparable. +top_level="$(cd "$top_level" && pwd -P)" + +git_dir="$(absolutize "$(git rev-parse --git-dir)")" +git_common_dir="$(absolutize "$(git rev-parse --git-common-dir)")" + +# A LINKED worktree has --git-dir == /worktrees/, so it differs +# from --git-common-dir. The MAIN worktree of a clone has them equal. This test +# is path-layout independent, which is what makes the guard correct on a host +# whose registry lives somewhere other than the documented path. +if [[ "$git_dir" != "$git_common_dir" ]]; then + is_linked_worktree=1 +else + is_linked_worktree=0 +fi + +# Registry root. Prefer the explicit env var (root CLAUDE.md contract); fall +# back to the clone's own position -- `//.git` -- which holds +# for a canonical clone AND for every worktree linked to it, because both share +# the same --git-common-dir. No hardcoded absolute paths (root CLAUDE.md #6). +omni_home="${OMNI_HOME:-}" +if [[ -z "$omni_home" ]]; then + omni_home="$(cd "$git_common_dir/../.." && pwd -P)" +fi + +# --- decision --------------------------------------------------------------- + +allowed=0 + +if [[ "$top_level" == "$omni_home" ]]; then + # The registry meta-repo itself commits directly to its docs branch. + allowed=1 +elif is_under "$top_level" "${ONEX_WORKTREES_ROOT:-}"; then + allowed=1 +elif is_under "$top_level" "$omni_home/omni_worktrees"; then + allowed=1 +elif is_under "$top_level" "$(dirname "$omni_home")/omni_worktrees"; then + allowed=1 +elif [[ "$is_linked_worktree" == "1" ]]; then + allowed=1 +fi + +if [[ "$allowed" == "0" ]] && is_under "$top_level" "$omni_home"; then + if [[ "${ALLOW_CANONICAL_CLONE_COMMIT:-}" != "1" ]]; then + cat >&2 </ + +Override only for an intentional emergency: + ALLOW_CANONICAL_CLONE_COMMIT=1 git ... +EOF + exit 1 + fi +fi + +# --- chain to the real hook (OMN-15071) ------------------------------------- +# +# Reaching here means the worktree-discipline guard permits the operation. It +# does NOT mean the operation is clean: the real hook chain still has to run, +# and `core.hooksPath` guarantees git will not run it for us. + +# Hooks in this fleet shell out to `uv`, `python3` and friends. A git hook can +# be invoked from a non-login shell -- e.g. a `ssh 'git commit ...'`, +# which is how agent lanes drive .200 -- whose PATH omits the package-manager +# prefixes. Without this, chaining would turn OMN-15071's silent pass into a +# blanket "Executable `uv` not found" refusal on every commit. Prefixes are +# APPENDED, so an explicitly-chosen toolchain earlier on PATH still wins. +for prefix in /opt/homebrew/bin /usr/local/bin; do + if [[ -d "$prefix" ]] && [[ ":$PATH:" != *":$prefix:"* ]]; then + PATH="$PATH:$prefix" + fi +done +export PATH + +self_real="$(resolve_path "$0")" +real_hook="$git_common_dir/hooks/$hook_name" + +if [[ -x "$real_hook" ]] && [[ "$(resolve_path "$real_hook")" != "$self_real" ]]; then + exec "$real_hook" "$@" +fi + +# No installed hook of this type. If the repo declares a pre-commit config, the +# stage may still have hooks bound to it (`pre-commit install` REFUSES to write +# hook files while core.hooksPath is set, so "not installed" says nothing about +# "not configured"). Invoke pre-commit's own hook entry point -- the exact call +# the generated hook file makes. +if [[ ! -f "$top_level/.pre-commit-config.yaml" ]]; then + exit 0 +fi + +precommit_cmd=() +if command -v pre-commit >/dev/null 2>&1; then + precommit_cmd=("$(command -v pre-commit)") +else + # A git hook can run under a non-login shell whose PATH omits the package + # manager prefixes (observed on .200: PATH lacks /opt/homebrew/bin). + for candidate in /opt/homebrew/bin/pre-commit /usr/local/bin/pre-commit; do + if [[ -x "$candidate" ]]; then + precommit_cmd=("$candidate") + break + fi + done +fi + +if [[ ${#precommit_cmd[@]} -eq 0 ]]; then + for candidate in /opt/homebrew/bin/python3 /usr/local/bin/python3 python3; do + if command -v "$candidate" >/dev/null 2>&1 && + "$candidate" -c 'import pre_commit' >/dev/null 2>&1; then + precommit_cmd=("$candidate" -m pre_commit) + break + fi + done +fi + +if [[ ${#precommit_cmd[@]} -eq 0 ]]; then + cat >&2 <=5 canary (OMN-15617) +# ============================================================================= +# On stickybeatz-studio (.200, the rule-11a default gate host), non-interactive +# ssh sessions resolve `bash` to the system 3.2.57 shell even though a modern +# bash 5.x sits at /opt/homebrew/bin/bash -- it just is not first on PATH for +# that session class. runner-monitor.sh (exercised end-to-end by +# tests/unit/observability/runner_health/test_runner_monitor_*.py) uses +# `declare -A`, which bash 3.2 does not support, so those tests fail SILENTLY +# on every push from this host class -- a bash syntax error deep inside a +# subprocess, not a resolvable "wrong interpreter" diagnostic. +# +# Resolve a bash>=5 interpreter EXPLICITLY here, independent of PATH order, +# via the same resolver the pytest harness uses (single source of truth -- +# scripts/ci/resolve_modern_bash.sh -- so the two can never drift apart), and +# export it so the harness does not have to re-discover it. Fail LOUD with a +# pointed remediation message if none is resolvable anywhere -- never a quiet +# skip, never a silent fallback to whatever "bash" happens to resolve first. +MODERN_BASH="$(bash "${REPO_ROOT}/scripts/ci/resolve_modern_bash.sh")" \ + || die "no bash>=5 interpreter resolvable on this host" \ + "install a modern bash (e.g. 'brew install bash') and/or set OMNIBASE_INFRA_BASH_BIN to its absolute path; see scripts/ci/resolve_modern_bash.sh" +export OMNIBASE_INFRA_BASH_BIN="$MODERN_BASH" +log "bash>=5 canary: resolved ${MODERN_BASH}" + BASE_REF="${PREPUSH_BASE_REF:-origin/dev}" # Deterministic diff base: fetch the base ref best-effort so an online push gets @@ -191,28 +276,97 @@ IS_FULL="$(read_sel is_full_suite)" \ "the selector emitted non-JSON; inspect $SELECTION_FILE" REASON="$(read_sel full_suite_reason 2> /dev/null || true)" -PATHS=() -PATHS_STR="" +# OMN-15245 SEAM: the selector now emits changed tests/integration/ paths -- a +# changed test module is never dropped by narrowing (fail-closed invariant). +# This hook is unit-scoped by design and passes --ignore=tests/integration to +# pytest below: handing pytest a path it also ignores collects nothing from it, +# and when it is the ONLY path pytest exits 5 ("no tests ran") and blocks the +# push. Filter those out here, visibly -- they are deferred to CI, which runs +# them. Keep this function self-contained (no globals): it is extracted and +# EXECUTED by tests/unit/scripts/test_prepush_smart_tests_seam.py. +filter_prepush_runnable_paths() { + local p + while IFS= read -r p; do + [ -n "$p" ] || continue + case "$p" in + tests/integration/*) continue ;; + esac + printf '%s\n' "$p" + done +} + +ALL_PATHS=() while IFS= read -r p; do if [ -n "$p" ]; then - PATHS+=("$p") - PATHS_STR="${PATHS_STR}${p} " + ALL_PATHS+=("$p") fi done < <(read_sel selected_paths) +PATHS=() +PATHS_STR="" +DEFERRED_STR="" +# Guard the array expansions: bash 3.2 (macOS system bash) errors on +# "${arr[@]}" for an empty array under `set -u`. +if [ "${#ALL_PATHS[@]}" -gt 0 ]; then + while IFS= read -r p; do + if [ -n "$p" ]; then + PATHS+=("$p") + PATHS_STR="${PATHS_STR}${p} " + fi + done < <(printf '%s\n' "${ALL_PATHS[@]}" | filter_prepush_runnable_paths) + for p in "${ALL_PATHS[@]}"; do + case " $PATHS_STR " in + *" $p "*) ;; + *) DEFERRED_STR="${DEFERRED_STR}${p} " ;; + esac + done +fi + log "selection: is_full_suite=${IS_FULL} reason=${REASON:-none} paths=[ ${PATHS_STR}] (feature-flag=${FLAG})" +if [ -n "$DEFERRED_STR" ]; then + log "deferred to CI (integration needs live services; this hook is unit-scoped): [ ${DEFERRED_STR}]" +fi # Assemble the pytest target set. tests/integration is always ignored -- it needs # real services and stays a CI-only concern. On a fail-closed escalation we run # the full UNIT suite (tests/unit/), NOT all of tests/, so the pre-push hook stays # unit-scoped and service-free (infra seam-match). RC=0 +# SINGLE SOURCE OF TRUTH for "what the heavy run is" (OMN-15408): the +# fail-closed escalation runs exactly this target, and `selection_is_whole_suite` +# measures the impacted-subset selection against this same value. Changing the +# escalation target automatically moves the guard predicate with it. +FULL_SUITE_TARGET="tests/unit/" + +# OMN-15071: git EXPORTS repo-scoping variables into hook processes -- a live +# `git push` from a worktree hands this hook +# `GIT_DIR=/worktrees/` -- and those variables OVERRIDE both `-C` +# and the cwd for every descendant `git` call (memory +# `reference_git_env_vars_override_c_and_cwd`). Tests that build throwaway +# repositories under `tmp_path` and commit into them therefore operate on THIS +# worktree instead, and fail at setup. Unset them for the test run only: the +# hook has already resolved everything it needs from git, and pytest must +# rediscover the repository from its own cwd like any ordinary invocation. +# +# This is not a latent nicety. Until OMN-15071 chained the canonical-clone +# guard into the real hook chain, `core.hooksPath` on `.200` meant this hook +# never executed there at all, so the leak had no observable effect on the +# documented default gate host. Turning the hook on without this unset would +# hand every `.200` push a fail-closed pre-push that cannot pass. +unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_COMMON_DIR GIT_PREFIX + if [ "$IS_FULL" = "True" ] || [ "$IS_FULL" = "true" ]; then guard_full_suite_host - log "running FULL unit suite (fail-closed escalation): uv run pytest tests/unit/ --ignore=tests/integration ${PREPUSH_PYTEST_ARGS:-}" + log "running FULL unit suite (fail-closed escalation): uv run pytest ${FULL_SUITE_TARGET} --ignore=tests/integration ${PREPUSH_PYTEST_ARGS:-}" # shellcheck disable=SC2086 - uv run pytest tests/unit/ --ignore=tests/integration --tb=short ${PREPUSH_PYTEST_ARGS:-} || RC=$? + uv run pytest "${FULL_SUITE_TARGET}" --ignore=tests/integration --tb=short ${PREPUSH_PYTEST_ARGS:-} || RC=$? elif [ "${#PATHS[@]}" -gt 0 ]; then + # OMN-15408: guard on the SELECTED WORK, not the is_full_suite flag. A + # selection that covers the whole full-suite target is the heavy run under + # another name and must be routed to .200 exactly as the flagged escalation is. + if selection_is_whole_suite "$FULL_SUITE_TARGET" "${PATHS[@]}"; then + guard_full_suite_host "whole-suite-equivalent impacted selection (is_full_suite=${IS_FULL}, selected paths [ ${PATHS_STR}] cover the entire '${FULL_SUITE_TARGET}' escalation target)" + fi log "running impacted subset: uv run pytest ${PATHS_STR}--ignore=tests/integration ${PREPUSH_PYTEST_ARGS:-}" # shellcheck disable=SC2086 uv run pytest "${PATHS[@]}" --ignore=tests/integration --tb=short ${PREPUSH_PYTEST_ARGS:-} || RC=$? diff --git a/scripts/lane-census-check.sh b/scripts/lane-census-check.sh index 25df8e09c7..6da8fcf224 100755 --- a/scripts/lane-census-check.sh +++ b/scripts/lane-census-check.sh @@ -25,7 +25,10 @@ # ./scripts/lane-census-check.sh --dry-run # print event/plan, do NOT publish # ./scripts/lane-census-check.sh --json # emit the plan JSON to stdout # -# Exit codes: 0 no drift, 30 drift detected (event emitted), 2 bad args, 3 missing deps. +# Exit codes: 0 no drift, 30 drift detected (event emitted), 2 bad args, 3 missing deps, +# 4 inventory unobservable (BOTH Engine API and bounded CLI failed — +# fail-loud, NO drift event; deliberately distinct from 30 so a host +# we cannot see is never reported as a host that is down. OMN-15466). # # Runs on .201 via the SHARED onex-disk-gc.timer (4th ExecStart — coordinated with # OMN-13008 rather than a second timer). Log: ~/.local/log/onex/lane-census.log @@ -38,6 +41,8 @@ DRY_RUN=false EMIT_JSON=false LOG_FILE="${HOME}/.local/log/onex/lane-census.log" DRIFT_TOPIC="onex.evt.infra.lane-census-drift.v1" +# Inventory unobservable — NOT drift. See the exit-code table above (OMN-15466). +EXIT_INVENTORY_UNAVAILABLE=4 while [[ $# -gt 0 ]]; do case "$1" in @@ -59,40 +64,42 @@ HOST="${LANE_CENSUS_HOST:-$(hostname)}" log "Starting (lane=${LANE:-ALL}, host=$HOST, $( [[ "$DRY_RUN" == true ]] && echo DRY-RUN || echo LIVE ))" # --------------------------------------------------------------------------- -# Gather actual state. Inventory goes to per-run scratch files (never /tmp) and -# is handed to the pure planner on stdin as a JSON envelope. No env-var size -# limit, no decision logic in bash. +# Gather actual state via the fail-loud collector (scripts/lane_census_inventory.py): +# Docker Engine API first, bounded docker-CLI fallback, hard failure if neither +# can see the host. No decision logic in bash. +# +# OMN-15466: this replaced `docker ps -a --format '{{json .}}' ... || : >file`, +# which (a) made the CLI request size=1, forcing daemon-side snapshotter.Usage +# per container — 90.363 s vs 0.150 s for the same inventory over the Engine API +# on .201's 111 containers — and (b) truncated the inventory to EMPTY on any +# docker failure, which the planner cannot distinguish from a genuine total +# outage (32 critical findings, published as real drift). A host we cannot +# observe must never be reported as a host that is down. # --------------------------------------------------------------------------- SCRATCH="$(mktemp -d "$(dirname "$LOG_FILE")/lane-census.XXXXXX")" trap 'rm -rf "$SCRATCH"' EXIT -docker ps -a --no-trunc --format '{{json .}}' >"$SCRATCH/ps.ndjson" 2>/dev/null || : >"$SCRATCH/ps.ndjson" -docker network ls --format '{{.Name}}' >"$SCRATCH/networks.txt" 2>/dev/null || : >"$SCRATCH/networks.txt" - # Resolve the runtime tag from the deploy-agent runtime version when available # (relaxes to the default pattern if unresolvable — see the planner). RUNTIME_TAG="${RUNTIME_TAG:-}" +set +e ENVELOPE_JSON="$( - SCRATCH_DIR="$SCRATCH" LANE="$LANE" RUNTIME_TAG="$RUNTIME_TAG" python3 -c ' -import json, os -d = os.environ["SCRATCH_DIR"] -lane = os.environ.get("LANE") or None -containers = [] -with open(os.path.join(d, "ps.ndjson")) as fh: - for line in fh: - line = line.strip() - if line: - containers.append(json.loads(line)) -networks = [n.strip() for n in open(os.path.join(d, "networks.txt")) if n.strip()] -print(json.dumps({ - "lane": lane, - "containers": containers, - "networks": networks, - "runtime_tag": os.environ.get("RUNTIME_TAG") or None, -})) -' + LANE="$LANE" RUNTIME_TAG="$RUNTIME_TAG" \ + python3 "${SCRIPT_DIR}/lane_census_inventory.py" 2>"$SCRATCH/inventory.err" )" +INVENTORY_RC=$? +set -e + +if [[ $INVENTORY_RC -ne 0 ]]; then + while IFS= read -r line; do [[ -n "$line" ]] && log "$line"; done <"$SCRATCH/inventory.err" + log "ABORT: docker inventory could not be observed (exit $INVENTORY_RC). \ +Publishing NO drift event — an unobservable host is not a drifted host." + exit "$EXIT_INVENTORY_UNAVAILABLE" +fi + +# Surface any fallback/degradation notices without changing the exit policy. +while IFS= read -r line; do [[ -n "$line" ]] && log "$line"; done <"$SCRATCH/inventory.err" PLAN_JSON="$(echo "$ENVELOPE_JSON" | python3 "${SCRIPT_DIR}/lane_census_plan.py")" diff --git a/scripts/lane_census_inventory.py b/scripts/lane_census_inventory.py new file mode 100644 index 0000000000..3b68439580 --- /dev/null +++ b/scripts/lane_census_inventory.py @@ -0,0 +1,334 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""lane_census_inventory.py — fail-loud docker inventory collector (OMN-15466). + +Collection counterpart to the pure planner (``lane_census_plan.py``). The planner +performs NO I/O; this module performs ALL of the census's docker I/O and emits the +planner's stdin envelope on stdout. + +WHY THIS EXISTS — two defects in the single line it replaces +------------------------------------------------------------ +``lane-census-check.sh`` previously gathered the inventory with:: + + docker ps -a --no-trunc --format '{{json .}}' >ps.ndjson 2>/dev/null || : >ps.ndjson + +D1 — ``{{json .}}`` silently opts into per-container SIZE computation. + The Docker CLI's ``{{json .}}`` context carries a ``Size`` field, so the CLI + sets ``size=1`` on ``GET /containers/json``. The daemon then runs + ``snapshotter.Usage`` for EVERY container. Measured on ``.201`` (111 + containers, 2026-07-30): **90.363 s** for that exact command versus + **0.150 s** for ``GET /containers/json?all=1`` without ``size``. The census + reads none of the size data — the planner consumes only Names/State/Status/ + Image/Labels — so the cost is pure waste on the critical path. + + Transport is NOT the variable: the Engine API *with* ``size=1`` is equally + slow (57.265 s) and the CLI *without* size is equally fast (0.128 s). That is + why the fallback below pins an explicit field list and never ``{{json .}}``. + +D2 — ``2>/dev/null || : >ps.ndjson`` converted any docker failure into a + fabricated EMPTY inventory. ``2>/dev/null`` discarded the error, ``|| :`` + defeated ``set -e``, and the truncation handed the planner a zero-container + envelope indistinguishable from a genuine total outage: 32 findings, all + ``critical``, across all four lanes, published as a real drift event. A probe + that cannot see is not a probe that saw nothing. Both paths here fail LOUD. + +Ordering: Engine API first (authoritative, cheapest, unambiguous label typing), +Docker CLI second under an explicit ``timeout``, then hard failure. Exit code +``4`` is reserved for "the inventory could not be observed" and is deliberately +distinct from the driver's drift code ``30`` so an unobservable host can never be +reported as a drifted host. + +Label typing note: the Engine API returns ``Labels`` as a real mapping. The CLI +returns a comma-joined ``k=v`` string in which a VALUE may itself contain commas +(``com.docker.compose.project.config_files`` routinely does), so the CLI form is +ambiguous by construction. The CLI parser here rejoins continuation segments so +both paths yield an identical envelope; the API path avoids the ambiguity +entirely. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import socket +import subprocess +import sys +from http.client import HTTPConnection +from typing import Any + +# Exit code for "inventory could not be observed". Distinct from the driver's +# drift code (30) and its bad-args (2) / missing-deps (3) codes. +EXIT_PROBE_FAILED = 4 + +DEFAULT_DOCKER_SOCKET = "/var/run/docker.sock" +DEFAULT_API_TIMEOUT_S = 15.0 +DEFAULT_CLI_TIMEOUT_S = 30.0 + +# Engine API inventory paths. NEITHER carries a `size` parameter — see D1. +API_CONTAINERS_PATH = "/containers/json?all=1" +API_NETWORKS_PATH = "/networks" + +# Docker CLI fallback format. Enumerates exactly the fields the planner reads. +# MUST NOT be '{{json .}}': that emits a Size field and triggers size=1 (D1). +CLI_CONTAINER_FORMAT = "{{.Names}}\t{{.State}}\t{{.Status}}\t{{.Image}}\t{{.Labels}}" +_CLI_FIELDS = ("Names", "State", "Status", "Image", "Labels") + + +class InventoryProbeError(RuntimeError): + """Raised when the container/network inventory could not be observed.""" + + +class _UnixHTTPConnection(HTTPConnection): + """HTTPConnection over an AF_UNIX socket (the Docker Engine API socket).""" + + def __init__(self, socket_path: str, timeout: float) -> None: + super().__init__("localhost", timeout=timeout) + self._socket_path = socket_path + self._timeout = timeout + + def connect(self) -> None: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.settimeout(self._timeout) + sock.connect(self._socket_path) + self.sock = sock + + +def api_get(socket_path: str, path: str, timeout: float) -> Any: + """GET a Docker Engine API path over the unix socket and decode the JSON body.""" + conn = _UnixHTTPConnection(socket_path, timeout) + try: + conn.request("GET", path) + response = conn.getresponse() + body = response.read() + if response.status != 200: + raise InventoryProbeError( + f"Docker Engine API GET {path} returned HTTP {response.status}: " + f"{body[:400].decode('utf-8', 'replace')}" + ) + return json.loads(body) + finally: + conn.close() + + +def parse_cli_labels(raw: str) -> dict[str, str]: + """Parse the Docker CLI's comma-joined ``k=v`` label string into a mapping. + + A label VALUE may contain commas (``...config_files=/a.yml,/b.yml``), so a + naive ``split(",")`` corrupts such values. Segments without ``=`` are treated + as continuations of the preceding value. + """ + labels: dict[str, str] = {} + current: str | None = None + for segment in raw.split(","): + if "=" in segment: + key, value = segment.split("=", 1) + current = key.strip() + labels[current] = value.strip() + elif current is not None and segment: + labels[current] = f"{labels[current]},{segment}" + return labels + + +def normalize_api_containers(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Normalize Engine API container rows to the planner's envelope shape.""" + out: list[dict[str, Any]] = [] + for row in rows: + names = row.get("Names") or [] + name = (names[0] if names else row.get("Name") or "").lstrip("/").strip() + if not name: + continue + labels = row.get("Labels") or {} + out.append( + { + "Names": name, + "State": str(row.get("State") or ""), + "Status": str(row.get("Status") or ""), + "Image": str(row.get("Image") or ""), + "Labels": {str(k): str(v) for k, v in labels.items()}, + } + ) + return sorted(out, key=lambda r: str(r["Names"])) + + +def normalize_cli_containers(text: str) -> list[dict[str, Any]]: + """Normalize tab-delimited Docker CLI rows to the planner's envelope shape.""" + out: list[dict[str, Any]] = [] + for line in text.splitlines(): + if not line.strip(): + continue + parts = line.split("\t") + if len(parts) < len(_CLI_FIELDS): + parts = parts + [""] * (len(_CLI_FIELDS) - len(parts)) + name = parts[0].lstrip("/").strip() + if not name: + continue + out.append( + { + "Names": name, + "State": parts[1].strip(), + "Status": parts[2].strip(), + "Image": parts[3].strip(), + "Labels": parse_cli_labels(parts[4]), + } + ) + return sorted(out, key=lambda r: str(r["Names"])) + + +def normalize_api_networks(rows: list[dict[str, Any]]) -> list[str]: + """Extract network names from an Engine API ``GET /networks`` response.""" + return sorted({str(r.get("Name") or "") for r in rows if r.get("Name")}) + + +def normalize_cli_networks(text: str) -> list[str]: + """Extract network names from ``docker network ls --format '{{.Name}}'``.""" + return sorted({line.strip() for line in text.splitlines() if line.strip()}) + + +def _run_cli(args: list[str], timeout_s: float) -> str: + """Run a docker CLI command under an explicit bound. Raises on any failure. + + The bound is ``subprocess.run(timeout=...)``, which is portable — coreutils + ``timeout(1)`` does not exist on macOS, and the gate/push host is a Mac. When + ``timeout(1)`` IS present it is layered underneath as defence in depth so the + docker client is reaped even if this process is itself wedged. + """ + if shutil.which("docker") is None: + raise InventoryProbeError("docker CLI not found on PATH") + + command = list(args) + timeout_bin = shutil.which("timeout") + if timeout_bin is not None: + command = [timeout_bin, str(int(timeout_s)), *args] + + try: + proc = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + timeout=timeout_s, + ) + except subprocess.TimeoutExpired as exc: + raise InventoryProbeError( + f"docker CLI fallback exceeded {timeout_s}s: {' '.join(command)}" + ) from exc + if proc.returncode == 124: + raise InventoryProbeError( + f"docker CLI fallback timed out after {timeout_s}s: {' '.join(command)}" + ) + if proc.returncode != 0: + raise InventoryProbeError( + f"docker CLI fallback failed (exit {proc.returncode}): " + f"{' '.join(command)}: {proc.stderr.strip()[:400]}" + ) + return proc.stdout + + +def collect_inventory( + *, + socket_path: str, + api_timeout_s: float, + cli_timeout_s: float, +) -> tuple[list[dict[str, Any]], list[str], str, list[str]]: + """Collect containers + networks, Engine API first, bounded CLI fallback. + + Returns ``(containers, networks, source, warnings)``. Raises + :class:`InventoryProbeError` when BOTH paths fail — never returns an empty + inventory to signal a failed probe (D2). + """ + warnings: list[str] = [] + + try: + containers = normalize_api_containers( + api_get(socket_path, API_CONTAINERS_PATH, api_timeout_s) + ) + networks = normalize_api_networks( + api_get(socket_path, API_NETWORKS_PATH, api_timeout_s) + ) + return containers, networks, "engine_api", warnings + except (OSError, InventoryProbeError, ValueError) as exc: + warnings.append( + f"engine_api path failed ({exc}); falling back to bounded docker CLI" + ) + + containers = normalize_cli_containers( + _run_cli( + ["docker", "ps", "-a", "--no-trunc", "--format", CLI_CONTAINER_FORMAT], + cli_timeout_s, + ) + ) + networks = normalize_cli_networks( + _run_cli(["docker", "network", "ls", "--format", "{{.Name}}"], cli_timeout_s) + ) + return containers, networks, "docker_cli", warnings + + +def build_envelope( + *, + lane: str | None, + runtime_tag: str | None, + containers: list[dict[str, Any]], + networks: list[str], + source: str, +) -> dict[str, Any]: + """Assemble the planner's stdin envelope.""" + return { + "lane": lane or None, + "containers": containers, + "networks": networks, + "runtime_tag": runtime_tag or None, + "inventory_source": source, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--lane", default=os.environ.get("LANE") or None) + parser.add_argument("--runtime-tag", default=os.environ.get("RUNTIME_TAG") or None) + args = parser.parse_args(argv) + + socket_path = os.environ.get("LANE_CENSUS_DOCKER_SOCKET", DEFAULT_DOCKER_SOCKET) + api_timeout_s = float( + os.environ.get("LANE_CENSUS_API_TIMEOUT_S", DEFAULT_API_TIMEOUT_S) + ) + cli_timeout_s = float( + os.environ.get("LANE_CENSUS_CLI_TIMEOUT_S", DEFAULT_CLI_TIMEOUT_S) + ) + + try: + containers, networks, source, warnings = collect_inventory( + socket_path=socket_path, + api_timeout_s=api_timeout_s, + cli_timeout_s=cli_timeout_s, + ) + except (OSError, InventoryProbeError, ValueError) as exc: + # FAIL LOUD. Never emit an envelope — an unobservable host must not be + # reported to the planner as an empty (i.e. totally-down) host. + print( + f"lane-census inventory probe FAILED (both Engine API and docker CLI): {exc}", + file=sys.stderr, + ) + return EXIT_PROBE_FAILED + + for warning in warnings: + print(f"lane-census inventory: {warning}", file=sys.stderr) + + json.dump( + build_envelope( + lane=args.lane, + runtime_tag=args.runtime_tag, + containers=containers, + networks=networks, + source=source, + ), + sys.stdout, + ) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/lane_census_plan.py b/scripts/lane_census_plan.py index 566e685a11..1b7a35c5e1 100644 --- a/scripts/lane_census_plan.py +++ b/scripts/lane_census_plan.py @@ -279,9 +279,22 @@ def reconcile_lane( return findings -def _labels_to_dict(labels: str) -> dict[str, str]: +def _labels_to_dict(labels: Any) -> dict[str, str]: + """Coerce a container's Labels to a mapping. + + The Docker Engine API returns Labels as a real mapping; the Docker CLI + returns a comma-joined ``k=v`` string. The collector + (``lane_census_inventory.py``) normalizes both to a mapping, so the mapping + branch is the live path — the string branch is retained for envelopes + produced by older callers and for direct CLI-shaped fixtures. Note the string + form is lossy by construction: a label VALUE may itself contain commas + (``com.docker.compose.project.config_files``), which a flat split cannot + recover. Prefer the mapping form (OMN-15466). + """ + if isinstance(labels, dict): + return {str(k): str(v) for k, v in labels.items()} out: dict[str, str] = {} - for pair in labels.split(","): + for pair in str(labels or "").split(","): if "=" in pair: key, value = pair.split("=", 1) out[key.strip()] = value.strip() diff --git a/scripts/monitor_logs.py b/scripts/monitor_logs.py index ce6ff50580..cb590142fc 100755 --- a/scripts/monitor_logs.py +++ b/scripts/monitor_logs.py @@ -52,6 +52,7 @@ import urllib.request import uuid from collections import deque +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path @@ -85,7 +86,83 @@ def _load_omnibase_env() -> None: pass -_load_omnibase_env() +@dataclass(frozen=True, slots=True) +class _MonitorEnvironmentConfig: + """Environment-derived configuration cached by monitor helpers.""" + + warning_cooldown_seconds: int + restart_hwm_file: Path + onex_state_dir: Path + default_file_logs: tuple[str, ...] + + +def _resolve_monitor_environment_config() -> _MonitorEnvironmentConfig: + """Resolve every module-level value derived from the process environment.""" + onex_state_dir = Path( + os.environ.get("ONEX_STATE_DIR", str(Path.home() / ".onex_state")) + ) + return _MonitorEnvironmentConfig( + warning_cooldown_seconds=int( + os.environ.get("MONITOR_WARNING_COOLDOWN", "1800") + ), + restart_hwm_file=Path( + os.environ.get( + "MONITOR_RESTART_HWM_FILE", + str(Path.home() / ".omnibase" / "monitor-restart-hwm.json"), + ) + ), + onex_state_dir=onex_state_dir, + default_file_logs=( + str(onex_state_dir / "logs" / "env-sync.log"), + str(onex_state_dir / "logs" / "hooks.log"), + str(onex_state_dir / "logs" / "pipeline-trace.log"), + ), + ) + + +_MONITOR_ENV_CONFIG: _MonitorEnvironmentConfig +WARNING_COOLDOWN_SECONDS: int +_RESTART_HWM_FILE: Path +_ONEX_STATE_DIR: Path +_DEFAULT_FILE_LOGS: list[str] + + +def _apply_monitor_environment_config(config: _MonitorEnvironmentConfig) -> None: + """Publish one resolved config to the compatibility module globals.""" + global WARNING_COOLDOWN_SECONDS # noqa: PLW0603 -- compatibility global + global _DEFAULT_FILE_LOGS, _MONITOR_ENV_CONFIG # noqa: PLW0603 + global _ONEX_STATE_DIR, _RESTART_HWM_FILE # noqa: PLW0603 + + _MONITOR_ENV_CONFIG = config + WARNING_COOLDOWN_SECONDS = config.warning_cooldown_seconds + _RESTART_HWM_FILE = config.restart_hwm_file + _ONEX_STATE_DIR = config.onex_state_dir + _DEFAULT_FILE_LOGS = list(config.default_file_logs) + + +_MONITOR_HOME_ENV_LOADED = False + + +def _bootstrap_monitor_environment() -> _MonitorEnvironmentConfig: + """Load operator env once and refresh every cached environment value.""" + global _MONITOR_HOME_ENV_LOADED # noqa: PLW0603 -- one-time bootstrap state + + if not _MONITOR_HOME_ENV_LOADED: + _load_omnibase_env() + _MONITOR_HOME_ENV_LOADED = True + config = _resolve_monitor_environment_config() + _apply_monitor_environment_config(config) + return config + + +if __name__ == "__main__": + # Direct execution still loads operator config before the module-level + # environment-derived constants below are evaluated. Library imports must + # remain side-effect-free in their caller's process. + _load_omnibase_env() + _MONITOR_HOME_ENV_LOADED = True + +_apply_monitor_environment_config(_resolve_monitor_environment_config()) # --------------------------------------------------------------------------- # Persistent cooldown (survives monitor restarts / launchd KeepAlive bounces) @@ -177,8 +254,6 @@ def _backoff_seconds(count: int) -> int: r")" ) -WARNING_COOLDOWN_SECONDS = int(os.environ.get("MONITOR_WARNING_COOLDOWN", "1800")) - # Backoff for warnings: 30m → 60m → cap at 60m _WARNING_BACKOFF_BASE = 1800 # 30 minutes _WARNING_BACKOFF_CAP = 3600 # 1 hour max @@ -1070,7 +1145,7 @@ def _load_monitor_alert_topic() -> str: # MonitorAlertEmitter._init_clients() will log the degradation. return "" try: - import yaml # type: ignore[import-untyped] + import yaml except ImportError as exc: raise RuntimeError( "PyYAML is required to load monitor_alert_contract.yaml " @@ -1232,12 +1307,6 @@ def emit( # Restart watcher (OMN-3596) # --------------------------------------------------------------------------- -_RESTART_HWM_FILE = Path( - os.environ.get( - "MONITOR_RESTART_HWM_FILE", - str(Path.home() / ".omnibase" / "monitor-restart-hwm.json"), - ) -) _restart_hwm_lock = threading.Lock() # Default restart-count delta that triggers an alert. @@ -1680,14 +1749,6 @@ def _maybe_warning_alert(self, label: str, lines: list[str]) -> None: # --------------------------------------------------------------------------- # Default file log sources (colon-separated env var MONITOR_FILE_LOGS overrides) -_ONEX_STATE_DIR = Path( - os.environ.get("ONEX_STATE_DIR", str(Path.home() / ".onex_state")) -) -_DEFAULT_FILE_LOGS = [ - str(_ONEX_STATE_DIR / "logs" / "env-sync.log"), - str(_ONEX_STATE_DIR / "logs" / "hooks.log"), - str(_ONEX_STATE_DIR / "logs" / "pipeline-trace.log"), -] # Default journal units (comma-separated env var MONITOR_JOURNALS overrides) _DEFAULT_JOURNALS = ["deploy-agent.service"] @@ -2359,6 +2420,7 @@ def run(self) -> None: def main() -> None: + _bootstrap_monitor_environment() parser = argparse.ArgumentParser( description="Monitor OmniNode container logs and post errors to Slack" ) diff --git a/scripts/omninode-ci-required-context-probe.py b/scripts/omninode-ci-required-context-probe.py new file mode 100755 index 0000000000..5bce7115cd --- /dev/null +++ b/scripts/omninode-ci-required-context-probe.py @@ -0,0 +1,588 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""omninode-ci-required-context-probe.py -- detect required checks that never reported. + +WHAT THIS IS (OMN-15550) + A read-only probe that answers one question nothing else in the fleet asks: + *which required status contexts did NOT show up?* + + It is executed on the ``.201`` host by + ``deploy/maintenance/omninode-system-slack-report.sh`` inside ``collect()``, + which means it rides the existing ``*/15`` alert cron, the existing Slack + poster, the existing state-change de-duplication and the existing + ``[OmniNode alert resolved]`` path. No new cron unit, no second Slack + integration, no new dashboard. + +THE DEFECT CLASS + A required status check that never reports is ABSENT, not RED. Branch + protection blocks the PR identically either way, but only one of the two is + visible: a RED check has a row in every list, an ABSENT check has no row at + all. So ``gh pr checks`` shows all-green, the CI Summary poller shows pass, + every dashboard shows "no failures" -- and the PR is permanently + unmergeable. + + Every existing surface iterates the checks that EXIST and grades them. This + probe iterates the REQUIRED set and subtracts what is present. That + set-difference is the whole point; without reading + ``branches//protection/required_status_checks`` the missing case is + not merely un-alarmed, it is unrepresentable. + +WHY IT DOES NOT RUN IN GITHUB ACTIONS + On 2026-07-30 (OMN-15536) ``omnibase_infra``'s ``ci.yml`` failed to + assemble. ``CI Summary`` is the SOLE required context on that repo's dev + branch, so it could not be produced for any PR and all 7 open PRs became + unmergeable at once, for ~2.5h, discovered by a human noticing. A detector + implemented inside the CI system it monitors would have failed to assemble + alongside it. This runs on ``.201``, which is independent of GitHub Actions. + +TRI-STATE, AND WHY ONLY ONE OF THE THREE ALARMS + ABSENT -- never reported. Invisible today. THIS is what alarms. + PENDING -- reported, not concluded. Visible today; ageing it out is + OMN-12560's job. Alarming here would page twice for one stall, + so PENDING never alarms at any age. + FAILED -- reported, concluded not-success. Already visible everywhere. + + Counted and printed in the heartbeat either way, so "we looked and found + none" is distinguishable from "we did not look" (the structural blindness + called out in ``reference_detection_shelf_structurally_blind``). + +TWO TRAPS THIS PROBE MUST NOT FALL INTO (both cost real time to find) + 1. PAGINATION. Heads in ``omnibase_infra`` carry 100-247 check-runs against + a GitHub default page size of 30. A single-page probe reports contexts + as ABSENT that are present on page 2 -- a false-positive generator that + would train the alert to be ignored. Every list endpoint here follows + ``Link: rel="next"`` to exhaustion. + 2. TWO SURFACES. A required context can be satisfied by a check-run OR by a + legacy commit status. On these repos ``CodeRabbit`` reports as a commit + status while everything else reports as a check-run, so a check-runs-only + probe reports ``CodeRabbit`` permanently absent. The present-set is the + UNION of both endpoints. + +FAIL-CLOSED, BUT NOT PAGE-ON-EVERY-BLIP + A GitHub API error is reported as a visible WARNING row -- never silently + skipped, because "we could not look" must not render as "nothing is wrong". + It is deliberately NOT CRITICAL: an unreachable API is not evidence that + PRs are stranded, and paging on every network blip is how an alert channel + gets muted. A genuinely stranded PR alarms on the next successful tick. + +OUTPUT CONTRACT + One ``ci|STATUS|key|detail`` row per finding on stdout, where STATUS is + OK / WARNING / CRITICAL. The reporter's ``row_status()`` reads column 2 for + this row shape and ``row_key()`` de-duplicates on ``ci|``, so ``key`` + must be stable across ticks -- no timestamps, no ages, no flapping counts + in it. Volatile numbers belong in ``detail``, which is not part of the + de-duplication identity. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Iterator +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +API_ROOT = "https://api.github.com" # url-authority-ok: GitHub's public API is an external SaaS control plane, not an ONEX service; it has no routing-authority/integration-catalog entry and cannot acquire one. Routing this probe through our own resolver would couple the outage detector to the infrastructure whose outages it exists to report. +OWNER = os.environ.get("OMNINODE_CI_PROBE_OWNER", "OmniNode-ai") + +# Repos whose merge path is gated by required contexts. A repo absent from this +# list is not probed -- same posture as the maintenance-sync MANIFEST: being in +# the list is what makes it governed. +DEFAULT_REPOS = ( + "omnibase_core", + "omnibase_infra", + "omnibase_spi", + "omnibase_compat", + "omniclaude", + "omnimarket", + "onex_change_control", + "omnidash", +) + +# Minimum age before an absence can alarm, measured from when CI demonstrably +# began on the head. Below this, "absent" is indistinguishable from "has not +# started yet" on a freshly-pushed head. +GRACE_MINUTES = int(os.environ.get("OMNINODE_CI_ABSENT_GRACE_MINUTES", "20")) + +# Above this age an absence alarms even while runs are still in flight. Without +# it, a permanently-hung run would hold a repo in a silent blind spot forever -- +# the in-flight suppression below would never lift. +CEILING_MINUTES = int(os.environ.get("OMNINODE_CI_ABSENT_CEILING_MINUTES", "180")) + +# How far back to scan workflow runs for the zero-job startup-failure signature. +ZEROJOB_WINDOW_MINUTES = int(os.environ.get("OMNINODE_CI_ZEROJOB_WINDOW_MINUTES", "90")) + +# Runs whose wall-clock duration exceeds this cannot be startup failures; used +# as a cheap pre-filter so the expensive per-run jobs call is rare. +ZEROJOB_MAX_DURATION_SECONDS = 5 + +HTTP_TIMEOUT_SECONDS = int(os.environ.get("OMNINODE_CI_PROBE_TIMEOUT_SECONDS", "20")) + +IN_FLIGHT_RUN_STATUSES = frozenset( + {"queued", "in_progress", "waiting", "requested", "pending"} +) + +# Conclusions that mean "this check reported and it is not a success". Present, +# therefore already visible, therefore not this probe's alarm. +FAILED_CONCLUSIONS = frozenset({"failure", "timed_out", "action_required", "stale"}) + + +class ProbeError(RuntimeError): + """A repo-scoped probe failure. Reported as WARNING, never silently dropped.""" + + +def _now() -> datetime: + """Current UTC time, overridable so grace arithmetic is deterministic in tests.""" + pinned = os.environ.get("OMNINODE_CI_PROBE_NOW") + if pinned: + return _parse_ts(pinned) or datetime.now(UTC) + return datetime.now(UTC) + + +def _parse_ts(value: str | None) -> datetime | None: + if not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + + +def emit(row: str) -> None: + """Write one snapshot row to stdout. + + Raw stdout rather than ``print`` because stdout IS this script's interface: + the reporter captures these lines verbatim into its snapshot, so no + formatting layer may sit between the row and the caller. + """ + sys.stdout.write(row + "\n") + + +def _sanitize(text: str) -> str: + """Strip the field delimiter and newlines out of anything bound for a row.""" + return text.replace("|", "/").replace("\n", " ").replace("\r", " ").strip() + + +class GitHub: + """Minimal paginating GitHub reader. + + Fixture mode (``OMNINODE_CI_PROBE_FIXTURE_DIR``) resolves each request to a + recorded JSON file instead of the network. That is what lets the hermetic + tests drive THIS file -- the artifact that actually runs on the host -- + against real recorded API payloads, rather than proving something about a + re-implementation (``feedback_test_the_artifact_that_runs``). + """ + + def __init__(self, token: str | None, fixture_dir: str | None = None) -> None: + self._token = token + self._fixture_dir = fixture_dir + + @staticmethod + def slug(path_and_query: str) -> str: + """Stable filename for a request. Mirrored by the fixture generator.""" + out = [] + for ch in path_and_query: + out.append(ch if ch.isalnum() else "_") + return "".join(out).strip("_")[:180] + + def get(self, path: str) -> Any: + if self._fixture_dir is not None: + return self._get_fixture(path) + return self._get_http(path) + + def _get_fixture(self, path: str) -> Any: + assert self._fixture_dir is not None + candidate = Path(self._fixture_dir) / (self.slug(path) + ".json") + if not candidate.exists(): + raise ProbeError(f"no fixture for {path} (expected {candidate})") + payload = json.loads(candidate.read_text(encoding="utf-8")) + if isinstance(payload, dict) and payload.get("__error__"): + raise ProbeError(str(payload["__error__"])) + # `__next__` is the fixture stand-in for a `Link: rel="next"` header, so + # multi-page reads exercise the same loop offline. Without it the + # pagination trap in the module docstring would be untestable, and an + # untested pagination path is how phantom absences ship. + if isinstance(payload, dict): + return _WithLink(payload, payload.get("__next__")) + return payload + + def _get_http(self, path: str) -> Any: + # S310: the scheme is fixed by API_ROOT above (https, literal), and + # `path` is built from repo/branch/sha values, never from user input. + request = urllib.request.Request(f"{API_ROOT}{path}") # noqa: S310 + request.add_header("Accept", "application/vnd.github+json") + request.add_header("X-GitHub-Api-Version", "2022-11-28") + request.add_header("User-Agent", "omninode-ci-required-context-probe") + if self._token: + request.add_header("Authorization", f"Bearer {self._token}") + try: + with urllib.request.urlopen( # noqa: S310 - fixed https API_ROOT + request, timeout=HTTP_TIMEOUT_SECONDS + ) as response: + body = response.read().decode("utf-8") + link = response.headers.get("Link", "") + except urllib.error.HTTPError as exc: + raise ProbeError(f"HTTP {exc.code} on {path}") from exc + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise ProbeError(f"{type(exc).__name__} on {path}") from exc + parsed = json.loads(body) if body else None + return _WithLink(parsed, _next_link(link)) + + def paginate(self, path: str, key: str | None = None) -> Iterator[Any]: + """Yield every item across all pages. + + Following ``Link: rel="next"`` to exhaustion is load-bearing, not + defensive: at a default page size of 30 against heads carrying 200+ + check-runs, a single-page read invents absences. + """ + current: str | None = path + seen_pages = 0 + while current: + payload = self.get(current) + nxt: str | None = None + if isinstance(payload, _WithLink): + nxt = payload.next_path + payload = payload.value + items = ( + payload.get(key, []) if (key and isinstance(payload, dict)) else payload + ) + if isinstance(items, list): + yield from items + seen_pages += 1 + if seen_pages > 50: # hard stop; no legitimate head has 5000 checks + break + current = nxt + + +class _WithLink: + """Carries a page payload plus the parsed ``rel="next"`` path.""" + + __slots__ = ("next_path", "value") + + def __init__(self, value: Any, next_path: str | None) -> None: + self.value = value + self.next_path = next_path + + def get(self, key: str, default: Any = None) -> Any: + return self.value.get(key, default) if isinstance(self.value, dict) else default + + +def _next_link(link_header: str) -> str | None: + for part in link_header.split(","): + section = part.split(";") + if len(section) < 2 or 'rel="next"' not in part: + continue + url = section[0].strip().lstrip("<").rstrip(">") + split = urllib.parse.urlsplit(url) + return split.path + ("?" + split.query if split.query else "") + return None + + +def _unwrap(payload: Any) -> Any: + return payload.value if isinstance(payload, _WithLink) else payload + + +def required_contexts(gh: GitHub, repo: str, branch: str) -> list[str]: + path = ( + f"/repos/{OWNER}/{repo}/branches/" + f"{urllib.parse.quote(branch, safe='')}/protection/required_status_checks" + ) + try: + payload = _unwrap(gh.get(path)) + except ProbeError as exc: + if "HTTP 404" in str(exc): + return [] # unprotected branch: nothing is required, nothing to miss + raise + contexts = payload.get("contexts", []) if isinstance(payload, dict) else [] + return [str(c) for c in contexts] + + +def observed_contexts( + gh: GitHub, repo: str, sha: str +) -> tuple[dict[str, str], datetime | None]: + """Return ``{context -> state}`` and the earliest moment CI reported on this head. + + ``state`` is one of ``pending`` / ``ok`` / ``failed``. The present-set is the + union of check-runs and commit statuses because the two surfaces carry + different producers -- see the module docstring. + """ + states: dict[str, str] = {} + earliest: datetime | None = None + # A context can carry several check-runs when a job is re-run. Branch + # protection honours the most recent one, so resolve by timestamp rather + # than by arrival order -- the API does not promise chronological ordering, + # and "whichever came last in the list wins" would make the verdict depend + # on pagination boundaries. Observed live: omnibase_infra#2582 head + # 144b2be7 carries both an in_progress and a completed `CI Summary`. + newest_seen: dict[str, datetime] = {} + + for run in gh.paginate( + f"/repos/{OWNER}/{repo}/commits/{sha}/check-runs?per_page=100", "check_runs" + ): + if not isinstance(run, dict): + continue + name = str(run.get("name", "")) + if not name: + continue + started = _parse_ts(run.get("started_at")) + if started and (earliest is None or started < earliest): + earliest = started + + prior = newest_seen.get(name) + if prior is not None and started is not None and started < prior: + continue + if started is not None: + newest_seen[name] = started + + if run.get("status") != "completed": + states[name] = "pending" + else: + conclusion = str(run.get("conclusion") or "") + states[name] = "failed" if conclusion in FAILED_CONCLUSIONS else "ok" + + combined = _unwrap( + gh.get(f"/repos/{OWNER}/{repo}/commits/{sha}/status?per_page=100") + ) + statuses = combined.get("statuses", []) if isinstance(combined, dict) else [] + for status in statuses: + if not isinstance(status, dict): + continue + context = str(status.get("context", "")) + if not context: + continue + state = str(status.get("state", "")) + if state == "pending": + states.setdefault(context, "pending") + elif state == "success": + states[context] = "ok" + else: + states[context] = "failed" + created = _parse_ts(status.get("created_at")) + if created and (earliest is None or created < earliest): + earliest = created + + return states, earliest + + +def has_in_flight_runs(gh: GitHub, repo: str, sha: str) -> bool: + """True when any workflow run on this head is still queued or executing. + + Suppressing an absence while runs are in flight is what keeps a long + fan-out pipeline (jobs gated behind ``needs:`` legitimately have no + check-run for tens of minutes) from generating false positives. The ceiling + above stops that suppression from becoming permanent. + """ + runs = _unwrap( + gh.get(f"/repos/{OWNER}/{repo}/actions/runs?head_sha={sha}&per_page=100") + ) + entries = runs.get("workflow_runs", []) if isinstance(runs, dict) else [] + return any( + isinstance(r, dict) and str(r.get("status", "")) in IN_FLIGHT_RUN_STATUSES + for r in entries + ) + + +def probe_repo( + gh: GitHub, repo: str, now: datetime +) -> tuple[list[str], dict[str, int]]: + rows: list[str] = [] + tally = {"prs": 0, "required": 0, "absent": 0, "pending": 0, "failed": 0} + + pulls = list(gh.paginate(f"/repos/{OWNER}/{repo}/pulls?state=open&per_page=100")) + protection_cache: dict[str, list[str]] = {} + + for pull in pulls: + if not isinstance(pull, dict) or pull.get("draft"): + continue + number = pull.get("number") + base = str((pull.get("base") or {}).get("ref", "")) + sha = str((pull.get("head") or {}).get("sha", "")) + if not (number and base and sha): + continue + + if base not in protection_cache: + protection_cache[base] = required_contexts(gh, repo, base) + required = protection_cache[base] + if not required: + continue + + tally["prs"] += 1 + tally["required"] += len(required) + + states, earliest = observed_contexts(gh, repo, sha) + missing = [ctx for ctx in required if ctx not in states] + tally["pending"] += sum(1 for c in required if states.get(c) == "pending") + tally["failed"] += sum(1 for c in required if states.get(c) == "failed") + + if not missing: + continue + + # Age from when CI demonstrably began on this head. When nothing has + # reported at all there is no such moment, so fall back to the head + # commit's own timestamp rather than treating the head as brand new. + anchor = earliest + if anchor is None: + anchor = _parse_ts( + ((pull.get("head") or {}).get("repo") or {}).get("pushed_at") + ) or _parse_ts(pull.get("created_at")) + if anchor is None: + continue + age_minutes = int((now - anchor).total_seconds() // 60) + + if age_minutes < GRACE_MINUTES: + continue + if age_minutes < CEILING_MINUTES and has_in_flight_runs(gh, repo, sha): + continue + + tally["absent"] += len(missing) + for context in missing: + key = _sanitize(f"absent/{repo}#{number}/{context}") + detail = _sanitize( + f"required context never reported: {age_minutes}m since CI started " + f"on {sha[:8]}, {len(states)} other contexts present, " + f"base={base} grace={GRACE_MINUTES}m" + ) + rows.append(f"ci|CRITICAL|{key}|{detail}") + + return rows, tally + + +def probe_zero_job_runs(gh: GitHub, repo: str, now: datetime) -> list[str]: + """Flag workflow runs that concluded having created zero jobs. + + This is the startup-failure signature and it has no benign reading: GitHub + could not assemble the workflow file, so it never parsed the file's + ``name:`` (the run renders as the raw path) and never created a job. It + fires upstream of the absent context and is unambiguous on its own. + """ + rows: list[str] = [] + runs = _unwrap(gh.get(f"/repos/{OWNER}/{repo}/actions/runs?per_page=50")) + entries = runs.get("workflow_runs", []) if isinstance(runs, dict) else [] + + for run in entries: + if not isinstance(run, dict) or run.get("status") != "completed": + continue + conclusion = str(run.get("conclusion") or "") + if conclusion in {"success", "skipped", "cancelled", "neutral"}: + continue + created = _parse_ts(run.get("created_at")) + updated = _parse_ts(run.get("updated_at")) + if ( + created is None + or (now - created).total_seconds() > ZEROJOB_WINDOW_MINUTES * 60 + ): + continue + + if conclusion != "startup_failure": + # Cheap pre-filter: a run that did real work cannot be a startup + # failure, so only near-instant runs are worth a jobs lookup. + if updated is None: + continue + if (updated - created).total_seconds() > ZEROJOB_MAX_DURATION_SECONDS: + continue + jobs = _unwrap( + gh.get( + f"/repos/{OWNER}/{repo}/actions/runs/{run.get('id')}/jobs?per_page=1" + ) + ) + total = jobs.get("total_count", -1) if isinstance(jobs, dict) else -1 + if total != 0: + continue + + key = _sanitize(f"zerojob/{repo}/{run.get('id')}") + detail = _sanitize( + f"workflow run concluded '{conclusion}' with 0 jobs (startup failure): " + f"{run.get('name', '?')} head={str(run.get('head_sha', ''))[:8]} " + f"branch={run.get('head_branch', '?')}" + ) + rows.append(f"ci|CRITICAL|{key}|{detail}") + + return rows + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--repos", + default=os.environ.get("OMNINODE_CI_PROBE_REPOS", ",".join(DEFAULT_REPOS)), + help="comma-separated repo names to probe", + ) + parser.add_argument( + "--skip-zero-job-scan", + action="store_true", + help="skip the workflow-run startup-failure scan", + ) + args = parser.parse_args(argv) + + repos = [r.strip() for r in args.repos.split(",") if r.strip()] + fixture_dir = os.environ.get("OMNINODE_CI_PROBE_FIXTURE_DIR") or None + token = os.environ.get("GH_PAT") or os.environ.get("GITHUB_TOKEN") + gh = GitHub(token, fixture_dir) + now = _now() + + rows: list[str] = [] + totals = {"prs": 0, "required": 0, "absent": 0, "pending": 0, "failed": 0} + scanned = 0 + degraded: list[str] = [] + + if not fixture_dir and not token: + # Unauthenticated reads cannot see branch protection at all, so the + # probe would report a clean board while being structurally blind. + emit( + "ci|WARNING|probe/credentials|no GH_PAT or GITHUB_TOKEN in environment; " + "required-context probe cannot read branch protection and did NOT run" + ) + return 0 + + for repo in repos: + try: + repo_rows, tally = probe_repo(gh, repo, now) + rows.extend(repo_rows) + for name, value in tally.items(): + totals[name] += value + if not args.skip_zero_job_scan: + rows.extend(probe_zero_job_runs(gh, repo, now)) + scanned += 1 + except ProbeError as exc: + degraded.append(repo) + rows.append( + f"ci|WARNING|probe/{_sanitize(repo)}|" + f"probe did not complete, state unknown: {_sanitize(str(exc))}" + ) + except Exception as exc: # noqa: BLE001 - one repo must not kill the tick + degraded.append(repo) + rows.append( + f"ci|WARNING|probe/{_sanitize(repo)}|" + f"unexpected probe error, state unknown: {_sanitize(type(exc).__name__)}" + ) + + if scanned == 0: + # "Scanned nothing" must never render as "found nothing wrong" -- that + # is the detection-shelf blindness this heartbeat exists to make loud. + rows.append( + f"ci|WARNING|required-contexts|probe scanned 0 of {len(repos)} repos; " + f"no required-context coverage this tick" + ) + else: + rows.append( + f"ci|OK|required-contexts|scanned {scanned}/{len(repos)} repos, " + f"{totals['prs']} open non-draft PRs, {totals['required']} required contexts; " + f"absent={totals['absent']} pending={totals['pending']} failed={totals['failed']}" + + (f" degraded={','.join(degraded)}" if degraded else "") + ) + + for row in rows: + emit(row) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/preflight_lane_deploy_attribution.py b/scripts/preflight_lane_deploy_attribution.py new file mode 100644 index 0000000000..698470830f --- /dev/null +++ b/scripts/preflight_lane_deploy_attribution.py @@ -0,0 +1,861 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Lane deploy ATTRIBUTION recorder + live-grant INTERLOCK (OMN-15218). + +Twice in two days the ``.201`` stability-test lane was rebuilt/restarted with +**no attributable trigger** while live, unconsumed prod-promotion grants were +pinned to the digests that rebuild replaced: + + * 2026-07-26T21:45:15Z — lane rebuilt to a local-workspace image while grant + ``grant-6dbeae94`` was live and pinned to the PREVIOUS digests. + * 2026-07-27T10:05:43-10:09:07Z — stability containers restarted by an + unknown actor while the three ``batch-b551aa00`` grants were live. + +Neither event named an actor, a reason, or a ticket, and neither was blocked or +even warned about. The prod-promotion gate (OMN-13418) resolves +"stability-proven" from the LIVE stability container at grant-issuance time, so +an unattributed stability rebuild between issuance and consumption silently +erodes the premise every live grant rests on. Two occurrences make this a +mechanism gap, not an incident — a rule is not a mechanism. + +This module is that mechanism. It is a **pre-mutation preflight** invoked from +the sanctioned deploy path (``scripts/deploy-runtime.sh`` and the lane refresh +tooling in ``scripts/runtime_build/``) BEFORE anything is tagged, built, +recreated, or restarted. It enforces two rules: + +1. **ATTRIBUTION (fail-fast).** Every deploy/restart of a governed lane + (``stability-test``, ``prod``, ``judge``) must declare *why* it is happening: + ``ONEX_DEPLOY_REASON`` is mandatory and must be a real sentence, not a + placeholder. Actor identity (user, uid, host, ssh peer, parent command), + the invoking command, the resolved ticket, and the grant verdict are written + to a durable JSONL deploy log plus a per-run attribution record, and are + folded into ``registry.json`` by the caller. An unattributed rebuild becomes + impossible on this path and traceable after the fact everywhere else. + +2. **GRANT INTERLOCK (fail-closed, refuse-by-default).** When the target lane is + ``stability-test`` and ``onex_change_control`` carries unconsumed, unexpired + prod-promotion grants at ``@main``, the deploy REFUSES and names every live + grant. The only override is an explicit acknowledgement + (``ONEX_DEPLOY_GRANT_ACK``) that must name **each** live ``grant_id``; the + acknowledgement itself is recorded in the attribution record, so overriding + is attributable rather than silent. A blanket ``true`` does not work — a + stale acknowledgement left in an environment cannot pre-authorize a grant + that did not exist when it was set. + + Grant state is resolved from ``onex_change_control@main`` (never a PR + branch), exactly like the OMN-13418 resolver's I/O boundary. If that state + cannot be established — no clone, fetch failure, unparseable YAML, malformed + entry — the verdict is ``UNREADABLE`` and the deploy REFUSES. Indeterminate + grant state is not a pass; it needs the ``unreadable-grant-state`` + acknowledgement token to proceed. + +Layering note: ``omnibase_infra`` must not import ``omnimarket`` (where the +OMN-13439 grant resolver EFFECT lives). The grant FILE is the contract surface +— this module parses that YAML directly and imports nothing from the governance +or market repos, the same way the resolver imports nothing from +``onex_change_control``. + +Test seam: every I/O boundary is injectable. ``--grants-file`` substitutes a +local file for the ``@main`` fetch and ``--now`` pins evaluation time, so the +whole verdict table is exercised hermetically — no lane contact, no network, no +real ``@main`` dependency (faithful dependency substitution, not mocks). + +Usage:: + + # From deploy-runtime.sh / refresh_stability_lane.sh, before any mutation: + ONEX_DEPLOY_REASON="OMN-15181 prod bootstrap rehearsal" \\ + uv run python scripts/preflight_lane_deploy_attribution.py \\ + --lane stability-test \\ + --compose-project omnibase-infra-stability-test \\ + --source deploy-runtime.sh \\ + --invoking-command "deploy-runtime.sh --execute --force --restart" + + # Evaluate without writing the durable record (dry-run/preview): + ... --check-only + +Exit codes: + 0 — allowed; attribution recorded (JSON record on stdout) + 1 — REFUSED (missing attribution, live grants, or unreadable grant state) + 2 — usage / environment error +""" + +from __future__ import annotations + +import argparse +import enum +import getpass +import hashlib +import json +import os +import socket +import subprocess +import sys +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime, timezone +from pathlib import Path +from typing import Any + +import yaml + +# --- policy constants -------------------------------------------------------- + +#: Lanes whose deploys/restarts require durable attribution. ``dev`` is +#: deliberately excluded: it is the fully-mutable test platform (CLAUDE.md lane +#: table) and gating it would push operators off the sanctioned path. +GOVERNED_LANES: frozenset[str] = frozenset({"stability-test", "prod", "judge"}) + +#: Lanes the live-grant interlock applies to. Prod promotion is already gated by +#: the OMN-13418 grant gate itself; the hole this closes is the PRE-prod lane +#: whose rebuild invalidates the "stability-proven" premise of a live grant. +GRANT_INTERLOCK_LANES: frozenset[str] = frozenset({"stability-test"}) + +#: The grant registry path inside the ``onex_change_control`` repo. +GRANTS_REPO_RELPATH = "grants/prod_promotion_grants.yaml" + +#: The ref the grant registry is resolved from. Never a PR branch (OMN-13418). +GRANTS_REF = "main" + +#: Env var carrying the mandatory human reason for the deploy. +ENV_REASON = "ONEX_DEPLOY_REASON" + +#: Env var carrying an explicit ticket id (otherwise extracted from the reason). +ENV_TICKET = "ONEX_DEPLOY_TICKET" + +#: Env var carrying the live-grant acknowledgement (must name each grant_id). +ENV_GRANT_ACK = "ONEX_DEPLOY_GRANT_ACK" + +#: Optional actor override for automation that knows its own identity better +#: than ``getpass.getuser()`` does (e.g. a CI runner naming its workflow run). +ENV_ACTOR = "ONEX_DEPLOY_ACTOR" + +#: Acknowledgement token required to proceed past an UNREADABLE grant state. +UNREADABLE_ACK_TOKEN = "unreadable-grant-state" + +#: A reason must clear this length AND not be one of the placeholders below. +MIN_REASON_LENGTH = 12 + +#: Placeholder reasons that carry no information. Rejected so the mandatory +#: field cannot be satisfied with a keystroke (the OMN-15218 failure mode is +#: *unattributed* rebuilds; "x" is unattributed with extra steps). +PLACEHOLDER_REASONS: frozenset[str] = frozenset( + { + "", + "-", + ".", + "n/a", + "na", + "none", + "null", + "tbd", + "todo", + "test", + "testing", + "deploy", + "redeploy", + "restart", + "rebuild", + "update", + "wip", + "x", + "xxx", + "asdf", + "reason", + } +) + +#: Required fields on a grant entry (OMN-13437 schema). A missing field makes +#: the registry unreadable rather than "no live grants". +REQUIRED_GRANT_FIELDS: frozenset[str] = frozenset( + { + "grant_id", + "runtime_lane", + "image_digest", + "promotion_batch_id", + "approved_by", + "expires_at", + "created_at", + "reason", + } +) + +#: Bound on the git calls used to resolve grant state. A hung fetch must fail +#: closed on a timer, not block a deploy forever. +GIT_TIMEOUT_SECONDS = 60 + +SCHEMA_VERSION = "1.0.0" + + +class EnumGrantVerdict(enum.Enum): + """Outcome of resolving live grant state for the target lane.""" + + #: Lane is outside :data:`GRANT_INTERLOCK_LANES`; interlock did not run. + NOT_APPLICABLE = "NOT_APPLICABLE" + #: Grant state read successfully; no unconsumed, unexpired grants. + CLEAR = "CLEAR" + #: Grant state read successfully; live grants exist and pin this lane's proof. + LIVE_GRANTS = "LIVE_GRANTS" + #: Grant state could NOT be established. Fails closed. + UNREADABLE = "UNREADABLE" + + +class PreflightError(RuntimeError): + """Usage / environment error (exit 2). Never a policy refusal.""" + + +# --- pure helpers ------------------------------------------------------------ + + +def lane_from_compose_project(compose_project: str) -> str: + """Derive the lane name from a compose project name. + + ``omnibase-infra`` -> ``dev``; ``omnibase-infra-`` -> ````. + Mirrors the derivation deploy-runtime.sh uses for its overlay and hot-patch + gates so one deploy cannot be attributed to two different lane names. + """ + lane = compose_project.removeprefix("omnibase-infra").removeprefix("-") + return lane or "dev" + + +def normalize_reason(raw: str | None) -> str: + """Collapse whitespace on a reason string (``None`` -> empty).""" + return " ".join((raw or "").split()) + + +def reason_is_meaningful(reason: str) -> bool: + """Whether a reason is a real justification rather than a placeholder.""" + collapsed = normalize_reason(reason) + if collapsed.strip(".- ").lower() in PLACEHOLDER_REASONS: + return False + return len(collapsed) >= MIN_REASON_LENGTH + + +def extract_ticket(*candidates: str | None) -> str: + """Return the first ``OMN-`` token found in the candidates.""" + for candidate in candidates: + if not candidate: + continue + for token in candidate.replace(",", " ").replace("/", " ").split(): + stripped = token.strip("()[]{}:;.,").upper() + if stripped.startswith("OMN-") and stripped[4:].isdigit(): + return stripped + return "" + + +def parse_ack_tokens(raw: str | None) -> tuple[str, ...]: + """Split an acknowledgement env value into normalized tokens.""" + if not raw: + return () + for separator in (",", ";"): + raw = raw.replace(separator, " ") + return tuple(token.strip().lower() for token in raw.split() if token.strip()) + + +def _coerce_datetime(value: Any) -> datetime: + """Coerce a grant timestamp (ISO-8601, ``Z`` allowed) to an aware datetime.""" + if isinstance(value, datetime): + parsed = value + elif isinstance(value, str): + parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00")) + else: + raise ValueError( + f"grant timestamp must be a datetime or ISO-8601 string; got {value!r}" + ) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed + + +def evaluate_grant_state(raw: bytes, *, now: datetime) -> dict[str, Any]: + """Evaluate raw grant-registry bytes into a verdict block. Pure — no I/O. + + Returns a dict with ``verdict``, ``live_grants``, ``grants_sha256`` and + ``errors``. Any structural surprise (non-mapping root, missing ``entries`` + list, entry missing a required field, unparseable timestamp) yields + ``UNREADABLE`` — an unparseable registry must never read as "no grants". + """ + block: dict[str, Any] = { + "verdict": EnumGrantVerdict.UNREADABLE.value, + "grants_sha256": hashlib.sha256(raw).hexdigest(), + "live_grants": [], + "errors": [], + } + + try: + document = yaml.safe_load(raw.decode("utf-8")) + except (UnicodeDecodeError, yaml.YAMLError) as exc: + block["errors"].append(f"grant registry does not parse as YAML: {exc}") + return block + + if document is None: + block["errors"].append("grant registry is empty (expected an 'entries' key)") + return block + if not isinstance(document, Mapping): + block["errors"].append( + f"grant registry root must be a mapping; got {type(document).__name__}" + ) + return block + + entries = document.get("entries") + if entries is None: + block["errors"].append("grant registry has no 'entries' key") + return block + if not isinstance(entries, Sequence) or isinstance(entries, (str, bytes)): + block["errors"].append( + f"grant registry 'entries' must be a list; got {type(entries).__name__}" + ) + return block + + live: list[dict[str, Any]] = [] + for index, entry in enumerate(entries): + if not isinstance(entry, Mapping): + block["errors"].append(f"entry[{index}] is not a mapping") + return block + missing = sorted(REQUIRED_GRANT_FIELDS - set(entry)) + if missing: + block["errors"].append( + f"entry[{index}] is missing required field(s): {', '.join(missing)}" + ) + return block + try: + expires_at = _coerce_datetime(entry["expires_at"]) + except ValueError as exc: + block["errors"].append( + f"entry[{index}] ({entry['grant_id']}) has an unparseable expires_at: {exc}" + ) + return block + + # A consumed grant is spent (OMN-13424) and no longer pins this lane. + if entry.get("consumed", False) is True: + continue + if expires_at <= now: + continue + + live.append( + { + "grant_id": str(entry["grant_id"]), + "runtime_lane": str(entry["runtime_lane"]), + "image_digest": str(entry["image_digest"]), + "promotion_batch_id": str(entry["promotion_batch_id"]), + "approved_by": str(entry["approved_by"]), + "expires_at": expires_at.isoformat().replace("+00:00", "Z"), + } + ) + + block["live_grants"] = live + block["verdict"] = ( + EnumGrantVerdict.LIVE_GRANTS if live else EnumGrantVerdict.CLEAR + ).value + return block + + +def apply_acknowledgement( + grant_block: Mapping[str, Any], ack_tokens: Sequence[str] +) -> dict[str, Any]: + """Decide whether an acknowledgement clears the grant verdict. + + ``LIVE_GRANTS`` clears only when every live ``grant_id`` is named. That is + what makes the override attributable rather than a blanket switch: an + acknowledgement set before a grant existed cannot name it, so it cannot + silently authorize the next rebuild. + + ``UNREADABLE`` clears only with the explicit + :data:`UNREADABLE_ACK_TOKEN` sentinel. + """ + verdict = str(grant_block.get("verdict")) + lowered = {token.lower() for token in ack_tokens} + result: dict[str, Any] = { + "acknowledged": False, + "acknowledgement_tokens": list(ack_tokens), + "unacknowledged_grant_ids": [], + } + + if verdict == EnumGrantVerdict.LIVE_GRANTS.value: + live_ids = [ + str(grant["grant_id"]) for grant in grant_block.get("live_grants", []) + ] + missing = [grant_id for grant_id in live_ids if grant_id.lower() not in lowered] + result["unacknowledged_grant_ids"] = missing + result["acknowledged"] = not missing + elif verdict == EnumGrantVerdict.UNREADABLE.value: + result["acknowledged"] = UNREADABLE_ACK_TOKEN in lowered + + return result + + +def collect_actor(env: Mapping[str, str]) -> dict[str, Any]: + """Collect the durable actor identity for this invocation. + + Records everything cheaply available that distinguishes "who did this": + login user, uid, host, the ssh peer (which is what an unattributed remote + rebuild would otherwise hide), and the parent process command line. + """ + try: + user = getpass.getuser() + except (KeyError, OSError): # pragma: no cover - only fails on exotic hosts + user = env.get("USER", "unknown") + + actor: dict[str, Any] = { + "user": env.get("SUDO_USER") or user, + "effective_user": user, + "uid": os.getuid(), + "host": socket.gethostname(), + "ssh_connection": env.get("SSH_CONNECTION", ""), + "ssh_client": env.get("SSH_CLIENT", ""), + "declared_actor": env.get(ENV_ACTOR, ""), + "parent_command": _parent_command(), + "ci": env.get("GITHUB_RUN_ID", ""), + } + actor["identity"] = actor["declared_actor"] or f"{actor['user']}@{actor['host']}" + return actor + + +def _parent_command() -> str: + """Best-effort parent-process command line (empty string when unavailable).""" + try: + completed = subprocess.run( + ["ps", "-o", "args=", "-p", str(os.getppid())], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + except ( + OSError, + subprocess.SubprocessError, + ): # pragma: no cover - platform dependent + return "" + return completed.stdout.strip() if completed.returncode == 0 else "" + + +# --- I/O boundary ------------------------------------------------------------ + + +def fetch_grant_bytes_from_main(grants_repo: Path) -> tuple[bytes, str]: + """Read the grant registry from ``onex_change_control@main``. + + Returns ``(raw_bytes, resolved_commit_sha)``. Raises :class:`OSError` / + :class:`subprocess.SubprocessError` derivatives on any failure so the caller + can fail closed. Never falls back to the working tree or a PR branch — the + ``@main`` anchor is the anti-self-issue property of the whole grant scheme. + """ + if not (grants_repo / ".git").exists(): + raise FileNotFoundError(f"not a git clone: {grants_repo}") + + def _git(*args: str) -> str: + completed = subprocess.run( + [ + "git", + "-c", + f"safe.directory={grants_repo}", + "-C", + str(grants_repo), + *args, + ], + capture_output=True, + text=True, + timeout=GIT_TIMEOUT_SECONDS, + check=False, + ) + if completed.returncode != 0: + raise ChildProcessError( + f"git {' '.join(args)} failed: {completed.stderr.strip()}" + ) + return completed.stdout + + # Refresh first: a stale local origin/main would hide a grant that landed + # minutes ago, which is exactly the window this interlock exists to cover. + _git("fetch", "origin", GRANTS_REF, "--quiet") + commit = _git("rev-parse", f"origin/{GRANTS_REF}").strip() + raw = subprocess.run( + [ + "git", + "-c", + f"safe.directory={grants_repo}", + "-C", + str(grants_repo), + "show", + f"origin/{GRANTS_REF}:{GRANTS_REPO_RELPATH}", + ], + capture_output=True, + timeout=GIT_TIMEOUT_SECONDS, + check=False, + ) + if raw.returncode != 0: + raise ChildProcessError( + f"git show origin/{GRANTS_REF}:{GRANTS_REPO_RELPATH} failed: {raw.stderr.decode(errors='replace').strip()}" + ) + return raw.stdout, commit + + +def resolve_grant_block( + *, + lane: str, + now: datetime, + grants_file: Path | None, + grants_repo: Path | None, +) -> dict[str, Any]: + """Resolve the grant verdict block for ``lane``, failing closed on any error.""" + if lane not in GRANT_INTERLOCK_LANES: + return { + "verdict": EnumGrantVerdict.NOT_APPLICABLE.value, + "source": "", + "grants_commit": "", + "grants_sha256": "", + "live_grants": [], + "errors": [], + } + + if grants_file is not None: + # Offline / test substitution: the exact same evaluation over a local + # file. Used by the unit suite and by an operator diagnosing a verdict. + try: + raw = grants_file.read_bytes() + except OSError as exc: + return { + "verdict": EnumGrantVerdict.UNREADABLE.value, + "source": str(grants_file), + "grants_commit": "", + "grants_sha256": "", + "live_grants": [], + "errors": [f"could not read grant registry file: {exc}"], + } + block = evaluate_grant_state(raw, now=now) + block["source"] = str(grants_file) + block["grants_commit"] = "" + return block + + if grants_repo is None: + return { + "verdict": EnumGrantVerdict.UNREADABLE.value, + "source": "", + "grants_commit": "", + "grants_sha256": "", + "live_grants": [], + "errors": [ + "no onex_change_control clone resolved (set OMNI_HOME or pass " + "--grants-repo) — cannot establish live grant state at " + f"onex_change_control@{GRANTS_REF}" + ], + } + + try: + raw, commit = fetch_grant_bytes_from_main(grants_repo) + except (OSError, subprocess.SubprocessError, ValueError) as exc: + # Fail closed on ANY resolution failure: missing clone (FileNotFoundError), + # failed fetch/show (ChildProcessError, an OSError), timeout + # (subprocess.TimeoutExpired), or a decode/format surprise. + return { + "verdict": EnumGrantVerdict.UNREADABLE.value, + "source": f"{grants_repo}@origin/{GRANTS_REF}", + "grants_commit": "", + "grants_sha256": "", + "live_grants": [], + "errors": [ + f"could not resolve grant registry at origin/{GRANTS_REF}: {exc}" + ], + } + + block = evaluate_grant_state(raw, now=now) + block["source"] = f"{grants_repo}@origin/{GRANTS_REF}:{GRANTS_REPO_RELPATH}" + block["grants_commit"] = commit + return block + + +# --- record assembly --------------------------------------------------------- + + +def build_record( + *, + lane: str, + compose_project: str, + source: str, + invoking_command: str, + mode: str, + env: Mapping[str, str], + now: datetime, + grant_block: Mapping[str, Any], +) -> dict[str, Any]: + """Assemble the attribution record and its ALLOW/REFUSE verdict. Pure.""" + reason = normalize_reason(env.get(ENV_REASON)) + ack_tokens = parse_ack_tokens(env.get(ENV_GRANT_ACK)) + ack = apply_acknowledgement(grant_block, ack_tokens) + + refusals: list[str] = [] + attribution_required = lane in GOVERNED_LANES + + if attribution_required and not reason: + refusals.append( + f"{ENV_REASON} is not set. Every {lane} deploy/restart must declare why it is " + "happening (OMN-15218: two unattributed stability rebuilds in two days)." + ) + elif attribution_required and not reason_is_meaningful(reason): + refusals.append( + f"{ENV_REASON}={reason!r} is a placeholder. Give a real justification " + f"(>= {MIN_REASON_LENGTH} chars, ideally naming a ticket)." + ) + + verdict = str(grant_block.get("verdict")) + live_grants = list(grant_block.get("live_grants", [])) + if verdict == EnumGrantVerdict.LIVE_GRANTS.value and not ack["acknowledged"]: + named = ", ".join( + f"{grant['grant_id']} (lane={grant['runtime_lane']}, digest={grant['image_digest'][:19]}…, " + f"batch={grant['promotion_batch_id']}, expires={grant['expires_at']})" + for grant in live_grants + ) + refusals.append( + f"{len(live_grants)} live prod-promotion grant(s) pin the current {lane} proof: {named}. " + f"Refreshing {lane} now invalidates the stability-proven premise those grants rest on " + "(OMN-15218 / OMN-13418). To proceed anyway, acknowledge every grant explicitly: " + f'{ENV_GRANT_ACK}="{",".join(str(g["grant_id"]) for g in live_grants)}" — the ' + "acknowledgement is recorded in the attribution record." + ) + elif verdict == EnumGrantVerdict.UNREADABLE.value and not ack["acknowledged"]: + refusals.append( + "live prod-promotion grant state is UNREADABLE, so this deploy cannot be proven safe: " + + "; ".join(str(err) for err in grant_block.get("errors", [])) + + f". Fail-closed (OMN-15218). Fix the grant source, or set {ENV_GRANT_ACK}=" + f"{UNREADABLE_ACK_TOKEN} to proceed on the record." + ) + + ticket = extract_ticket(env.get(ENV_TICKET), reason) + + return { + "schema_version": SCHEMA_VERSION, + "mechanism_ticket": "OMN-15218", + "ts_utc": now.isoformat().replace("+00:00", "Z"), + "lane": lane, + "compose_project": compose_project, + "source": source, + "mode": mode, + "invoking_command": invoking_command, + "reason": reason, + "ticket": ticket, + "attribution_required": attribution_required, + "actor": collect_actor(env), + "grant_guard": { + "applies": lane in GRANT_INTERLOCK_LANES, + "verdict": verdict, + "grants_ref": f"onex_change_control@{GRANTS_REF}", + "source": grant_block.get("source", ""), + "grants_commit": grant_block.get("grants_commit", ""), + "grants_sha256": grant_block.get("grants_sha256", ""), + "live_grants": live_grants, + "errors": list(grant_block.get("errors", [])), + **ack, + }, + "result": "REFUSE" if refusals else "ALLOW", + "refusal_reasons": refusals, + } + + +def write_record(record: Mapping[str, Any], record_dir: Path) -> dict[str, str]: + """Persist the record: one JSONL deploy-log line + one per-run JSON file. + + The JSONL log is append-only and is the surface a future session reads to + answer "who rebuilt this lane, when, and why" without re-deriving a forensic + chain by hand. Both REFUSE and ALLOW records are written — a refused deploy + attempt is itself attribution-worthy. + """ + record_dir.mkdir(parents=True, exist_ok=True) + attribution_dir = record_dir / "deploy-attribution" + attribution_dir.mkdir(parents=True, exist_ok=True) + + stamp = str(record["ts_utc"]).replace(":", "").replace("-", "") + record_path = ( + attribution_dir / f"{stamp}-{record['lane']}-{record['result'].lower()}.json" + ) + record_path.write_text( + json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + log_path = record_dir / "deploy-log.jsonl" + with log_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + + return {"record_path": str(record_path), "log_path": str(log_path)} + + +# --- CLI --------------------------------------------------------------------- + + +def default_grants_repo(env: Mapping[str, str]) -> Path | None: + """Resolve the ``onex_change_control`` clone from ``OMNI_HOME`` (may be None).""" + omni_home = env.get("OMNI_HOME", "").strip() + if not omni_home: + return None + return Path(omni_home) / "onex_change_control" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Record lane-deploy attribution and enforce the live-grant interlock (OMN-15218).", + ) + parser.add_argument( + "--lane", + default="", + help="Target lane (derived from --compose-project when omitted).", + ) + parser.add_argument( + "--compose-project", default="", help="Compose project of the target lane." + ) + parser.add_argument( + "--source", default="unknown", help="Which entrypoint invoked this preflight." + ) + parser.add_argument( + "--invoking-command", default="", help="The command line being guarded." + ) + parser.add_argument( + "--record-dir", + default="", + help="Directory for deploy-log.jsonl (default: ~/.omnibase/infra).", + ) + parser.add_argument( + "--grants-repo", + default="", + help="onex_change_control clone (default: $OMNI_HOME/onex_change_control).", + ) + parser.add_argument( + "--grants-file", + default="", + help="Read grant state from this file instead of @main (offline/test).", + ) + parser.add_argument( + "--now", default="", help="ISO-8601 evaluation time (default: now, UTC)." + ) + parser.add_argument( + "--check-only", + action="store_true", + help="Evaluate and print; do not write the durable record.", + ) + parser.add_argument( + "--json", + action="store_true", + help="Print only the JSON record on stdout (no human summary).", + ) + return parser + + +def _print_human_summary(record: Mapping[str, Any], stream: Any) -> None: + grant_guard = record["grant_guard"] + print( + f"[lane-deploy-attribution] lane : {record['lane']} ({record['compose_project']})", + file=stream, + ) + print( + f"[lane-deploy-attribution] actor : {record['actor']['identity']} (uid={record['actor']['uid']})", + file=stream, + ) + if record["actor"]["ssh_connection"]: + print( + f"[lane-deploy-attribution] ssh : {record['actor']['ssh_connection']}", + file=stream, + ) + print( + f"[lane-deploy-attribution] reason : {record['reason'] or ''}", + file=stream, + ) + print( + f"[lane-deploy-attribution] ticket : {record['ticket'] or ''}", + file=stream, + ) + print( + f"[lane-deploy-attribution] invoked by : {record['source']}", file=stream + ) + print( + f"[lane-deploy-attribution] grant verdict : {grant_guard['verdict']} ({len(grant_guard['live_grants'])} live)", + file=stream, + ) + for grant in grant_guard["live_grants"]: + print( + f"[lane-deploy-attribution] - {grant['grant_id']} lane={grant['runtime_lane']} " + f"digest={grant['image_digest']} batch={grant['promotion_batch_id']} expires={grant['expires_at']}", + file=stream, + ) + if grant_guard["acknowledged"]: + print( + f"[lane-deploy-attribution] acknowledged : {', '.join(grant_guard['acknowledgement_tokens'])}", + file=stream, + ) + print( + f"[lane-deploy-attribution] result : {record['result']}", file=stream + ) + for reason in record["refusal_reasons"]: + print(f"[lane-deploy-attribution] REFUSED: {reason}", file=stream) + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + env = os.environ + + compose_project = args.compose_project.strip() + lane = args.lane.strip() + if not lane and not compose_project: + print("ERROR: one of --lane / --compose-project is required.", file=sys.stderr) + return 2 + if not lane: + lane = lane_from_compose_project(compose_project) + if not compose_project: + compose_project = ( + "omnibase-infra" if lane == "dev" else f"omnibase-infra-{lane}" + ) + + if args.now: + try: + now = _coerce_datetime(args.now) + except ValueError as exc: + print(f"ERROR: --now is not an ISO-8601 datetime: {exc}", file=sys.stderr) + return 2 + else: + now = datetime.now(UTC) + + grants_file = Path(args.grants_file).expanduser() if args.grants_file else None + grants_repo = ( + Path(args.grants_repo).expanduser() + if args.grants_repo + else default_grants_repo(env) + ) + + grant_block = resolve_grant_block( + lane=lane, + now=now, + grants_file=grants_file, + grants_repo=grants_repo, + ) + + record = build_record( + lane=lane, + compose_project=compose_project, + source=args.source, + invoking_command=args.invoking_command, + mode="check-only" if args.check_only else "execute", + env=env, + now=now, + grant_block=grant_block, + ) + + if not args.check_only: + record_dir = ( + Path(args.record_dir).expanduser() + if args.record_dir + else Path.home() / ".omnibase" / "infra" + ) + try: + record["written"] = write_record(record, record_dir) + except OSError as exc: + # A record we cannot persist is not attribution. Fail, do not warn. + print( + f"ERROR: could not write the attribution record under {record_dir}: {exc}", + file=sys.stderr, + ) + return 2 + + if not args.json: + _print_human_summary(record, sys.stderr) + print(json.dumps(record, sort_keys=True)) + + return 1 if record["result"] == "REFUSE" else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/proof/e2e_cloud_workflow_harness.py b/scripts/proof/e2e_cloud_workflow_harness.py new file mode 100644 index 0000000000..09c3907d36 --- /dev/null +++ b/scripts/proof/e2e_cloud_workflow_harness.py @@ -0,0 +1,388 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# Copyright (c) 2026 OmniNode Team +"""OMN-10858 -- end-to-end cloud workflow proof harness (skeleton). + +Chain proven: **login -> tenant -> submit -> terminal readback -> cross-tenant +denial**, against the managed-staging one-gateway / one-synthetic-tenant surface +frozen by OMN-15123. + +Status of this file +------------------- +This is a **skeleton**, and it says so out loud rather than pretending otherwise. +Every stage function names *exactly* which endpoint or topic it asserts against +(see ``STAGE_SURFACES``, matched field-by-field against +``docs/runbooks/managed-staging-proof-kit/fields.yaml`` by +``tests/ci/test_managed_staging_proof_kit_seam.py``), but the live bodies raise +:class:`StageNotImplementedError`. They are filled in when the canary lane is up; the +skeleton exists so the *shape* of the proof is reviewable and seam-checked now, +not invented under time pressure on the day of the run. + +Safety posture +-------------- +* ``--live`` **defaults OFF**. With no flags this program performs **no network, + no broker, and no database I/O**: it prints the plan -- each stage and the + surface it would assert against -- and exits 0. +* ``--live`` requires a fully resolved :class:`HarnessConfig`; anything missing + raises :class:`HarnessConfigError`. It **fails closed**, never degrades to a + partial run, and there is no default/fallback endpoint anywhere in this file. +* Nothing here targets prod. Gateway, topic prefix, brokers, and DSN all come + from explicit flags or ``ONEX_E2E_*`` environment variables. + +Usage +----- +:: + + # dry run -- safe anywhere, and the default + uv run python scripts/proof/e2e_cloud_workflow_harness.py + + # live run -- HELD FOR OPERATOR; every value must be supplied + uv run python scripts/proof/e2e_cloud_workflow_harness.py --live \\ + --gateway-base-url https:// \\ + --tenant-id --other-tenant-id \\ + --topic-prefix onex.mstg1. \\ + --bootstrap-servers \\ + --projection-dsn "$CANARY_DSN" + +A completed live run emits the per-stage evidence that OMN-10858's +``workflow_receipt.json`` is assembled from; the receipt's verifier must not be +the runner. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# --- errors ----------------------------------------------------------------- + + +class HarnessConfigError(RuntimeError): + """A live run was requested without a fully resolved configuration. + + Raised *before* any I/O. This is the fail-closed boundary: the harness never + substitutes a default endpoint, a plaintext broker, or a relaxed sslmode. + """ + + +class StageNotImplementedError(NotImplementedError): + """A live stage body is not implemented yet in this skeleton.""" + + +# --- config ----------------------------------------------------------------- + + +@dataclass(frozen=True) +class HarnessConfig: + """Fully resolved live-run configuration. Every field is required.""" + + gateway_base_url: str + tenant_id: str + other_tenant_id: str + topic_prefix: str + bootstrap_servers: str + projection_dsn: str + + @classmethod + def from_args(cls, args: argparse.Namespace) -> HarnessConfig: + """Resolve config from flags, falling back to ``ONEX_E2E_*`` env vars. + + Missing or blank values raise :class:`HarnessConfigError` naming every + missing key at once -- never one-at-a-time discovery mid-run. + """ + env_keys = { + "gateway_base_url": "ONEX_E2E_GATEWAY_BASE_URL", + "tenant_id": "ONEX_E2E_TENANT_ID", + "other_tenant_id": "ONEX_E2E_OTHER_TENANT_ID", + "topic_prefix": "ONEX_E2E_TOPIC_PREFIX", + "bootstrap_servers": "ONEX_E2E_BOOTSTRAP_SERVERS", + "projection_dsn": "ONEX_E2E_PROJECTION_DSN", + } + resolved: dict[str, str] = {} + missing: list[str] = [] + for name, env_key in env_keys.items(): + value = (getattr(args, name, None) or os.environ.get(env_key) or "").strip() + if not value: + missing.append(f"--{name.replace('_', '-')} (or ${env_key})") + resolved[name] = value + if missing: + raise HarnessConfigError( + "live run refused -- unresolved configuration: " + ", ".join(missing) + ) + if resolved["tenant_id"] == resolved["other_tenant_id"]: + raise HarnessConfigError( + "live run refused -- --other-tenant-id must differ from --tenant-id, " + "or the cross-tenant denial stage is vacuous" + ) + return cls(**resolved) + + +# --- results ---------------------------------------------------------------- + + +@dataclass +class StageResult: + """One stage outcome. ``PLANNED`` means "not executed", never "passed".""" + + stage_id: str + status: str # PLANNED | PASS | FAIL | ERROR + surface: str + detail: str = "" + evidence: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class HarnessPlan: + live: bool + results: list[StageResult] + + @property + def ok(self) -> bool: + return all(r.status in {"PLANNED", "PASS"} for r in self.results) + + +# --- stage surfaces (mirrored in the proof-kit manifest) -------------------- + +STAGE_SURFACES: dict[str, str] = { + "login": ( + "gateway: POST {gateway_base_url}/auth/token (Keycloak-backed); " + "tenant-scoped claim required" + ), + "tenant": "gateway: GET {gateway_base_url}/v1/tenants/{tenant_id}", + "submit": "topic: {topic_prefix}onex.cmd.omnimarket.alpha-text-analysis-requested.v1", + "terminal_readback": ( + "topic: {topic_prefix}onex.evt.omnimarket.alpha-text-analysis-completed.v1 " + "(or ...-failed.v1); table: alpha_workflow_results" + ), + "cross_tenant_denial": ( + "gateway: GET {gateway_base_url}/v1/workflows/{workflow_id} with tenant B's token" + ), +} + + +def _surface(stage_id: str, config: HarnessConfig | None) -> str: + """Render a stage surface, with placeholders left intact on a dry run.""" + template = STAGE_SURFACES[stage_id] + if config is None: + return template + return template.format( + gateway_base_url=config.gateway_base_url, + tenant_id=config.tenant_id, + topic_prefix=config.topic_prefix, + workflow_id="", + ) + + +# --- stages ----------------------------------------------------------------- +# +# Each stage takes the resolved config plus the mutable run context (tokens, +# correlation_id, workflow_id produced by earlier stages) and returns a +# StageResult. The assertions are written out in each docstring so the skeleton +# is reviewable against OMN-10858's DoD before the bodies exist. + + +def stage_login(config: HarnessConfig, ctx: dict[str, Any]) -> StageResult: + """Assert: the gateway issues a token whose claims are scoped to ``tenant_id``. + + FAIL if: no token, or a token whose tenant claim is absent, wildcard, or + mismatched -- a wildcard-tenant token would make ``cross_tenant_denial`` + vacuous. + Evidence: token claim set (never the raw token), gateway auth context. + """ + raise StageNotImplementedError( + "stage_login: live body pending canary lane bring-up" + ) + + +def stage_tenant(config: HarnessConfig, ctx: dict[str, Any]) -> StageResult: + """Assert: the synthetic tenant resolves and equals the frozen tuple's tenant. + + FAIL if: 404, or the returned tenant id differs from OMN-15123's + ``synthetic_tenant_id`` -- proving against a different tenant proves nothing + about the canary. + Evidence: tenant id, tenant status, gateway response hash. + """ + raise StageNotImplementedError( + "stage_tenant: live body pending canary lane bring-up" + ) + + +def stage_submit(config: HarnessConfig, ctx: dict[str, Any]) -> StageResult: + """Assert: one reference command is published and accepted. + + Payload ``{prompt, max_tokens, tenant_id, correlation_id}`` onto + ``{topic_prefix}onex.cmd.omnimarket.alpha-text-analysis-requested.v1``. + FAIL if: an invalid schema is rejected *after* publication rather than before + it (OMN-10858 negative case 2 requires pre-publication rejection), or the + broker auto-created the topic (auto-create must be OFF -- see OMN-15124). + Evidence: the full submitted envelope, correlation_id, publish offset. + """ + raise StageNotImplementedError( + "stage_submit: live body pending canary lane bring-up" + ) + + +def stage_terminal_readback(config: HarnessConfig, ctx: dict[str, Any]) -> StageResult: + """Assert: a terminal event AND a projection row, correlation-linked. + + Reads ``...-completed.v1`` / ``...-failed.v1`` and the + ``alpha_workflow_results`` row carrying the same correlation/causation chain. + FAIL if: no terminal event inside the deadline; a terminal event with no + projection row (or the reverse); a correlation/causation break; or total + latency over OMN-10858's 30s budget for the reference workflow. + Evidence: every emitted event with correlation + causation links, projection + row before and after, SHA-256 of the result payload, latency_ms. + """ + raise StageNotImplementedError( + "stage_terminal_readback: live body pending canary lane bring-up" + ) + + +def stage_cross_tenant_denial( + config: HarnessConfig, ctx: dict[str, Any] +) -> StageResult: + """NEGATIVE CONTROL. Assert: tenant B cannot read tenant A's workflow. + + Re-requests the same ``workflow_id`` with a token scoped to + ``other_tenant_id``. + PASS only on 403/404. **200 is a FAIL, and a transport error is an ERROR, not + a pass** -- a connection refusal proves nothing about authorization. + Evidence: status code, response body hash, both tenant ids. + """ + raise StageNotImplementedError( + "stage_cross_tenant_denial: live body pending canary lane bring-up" + ) + + +StageFn = Callable[[HarnessConfig, dict[str, Any]], StageResult] + +STAGES: dict[str, StageFn] = { + "login": stage_login, + "tenant": stage_tenant, + "submit": stage_submit, + "terminal_readback": stage_terminal_readback, + "cross_tenant_denial": stage_cross_tenant_denial, +} + + +# --- driver ----------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="e2e_cloud_workflow_harness", + description=( + "OMN-10858 end-to-end cloud workflow proof harness. " + "Dry run by default; --live is required to touch anything." + ), + ) + parser.add_argument( + "--live", + action="store_true", + default=False, + help="execute the assertions against the real surfaces (default: OFF, plan only)", + ) + parser.add_argument("--gateway-base-url", dest="gateway_base_url", default=None) + parser.add_argument("--tenant-id", dest="tenant_id", default=None) + parser.add_argument( + "--other-tenant-id", + dest="other_tenant_id", + default=None, + help="tenant B, used only by the cross-tenant denial negative control", + ) + parser.add_argument("--topic-prefix", dest="topic_prefix", default=None) + parser.add_argument("--bootstrap-servers", dest="bootstrap_servers", default=None) + parser.add_argument("--projection-dsn", dest="projection_dsn", default=None) + parser.add_argument( + "--out-dir", + dest="out_dir", + default=None, + help="directory to write per-stage evidence into (live runs only)", + ) + return parser + + +def run(args: argparse.Namespace) -> HarnessPlan: + """Plan (default) or execute (``--live``) the proof chain. + + A dry run returns one ``PLANNED`` result per stage and performs no I/O. + """ + if not args.live: + return HarnessPlan( + live=False, + results=[ + StageResult(stage_id=sid, status="PLANNED", surface=_surface(sid, None)) + for sid in STAGES + ], + ) + + config = HarnessConfig.from_args(args) + ctx: dict[str, Any] = {} + results: list[StageResult] = [] + for stage_id, fn in STAGES.items(): + try: + results.append(fn(config, ctx)) + except StageNotImplementedError as exc: + results.append( + StageResult( + stage_id=stage_id, + status="ERROR", + surface=_surface(stage_id, config), + detail=str(exc), + ) + ) + break + return HarnessPlan(live=True, results=results) + + +def _render(plan: HarnessPlan) -> str: + mode = "LIVE" if plan.live else "DRY RUN (no I/O performed; pass --live to execute)" + lines = [f"OMN-10858 e2e cloud workflow proof harness -- {mode}", ""] + for result in plan.results: + lines.append(f" [{result.status:<7}] {result.stage_id}") + lines.append(f" surface: {result.surface}") + if result.detail: + lines.append(f" detail: {result.detail}") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + plan = run(args) + except HarnessConfigError as exc: + print(f"REFUSED: {exc}", file=sys.stderr) + return 2 + print(_render(plan)) + if plan.live and args.out_dir: + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "harness_stage_results.json").write_text( + json.dumps( + [ + { + "stage_id": r.stage_id, + "status": r.status, + "surface": r.surface, + "detail": r.detail, + "evidence": r.evidence, + } + for r in plan.results + ], + indent=2, + sort_keys=True, + ), + encoding="utf-8", + ) + return 0 if plan.ok else 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/scripts/proof/gateway_attach_e2e_proof.py b/scripts/proof/gateway_attach_e2e_proof.py new file mode 100644 index 0000000000..64b373ea56 --- /dev/null +++ b/scripts/proof/gateway_attach_e2e_proof.py @@ -0,0 +1,213 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# Copyright (c) 2026 OmniNode Team +"""OMN-15753 -- attach ingress end-to-end slice proof (skeleton). + +Chain proven: **attach (client-credentials token) -> ACTIVE session -> +heartbeat -> operator disables the tenant's Keycloak client -> next heartbeat +observes revocation -> session torn down**, against +``node_gateway_attach_effect`` (OMN-15750) once it is deployed to onex-dev +(OMN-15754). + +Status of this file +-------------------- +This is a **skeleton**, matching the convention set by +``scripts/proof/e2e_cloud_workflow_harness.py`` (OMN-10858): every stage +function names exactly which topic/endpoint it asserts against, but the live +bodies raise ``StageNotImplementedError`` until run against a deployed node. +The handler-level equivalent of every stage below is already proven at the +unit level in +``tests/unit/nodes/node_gateway_attach_effect/test_handlers.py`` (see +``test_heartbeat_after_keycloak_revocation_tears_down_session``) -- what this +harness adds is the live bus + live Keycloak round trip, which requires the +node to actually be running somewhere reachable. + +Safety posture +--------------- +* ``--live`` defaults OFF. With no flags this program prints the stage plan + and exits 0 -- no network, no Kafka, no Keycloak admin call. +* ``--live`` requires every value below explicitly; nothing defaults or + falls back silently. +* This never targets prod. onex-dev only, and only via the tenant's own + client-credentials token -- this harness never uses a Keycloak *admin* + credential except for the one explicit ``--revoke`` step, which is opt-in + and logs exactly what it is about to disable before doing it. + +Usage +----- +:: + + # dry run -- safe anywhere, and the default + uv run python scripts/proof/gateway_attach_e2e_proof.py + + # live run against onex-dev, house tenant + uv run python scripts/proof/gateway_attach_e2e_proof.py --live \\ + --edge-instance-id 201-house-tenant-test-lane \\ + --bootstrap-servers \\ + --attach-request-topic onex.cmd.omnibase-infra.gateway-attach-request.v1 \\ + --heartbeat-request-topic onex.cmd.omnibase-infra.gateway-heartbeat-request.v1 \\ + --detach-request-topic onex.cmd.omnibase-infra.gateway-detach-request.v1 \\ + --session-event-topic onex.evt.omnibase-infra.gateway-session.v1 \\ + --revoke +""" + +from __future__ import annotations + +import argparse +import sys +from dataclasses import dataclass +from datetime import UTC, datetime + + +class StageNotImplementedError(NotImplementedError): + """Raised by a live stage body that has not been wired to a real transport yet.""" + + +@dataclass(frozen=True) +class HarnessConfig: + edge_instance_id: str + bootstrap_servers: str + attach_request_topic: str + heartbeat_request_topic: str + detach_request_topic: str + session_event_topic: str + revoke: bool + + +STAGE_SURFACES: tuple[tuple[str, str], ...] = ( + ( + "attach", + "publish ModelGatewayAttachRequest (client-credentials token) to " + "{attach_request_topic}; read back ModelGatewayAttachResponse from " + "{session_event_topic}, assert status == ACTIVE", + ), + ( + "heartbeat_active", + "publish ModelGatewayHeartbeatRequest to {heartbeat_request_topic}; " + "assert session_event.event_type == HEARTBEAT_OK, revoked == False", + ), + ( + "revoke", + "Keycloak Admin API: disable the tenant's confidential client " + "(enabled=false) -- the sole admin-credential call in this harness; " + "opt-in via --revoke", + ), + ( + "heartbeat_after_revoke", + "publish another ModelGatewayHeartbeatRequest to " + "{heartbeat_request_topic}; assert session_event.event_type == " + "REVOKED, revoked == True, and that the session no longer answers a " + "subsequent detach with anything but SessionNotFoundError", + ), +) + + +def _print_plan(config: HarnessConfig) -> None: + print(f"[{datetime.now(UTC).isoformat()}] gateway attach e2e proof -- PLAN ONLY") + for name, template in STAGE_SURFACES: + if name == "revoke" and not config.revoke: + print(f" - {name}: SKIPPED (--revoke not set)") + continue + surface = template.format( + attach_request_topic=config.attach_request_topic, + heartbeat_request_topic=config.heartbeat_request_topic, + session_event_topic=config.session_event_topic, + ) + print(f" - {name}: {surface}") + + +def stage_attach(config: HarnessConfig) -> None: + raise StageNotImplementedError( + "attach: requires a live Kafka producer against " + f"{config.bootstrap_servers} / {config.attach_request_topic} -- not " + "wired until node_gateway_attach_effect is deployed and reachable " + "(OMN-15754)." + ) + + +def stage_heartbeat(config: HarnessConfig, *, expect_revoked: bool) -> None: + raise StageNotImplementedError( + f"heartbeat(expect_revoked={expect_revoked}): requires a live Kafka " + f"round trip against {config.heartbeat_request_topic} / " + f"{config.session_event_topic}." + ) + + +def stage_revoke(config: HarnessConfig) -> None: + raise StageNotImplementedError( + "revoke: requires a live Keycloak Admin API call disabling the " + "tenant's confidential client -- deliberately not implemented in " + "this skeleton pass; the admin credential this needs is a distinct, " + "higher-privilege ref from the tenant's own attach credential." + ) + + +def run_live(config: HarnessConfig) -> int: + stage_attach(config) + stage_heartbeat(config, expect_revoked=False) + if config.revoke: + stage_revoke(config) + stage_heartbeat(config, expect_revoked=True) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--live", action="store_true", default=False) + parser.add_argument("--edge-instance-id", default=None) + parser.add_argument("--bootstrap-servers", default=None) + parser.add_argument("--attach-request-topic", default=None) + parser.add_argument("--heartbeat-request-topic", default=None) + parser.add_argument("--detach-request-topic", default=None) + parser.add_argument("--session-event-topic", default=None) + parser.add_argument("--revoke", action="store_true", default=False) + args = parser.parse_args(argv) + + if not args.live: + config = HarnessConfig( + edge_instance_id=args.edge_instance_id or "", + bootstrap_servers=args.bootstrap_servers or "", + attach_request_topic=args.attach_request_topic + or "onex.cmd.omnibase-infra.gateway-attach-request.v1", + heartbeat_request_topic=args.heartbeat_request_topic + or "onex.cmd.omnibase-infra.gateway-heartbeat-request.v1", + detach_request_topic=args.detach_request_topic + or "onex.cmd.omnibase-infra.gateway-detach-request.v1", + session_event_topic=args.session_event_topic + or "onex.evt.omnibase-infra.gateway-session.v1", + revoke=args.revoke, + ) + _print_plan(config) + return 0 + + missing = [ + name + for name, value in ( + ("--edge-instance-id", args.edge_instance_id), + ("--bootstrap-servers", args.bootstrap_servers), + ("--attach-request-topic", args.attach_request_topic), + ("--heartbeat-request-topic", args.heartbeat_request_topic), + ("--detach-request-topic", args.detach_request_topic), + ("--session-event-topic", args.session_event_topic), + ) + if not value + ] + if missing: + print(f"--live requires all of: {', '.join(missing)}", file=sys.stderr) + return 2 + + config = HarnessConfig( + edge_instance_id=args.edge_instance_id, + bootstrap_servers=args.bootstrap_servers, + attach_request_topic=args.attach_request_topic, + heartbeat_request_topic=args.heartbeat_request_topic, + detach_request_topic=args.detach_request_topic, + session_event_topic=args.session_event_topic, + revoke=args.revoke, + ) + return run_live(config) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/provision-infisical.py b/scripts/provision-infisical.py index 48e6e25114..087ce0e938 100644 --- a/scripts/provision-infisical.py +++ b/scripts/provision-infisical.py @@ -422,7 +422,7 @@ def main() -> int: "--addr", default=os.environ.get( "INFISICAL_ADDR", - "http://localhost:8880", # fallback-ok: local provisioning default + "http://localhost:8880", # url-authority-ok: INFISICAL_ADDR is bootstrap-only by registry contract, and a resolver cannot reach Infisical before this provisioning tool creates it. ), help="Infisical server address (default: http://localhost:8880)", ) diff --git a/scripts/provision-keycloak.py b/scripts/provision-keycloak.py index 7d72b3d2fe..4ab62b92f8 100644 --- a/scripts/provision-keycloak.py +++ b/scripts/provision-keycloak.py @@ -586,14 +586,15 @@ def write_env_credentials( realm: str, onex_admin_secret: str, onex_service_secret: str, + issuer_base_url: str, dry_run: bool = False, ) -> None: """Write the 7 KEYCLOAK_*/ONEX_SERVICE_* vars to the env file. ``KEYCLOAK_ADMIN_URL`` is always the INTERNAL Docker DNS address - (``http://keycloak:8080``), regardless of the ``--kc-url`` argument. - The ``--kc-url`` arg is the external URL used only by this script and - must NEVER be written to the env file. + (``http://keycloak:8080``). ``KEYCLOAK_ISSUER`` is derived from the + operator-selected external ``--kc-url`` so clients receive an issuer they + can actually resolve. ``KEYCLOAK_ADMIN_PASSWORD`` is NOT written — it is consumed transiently during bootstrap and must not persist in env files. @@ -607,7 +608,7 @@ def write_env_credentials( "KEYCLOAK_REALM": realm, "KEYCLOAK_ADMIN_CLIENT_ID": "onex-admin", "KEYCLOAK_ADMIN_CLIENT_SECRET": onex_admin_secret, - "KEYCLOAK_ISSUER": f"http://localhost:28080/realms/{realm}", + "KEYCLOAK_ISSUER": f"{issuer_base_url.rstrip('/')}/realms/{realm}", "ONEX_SERVICE_CLIENT_ID": "onex-service", "ONEX_SERVICE_CLIENT_SECRET": onex_service_secret, } @@ -638,6 +639,7 @@ def seed_infisical( realm: str, onex_admin_secret: str, onex_service_secret: str, + issuer_base_url: str, dry_run: bool = False, ) -> None: """Seed Keycloak config and secrets into Infisical if INFISICAL_ADDR is set. @@ -661,7 +663,7 @@ def seed_infisical( shared_vars = { "KEYCLOAK_ADMIN_URL": "http://keycloak:8080", "KEYCLOAK_REALM": realm, - "KEYCLOAK_ISSUER": f"http://localhost:28080/realms/{realm}", + "KEYCLOAK_ISSUER": f"{issuer_base_url.rstrip('/')}/realms/{realm}", "KEYCLOAK_ADMIN_CLIENT_ID": "onex-admin", "ONEX_SERVICE_CLIENT_ID": "onex-service", } @@ -765,7 +767,7 @@ def _build_parser() -> argparse.ArgumentParser: ) p.add_argument( "--kc-url", - default="http://localhost:28080", # fallback-ok: local Keycloak provisioning default + default="http://localhost:28080", # fallback-ok: operator-facing bootstrap CLI default; # url-authority-ok: no runtime routing authority exists before Keycloak is provisioned. help=( "External base URL for Keycloak (no path prefix). " "Default: http://localhost:28080. " @@ -907,6 +909,7 @@ def main(argv: list[str] | None = None) -> int: args.realm, onex_admin_secret, onex_service_secret, + args.kc_url, dry_run=args.dry_run, ) @@ -916,6 +919,7 @@ def main(argv: list[str] | None = None) -> int: args.realm, onex_admin_secret, onex_service_secret, + args.kc_url, dry_run=args.dry_run, ) else: diff --git a/scripts/pull-all.sh b/scripts/pull-all.sh index 891363005c..eb9ffcfebe 100755 --- a/scripts/pull-all.sh +++ b/scripts/pull-all.sh @@ -33,6 +33,160 @@ if [[ $# -gt 0 ]]; then REPOS=("$@") fi + +RESULTS_DIR=$(mktemp -d) + +# === Stage tracking + terminal completion signal (OMN-15590) === +# Field failure this closes (remote-gate-readiness run wf_c69db51c-74d, +# 2026-07-31, host stickybeatz-studio): the drift-repair stage below ran +# unbounded, overran the caller's 3-minute timeout, orphaned three bash +# processes, and never reached the plugin-cache / pre-commit / summary stages. +# A caller that runs the documented "sync first" step and sees no error got a +# PARTIALLY-EXECUTED sync with nothing surfaced. Independently, the stage's +# failure path printed a banner and fell through -- a clean drift-repair +# FAILURE also produced a green-looking, exit-0 run. +# +# Every stage now carries an explicit status, the summary is emitted from an +# EXIT trap (so it appears on every exit path, including early aborts), and one +# machine-parseable line lets a caller distinguish a complete run from a +# stopped-or-failed one without reading prose. +STAGE_REPOS="PENDING" +STAGE_DRIFT_REPAIR="PENDING" +STAGE_PLUGIN_CACHE="PENDING" +STAGE_PRECOMMIT_HOOKS="PENDING" +STAGE_FAILURES=() +SUMMARY_EMITTED=0 + +# Aggregates are declared here (not at the aggregation step) so the EXIT-trap +# summary can read them even when the script aborts before that step. +OK=0 +FAILED=() +WARNED=() + +# The bound for the drift-repair stage, in seconds. Explicit and declared; +# overridable per-invocation for tests and for hosts with a slower canonical +# venv. 300s is ~150x the measured healthy cost of the repair's two uv steps on +# the gate host (1.93s resolve + 43ms leaf check, measured 2026-08-02), so it +# only fires on a genuine stall, never on a slow-but-progressing install. +DRIFT_REPAIR_TIMEOUT_SECONDS="${PULL_ALL_DRIFT_REPAIR_TIMEOUT_SECONDS:-300}" + +# Same bound, applied to the other unbounded network call on this script: the +# best-effort `pre-commit install-hooks` env pre-build. Non-fatal either way +# (the hook script is already written by then), but it must not be able to +# stall the run past its terminal summary. +HOOK_ENV_TIMEOUT_SECONDS="${PULL_ALL_HOOK_ENV_TIMEOUT_SECONDS:-300}" + +# Same bound for the plugin-cache content hash, which walks a 53k-file tree on +# the gate host. See _plugin_content_hash for the process-per-file defect that +# made this stage the SECOND unbounded stall found while proving OMN-15590. +PLUGIN_HASH_TIMEOUT_SECONDS="${PULL_ALL_PLUGIN_HASH_TIMEOUT_SECONDS:-300}" + +_emit_summary() { + [[ "$SUMMARY_EMITTED" -eq 1 ]] && return 0 + SUMMARY_EMITTED=1 + + local overall + if [[ "$STAGE_REPOS" == "PENDING" ]]; then + overall="ABORTED" + elif [[ ${#FAILED[@]} -gt 0 || ${#STAGE_FAILURES[@]} -gt 0 ]]; then + overall="FAILED" + elif [[ "$STAGE_PLUGIN_CACHE" == "PENDING" || "$STAGE_PRECOMMIT_HOOKS" == "PENDING" ]]; then + overall="INCOMPLETE" + else + overall="OK" + fi + + echo "" + if [[ ${#WARNED[@]} -gt 0 ]]; then + echo "WARN: ${#WARNED[@]} repo(s) not found locally and were skipped:" + local w + for w in "${WARNED[@]}"; do + echo " WARN $w" + done + fi + echo "${OK} repo(s) up to date. ${#FAILED[@]} failed. ${#WARNED[@]} absent (skipped)." + + echo "" + echo "== pull-all.sh stage summary ==" + echo " repos : $STAGE_REPOS" + echo " omnimarket-drift-repair: $STAGE_DRIFT_REPAIR" + echo " plugin-cache-refresh : $STAGE_PLUGIN_CACHE" + echo " pre-commit-hooks : $STAGE_PRECOMMIT_HOOKS" + if [[ ${#STAGE_FAILURES[@]} -gt 0 ]]; then + local f + for f in "${STAGE_FAILURES[@]}"; do + echo " STAGE FAILED : $f" + done + fi + + # Single machine-parseable completion signal. A caller checks this ONE line + # instead of parsing prose; its absence means the run did not reach the end. + echo "PULL-ALL-RESULT: overall=${overall} repos_ok=${OK} repos_failed=${#FAILED[@]} repos_absent=${#WARNED[@]} drift_repair=${STAGE_DRIFT_REPAIR} plugin_cache=${STAGE_PLUGIN_CACHE} precommit_hooks=${STAGE_PRECOMMIT_HOOKS}" +} + +_on_exit() { + local rc=$? + _emit_summary || true + rm -rf "$RESULTS_DIR" || true + return "$rc" +} +trap _on_exit EXIT + +# Run a command under a hard wall-clock bound, in its OWN process group, so a +# timeout kills the entire descendant tree. Returns the command's exit status, +# or 124 on timeout (GNU timeout's convention). +# +# Why not `timeout(1)`: macOS ships neither coreutils `timeout`/`gtimeout` nor +# `setsid` (verified on the .200 gate host, 2026-08-02), and GNU `timeout` +# signals only its direct child by default -- which on this stage would leave +# the git/uv/python grandchildren running. Those grandchildren ARE the field +# symptom (orphaned PIDs 61908/62077/62078). Enabling job control (`set -m`) +# makes the backgrounded job a process-group leader, so `kill -- -PGID` reaches +# the whole tree. +_run_bounded() { + local bound="$1" + shift + + local marker cmd_pid watchdog_pid rc waited + marker=$(mktemp) + + set -m + "$@" & + cmd_pid=$! + set +m + + ( + waited=0 + while [[ "$waited" -lt "$bound" ]]; do + sleep 1 + waited=$((waited + 1)) + kill -0 "$cmd_pid" 2>/dev/null || exit 0 + done + echo "timeout" > "$marker" + kill -TERM -"$cmd_pid" 2>/dev/null || kill -TERM "$cmd_pid" 2>/dev/null || true + sleep 5 + kill -KILL -"$cmd_pid" 2>/dev/null || kill -KILL "$cmd_pid" 2>/dev/null || true + ) & + watchdog_pid=$! + + rc=0 + wait "$cmd_pid" || rc=$? + + kill "$watchdog_pid" 2>/dev/null || true + wait "$watchdog_pid" 2>/dev/null || true + + # Sweep the group unconditionally: a command can exit while leaving a + # backgrounded descendant behind, which is the same orphan class. + kill -KILL -"$cmd_pid" 2>/dev/null || true + + if [[ -s "$marker" ]]; then + rc=124 + fi + rm -f "$marker" + return "$rc" +} +# === End stage tracking === + # === Pre-pull validation: detect bare repo corruption (OMN-7600) === # If core.bare=true, git pull updates refs but NOT the working tree, causing # stale files. This is corruption in omni_home — repos must be non-bare clones. @@ -64,9 +218,6 @@ if [[ ${#BARE_REPOS[@]} -gt 0 ]]; then fi # === End bare repo validation === -RESULTS_DIR=$(mktemp -d) -trap 'rm -rf "$RESULTS_DIR"' EXIT - # Switch to a branch, creating it from origin/ when needed, then # fast-forward it to the fetched remote branch. _checkout_and_ff() { @@ -182,11 +333,8 @@ done wait -# Aggregate results -OK=0 -FAILED=() -WARNED=() - +# Aggregate results (OK / FAILED / WARNED are declared with the stage tracking +# block above so the EXIT-trap summary can read them on any exit path). for repo in "${REPOS[@]}"; do result_file="$RESULTS_DIR/$repo" if [[ -f "$result_file" ]]; then @@ -200,6 +348,124 @@ for repo in "${REPOS[@]}"; do fi done +if [[ ${#FAILED[@]} -gt 0 ]]; then + STAGE_REPOS="FAILED" +else + STAGE_REPOS="OK" +fi + +# === Omnimarket venv drift auto-repair (OMN-15242) === +# The OMN-14060 pre-flight guard (src/omnibase_infra/cli/omnimarket_drift_guard.py) +# detects when the canonical omnibase_infra venv's installed omnimarket has +# fallen behind the just-advanced $OMNI_HOME/omnimarket clone -- but it only +# detects and instructs, it never repairs. A canonical omnimarket pull (right +# here, above) is the EXACT event that creates that drift. Two same-day +# 2026-07-27 incidents (13:04Z and 19:23Z) bricked every onex CLI/skill +# dispatch on this Mac until a human ran the repair by hand. +# +# INTERACTIVE-SESSION SCOPE ONLY. Preregistered battery runs use the frozen +# execution-environment mechanism (OMN-15265) and must NEVER be auto-repaired +# mid-run -- that would change the delegation stack version between seeds and +# contaminate the run. This hook lives only in pull-all.sh (the interactive/ +# session sync entrypoint), never in a battery driver, so that boundary holds +# structurally rather than by convention. +# +# Design guarantees: +# * Only triggers when omnimarket was part of THIS run and its pull result +# was OK -- a FAILED/MISSING/SKIPPED/not-requested omnimarket is untouched. +# * Skip-guarded -- a missing local omnibase_infra clone, missing drift +# script, or missing canonical venv is a clean no-op (nothing to repair +# against). +# * BOUNDED and FATAL (OMN-15590, revising OMN-15242's +# "fail-loud-but-not-fatal") -- the invocation runs under an explicit +# wall-clock bound in its own process group, and BOTH a timeout and a +# non-zero exit are carried into the terminal summary AND the exit code. +# The original non-fatal design meant a caller could not distinguish +# "sync completed" from "sync stopped or failed partway" -- and the +# unbounded call meant the run never reached the later stages at all. +# * Attributable -- the check/repair invocation and outcome are echoed +# inline in pull-all.sh's own stdout, the same log surface as every other +# step here. +# +# ROOT CAUSE of the observed hang (OMN-15590 AC6), established by controlled +# experiment on stickybeatz-studio 2026-08-02, not inferred: +# The repair chain ends in `uv pip install --python `. +# uv takes an EXCLUSIVE flock on `/.lock` for the duration of an +# install, has no lock-acquisition timeout, and prints nothing at default +# verbosity while it waits. Holding that lock from a second process made an +# otherwise 2-second install sit silent past 30s and then finish in +# milliseconds the instant the lock was released. `.200` runs many parallel +# sessions against ONE shared canonical venv -- including the readiness probe +# that itself runs pull-all.sh -- so a peer's uv operation blocks this stage +# for the peer's entire duration, unbounded. Resolve/network was excluded by +# measurement on the same host: step-1 git+HTTPS resolve 1.93s, step-2 leaf +# resolve 43ms, `git ls-remote` 0.19s. +_omnimarket_result_file="$RESULTS_DIR/omnimarket" +if [[ ! -f "$_omnimarket_result_file" || "$(cat "$_omnimarket_result_file")" != "OK" ]]; then + STAGE_DRIFT_REPAIR="SKIPPED" # omnimarket not in this run, or its pull did not succeed +else + _infra_dir="$OMNI_HOME/omnibase_infra" + _drift_script="$_infra_dir/scripts/check-omnimarket-venv-drift.sh" + _infra_venv_python="$_infra_dir/.venv/bin/python" + + if [[ ! -d "$_infra_dir" || ! -x "$_drift_script" ]]; then + STAGE_DRIFT_REPAIR="SKIPPED" # no local omnibase_infra clone (or drift script) + elif [[ ! -x "$_infra_venv_python" ]]; then + STAGE_DRIFT_REPAIR="SKIPPED" # no canonical omnibase_infra venv to repair + else + echo "" + echo "== checking omnimarket venv drift against canonical omnibase_infra venv ==" + echo " (bounded at ${DRIFT_REPAIR_TIMEOUT_SECONDS}s -- OMN-15590; override with PULL_ALL_DRIFT_REPAIR_TIMEOUT_SECONDS)" + _drift_rc=0 + _run_bounded "$DRIFT_REPAIR_TIMEOUT_SECONDS" \ + env OMNI_HOME="$OMNI_HOME" bash "$_drift_script" --repair "$_infra_venv_python" \ + || _drift_rc=$? + + if [[ "$_drift_rc" -eq 0 ]]; then + STAGE_DRIFT_REPAIR="OK" + echo " DRIFT-REPAIR omnimarket venv OK (canonical omnibase_infra venv)" + else + if [[ "$_drift_rc" -eq 124 ]]; then + STAGE_DRIFT_REPAIR="TIMEOUT" + STAGE_FAILURES+=("omnimarket-drift-repair timed out after ${DRIFT_REPAIR_TIMEOUT_SECONDS}s") + else + STAGE_DRIFT_REPAIR="FAILED" + STAGE_FAILURES+=("omnimarket-drift-repair exited ${_drift_rc}") + fi + echo "" + echo "############################################################" + if [[ "$STAGE_DRIFT_REPAIR" == "TIMEOUT" ]]; then + echo "# OMN-15590: omnimarket venv drift-repair TIMED OUT after ${DRIFT_REPAIR_TIMEOUT_SECONDS}s" + echo "#" + echo "# The stage was killed (whole process group) and this run is a" + echo "# FAILURE. Most likely cause: another process holds the exclusive" + echo "# uv lock on the canonical venv" + echo "# $_infra_dir/.venv/.lock" + echo "# uv waits on that lock forever and prints nothing while waiting." + echo "# Check for a concurrent uv/pull-all/session on this host, then" + echo "# re-run. Raise PULL_ALL_DRIFT_REPAIR_TIMEOUT_SECONDS only if the" + echo "# repair is genuinely slow rather than blocked." + else + echo "# OMN-15242: omnimarket venv drift-repair FAILED (exit ${_drift_rc})" + fi + echo "#" + echo "# pull-all.sh just advanced the canonical omnimarket clone, but" + echo "# could not repair the canonical omnibase_infra venv against it." + echo "# Every onex CLI / skill dispatch command is now at risk of the" + echo "# OMN-14060 OmnimarketDriftError until this is fixed BY HAND:" + echo "#" + echo "# OMNI_HOME=$OMNI_HOME bash $_drift_script --repair $_infra_venv_python" + echo "#" + echo "# See OMN-15590 / OMN-15242 / OMN-14060 for context." + echo "############################################################" + echo "" + echo " (continuing to the remaining stages -- their disposition is" + echo " reported in the terminal summary; nothing is silently skipped)" + fi + fi +fi +# === End omnimarket venv drift auto-repair === + # === Plugin cache refresh (Layer 2, OMN-7369) === # When omniclaude was updated, refresh the Claude Code plugin cache. # @@ -232,6 +498,15 @@ fi # The hash is computed against RELATIVE paths so a repo-side and cache-side # computation of the same plugin tree yield the same hash (shasum emits # `hash path` — absolute paths would otherwise break comparability). +# +# `-exec ... +` NOT `-exec ... \;` (OMN-15590). The per-file form spawned ONE +# `shasum` (a perl script) process per file. Measured on the gate host +# 2026-08-02: the live plugin cache holds 53,057 files, so the per-file form +# forks ~53k perl interpreters and the stage ran >10 minutes without finishing +# on a loaded host -- a second unbounded stall in this same script, hit while +# proving the drift-repair bound. The batched form produces byte-identical +# output (same `hash ./path` lines, re-sorted downstream anyway) and completes +# the same tree in 7.9s. _plugin_content_hash() { local root="$1" ( cd "${root}" && find . -type f \ @@ -239,10 +514,12 @@ _plugin_content_hash() { ! -path "*/__pycache__/*" \ ! -name ".deployed-commit" \ ! -name ".content-hash" \ - -exec shasum {} \; 2>/dev/null | sort | shasum | cut -d' ' -f1 ) + -exec shasum {} + 2>/dev/null | sort | shasum | cut -d' ' -f1 ) } +STAGE_PLUGIN_CACHE="SKIPPED" # no cache and/or no omniclaude clone -- nothing to refresh if [[ -n "${_plugin_cache}" && -d "${_omniclaude_dir}" && -d "${_plugin_cache}" ]]; then + STAGE_PLUGIN_CACHE="UP-TO-DATE" _current=$(git -C "${_omniclaude_dir}" rev-parse HEAD 2>/dev/null) _deployed="" [[ -f "${_plugin_cache}/.deployed-commit" ]] && _deployed=$(cat "${_plugin_cache}/.deployed-commit" 2>/dev/null) @@ -250,7 +527,14 @@ if [[ -n "${_plugin_cache}" && -d "${_omniclaude_dir}" && -d "${_plugin_cache}" # Compare against repo content hash as a second signal beyond commit SHA. _repo_hash="" if [[ -d "${_omniclaude_dir}/plugins/onex" ]]; then - _repo_hash=$(_plugin_content_hash "${_omniclaude_dir}/plugins/onex") + # Bounded for the same reason as the drift stage: a stall here used to hang + # the run past its terminal summary. An empty hash on timeout fails toward + # "cache looks stale" (a refresh), never toward a silent skip. + _repo_hash=$(_run_bounded "$PLUGIN_HASH_TIMEOUT_SECONDS" _plugin_content_hash "${_omniclaude_dir}/plugins/onex") || { + echo "WARN: plugin content hash (repo) exceeded ${PLUGIN_HASH_TIMEOUT_SECONDS}s -- treating cache as stale." + STAGE_PLUGIN_CACHE="WARN" + _repo_hash="" + } fi _cache_hash="" [[ -f "${_plugin_cache}/.content-hash" ]] && _cache_hash=$(cat "${_plugin_cache}/.content-hash" 2>/dev/null) @@ -270,14 +554,17 @@ if [[ -n "${_plugin_cache}" && -d "${_omniclaude_dir}" && -d "${_plugin_cache}" cp -R "${_tmpdir}/plugins/onex/." "${_plugin_cache}/" echo "${_current}" > "${_plugin_cache}/.deployed-commit" # Recompute hash against the cache after refresh and persist. - _new_hash=$(_plugin_content_hash "${_plugin_cache}") + _new_hash=$(_run_bounded "$PLUGIN_HASH_TIMEOUT_SECONDS" _plugin_content_hash "${_plugin_cache}") || _new_hash="" echo "${_new_hash}" > "${_plugin_cache}/.content-hash" echo "Plugin cache refreshed (content hash ${_new_hash:0:8})." + STAGE_PLUGIN_CACHE="REFRESHED" else echo "WARN: Plugin cache refresh failed (archive missing plugins/onex/)." + STAGE_PLUGIN_CACHE="WARN" fi else echo "WARN: Plugin cache refresh failed (git archive error)." + STAGE_PLUGIN_CACHE="WARN" fi rm -rf "${_tmpdir}" fi @@ -309,12 +596,20 @@ if ! command -v pre-commit >/dev/null 2>&1; then echo "WARN: 'pre-commit' not found on PATH -- local git hooks were NOT installed." echo " Install it (e.g. 'brew install pre-commit') so pattern/static" echo " defects fail at commit time instead of first failing in CI." + STAGE_PRECOMMIT_HOOKS="UNAVAILABLE" else + STAGE_PRECOMMIT_HOOKS="OK" for repo in "${REPOS[@]}"; do _pc_dir="$OMNI_HOME/$repo" [[ -d "$_pc_dir" ]] || continue [[ -f "$_pc_dir/.pre-commit-config.yaml" ]] || continue - ( + # The install runs in a subshell (it `cd`s), so a failure inside it cannot + # assign to STAGE_PRECOMMIT_HOOKS directly. It signals with exit 3 and the + # parent downgrades the stage -- otherwise the summary would report + # `precommit_hooks=OK` on a run that printed a screenful of WARN lines, + # which is the same "green-looking incomplete run" defect this ticket + # exists to close, just moved one stage over. (OMN-15590) + if ! ( cd "$_pc_dir" || exit 0 # `git rev-parse --git-path hooks` resolves to the SHARED hooks dir (the # common git dir), so installing in the canonical clone covers all of its @@ -329,22 +624,34 @@ else # Best-effort env pre-build so the first real commit is not slow. Runs # at most once per repo (guarded above); a failure here is non-fatal # because the hook script is already written and will still fire. - pre-commit install-hooks >/dev/null 2>&1 || true + # + # BOUNDED (OMN-15590): this is a network-bound call with no client + # timeout -- the same unbounded class as the drift-repair stage above, + # on the same script. It is bounded with the same mechanism so a hung + # hook-env build cannot stall the sync past its terminal summary. + _run_bounded "$HOOK_ENV_TIMEOUT_SECONDS" \ + bash -c 'pre-commit install-hooks >/dev/null 2>&1' || true echo " HOOK $repo (pre-commit git hook installed)" else echo " WARN $repo (pre-commit install failed -- commit-time enforcement inactive)" + exit 3 fi - ) + ); then + STAGE_PRECOMMIT_HOOKS="WARN" + fi done fi # === End pre-commit hook installation === -echo "" -if [[ ${#WARNED[@]} -gt 0 ]]; then - echo "WARN: ${#WARNED[@]} repo(s) not found locally and were skipped:" - for w in "${WARNED[@]}"; do - echo " WARN $w" - done +# The terminal summary is emitted by the EXIT trap (_on_exit -> _emit_summary) +# so it appears on EVERY exit path, including the early bare-repo abort and any +# unexpected `set -e` failure -- not only on this happy path. All that remains +# here is the exit code. +# +# A repo failure OR a failed/timed-out stage is a failed run (OMN-15590): a +# caller running the documented "sync first" step must not read exit 0 off a +# sync that stopped or failed partway. +if [[ ${#FAILED[@]} -gt 0 || ${#STAGE_FAILURES[@]} -gt 0 ]]; then + exit 1 fi -echo "${OK} repo(s) up to date. ${#FAILED[@]} failed. ${#WARNED[@]} absent (skipped)." -[[ ${#FAILED[@]} -eq 0 ]] || exit 1 +exit 0 diff --git a/scripts/render_application_database_topology.py b/scripts/render_application_database_topology.py new file mode 100644 index 0000000000..c6f902b41e --- /dev/null +++ b/scripts/render_application_database_topology.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Render or parity-check secret-free application database topology projections.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import yaml + +from omnibase_infra.topology import ( + SUPPORTED_ENVIRONMENTS, + render_database_projection, + validate_database_projection, + validate_docker_catalog_parity, +) +from omnibase_infra.topology.application_database import write_database_projection + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--environment", + choices=sorted(SUPPORTED_ENVIRONMENTS), + required=True, + ) + output_group = parser.add_mutually_exclusive_group() + output_group.add_argument("--output", type=Path) + output_group.add_argument("--check", type=Path) + parser.add_argument( + "--validate-docker", + action="store_true", + help="Also parity-check Docker catalog consumers (local only).", + ) + return parser + + +def main() -> int: + args = _parser().parse_args() + environment = str(args.environment) + output = args.output + check = args.check + + if output is not None: + write_database_projection(environment, output) + elif check is not None: + validate_database_projection(environment, check) + else: + print( + yaml.safe_dump( + render_database_projection(environment), + sort_keys=True, + ), + end="", + ) + + if args.validate_docker: + if environment != "local": + raise ValueError("Docker catalog parity is defined only for local topology") + validate_docker_catalog_parity() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_runtime_policy_env.py b/scripts/render_runtime_policy_env.py index c8d7e39cec..5591205272 100755 --- a/scripts/render_runtime_policy_env.py +++ b/scripts/render_runtime_policy_env.py @@ -116,8 +116,9 @@ def render_env(contract: ModelRuntimePolicyContract) -> dict[str, str]: if process_name == "worker": # OMN-12990: pin the worker replica count in the ledgered config # surface so a plain compose recreate cannot silently drop the - # worker via the base ${WORKER_REPLICAS:-0} default. The lane - # overrides reference ${_WORKER_REPLICAS:?...} fail-fast. + # worker. Every lane compose surface -- the base infra file for + # dev (OMN-14968) and the stability-test/prod overlays -- resolves + # ${_WORKER_REPLICAS:?...} fail-fast against these values. env[f"{profile_prefix}_WORKER_REPLICAS"] = str(process.replicas) env[f"{prefix}_CAPABILITIES"] = ",".join(process.capabilities) env[f"{prefix}_BIFROST_VERIFY_ENDPOINTS"] = _bifrost_text( diff --git a/scripts/run-forward-migrations.sh b/scripts/run-forward-migrations.sh index 3f11dbe90f..31fee29b6b 100755 --- a/scripts/run-forward-migrations.sh +++ b/scripts/run-forward-migrations.sh @@ -4,7 +4,8 @@ # run-forward-migrations.sh — Apply omnibase_infra forward migrations on warm Postgres volumes # -# Tracks applied migrations in public.schema_migrations and applies any +# Tracks service-owned flat migrations in public.schema_migrations and +# application/node migrations in platform_catalog.schema_migrations. Applies # pending files from /migrations/forward in sorted order. Safe to run on # both fresh volumes (no-op for files already applied via docker-entrypoint-initdb.d) # and warm volumes (applies any new files not yet recorded). @@ -52,6 +53,17 @@ # migration numbered e.g. 076 NEVER collides with infra's flat 076 file — # the renumber-as-operational-pattern is eliminated. # +# --------------------------------------------------------------------------- +# Operator fence for node migrations (OMN-15336 — parity with the k8s runner) +# --------------------------------------------------------------------------- +# The node-migration id space above is SHARED with omninode_infra's k8s Job +# runner (k8s/migrations/omnibase-infra-migrate.yaml): both walk the same +# vendored nodes//*.sql tree and both mint the id +# node::. Until OMN-15336 only the k8s runner carried the +# operator fence, so the two sanctioned paths disagreed about what is gated and +# every compose lane applied the gated migrations unattended. See +# FENCED_NODE_MIGRATION_IDS below for the semantics and the seam. +# # Environment: # POSTGRES_USER (default: postgres) # POSTGRES_PASSWORD (required) @@ -62,6 +74,15 @@ # NODE_MIGRATIONS_DIR (default: ${MIGRATIONS_DIR}/nodes) # NODE_POSTGRES_DB (default: POSTGRES_DB; compose sets omnidash_analytics) # PG_WAIT_RETRIES (default: 30 — number of 2s waits for postgres ready) +# FORWARD_MIGRATION_LOCK_ID (default: 100010 — advisory lock id, OMN-15291) +# MIGRATION_LOCK_WAIT_SECONDS (default: 300 — bounded wait for that lock) +# ONEX_MIGRATION_LANE (default: unset = FULL operator fence, OMN-15379). +# Lane indicator for the lane-scoped fence release. The ONLY recognised +# value is `dev` (the lab compose lane); unset or unknown means every fenced +# node migration is skipped. Set by docker/docker-compose.dev-lane.yml, +# which only the dev/lab project loads — deliberately NOT set in +# docker-compose.infra.yml, which every lane overlay merges. See the +# LANE-SCOPED FENCE RELEASE block below. set -e @@ -73,6 +94,11 @@ MIGRATIONS_DIR="${MIGRATIONS_DIR:-/migrations/forward}" NODE_MIGRATIONS_DIR="${NODE_MIGRATIONS_DIR:-${MIGRATIONS_DIR}/nodes}" NODE_PGDB="${NODE_POSTGRES_DB:-${PGDB}}" PG_WAIT_RETRIES="${PG_WAIT_RETRIES:-30}" +LEDGER_BOOTSTRAP="${MIGRATIONS_DIR}/_ledger/bootstrap.sql" +APPLICATION_MIGRATION_MANIFEST="${MIGRATIONS_DIR}/_ledger/application-migrations.tsv" +APPLICATION_MIGRATION_BLOCKS="${MIGRATIONS_DIR}/_ledger/application-migration-blocks.tsv" +LEGACY_NODE_MIGRATION_DECLARATIONS="${MIGRATIONS_DIR}/_ledger/legacy-node-migrations.tsv" +CLOUD_MIGRATION_ALIASES="${MIGRATIONS_DIR}/_ledger/cloud-migration-aliases.tsv" export PGPASSWORD="${POSTGRES_PASSWORD}" @@ -100,6 +126,224 @@ is_skipped_by_manifest() { echo "${SKIPPED_IDS}" | grep -Fxq "${migration_id}" } +# ---- BEGIN operator fence — node migration ids (OMN-15336) ---- +# SINGLE-SOURCED from docker/migrations/forward/fenced-node-migrations.yaml +# (OMN-15349). That file is the baseline operator fence over the id space +# this runner and omninode_infra/k8s/migrations/omnibase-infra-migrate.yaml +# share (node::, minted over the same vendored SQL tree) — a +# fence in only one of them is not a fence: OMN-15336 found the .201 dev lane +# and the stability-test lane had applied all of the gated ids and were +# running FORCE ROW LEVEL SECURITY on six tables, while the cloud RDS copy +# the k8s runner drives was clean. +# +# Before OMN-15349 this list was a hand-maintained literal here AND in the +# k8s manifest — two copies that had already drifted once (k8s durably +# released the registration trio under operator ruling 21 while this runner +# stayed at the full baseline; see the LANE-SCOPED FENCE RELEASE block below +# for why that is a *different* release mechanism, not a parity break). +# Loading the same manifest file both runners read removes that drift class +# for the baseline; each runner's release policy on top of the baseline is +# still its own, because the release is an environment-specific operator +# decision, not fence data. +# +# Loaded from the manifest, NOT `${FENCED_NODE_MIGRATION_IDS:-...}`: only a +# COMMITTED manifest is honoured, exactly as with the skip-manifest above. An +# operator env var must not be able to supply or empty this list — see +# test_fence_is_not_overridable_by_environment. +FENCE_MANIFEST="${MIGRATIONS_DIR}/fenced-node-migrations.yaml" +if [ ! -f "${FENCE_MANIFEST}" ]; then + echo "FATAL: operator fence manifest not found: ${FENCE_MANIFEST}" >&2 + exit 1 +fi +FENCED_NODE_MIGRATION_IDS="$(sed -n \ + 's/^[[:space:]]*-[[:space:]]*id:[[:space:]]*"\([^"]*\)".*/\1/p' \ + "${FENCE_MANIFEST}")" +if [ -z "${FENCED_NODE_MIGRATION_IDS}" ]; then + # Not FATAL: an empty fence is a legitimate future state (every id + # eventually released) as well as the symptom of a malformed manifest, and + # this sed grammar cannot tell the two apart. Blocking the whole migration + # run on an empty fence would make "no ids currently need gating" a worse + # outage than the fence gap it replaces. The committed-content checks + # (test_manifest_pins_the_known_baseline_fence, + # test_manifest_shell_parse_matches_yaml_parse) are what catch an + # accidentally-emptied manifest, at PR time, before it ships. + echo "[forward-migration] WARNING: operator fence manifest ${FENCE_MANIFEST}" \ + "parsed to an empty list — no node migrations are currently fenced" >&2 +fi + +is_fenced_node_migration() { + candidate="$1" + printf '%s\n' "${FENCED_NODE_MIGRATION_IDS}" | grep -Fxq "${candidate}" +} + +# ---- BEGIN FORCE ROW LEVEL SECURITY grandfather snapshot (OMN-15336 item 4 repair) ---- +# What this is, and why it is a SEPARATE file from ${FENCE_MANIFEST}: see +# grandfathered-force-rls-migrations.yaml's own header. Short version — it is +# a frozen snapshot of the FORCE-enabling node migrations that were already +# vendored and already applying ungated, on EVERY lane, before the guard +# below existed. It is a grandfather RECORD (a fact about the tree at guard- +# introduction time), never an operator GATE (the fence manifest above stays +# the only place an operator holds a migration back) -- conflating the two +# would let a future editor "release" a genuinely new hazard by mislabeling +# it grandfathered instead of routing it through fence review. +# +# Same committed-file-only discipline as the fence manifest: only a file +# checked into this repo is honoured, never an operator env var (parity with +# test_fence_is_not_overridable_by_environment's reasoning, mirrored here by +# test_grandfather_is_not_overridable_by_environment). +GRANDFATHER_MANIFEST="${MIGRATIONS_DIR}/grandfathered-force-rls-migrations.yaml" +if [ ! -f "${GRANDFATHER_MANIFEST}" ]; then + echo "FATAL: FORCE-RLS grandfather manifest not found: ${GRANDFATHER_MANIFEST}" >&2 + exit 1 +fi +GRANDFATHERED_FORCE_RLS_IDS="$(sed -n \ + 's/^[[:space:]]*-[[:space:]]*id:[[:space:]]*"\([^"]*\)".*/\1/p' \ + "${GRANDFATHER_MANIFEST}")" + +is_grandfathered_force_rls_migration() { + candidate="$1" + printf '%s\n' "${GRANDFATHERED_FORCE_RLS_IDS}" | grep -Fxq "${candidate}" +} +# ---- END FORCE ROW LEVEL SECURITY grandfather snapshot (OMN-15336 item 4 repair) ---- + +# ---- BEGIN unclassified FORCE ROW LEVEL SECURITY guard (OMN-15336 item 4) ---- +# What this closes: the fence above only gates ids someone already listed in +# ${FENCE_MANIFEST}. OMN-15336's own required-fix item 4 found three ids that +# carry the identical FORCE hazard (write-lockout for the table owner, the +# OMN-15301 condition) and were NEVER in that list on ANY runner: +# node_projection_registration/0002 (since added to the manifest), +# node_projection_delegation_inference_response/0003, and +# node_projection_savings/081. A manifest is only as good as remembering to +# add to it -- this is the mechanism that stops relying on memory: any node +# migration that is about to apply FOR THE FIRST TIME (never recorded in the +# ledger) and whose DDL enables FORCE ROW LEVEL SECURITY, but whose id is +# nowhere in ${FENCE_MANIFEST}, is refused. Classification (add a fence entry +# citing the owning ticket, with or without a lane release) is what lets it +# proceed -- silent, unattended FORCE is what this removes. +# +# Deliberately does NOT fire for: +# - an id already in the fence manifest (classified, whether currently +# released on this lane or not) -- that migration already went through +# operator review; re-litigating it here would be noise, not a gate. +# - an id in ${GRANDFATHER_MANIFEST} -- a FORCE-enabling migration that was +# already vendored and already applying ungated on every lane BEFORE this +# guard existed (see that file's own header for the entry criteria and +# why this is a frozen snapshot, not a rolling allowlist). Repair for the +# defect found empirically 2026-08-05: the guard as first shipped fired +# for ALL 13 vendored FORCE-enabling node migrations except the 4 the +# operator fence happened to already cover, FATALing on a virgin database +# at the FIRST of the other 9 it reached and applying NOTHING -- a cold +# lane bring-up (CI, a fresh compose volume) could never converge. The +# fence and the grandfather list are checked as two independent +# conditions (see the call site), never merged into one list: the fence +# is operator-editable data that can gate a BRAND NEW migration; the +# grandfather list is a closed historical fact that cannot. +# - a migration already recorded in the ledger -- the guard is checked at +# the call site ONLY after migration_is_applied returns false, so it can +# never retroactively block a lane (e.g. .201 dev) where an unclassified +# id already applied before this guard existed. Fencing forward, not +# rewriting history: matches the baseline manifest's own +# "BASELINE, NOT PER-LANE EFFECTIVE FENCE" contract. +# - `ALTER TABLE ... NO FORCE ROW LEVEL SECURITY` (a disabling statement, +# the opposite hazard) -- excluded explicitly below, not just by the +# enabling-keyword match, so a future FORCE-strip migration is never +# blocked by the guard it is written to route around. +# +# Comment-blind, not byte-blind: `--` line comments are stripped before the +# match so a FORCE ROW LEVEL SECURITY mention in prose (this very file +# demonstrates why that matters) does not false-positive. This is a narrower +# lexer than the k8s runner's OMN-15345 comment-blind matcher (no block +# comments / dollar-quoting) because committed node migrations here are plain +# DDL files with no PL/pgSQL bodies; a file that needed the full lexer would +# be a first, and the narrower one still fails closed (retaining more text on +# any parse ambiguity, never less). +migration_declares_unclassified_force_rls() { + candidate_file="$1" + sed 's/--.*$//' "${candidate_file}" \ + | grep -Ei 'FORCE[[:space:]]+ROW[[:space:]]+LEVEL[[:space:]]+SECURITY' \ + | grep -Eviq 'NO[[:space:]]+FORCE[[:space:]]+ROW[[:space:]]+LEVEL[[:space:]]+SECURITY' +} +# ---- END unclassified FORCE ROW LEVEL SECURITY guard (OMN-15336 item 4) ---- + +# --- LANE-SCOPED FENCE RELEASE (OMN-15379 — operator ruling 15, 2026-07-29) --- +# Operator ruling 15: node_service_registry FORCE ROW LEVEL SECURITY extends to +# the LAB LANE ONLY. The lab (compose dev lane, project `omnibase-infra`) applies +# the registration trio IN FULL — CREATE + heartbeat columns + ENABLE/FORCE RLS — +# as the proving ground that generates the evidence the staging un-fence is +# waiting on. +# +# CORRECTION (OMN-15349, 2026-08-05): this comment previously claimed "the +# staging k8s fence is UNCHANGED and stays at all seven ids." That was true +# when ruling 15 landed (2026-07-29) and stale within two days: operator +# ruling 21 (OMN-15332 comment 1a067542, 2026-07-31T14:05Z GO) durably +# released the registration trio on the k8s/staging side too — permanently, +# not env-gated like this lab-lane release, because that Job serves exactly +# one environment. So THIS runner's release (below) and the k8s runner's +# release are two independently-ruled policies over the same shared baseline +# manifest (docker/migrations/forward/fenced-node-migrations.yaml), not one +# parity relationship — do not re-derive "the two runners must show the same +# effective fence" from this block; they intentionally do not right now. +# +# FAIL-CLOSED BY CONSTRUCTION. Three independent properties, each tested: +# +# 1. DEFAULT IS FULLY FENCED. ${ONEX_MIGRATION_LANE} unset -> empty release set +# -> every one of the seven ids is skipped, exactly as before this change. +# An UNKNOWN value is treated the same as unset (and warns). There is no +# value that widens the fence relative to today; a lane can only ever +# release a SUBSET of it. +# 2. THE RELEASE SET IS COMMITTED, NOT SUPPLIED. The env var selects among +# policies that are literal in this file; it never carries ids. No env +# value can release an id that is not written below, and the release is +# only ever consulted for an id that is already fenced (see the node loop), +# so a lane cannot "release" something the fence does not cover. +# 3. NOT INHERITABLE. The indicator is NOT set in docker-compose.infra.yml. +# Every non-dev lane overlay (stability-test / prod / judge) MERGES that +# base file, so anything set there would be inherited by all of them — +# fail-OPEN, and silently so for any lane added later. It is instead set by +# docker/docker-compose.dev-lane.yml, an overlay that ONLY the dev/lab +# project loads. A lane that does not load that file — stability-test, +# prod, judge, CI, a raw `docker compose -f docker-compose.infra.yml up` on +# a fresh volume, and any future lane — gets no indicator and the full +# fence. Asserted over the RENDERED compose config in +# tests/scripts/test_node_migration_fence_parity.py. +# +# HONEST LIMIT: a deliberate `-e ONEX_MIGRATION_LANE=dev` on another lane's +# forward-migration container would release the trio there. That is an explicit +# operator act on the same footing as editing the fence list itself, and it is +# not defended against — the container has no un-forgeable lane fact available +# to corroborate against (compose does not inject the project name, and the +# service's own container_name is not readable from inside it). +# +# Delegation 0023-0026 are NOT releasable on ANY lane. Ruling 15 is scoped to +# node_service_registry; the delegation tenant-RLS hold is a separate ruling +# still pending, and no case arm below names those ids. +ONEX_MIGRATION_LANE="${ONEX_MIGRATION_LANE:-}" +case "${ONEX_MIGRATION_LANE}" in + dev) + # The lab/dev compose lane. Ruling 15 proving ground. + LANE_RELEASED_NODE_MIGRATION_IDS="\ +node:node_projection_registration:0000_create_node_service_registry.sql +node:node_projection_registration:0001_add_heartbeat_columns.sql +node:node_projection_registration:0002_node_service_registry_tenant_rls.sql" + ;; + "") + LANE_RELEASED_NODE_MIGRATION_IDS="" + ;; + *) + echo "[forward-migration] WARNING: unknown ONEX_MIGRATION_LANE='${ONEX_MIGRATION_LANE}' — applying the FULL operator fence (fail-closed)." >&2 + LANE_RELEASED_NODE_MIGRATION_IDS="" + ;; +esac + +is_lane_released_node_migration() { + candidate="$1" + if [ -z "${LANE_RELEASED_NODE_MIGRATION_IDS}" ]; then + return 1 + fi + printf '%s\n' "${LANE_RELEASED_NODE_MIGRATION_IDS}" | grep -Fxq "${candidate}" +} +# ---- END operator fence — node migration ids (OMN-15336) ---- + # --------------------------------------------------------------------------- # 0. Wait for Postgres to be ready (first-boot initdb race guard, OMN-13062) # --------------------------------------------------------------------------- @@ -116,6 +360,135 @@ until psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" -c "SELECT 1" >/dev done echo "[forward-migration] Postgres is ready." +# ---- BEGIN canonical forward-migration advisory lock (OMN-15291) ---- +# Port of the OMN-15254 single-session lock to this runner, in POSIX sh (this +# script is #!/bin/sh and runs under busybox ash in the migration container -- +# no `local`, no arrays, no bashisms). +# +# What was wrong: this runner had NO lock of any kind. Every migration was +# applied through an unsynchronized check-then-act (SELECT from +# schema_migrations -> psql -f -> INSERT ... ON CONFLICT DO NOTHING), so two +# concurrent runners both read "not applied" and both executed the same file. +# Non-idempotent DDL then errored in the loser, and the ON CONFLICT hid the +# double-apply so schema_migrations still looked clean afterwards. +# +# Why this shape: pg_advisory_lock() is SESSION-scoped. Acquiring it with a +# one-shot `psql -c` releases it the instant psql exits, and a later +# pg_advisory_unlock() from a different session returns false without ever +# having held anything. The lock is therefore held by ONE dedicated psql +# session that stays alive for the whole run; release is that session ending, +# never a cross-session unlock. +# +# Scope: advisory locks are per-DATABASE. This lock is taken in ${PGDB} and +# held across BOTH the infra phase (${PGDB}) and the node phase (${NODE_PGDB}), +# so two instances of THIS runner are fully serialized against each other. It +# does not serialize against unrelated writers of ${NODE_PGDB}. +# +# Lock id 100010 is deliberately outside omninode_infra's k8s Job registry +# (100001-100006, see omninode_infra/scripts/run-migrations.sh) so the two +# runners never contend on a shared id by accident. +FORWARD_MIGRATION_LOCK_ID="${FORWARD_MIGRATION_LOCK_ID:-100010}" +MIGRATION_LOCK_WAIT_SECONDS="${MIGRATION_LOCK_WAIT_SECONDS:-300}" +MIGRATION_LOCK_TAG="forward-migration-lock-${FORWARD_MIGRATION_LOCK_ID}-$$" +MIGRATION_LOCK_OWNER_PID="" + +# True only when the lock is granted to OUR holder session. Matching on +# application_name is what makes this specific: "somebody holds it" is not +# proof that WE hold it. A psql error is reported as not-held. +migration_lock_held() { + _mlh_granted="$(psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" -tAc \ + "SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid + WHERE l.locktype = 'advisory' AND l.granted AND l.objsubid = 1 + AND l.classid::bigint * 4294967296 + l.objid::bigint = ${FORWARD_MIGRATION_LOCK_ID} + AND a.application_name = '${MIGRATION_LOCK_TAG}';" 2>/dev/null)" || return 1 + [ "$_mlh_granted" = "1" ] +} + +# Release by ending the holder session: Postgres drops session-level advisory +# locks on disconnect. No cross-session unlock call anywhere. Confirmation +# polls the lock itself instead of using `wait`: the holder is a background +# PIPELINE, and `wait` on it blocks until every member exits -- a deadlock +# when this runs from the EXIT trap. +release_migration_lock() { + [ -n "$MIGRATION_LOCK_OWNER_PID" ] || return 0 + kill "$MIGRATION_LOCK_OWNER_PID" 2>/dev/null || true + MIGRATION_LOCK_OWNER_PID="" + _rml_waited=0 + while migration_lock_held && [ "$_rml_waited" -lt 10 ]; do + sleep 1 + _rml_waited=$((_rml_waited + 1)) + done +} + +acquire_migration_lock() { + echo "[forward-migration] Acquiring advisory lock ${FORWARD_MIGRATION_LOCK_ID} (single session, bounded wait ${MIGRATION_LOCK_WAIT_SECONDS}s)..." + { + # statement_timeout bounds the blocking acquire IN THE SERVER, so a + # contended lock fails loud instead of hanging until the deploy times out. + echo "SET statement_timeout = '${MIGRATION_LOCK_WAIT_SECONDS}s';" + echo "SELECT pg_advisory_lock(${FORWARD_MIGRATION_LOCK_ID});" + echo "SET statement_timeout = 0;" + # Hold this session's stdin -- and therefore the session, and therefore the + # lock -- open for the rest of the run. The payload is a SQL comment: psql + # reads and discards it with no server round trip, but the write fails with + # EPIPE the moment the holder session is gone, which is how this loop + # learns to stop. The tick cap is a backstop so an orphaned keepalive can + # never outlive the day. + _aml_ticks=0 + while [ "$_aml_ticks" -lt 86400 ]; do + sleep 1 + echo "-- forward migration advisory lock keepalive" || break + _aml_ticks=$((_aml_ticks + 1)) + done + # 2>/dev/null is load-bearing: this group must not inherit the runner's + # stderr. It can outlive the runner by up to a second, and any parent reading + # the runner's output to EOF would otherwise block on the inherited pipe. + } 2>/dev/null | PGAPPNAME="$MIGRATION_LOCK_TAG" psql -h "$PGHOST" -p "$PGPORT" \ + -U "$PGUSER" -d "$PGDB" -v ON_ERROR_STOP=1 -q -o /dev/null & + MIGRATION_LOCK_OWNER_PID=$! + + _aml_deadline=$(( $(date +%s) + MIGRATION_LOCK_WAIT_SECONDS + 5 )) + until migration_lock_held; do + if ! kill -0 "$MIGRATION_LOCK_OWNER_PID" 2>/dev/null; then + MIGRATION_LOCK_OWNER_PID="" + echo "[forward-migration] FATAL: could not acquire advisory lock ${FORWARD_MIGRATION_LOCK_ID} -- holder session exited before the lock was granted (another forward-migration run likely holds it, or the ${MIGRATION_LOCK_WAIT_SECONDS}s statement_timeout fired)" >&2 + exit 1 + fi + if [ "$(date +%s)" -ge "$_aml_deadline" ]; then + echo "[forward-migration] FATAL: could not acquire advisory lock ${FORWARD_MIGRATION_LOCK_ID} within ${MIGRATION_LOCK_WAIT_SECONDS}s -- refusing to apply migrations unserialized" >&2 + release_migration_lock + exit 1 + fi + sleep 1 + done + echo "[forward-migration] Advisory lock ${FORWARD_MIGRATION_LOCK_ID} acquired; held by session '${MIGRATION_LOCK_TAG}' for the whole run." +} + +# Called as the last act before the sentinel is set TRUE: a holder session that +# died mid-run means the migrations above may have run unserialized, so +# reporting success (and flipping the migration gate HEALTHY) would be a lie. +assert_migration_lock_still_held() { + if ! kill -0 "$MIGRATION_LOCK_OWNER_PID" 2>/dev/null || ! migration_lock_held; then + echo "[forward-migration] FATAL: advisory lock ${FORWARD_MIGRATION_LOCK_ID} was NOT held for the whole run -- the holder session died mid-run and migrations above may have run unserialized" >&2 + exit 1 + fi +} + +# Traps set BEFORE acquisition so a partially-started holder is still reaped on +# set -e failures and signals. +# +# EXIT and the signals are deliberately SEPARATE traps. In POSIX sh only the +# EXIT trap is terminal: a HUP/INT/TERM handler that returns normally RESUMES +# the script, so a single combined trap would release the lock mid-run and then +# keep applying migrations unserialized until the final held-ness assertion +# noticed. The signal handler therefore exits non-zero itself; the EXIT trap +# then re-runs release_migration_lock, which is idempotent (it returns +# immediately once MIGRATION_LOCK_OWNER_PID is cleared). +trap 'release_migration_lock' EXIT +trap 'release_migration_lock; echo "[forward-migration] FATAL: terminated by signal before completion" >&2; exit 1' HUP INT TERM +acquire_migration_lock +# ---- END canonical forward-migration advisory lock (OMN-15291) ---- + validate_database_identifier() { database="$1" if ! printf '%s' "$database" | grep -Eq '^[A-Za-z_][A-Za-z0-9_-]*$'; then @@ -146,9 +519,634 @@ EOSQL } # --------------------------------------------------------------------------- -# 1. Ensure schema_migrations tracking table exists (idempotent) +# Canonical application migration ledger (OMN-15413) # --------------------------------------------------------------------------- -echo "[forward-migration] Ensuring schema_migrations table exists..." +# The approved topology contract selects platform_catalog.schema_migrations +# with explicit stream/domain/version/checksum columns. bootstrap.sql evolves +# the checksum-capable node relation in place before any already-applied probe; +# the filename-only relation remains immutable import evidence. + +validate_migration_identity() { + identity="$1" + if ! printf '%s' "$identity" | grep -Eq '^[A-Za-z0-9_./:-]+$'; then + echo "[forward-migration] FATAL: invalid migration identity '${identity}'" >&2 + exit 1 + fi +} + +validate_client_file_path() { + client_file_path="$1" + case "$client_file_path" in + ""|*[!A-Za-z0-9_./-]*) + echo "[forward-migration] FATAL: unsafe psql client file path '${client_file_path}'" >&2 + exit 1 + ;; + esac + if [ ! -f "$client_file_path" ]; then + echo "[forward-migration] FATAL: psql client file is missing: ${client_file_path}" >&2 + exit 1 + fi +} + +file_sha256() { + sha256sum "$1" | awk '{print $1}' +} + +validate_application_migration_manifest() { + for manifest_file in \ + "$APPLICATION_MIGRATION_MANIFEST" \ + "$APPLICATION_MIGRATION_BLOCKS" \ + "$LEGACY_NODE_MIGRATION_DECLARATIONS" \ + "$CLOUD_MIGRATION_ALIASES" + do + if [ ! -f "$manifest_file" ]; then + echo "[forward-migration] FATAL: application migration declaration missing: ${manifest_file}" >&2 + exit 1 + fi + done + + if ! awk -F '\t' ' + NF != 6 { exit 1 } + { + path_count = split($1, path_parts, "/") + expected_owner = "node:" path_parts[2] + expected_version = expected_owner ":" path_parts[3] + } + path_count != 3 || path_parts[1] != "nodes" { exit 1 } + path_parts[2] !~ /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/ { exit 1 } + path_parts[3] !~ /^[A-Za-z0-9_][A-Za-z0-9_.-]*[.]sql$/ { exit 1 } + $2 != expected_owner || $3 != expected_owner || $5 != expected_version { exit 1 } + $4 != "tenant" && $4 != "omninode_internal" { exit 1 } + $6 !~ /^[0-9a-f]{64}$/ { exit 1 } + ' "$APPLICATION_MIGRATION_MANIFEST"; then + echo "[forward-migration] FATAL: malformed or unknown stream/owner/domain in application migration manifest" >&2 + exit 1 + fi + if ! awk -F '\t' ' + NF != 5 { exit 1 } + { + path_count = split($1, path_parts, "/") + expected_version = "node:" path_parts[2] ":" path_parts[3] + } + path_count != 3 || path_parts[1] != "nodes" { exit 1 } + path_parts[2] !~ /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/ { exit 1 } + path_parts[3] !~ /^[A-Za-z0-9_][A-Za-z0-9_.-]*[.]sql$/ { exit 1 } + $2 != expected_version || $3 !~ /^[0-9a-f]{64}$/ { exit 1 } + $4 !~ /^OMN-[0-9]+$/ || $5 == "" { exit 1 } + ' "$APPLICATION_MIGRATION_BLOCKS"; then + echo "[forward-migration] FATAL: malformed application migration block declaration" >&2 + exit 1 + fi + if ! awk -F '\t' ' + NF != 6 { exit 1 } + { + version_count = split($4, version_parts, ":") + expected_owner = "node:" version_parts[2] + } + version_count != 3 || version_parts[1] != "node" { exit 1 } + version_parts[2] !~ /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/ { exit 1 } + version_parts[3] !~ /^[A-Za-z0-9_][A-Za-z0-9_.-]*[.]sql$/ { exit 1 } + $1 != expected_owner || $2 != expected_owner { exit 1 } + $3 != "tenant" && $3 != "omninode_internal" { exit 1 } + $5 !~ /^[A-Za-z0-9_.:-]+$/ || $6 !~ /^OMN-[0-9]+$/ { exit 1 } + ' "$LEGACY_NODE_MIGRATION_DECLARATIONS"; then + echo "[forward-migration] FATAL: malformed historical node migration declaration" >&2 + exit 1 + fi + if ! awk -F '\t' 'NF != 2 || $1 !~ /^[A-Za-z0-9_.-]+$/ || $2 !~ /^[A-Za-z0-9_.-]+[.]sql$/ { exit 1 }' \ + "$CLOUD_MIGRATION_ALIASES"; then + echo "[forward-migration] FATAL: malformed cloud migration alias declaration" >&2 + exit 1 + fi + if [ -n "$(cut -f 1 "$CLOUD_MIGRATION_ALIASES" | sort | uniq -d | head -n 1)" ] \ + || [ -n "$(cut -f 2 "$CLOUD_MIGRATION_ALIASES" | sort | uniq -d | head -n 1)" ]; then + echo "[forward-migration] FATAL: duplicate cloud migration alias declaration" >&2 + exit 1 + fi + + duplicate_artifact="$(cut -f 1 "$APPLICATION_MIGRATION_MANIFEST" | sort | uniq -d | head -n 1)" + duplicate_identity="$(cut -f 2,4,5 "$APPLICATION_MIGRATION_MANIFEST" | sort | uniq -d | head -n 1)" + if [ -n "$duplicate_artifact" ]; then + echo "[forward-migration] FATAL: double migration declaration for artifact ${duplicate_artifact}" >&2 + exit 1 + fi + if [ -n "$duplicate_identity" ]; then + echo "[forward-migration] FATAL: duplicate migration version ${duplicate_identity}" >&2 + exit 1 + fi + duplicate_legacy_version="$(cut -f 4 "$LEGACY_NODE_MIGRATION_DECLARATIONS" | sort | uniq -d | head -n 1)" + legacy_active_overlap="$( { cut -f 5 "$APPLICATION_MIGRATION_MANIFEST"; cut -f 2 "$APPLICATION_MIGRATION_BLOCKS"; cut -f 4 "$LEGACY_NODE_MIGRATION_DECLARATIONS"; } | sort | uniq -d | head -n 1)" + if [ -n "$duplicate_legacy_version" ] || [ -n "$legacy_active_overlap" ]; then + echo "[forward-migration] FATAL: ambiguous historical node migration declaration" >&2 + exit 1 + fi + + while IFS=' ' read -r legacy_stream legacy_owner _ legacy_version _ _; do + legacy_node="$(printf '%s' "$legacy_version" | cut -d ':' -f 2)" + legacy_filename="$(printf '%s' "$legacy_version" | cut -d ':' -f 3)" + if [ -f "${NODE_MIGRATIONS_DIR}/${legacy_node}/${legacy_filename}" ]; then + echo "[forward-migration] FATAL: historical declaration has active artifact ${legacy_version}" >&2 + exit 1 + fi + done <"$LEGACY_NODE_MIGRATION_DECLARATIONS" + + while IFS=' ' read -r artifact_path _ _ _ declared_version declared_checksum; do + migration_file="${MIGRATIONS_DIR}/${artifact_path}" + if [ ! -f "$migration_file" ]; then + echo "[forward-migration] FATAL: declared migration artifact missing: ${artifact_path}" >&2 + exit 1 + fi + actual_checksum="$(file_sha256 "$migration_file")" + if [ "$actual_checksum" != "$declared_checksum" ]; then + echo "[forward-migration] FATAL: conflicting migration checksum for ${declared_version}" >&2 + exit 1 + fi + done <"$APPLICATION_MIGRATION_MANIFEST" + + while IFS=' ' read -r artifact_path blocked_version blocked_checksum blocked_ticket blocked_reason; do + migration_file="${MIGRATIONS_DIR}/${artifact_path}" + if [ ! -f "$migration_file" ] || [ "$(file_sha256 "$migration_file")" != "$blocked_checksum" ]; then + echo "[forward-migration] FATAL: conflicting migration checksum for blocked ${blocked_version}" >&2 + exit 1 + fi + if ! is_fenced_node_migration "$blocked_version" \ + || is_lane_released_node_migration "$blocked_version"; then + echo "[forward-migration] FATAL: unresolved migration domain for ${blocked_version} (${blocked_ticket}: ${blocked_reason})" >&2 + exit 1 + fi + done <"$APPLICATION_MIGRATION_BLOCKS" + + for migration_file in $(find "$NODE_MIGRATIONS_DIR" -mindepth 2 -maxdepth 2 -type f -name '*.sql' | sort); do + artifact_path="nodes/${migration_file#"${NODE_MIGRATIONS_DIR}"/}" + declared_count="$(awk -F '\t' -v path="$artifact_path" '$1 == path { count++ } END { print count + 0 }' \ + "$APPLICATION_MIGRATION_MANIFEST")" + blocked_count="$(awk -F '\t' -v path="$artifact_path" '$1 == path { count++ } END { print count + 0 }' \ + "$APPLICATION_MIGRATION_BLOCKS")" + if [ $((declared_count + blocked_count)) -ne 1 ]; then + echo "[forward-migration] FATAL: migration ${artifact_path} must have exactly one declaration or explicit block" >&2 + exit 1 + fi + done +} + +resolve_application_migration() { + artifact_path="$1" + expected_version="$2" + declaration="$(awk -F '\t' -v path="$artifact_path" '$1 == path { print }' \ + "$APPLICATION_MIGRATION_MANIFEST")" + if [ -z "$declaration" ]; then + block="$(awk -F '\t' -v path="$artifact_path" '$1 == path { print }' \ + "$APPLICATION_MIGRATION_BLOCKS")" + if [ -n "$block" ]; then + block_ticket="$(printf '%s\n' "$block" | cut -f 4)" + block_reason="$(printf '%s\n' "$block" | cut -f 5)" + echo "[forward-migration] FATAL: unresolved migration domain for ${expected_version} (${block_ticket}: ${block_reason})" >&2 + else + echo "[forward-migration] FATAL: unknown application migration ${artifact_path}" >&2 + fi + exit 1 + fi + + DECLARED_STREAM="$(printf '%s\n' "$declaration" | cut -f 2)" + DECLARED_OWNER="$(printf '%s\n' "$declaration" | cut -f 3)" + DECLARED_DOMAIN="$(printf '%s\n' "$declaration" | cut -f 4)" + DECLARED_VERSION="$(printf '%s\n' "$declaration" | cut -f 5)" + DECLARED_CHECKSUM="$(printf '%s\n' "$declaration" | cut -f 6)" + if [ "$DECLARED_VERSION" != "$expected_version" ]; then + echo "[forward-migration] FATAL: migration version mismatch for ${artifact_path}" >&2 + exit 1 + fi +} + +prepare_canonical_ledger() { + ledger_database="$1" + validate_database_identifier "$ledger_database" + if [ ! -f "$LEDGER_BOOTSTRAP" ]; then + echo "[forward-migration] FATAL: canonical ledger bootstrap missing: ${LEDGER_BOOTSTRAP}" >&2 + exit 1 + fi + validate_client_file_path "$APPLICATION_MIGRATION_MANIFEST" + validate_client_file_path "$LEGACY_NODE_MIGRATION_DECLARATIONS" + echo "[forward-migration] Converging canonical ledger in ${ledger_database}..." + psql -X -q -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$ledger_database" \ + -v ON_ERROR_STOP=1 \ + -c "CREATE TEMP TABLE onex_application_migration_manifest ( + artifact_path TEXT NOT NULL, + migration_stream TEXT NOT NULL, + owner TEXT NOT NULL, + domain TEXT NOT NULL, + version TEXT NOT NULL, + checksum TEXT NOT NULL, + PRIMARY KEY (artifact_path), + UNIQUE (migration_stream, domain, version) + ); CREATE TEMP TABLE onex_legacy_node_migration_declarations ( + migration_stream TEXT NOT NULL, + owner TEXT NOT NULL, + domain TEXT NOT NULL, + version TEXT NOT NULL PRIMARY KEY, + source_checksum TEXT NOT NULL, + ticket TEXT NOT NULL + )" \ + -c "\copy onex_application_migration_manifest FROM '${APPLICATION_MIGRATION_MANIFEST}' WITH (FORMAT text, DELIMITER E'\t')" \ + -c "\copy onex_legacy_node_migration_declarations FROM '${LEGACY_NODE_MIGRATION_DECLARATIONS}' WITH (FORMAT text, DELIMITER E'\t')" \ + -f "$LEDGER_BOOTSTRAP" +} + +# Return 0 only when the version is already present and its canonical metadata +# is safe to skip. Content hashes must match byte-for-byte. A +# legacy_attestation row is deliberately distinguishable and can never satisfy +# an active node migration probe: it proves a source record, not file bytes. +migration_is_applied() { + ledger_database="$1" + ledger_stream="$2" + ledger_owner="$3" + ledger_domain="$4" + ledger_version="$5" + expected_checksum="$6" + validate_migration_identity "$ledger_version" + + ledger_row="$( + psql -X -qAt -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$ledger_database" \ + -v ON_ERROR_STOP=1 \ + -v ledger_stream="$ledger_stream" \ + -v ledger_version="$ledger_version" \ + -v ledger_domain="$ledger_domain" \ + -f - <<'EOSQL' +SELECT checksum || '|' || checksum_kind || '|' || owner || '|' || provenance + FROM platform_catalog.schema_migrations + WHERE migration_stream = :'ledger_stream' + AND domain = :'ledger_domain' + AND version = :'ledger_version'; +EOSQL + )" + if [ -z "$ledger_row" ]; then + return 1 + fi + + recorded_checksum="$(printf '%s\n' "$ledger_row" | cut -d '|' -f 1)" + recorded_kind="$(printf '%s\n' "$ledger_row" | cut -d '|' -f 2)" + recorded_owner="$(printf '%s\n' "$ledger_row" | cut -d '|' -f 3)" + + if [ "$recorded_owner" != "$ledger_owner" ]; then + echo "[forward-migration] FATAL: double migration declaration for ${ledger_stream}:${ledger_domain}:${ledger_version} (recorded owner ${recorded_owner}, declared ${ledger_owner})" >&2 + exit 1 + fi + case "$recorded_kind" in + content_sha256) + if [ "$recorded_checksum" != "$expected_checksum" ]; then + echo "[forward-migration] FATAL: conflicting migration checksum for ${ledger_stream}:${ledger_domain}:${ledger_version}" >&2 + exit 1 + fi + ;; + legacy_attestation) + echo "[forward-migration] FATAL: active migration ${ledger_version} has only a legacy checksum attestation" >&2 + exit 1 + ;; + *) + echo "[forward-migration] FATAL: unknown checksum kind '${recorded_kind}' for ${ledger_version}" >&2 + exit 1 + ;; + esac + return 0 +} + +record_migration() { + ledger_database="$1" + ledger_stream="$2" + ledger_owner="$3" + ledger_domain="$4" + ledger_version="$5" + ledger_checksum="$6" + ledger_provenance="$7" + validate_migration_identity "$ledger_version" + if ! printf '%s' "$ledger_checksum" | grep -Eq '^[0-9a-f]{64}$'; then + echo "[forward-migration] FATAL: malformed SHA-256 for ${ledger_version}" >&2 + exit 1 + fi + + psql -X -q -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$ledger_database" \ + -v ON_ERROR_STOP=1 \ + -v ledger_stream="$ledger_stream" \ + -v ledger_owner="$ledger_owner" \ + -v ledger_domain="$ledger_domain" \ + -v ledger_version="$ledger_version" \ + -v ledger_checksum="$ledger_checksum" \ + -v ledger_provenance="$ledger_provenance" \ + -f - <<'EOSQL' +INSERT INTO platform_catalog.schema_migrations ( + migration_stream, owner, domain, version, checksum, checksum_kind, provenance + ) VALUES ( + :'ledger_stream', :'ledger_owner', :'ledger_domain', :'ledger_version', + :'ledger_checksum', 'content_sha256', :'ledger_provenance' + ); +EOSQL +} + +database_exists() { + candidate_database="$1" + validate_database_identifier "$candidate_database" + [ "$( + psql -X -qAt -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" \ + -v ON_ERROR_STOP=1 -v candidate_database="$candidate_database" \ + -f - <<'EOSQL' +SELECT count(*) FROM pg_database WHERE datname = :'candidate_database'; +EOSQL + )" = "1" ] +} + +import_ledger_stage() { + target_database="$1" + stage_file="$2" + if [ ! -s "$stage_file" ]; then + return 0 + fi + validate_client_file_path "$stage_file" + + psql -X -q -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$target_database" \ + -v ON_ERROR_STOP=1 \ + -c "BEGIN; CREATE TEMP TABLE onex_migration_import_stage ( + migration_stream TEXT NOT NULL, + owner TEXT NOT NULL, + domain TEXT NOT NULL, + version TEXT NOT NULL, + checksum TEXT NOT NULL, + checksum_kind TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL, + provenance TEXT NOT NULL + ) ON COMMIT DROP" \ + -c "\copy onex_migration_import_stage FROM '${stage_file}' WITH (FORMAT csv)" \ + -f - <<'EOSQL' +DO $import_validation$ +BEGIN + IF EXISTS ( + SELECT 1 FROM onex_migration_import_stage + WHERE NOT ( + migration_stream = 'omninode-cloud' + AND owner = 'service:onex_api' + AND domain = 'legacy_unclassified' + ) + ) THEN + RAISE EXCEPTION 'unknown migration stream/domain declaration in import'; + END IF; + IF EXISTS ( + SELECT 1 FROM onex_migration_import_stage + WHERE checksum !~ '^[0-9a-f]{64}$' + OR checksum_kind NOT IN ('content_sha256', 'legacy_attestation') + ) THEN + RAISE EXCEPTION 'malformed migration checksum in import'; + END IF; + IF EXISTS ( + SELECT 1 + FROM onex_migration_import_stage + GROUP BY migration_stream, domain, version + HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION 'duplicate migration version in import'; + END IF; + IF EXISTS ( + SELECT 1 + FROM onex_migration_import_stage incoming + JOIN platform_catalog.schema_migrations existing + USING (migration_stream, domain, version) + WHERE incoming.checksum <> existing.checksum + ) THEN + RAISE EXCEPTION 'conflicting migration checksum in import'; + END IF; + IF EXISTS ( + SELECT 1 + FROM onex_migration_import_stage incoming + JOIN platform_catalog.schema_migrations existing + USING (migration_stream, domain, version) + WHERE incoming.checksum = existing.checksum + AND ( + incoming.owner <> existing.owner + OR incoming.checksum_kind <> existing.checksum_kind + OR incoming.applied_at <> existing.applied_at + OR incoming.provenance <> existing.provenance + ) + ) THEN + RAISE EXCEPTION 'double migration declaration in import'; + END IF; +END +$import_validation$; + +INSERT INTO platform_catalog.schema_migrations ( + migration_stream, owner, domain, version, checksum, checksum_kind, + applied_at, provenance +) +SELECT incoming.migration_stream, incoming.owner, incoming.domain, incoming.version, + incoming.checksum, incoming.checksum_kind, incoming.applied_at, + incoming.provenance +FROM onex_migration_import_stage incoming +WHERE NOT EXISTS ( + SELECT 1 + FROM platform_catalog.schema_migrations existing + WHERE existing.migration_stream = incoming.migration_stream + AND existing.domain = incoming.domain + AND existing.version = incoming.version +); +COMMIT; +EOSQL +} + +import_cloud_history() { + target_database="$1" + cloud_database="${OMNINODE_CLOUD_HISTORY_DB:-omninode_cloud}" + validate_database_identifier "$cloud_database" + if ! database_exists "$cloud_database"; then + echo "[forward-migration] Historical cloud database ${cloud_database} absent; no cloud history to import." + return 0 + fi + + stage_file="$(mktemp)" + validate_client_file_path "$CLOUD_MIGRATION_ALIASES" + psql -X -q -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$cloud_database" \ + -v ON_ERROR_STOP=1 \ + -c "CREATE TEMP TABLE onex_cloud_migration_alias ( + migration_name TEXT PRIMARY KEY, + runner_version TEXT NOT NULL UNIQUE + )" \ + -c "CREATE TEMP TABLE onex_cloud_migration_export ( + migration_stream TEXT NOT NULL, + owner TEXT NOT NULL, + domain TEXT NOT NULL, + version TEXT NOT NULL, + checksum TEXT NOT NULL, + checksum_kind TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL, + provenance TEXT NOT NULL + )" \ + -c "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY" \ + -c "\copy onex_cloud_migration_alias FROM '${CLOUD_MIGRATION_ALIASES}' WITH (FORMAT text, DELIMITER E'\t')" \ + -f - <<'EOSQL' >"$stage_file" +DO $cloud_history_export$ +DECLARE + schema_columns TEXT; + log_columns TEXT; +BEGIN + SELECT coalesce(string_agg(column_name, ',' ORDER BY column_name), '') + INTO schema_columns + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations'; + SELECT coalesce(string_agg(column_name, ',' ORDER BY column_name), '') + INTO log_columns + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'migrations_log'; + + IF schema_columns = '' THEN + IF log_columns <> '' THEN + RAISE EXCEPTION + 'cloud migrations_log is audit-only: applied-set ledger is absent'; + END IF; + RETURN; + END IF; + IF schema_columns <> 'applied_at,checksum,version' OR EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'schema_migrations' + AND ( + (column_name = 'version' + AND (udt_name <> 'text' OR is_nullable <> 'NO')) + OR (column_name = 'checksum' AND udt_name <> 'text') + OR (column_name = 'applied_at' + AND (udt_name <> 'timestamptz' OR is_nullable <> 'NO')) + ) + ) THEN + RAISE EXCEPTION + 'unknown cloud applied-set ledger shape: public.schema_migrations (%)', + schema_columns; + END IF; + + IF log_columns <> '' AND ( + NOT ('migration_name' = ANY (string_to_array(log_columns, ','))) + OR NOT ('direction' = ANY (string_to_array(log_columns, ','))) + OR NOT ('executed_at' = ANY (string_to_array(log_columns, ','))) + OR EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'migrations_log' + AND ( + (column_name IN ('migration_name', 'direction') + AND (udt_name <> 'text' OR is_nullable <> 'NO')) + OR (column_name = 'executed_at' + AND (udt_name <> 'timestamptz' OR is_nullable <> 'NO')) + ) + ) + ) THEN + RAISE EXCEPTION + 'unknown cloud audit ledger shape: public.migrations_log (%)', log_columns; + END IF; + + IF log_columns = '' THEN + INSERT INTO onex_cloud_migration_export + SELECT + 'omninode-cloud', + 'service:onex_api', + 'legacy_unclassified', + applied.version, + CASE + WHEN applied.checksum ~ '^[0-9a-f]{64}$' THEN applied.checksum + ELSE encode(sha256(convert_to( + 'omninode-cloud|legacy_unclassified|' || applied.version || '|' || + coalesce(applied.checksum, ''), 'UTF8' + )), 'hex') + END, + CASE + WHEN applied.checksum ~ '^[0-9a-f]{64}$' THEN 'content_sha256' + ELSE 'legacy_attestation' + END, + applied.applied_at, + format( + 'legacy:%s:public.schema_migrations:version:%s:raw-checksum=%s', + current_database(), applied.version, coalesce(applied.checksum, '') + ) + FROM public.schema_migrations applied + ORDER BY applied.version; + RETURN; + END IF; + + IF EXISTS ( + SELECT 1 FROM public.migrations_log + WHERE direction IS NULL OR direction NOT IN ('forward', 'rollback') + ) THEN + RAISE EXCEPTION 'unknown cloud migrations_log direction'; + END IF; + IF EXISTS ( + SELECT 1 + FROM public.migrations_log log + LEFT JOIN onex_cloud_migration_alias alias + ON alias.migration_name = log.migration_name + WHERE alias.migration_name IS NULL + ) THEN + RAISE EXCEPTION 'unknown cloud migrations_log alias'; + END IF; + IF EXISTS ( + SELECT 1 + FROM public.migrations_log log + JOIN onex_cloud_migration_alias alias + ON alias.migration_name = log.migration_name + LEFT JOIN public.schema_migrations applied + ON applied.version = alias.runner_version + WHERE applied.version IS NULL + ) THEN + RAISE EXCEPTION + 'cloud migrations_log is audit-only: log-only alias cannot be imported as applied'; + END IF; + + INSERT INTO onex_cloud_migration_export + SELECT + 'omninode-cloud', + 'service:onex_api', + 'legacy_unclassified', + applied.version, + CASE + WHEN applied.checksum ~ '^[0-9a-f]{64}$' THEN applied.checksum + ELSE encode(sha256(convert_to( + 'omninode-cloud|legacy_unclassified|' || applied.version || '|' || + coalesce(applied.checksum, ''), 'UTF8' + )), 'hex') + END, + CASE + WHEN applied.checksum ~ '^[0-9a-f]{64}$' THEN 'content_sha256' + ELSE 'legacy_attestation' + END, + applied.applied_at, + format( + 'legacy:%s:public.schema_migrations:version:%s:raw-checksum=%s%s', + current_database(), applied.version, coalesce(applied.checksum, ''), + CASE WHEN bool_or(log.migration_name IS NOT NULL) + THEN ';migrations_log:' || max(log.migration_name) + ELSE '' + END + ) + FROM public.schema_migrations applied + LEFT JOIN onex_cloud_migration_alias alias + ON alias.runner_version = applied.version + LEFT JOIN public.migrations_log log + ON log.migration_name = alias.migration_name + GROUP BY applied.version, applied.checksum, applied.applied_at + ORDER BY applied.version; +END +$cloud_history_export$; + +COPY onex_cloud_migration_export TO STDOUT WITH (FORMAT csv); +COMMIT; +EOSQL + import_ledger_stage "$target_database" "$stage_file" + rm -f "$stage_file" +} + +# Validate the complete checked-in application declaration surface before +# either database is mutated. Service-only invocations with no node tree keep +# using only the separate flat ledger and do not bootstrap an application DB. +if [ -d "$NODE_MIGRATIONS_DIR" ]; then + validate_application_migration_manifest +fi + +# --------------------------------------------------------------------------- +# 1. Ensure service-owned schema_migrations tracking table exists (idempotent) +# --------------------------------------------------------------------------- +echo "[forward-migration] Ensuring service migration ledger exists in ${PGDB}..." psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" -c " CREATE TABLE IF NOT EXISTS public.schema_migrations ( @@ -202,7 +1200,7 @@ for migration_file in $(ls "${MIGRATIONS_DIR}"/*.sql | sort); do if is_skipped_by_manifest "${migration_id}"; then echo "[forward-migration] skip ${filename} (skip-manifest)" SKIPPED=$((SKIPPED + 1)) - # Record in schema_migrations so the table stays consistent. + # Record in the service-owned ledger so the table stays consistent. psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" \ -c "INSERT INTO public.schema_migrations (migration_id, checksum, source_set) VALUES ('${migration_id}', 'skip-manifest', 'docker') @@ -210,7 +1208,8 @@ for migration_file in $(ls "${MIGRATIONS_DIR}"/*.sql | sort); do continue fi - # Check if already applied + # The flat set belongs to the separate omnibase_infra service database. Its + # runner/ledger remain out of the unified application-database scope. already_applied=$(psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" \ -tAc "SELECT 1 FROM public.schema_migrations WHERE migration_id = '${migration_id}'" 2>/dev/null || true) @@ -236,6 +1235,14 @@ for migration_file in $(ls "${MIGRATIONS_DIR}"/*.sql | sort); do APPLIED=$((APPLIED + 1)) done +# Converge only the unified application database when this invocation actually +# carries the node migration tree. omnibase_infra remains a separate +# service-owned database under plan section 0.1. +if [ -d "$NODE_MIGRATIONS_DIR" ]; then + prepare_canonical_ledger "$NODE_PGDB" + import_cloud_history "$NODE_PGDB" +fi + # --------------------------------------------------------------------------- # 3. Auto-discover and apply node-owned migrations (OMN-12559) # --------------------------------------------------------------------------- @@ -247,17 +1254,6 @@ NODE_APPLIED=0 NODE_SKIPPED=0 if [ -d "${NODE_MIGRATIONS_DIR}" ]; then - echo "[forward-migration] Ensuring schema_migrations table exists in node projection database ${NODE_PGDB}..." - - psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$NODE_PGDB" -c " -CREATE TABLE IF NOT EXISTS public.schema_migrations ( - migration_id TEXT PRIMARY KEY, - applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - checksum TEXT NOT NULL, - source_set TEXT NOT NULL -); -" - echo "[forward-migration] Scanning ${NODE_MIGRATIONS_DIR} for node-owned migrations in ${NODE_PGDB}..." # Iterate node directories in sorted order for deterministic application. @@ -272,25 +1268,73 @@ CREATE TABLE IF NOT EXISTS public.schema_migrations ( for migration_file in $(ls "${node_dir}"*.sql | sort); do filename=$(basename "$migration_file") migration_id="node:${node_name}:${filename}" + migration_checksum="$(file_sha256 "$migration_file")" + + # ---- BEGIN fenced-id skip (OMN-15336) ---- + # FIRST, ahead of the ledger probe and the apply. Skip and count it; + # never apply it, never record it in schema_migrations. + # + # OMN-15379: the lane-scoped release is checked INSIDE the fenced branch, + # never beside it. An id that is not fenced never consults the release + # set, and the release set can therefore only ever un-gate a strict subset + # of the fence — it can never gate or un-gate anything else. + if is_fenced_node_migration "${migration_id}"; then + if is_lane_released_node_migration "${migration_id}"; then + echo "[forward-migration] RELEASED on lane '${ONEX_MIGRATION_LANE}' (operator ruling 15, OMN-15379): ${migration_id}" + # Fall through to the normal already-applied probe + apply + record. + else + echo "[forward-migration] SKIP (operator-gated, see OMN-14974/OMN-15313/OMN-15335/OMN-15343): ${migration_id}" + NODE_SKIPPED=$((NODE_SKIPPED + 1)) + continue + fi + fi + # ---- END fenced-id skip (OMN-15336) ---- - already_applied=$(psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$NODE_PGDB" \ - -tAc "SELECT 1 FROM public.schema_migrations WHERE migration_id = '${migration_id}'" 2>/dev/null || true) + artifact_path="nodes/${node_name}/${filename}" + resolve_application_migration "$artifact_path" "$migration_id" + if [ "$DECLARED_CHECKSUM" != "$migration_checksum" ]; then + echo "[forward-migration] FATAL: conflicting migration checksum for ${migration_id}" >&2 + exit 1 + fi - if [ "$already_applied" = "1" ]; then + if migration_is_applied \ + "$NODE_PGDB" "$DECLARED_STREAM" "$DECLARED_OWNER" "$DECLARED_DOMAIN" \ + "$migration_id" "$migration_checksum"; then echo "[forward-migration] skip ${migration_id} (already applied)" NODE_SKIPPED=$((NODE_SKIPPED + 1)) continue fi + # ---- BEGIN unclassified FORCE ROW LEVEL SECURITY guard call (OMN-15336 item 4) ---- + # Reached only for a migration that (a) is not in the fence manifest at + # all -- an already-fenced id was already handled above, released or + # not -- (b) is not in the grandfather snapshot -- already vendored and + # already applying, before this guard existed, see + # GRANDFATHER_MANIFEST's definition above -- and (c) has never applied + # on this database -- the ledger probe just above returned false. See + # the guard's own definition for why all three conditions are required. + if ! is_fenced_node_migration "${migration_id}" \ + && ! is_grandfathered_force_rls_migration "${migration_id}" \ + && migration_declares_unclassified_force_rls "$migration_file"; then + echo "[forward-migration] FATAL: ${migration_id} enables FORCE ROW LEVEL SECURITY" \ + "but is not in the operator fence manifest (${FENCE_MANIFEST})." \ + "This migration has never applied on this database. Classify it" \ + "before it may run: add a fence entry citing the owning ticket to" \ + "${FENCE_MANIFEST} (with a lane release only if an operator ruling" \ + "authorizes one). NOTHING was applied by this migration." >&2 + exit 1 + fi + # ---- END unclassified FORCE ROW LEVEL SECURITY guard call (OMN-15336 item 4) ---- + echo "[forward-migration] apply ${migration_id}..." psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$NODE_PGDB" \ -v ON_ERROR_STOP=1 -f "$migration_file" - psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$NODE_PGDB" \ - -c "INSERT INTO public.schema_migrations (migration_id, checksum, source_set) - VALUES ('${migration_id}', 'applied-by-runner', 'node') - ON CONFLICT (migration_id) DO NOTHING;" + record_migration \ + "$NODE_PGDB" "$DECLARED_STREAM" "$DECLARED_OWNER" "$DECLARED_DOMAIN" \ + "$migration_id" "$migration_checksum" \ + "file:nodes/${node_name}/${filename}" echo "[forward-migration] done ${migration_id}" NODE_APPLIED=$((NODE_APPLIED + 1)) @@ -307,6 +1351,11 @@ echo "[forward-migration] Complete: ${APPLIED} infra applied, ${SKIPPED} infra s # --------------------------------------------------------------------------- # This is the FINAL act. Any earlier failure leaves migrations_complete=FALSE. # runner_completed_at records the timestamp of this successful completion. +# +# The lock must still be ours at this point (OMN-15291): flipping the gate +# HEALTHY after a run that may have been unserialized is exactly the false +# green this lock exists to prevent. +assert_migration_lock_still_held echo "[forward-migration] All migrations complete. Setting sentinel TRUE..." psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" -v ON_ERROR_STOP=1 -c " UPDATE public.db_metadata diff --git a/scripts/runtime_build/consumer_groups_stability.yaml b/scripts/runtime_build/consumer_groups_stability.yaml index b18a0cd327..b08411feac 100644 --- a/scripts/runtime_build/consumer_groups_stability.yaml +++ b/scripts/runtime_build/consumer_groups_stability.yaml @@ -41,10 +41,24 @@ # refresh is also proving live, demand-driven). Extend this list as new # load-bearing consumer groups come online; never silently drop an entry # without a reason recorded in the PR that removes it. +# +# OMN-15838: node_ledger_projection_compute's `.consume..` segment +# had drifted to a stale `1.0.2` (its contract's version at file-authoring +# time, OMN-14823) while the live contract +# (src/omnibase_infra/nodes/node_ledger_projection_compute/contract.yaml +# `contract_version`) had moved on to 1.1.0 (OMN-15006) then 1.2.0 +# (OMN-15168) -- this file was never updated in step. `rpk group describe` +# against a bare-truncated OR version-stale group name resolves to a +# DISTINCT, nonexistent group (STATE=Dead, per the warning above), so a stale +# pin here silently turns this entry into a false Dead-group health-gate +# failure on the very next refresh once the runtime image carries the newer +# contract version. Re-pinned to the live 1.2.0. This is exactly the +# recurring class the warning above already names -- no mechanized check +# exists yet to catch this drift automatically (filed OMN-15844). consumer_groups: - name: stability-test.omnibase_infra.node_registration_orchestrator.consume.1.1.1.__i.stability-test-main.__t.onex.evt.platform.node-heartbeat.v1 description: kernel-owned node-registration orchestrator, node-heartbeat fan-in (continuously active, always Stable) - - name: stability-test.omnibase_infra.node_ledger_projection_compute.consume.1.0.2.__i.stability-test-main.__t.onex.evt.platform.node-heartbeat.v1 + - name: stability-test.omnibase_infra.node_ledger_projection_compute.consume.1.2.0.__i.stability-test-main.__t.onex.evt.platform.node-heartbeat.v1 description: node-registration ledger projection, node-heartbeat fan-in (continuously active, always Stable) - name: stability-test.omnimarket.projection_delegation.consume.1.0.0.__i.stability-test-main.__t.onex.evt.omnibase-infra.delegation-completed.v1 description: delegation projection consumer (carries the OMN-14855 judge-verdict DLQ fix this refresh lands; demand-driven, may show Empty when idle) diff --git a/scripts/runtime_build/refresh_stability_lane.sh b/scripts/runtime_build/refresh_stability_lane.sh index da2889c718..e992346d34 100755 --- a/scripts/runtime_build/refresh_stability_lane.sh +++ b/scripts/runtime_build/refresh_stability_lane.sh @@ -40,6 +40,20 @@ # the canonical omnibase_infra clone, the same way deploy-runtime.sh and # cut-lab-ref.sh already do -- NOT from a worktree, NOT over ssh wrapping. # +# Required environment (OMN-15218 lane-deploy attribution + grant interlock): +# ONEX_DEPLOY_REASON MANDATORY. Why this refresh is happening (a real +# justification, ideally naming a ticket). Recorded with +# the actor (user/uid/host/ssh peer/parent command) in +# ~/.omnibase/infra/deploy-log.jsonl and in this script's +# own receipt. Two unattributed stability rebuilds in two +# days (2026-07-26, 2026-07-27) are why this is required. +# ONEX_DEPLOY_GRANT_ACK Comma-separated grant ids. This refresh is REFUSED +# while unconsumed, unexpired prod-promotion grants at +# onex_change_control@main pin the lane's proof — +# refreshing invalidates the stability-proven premise +# those grants rest on. Proceeding requires naming EVERY +# live grant id (the acknowledgement is recorded). +# # Usage: # refresh_stability_lane.sh [--ref ] [--min-contracts ] [--execute] # @@ -178,7 +192,7 @@ HEALTH_URL="http://${LANE_PROBE_HOST}:${STABILITY_TEST_RUNTIME_MAIN_PORT}/health MODE="plan" usage() { - sed -n '4,52p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + sed -n '4,65p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' exit "${1:-0}" } @@ -299,6 +313,53 @@ log "min contracts : ${MIN_CONTRACTS}" log "core services : ${CORE_SERVICES[*]}" log "mode : ${MODE}" +# ============================================================================= +# 0a. Lane-deploy ATTRIBUTION + live-grant INTERLOCK (OMN-15218) +# +# Runs BEFORE the ambient-clone checkout, the preflight image tags, and the +# build — i.e. before this script's FIRST mutation, not just before +# deploy-runtime.sh's. deploy-runtime.sh runs the same preflight itself, but +# steps 2 and 3 below (docker tag / git checkout --force) happen earlier and +# would otherwise be unattributed and un-interlocked. +# +# Refuses when ONEX_DEPLOY_REASON is absent/placeholder, or when unconsumed, +# unexpired prod-promotion grants at onex_change_control@main pin this lane's +# proof (acknowledge each grant id via ONEX_DEPLOY_GRANT_ACK to proceed on the +# record). Fails closed when grant state cannot be resolved. +# ============================================================================= +ATTRIBUTION_PREFLIGHT="${REPO_ROOT}/scripts/preflight_lane_deploy_attribution.py" +ATTRIBUTION_RECORD_JSON="null" +if [[ ! -f "${ATTRIBUTION_PREFLIGHT}" ]]; then + err "lane-deploy attribution preflight not found: ${ATTRIBUTION_PREFLIGHT}" + err " Refusing to refresh ${LANE} with no attribution mechanism (OMN-15218)." + exit 64 +fi +ATTRIBUTION_ARGS=( + --lane "${LANE}" + --compose-project "${COMPOSE_PROJECT}" + --source "refresh_stability_lane.sh" + --invoking-command "refresh_stability_lane.sh --ref ${REF} (mode=${MODE})" +) +if [[ "${MODE}" != "execute" ]]; then + ATTRIBUTION_ARGS+=(--check-only) +fi +ATTRIBUTION_EXIT=0 +if [[ "${PYTHON_BIN}" == "uv-run" ]]; then + ATTRIBUTION_RECORD_JSON="$(uv run --project "${REPO_ROOT}" python "${ATTRIBUTION_PREFLIGHT}" \ + "${ATTRIBUTION_ARGS[@]}")" || ATTRIBUTION_EXIT=$? +else + ATTRIBUTION_RECORD_JSON="$("${PYTHON_BIN}" "${ATTRIBUTION_PREFLIGHT}" \ + "${ATTRIBUTION_ARGS[@]}")" || ATTRIBUTION_EXIT=$? +fi +if [[ "${ATTRIBUTION_EXIT}" -ne 0 ]]; then + err "lane-deploy attribution preflight REFUSED this refresh (exit ${ATTRIBUTION_EXIT})." + err " Nothing was tagged, checked out, built, or restarted." + exit 64 +fi +if ! jq -e . >/dev/null 2>&1 <<<"${ATTRIBUTION_RECORD_JSON}"; then + ATTRIBUTION_RECORD_JSON="null" +fi + if [[ "${MODE}" != "execute" ]]; then log "dry-run: no fetch/checkout/build/restart performed. Re-run with --execute." log "would run: OMNI_HOME=${OMNI_HOME} OMNIBASE_INFRA_COMPOSE_PROJECT=${COMPOSE_PROJECT} \\" @@ -590,6 +651,7 @@ jq -n \ --slurpfile health_gate "${GATE1_JSON}" \ --argjson rollback_triggered "${ROLLBACK_TRIGGERED}" \ --argjson rollback_gate "${ROLLBACK_GATE_JSON}" \ + --argjson attribution "${ATTRIBUTION_RECORD_JSON}" \ --arg result "${RESULT}" \ '{ ts_utc: $ts, @@ -600,6 +662,7 @@ jq -n \ build_scope: $build_scope, health_gate: $health_gate[0], rollback: {triggered: $rollback_triggered, gate: $rollback_gate}, + attribution: $attribution, result: $result }' > "${RECEIPT_PATH}" diff --git a/scripts/runtime_build/stage_workspace.sh b/scripts/runtime_build/stage_workspace.sh index 0ef888a56a..63fcdd2ed6 100644 --- a/scripts/runtime_build/stage_workspace.sh +++ b/scripts/runtime_build/stage_workspace.sh @@ -206,7 +206,7 @@ if [[ -f "${CONSUMER_LOCK}" ]]; then "${PREFLIGHT_REPO_ARGS[@]}" \ --output "${PIN_COMPARISON_OUT}" \ --build-source workspace \ - "${preflight_extra[@]}" || preflight_status=$? + ${preflight_extra[@]+"${preflight_extra[@]}"} || preflight_status=$? else preflight_status=0 "${PREFLIGHT_PYTHON}" "${SCRIPT_DIR}/check_sibling_lock_pins.py" \ @@ -214,7 +214,7 @@ if [[ -f "${CONSUMER_LOCK}" ]]; then "${PREFLIGHT_REPO_ARGS[@]}" \ --output "${PIN_COMPARISON_OUT}" \ --build-source workspace \ - "${preflight_extra[@]}" || preflight_status=$? + ${preflight_extra[@]+"${preflight_extra[@]}"} || preflight_status=$? fi if [[ "${preflight_status}" != "0" ]]; then echo "ERROR: sibling-pin preflight failed against ${CONSUMER_LOCK}" >&2 diff --git a/scripts/runtime_build/tests/test_verify_stability_refresh.py b/scripts/runtime_build/tests/test_verify_stability_refresh.py index 87adcedf08..5b5de1e84c 100644 --- a/scripts/runtime_build/tests/test_verify_stability_refresh.py +++ b/scripts/runtime_build/tests/test_verify_stability_refresh.py @@ -30,14 +30,26 @@ check_manifest_count = _mod.check_manifest_count check_health = _mod.check_health check_cluster_health = _mod.check_cluster_health +check_partition_headroom = _mod.check_partition_headroom check_consumer_group = _mod.check_consumer_group check_consumer_group_with_retry = _mod.check_consumer_group_with_retry check_service_digest = _mod.check_service_digest run_health_gate = _mod.run_health_gate build_receipt = _mod.build_receipt HealthGateReport = _mod.HealthGateReport +PartitionHeadroomCheck = _mod.PartitionHeadroomCheck CORE_SERVICES = _mod.CORE_SERVICES DEFAULT_MIN_CONTRACTS = _mod.DEFAULT_MIN_CONTRACTS +DEFAULT_PARTITION_WARN_THRESHOLD = _mod.DEFAULT_PARTITION_WARN_THRESHOLD + + +def _topic_list_output(partitions: list[int]) -> str: + """Fixed-width `rpk topic list` output -- NAME / PARTITIONS / REPLICAS.""" + header = "NAME PARTITIONS REPLICAS" + rows = [ + f"topic-{i:04d} {p} 1" for i, p in enumerate(partitions) + ] + return "\n".join([header, *rows]) + "\n" # ─── helpers ───────────────────────────────────────────────────────────────── @@ -193,6 +205,111 @@ def test_cluster_health_not_ok_nonzero_exit(): assert ok is False +# ─── partition headroom [OMN-14013] ───────────────────────────────────────── +# +# `rpk cluster health` never surfaces partition-allocation headroom -- these +# checks prove the NEW dedicated probe does, and that it distinguishes +# "healthy headroom" / "crossed the visibility warn threshold" / "at or over +# cap (the literal live incident: 7046/7000, `rpk cluster health` green +# throughout)" as three genuinely different states. + + +def test_partition_headroom_ok_well_below_threshold(): + runner = MagicMock( + side_effect=[ + _completed(stdout="15000\n"), # cluster config get + _completed(stdout=_topic_list_output([1] * 1529)), # topic list + ] + ) + result = check_partition_headroom("redpanda-container", runner=runner) + assert result.cap == 15000 + assert result.total_partitions == 1529 + assert result.at_or_over_cap is False + assert result.crossed_warn_threshold is False + assert result.error is None + + +def test_partition_headroom_crosses_warn_threshold_but_not_at_cap(): + runner = MagicMock( + side_effect=[ + _completed(stdout="8000\n"), + _completed(stdout=_topic_list_output([1] * 7047)), + ] + ) + result = check_partition_headroom( + "redpanda-container", runner=runner, warn_threshold=0.8 + ) + assert result.usage_ratio is not None + assert result.usage_ratio > 0.8 + assert result.crossed_warn_threshold is True + assert result.at_or_over_cap is False + + +def test_partition_headroom_at_or_over_cap_is_a_real_failure(): + """The literal OMN-14013 live incident: 7046/7000, `rpk cluster health` + reported Healthy: true throughout. This check must call it a failure.""" + runner = MagicMock( + side_effect=[ + _completed(stdout="7000\n"), + _completed(stdout=_topic_list_output([1] * 7046)), + ] + ) + result = check_partition_headroom("redpanda-container", runner=runner) + assert result.at_or_over_cap is True + assert "AT OR OVER CAP" in result.detail + + +def test_partition_headroom_exactly_at_cap_is_at_or_over(): + runner = MagicMock( + side_effect=[ + _completed(stdout="7000\n"), + _completed(stdout=_topic_list_output([1] * 7000)), + ] + ) + result = check_partition_headroom("redpanda-container", runner=runner) + assert result.at_or_over_cap is True + + +def test_partition_headroom_cap_probe_failure_is_not_silently_ok(): + runner = MagicMock( + side_effect=[ + _completed(returncode=1, stderr="no such config"), + _completed(stdout=_topic_list_output([1] * 100)), + ] + ) + result = check_partition_headroom("redpanda-container", runner=runner) + assert result.cap is None + assert result.at_or_over_cap is False # unknown, not silently healthy + assert result.error is not None + + +def test_partition_headroom_topic_list_probe_failure_is_not_silently_ok(): + runner = MagicMock( + side_effect=[ + _completed(stdout="15000\n"), + _completed(returncode=1, stderr="connection refused"), + ] + ) + result = check_partition_headroom("redpanda-container", runner=runner) + assert result.total_partitions is None + assert result.error is not None + + +def test_partition_headroom_parses_partitions_column_by_header_not_position(): + """The PARTITIONS column must be located by its header label, not a + hardcoded index -- a reordered rpk table (NAME / REPLICAS / PARTITIONS) + must still sum the right column.""" + reordered = "NAME REPLICAS PARTITIONS\ntopic-a 1 3\ntopic-b 1 4\n" + runner = MagicMock( + side_effect=[ + _completed(stdout="100\n"), + _completed(stdout=reordered), + ] + ) + result = check_partition_headroom("redpanda-container", runner=runner) + assert result.total_partitions == 7 + + def _group_describe_output(state: str) -> str: """rpk group describe has NO -f json mode -- fixed-width plain text.""" return ( @@ -299,6 +416,10 @@ def _run(cmd, capture_output=True, text=True, timeout=30, check=False): if "Image" in fmt: return _completed(stdout="sha256:new-image") return _completed(stdout="newrevision1234") + if "config" in cmd and "get" in cmd: + return _completed(stdout="15000\n") + if "topic" in cmd and "list" in cmd: + return _completed(stdout=_topic_list_output([1] * 1529)) if cmd[:3] == ["docker", "exec", "redpanda-container"] and "cluster" in cmd: return _completed(stdout="Healthy: true\n") if "group" in cmd and "describe" in cmd: @@ -345,6 +466,10 @@ def runner(cmd, capture_output=True, text=True, timeout=30, check=False): if "Image" in fmt: return _completed(stdout="sha256:new-image") return _completed(stdout="newrevision1234") + if "config" in cmd and "get" in cmd: + return _completed(stdout="15000\n") + if "topic" in cmd and "list" in cmd: + return _completed(stdout=_topic_list_output([1] * 1529)) if "cluster" in cmd: return _completed(stdout="Healthy: true\n") if "group" in cmd and "describe" in cmd: @@ -377,6 +502,101 @@ def opener(url, timeout=10): assert report.groups_stable is False +def test_health_gate_overall_fail_when_partition_cap_reached(): + """OMN-14013: everything else passes, but the broker is AT its partition + cap -- overall must be FAIL (a real, checked defect `rpk cluster health` + alone would have missed).""" + pre_image_ids = dict.fromkeys(CORE_SERVICES, "sha256:old-image") + + def runner(cmd, capture_output=True, text=True, timeout=30, check=False): + if cmd[:2] == ["docker", "inspect"]: + fmt = cmd[-1] + if "Image" in fmt: + return _completed(stdout="sha256:new-image") + return _completed(stdout="newrevision1234") + if "config" in cmd and "get" in cmd: + return _completed(stdout="7000\n") + if "topic" in cmd and "list" in cmd: + return _completed(stdout=_topic_list_output([1] * 7046)) + if "cluster" in cmd: + return _completed(stdout="Healthy: true\n") + if "group" in cmd and "describe" in cmd: + return _completed(stdout=_group_describe_output("Stable")) + raise AssertionError(f"unexpected command: {cmd}") + + def opener(url, timeout=10): + if "manifest" in url: + return _FakeHTTPResponse( + json.dumps({"contracts": list(range(DEFAULT_MIN_CONTRACTS))}).encode() + ) + return _FakeHTTPResponse(json.dumps({"status": "healthy"}).encode()) + + report = run_health_gate( + lane="stability-test", + pre_image_ids=pre_image_ids, + expected_revision="newrevision1234", + manifest_url="http://x/manifest", + health_url="http://x/health", + broker_container="redpanda-container", + min_contracts=DEFAULT_MIN_CONTRACTS, + consumer_groups=["group.a"], + runner=runner, + opener=opener, + sleep_fn=lambda _s: None, + ) + assert report.overall == "FAIL" + assert report.partition_headroom is not None + assert report.partition_headroom.at_or_over_cap is True + assert report.partition_headroom_ok is False + + +def test_health_gate_overall_pass_when_partition_headroom_only_crosses_warn(): + """OMN-14013: crossing the warn threshold (but not the cap) is visibility + only -- it must NOT retroactively fail an otherwise-healthy refresh.""" + pre_image_ids = dict.fromkeys(CORE_SERVICES, "sha256:old-image") + + def runner(cmd, capture_output=True, text=True, timeout=30, check=False): + if cmd[:2] == ["docker", "inspect"]: + fmt = cmd[-1] + if "Image" in fmt: + return _completed(stdout="sha256:new-image") + return _completed(stdout="newrevision1234") + if "config" in cmd and "get" in cmd: + return _completed(stdout="8000\n") + if "topic" in cmd and "list" in cmd: + return _completed(stdout=_topic_list_output([1] * 7047)) # ~88% + if "cluster" in cmd: + return _completed(stdout="Healthy: true\n") + if "group" in cmd and "describe" in cmd: + return _completed(stdout=_group_describe_output("Stable")) + raise AssertionError(f"unexpected command: {cmd}") + + def opener(url, timeout=10): + if "manifest" in url: + return _FakeHTTPResponse( + json.dumps({"contracts": list(range(DEFAULT_MIN_CONTRACTS))}).encode() + ) + return _FakeHTTPResponse(json.dumps({"status": "healthy"}).encode()) + + report = run_health_gate( + lane="stability-test", + pre_image_ids=pre_image_ids, + expected_revision="newrevision1234", + manifest_url="http://x/manifest", + health_url="http://x/health", + broker_container="redpanda-container", + min_contracts=DEFAULT_MIN_CONTRACTS, + consumer_groups=["group.a"], + runner=runner, + opener=opener, + sleep_fn=lambda _s: None, + ) + assert report.overall == "PASS" + assert report.partition_headroom is not None + assert report.partition_headroom.crossed_warn_threshold is True + assert report.partition_headroom.at_or_over_cap is False + + # ─── receipt: ancestry true/false + rollback re-verification ─────────────── diff --git a/scripts/runtime_build/verify_stability_refresh.py b/scripts/runtime_build/verify_stability_refresh.py index 6543eac6b0..4d4a61c9cc 100644 --- a/scripts/runtime_build/verify_stability_refresh.py +++ b/scripts/runtime_build/verify_stability_refresh.py @@ -21,6 +21,15 @@ 6. **Revision readback** -- the ``org.opencontainers.image.revision`` label on each core container equals the intended new ref (or a prefix thereof, tolerating short/full SHA differences). + 7. **Partition headroom** [OMN-14013] -- live ``topic_partitions_per_shard`` + cluster config vs. the live total partition count across all topics. + ``rpk cluster health`` has NO concept of partition-allocation headroom -- + it reported ``Healthy: true`` right up to (and past) the lane's + allocation ceiling, silently blocking every new topic create. At/over the + cap is a genuine FAIL (a new topic create WILL fail); crossing the warn + threshold (default 80%) is surfaced in the report but does not flip + ``overall`` -- it is an early-warning signal, not by itself a broken + lane. ``refresh_stability_lane.sh`` calls this script both for the post-refresh gate and, on a FAIL, again against the rolled-back state to confirm rollback @@ -29,8 +38,8 @@ Exit codes: 0 - PASS (all requested checks succeeded) 1 - FAIL (a genuine check failure -- digest not changed, manifest floor - not met, unhealthy, cluster unhealthy, a group not Stable, or a - revision mismatch) + not met, unhealthy, cluster unhealthy, a group not Stable, a revision + mismatch, or partition usage at/over the cap) 2 - INFRA_ERROR (could not run a check at all -- docker/curl/rpk unavailable, container missing, etc.) -- distinguished from a genuine FAIL so the caller does not conflate "couldn't check" with @@ -57,6 +66,10 @@ _REVISION_LABEL = "org.opencontainers.image.revision" DEFAULT_MIN_CONTRACTS = 288 +# OMN-14013: fraction of topic_partitions_per_shard in use that triggers a +# visible (non-blocking) WARN. `rpk cluster health` never surfaces this on its +# own -- see the module docstring's check #7. +DEFAULT_PARTITION_WARN_THRESHOLD = 0.8 # ─── Data structures ──────────────────────────────────────────────────────── @@ -87,6 +100,37 @@ class ConsumerGroupCheck: error: str | None = None +@dataclass +class PartitionHeadroomCheck: + """Partition-allocation headroom -- invisible to `rpk cluster health` (OMN-14013). + + ``rpk cluster health`` reports on broker/replica state only; it has no + concept of "no partition-allocation headroom left" and stays green right + up to (and past) a single-shard broker's ``topic_partitions_per_shard`` + ceiling (OMN-14013: the stability-test lane was observed at 6994/7000, + and later 7046/7000 -- OVER cap -- with `rpk cluster health` reporting + `Healthy: true` throughout both times). This check queries the live cap + and the live total partition count directly so that condition is visible + BEFORE the next topic-create silently fails. + + ``at_or_over_cap`` participates in the overall health-gate verdict (a + genuine, checked FAIL: new topic creates WILL fail). ``crossed_warn_threshold`` + is visibility-only and never flips ``overall`` on its own -- unrelated + historical topic accumulation crossing 80% headroom usage should not + retroactively fail an otherwise-healthy refresh; it is an early-warning + signal for a separate topic-retirement/cap-raise action. + """ + + cap: int | None + total_partitions: int | None + usage_ratio: float | None + warn_threshold: float + at_or_over_cap: bool + crossed_warn_threshold: bool + detail: str + error: str | None = None + + @dataclass class HealthGateReport: """Aggregate health-gate report. ``overall`` is the AND of every check. @@ -109,6 +153,7 @@ class HealthGateReport: cluster_healthy: bool = False cluster_detail: str | None = None consumer_groups: list[ConsumerGroupCheck] = field(default_factory=list) + partition_headroom: PartitionHeadroomCheck | None = None errors: list[str] = field(default_factory=list) require_digest_change: bool = True @@ -126,6 +171,20 @@ def groups_stable(self) -> bool: g.stable for g in self.consumer_groups ) + @property + def partition_headroom_ok(self) -> bool: + """True unless the headroom check ran AND found the lane at/over cap. + + A crossed-warn-threshold-but-below-cap state is still "ok" here by + design (visibility only, see ``PartitionHeadroomCheck`` docstring); a + probe failure (``error is not None``) is surfaced via ``errors`` -> + ``INFRA_ERROR`` instead, mirroring ``check_manifest_count``. + """ + return ( + self.partition_headroom is None + or not self.partition_headroom.at_or_over_cap + ) + @property def overall(self) -> str: if self.errors: @@ -138,6 +197,7 @@ def overall(self) -> str: and self.cluster_healthy and self.groups_stable and self.revisions_match + and self.partition_headroom_ok ): return "PASS" return "FAIL" @@ -157,6 +217,12 @@ def to_dict(self) -> dict[str, object]: "consumer_groups": {g.group: g.state for g in self.consumer_groups}, "consumer_groups_stable": self.groups_stable, "revision_readback_ok": self.revisions_match, + "partition_headroom": ( + asdict(self.partition_headroom) + if self.partition_headroom is not None + else None + ), + "partition_headroom_ok": self.partition_headroom_ok, "services": [asdict(s) for s in self.services], "errors": self.errors, "overall": self.overall, @@ -348,6 +414,141 @@ def check_cluster_health( return healthy, stdout.strip().splitlines()[0] if stdout.strip() else "no output" +def _get_partition_cap( + broker_container: str, *, runner: object | None = None +) -> tuple[int | None, str | None]: + """Read the live ``topic_partitions_per_shard`` cluster config value.""" + try: + result = _run( + [ + "docker", + "exec", + broker_container, + "rpk", + "cluster", + "config", + "get", + "topic_partitions_per_shard", + ], + runner=runner, + ) + except subprocess.TimeoutExpired: + return ( + None, + f"timed out reading topic_partitions_per_shard via {broker_container}", + ) + except FileNotFoundError: + return None, "docker command not found" + if result.returncode != 0: + return ( + None, + f"rpk cluster config get failed (exit {result.returncode}): {(result.stderr or '').strip()}", + ) + stdout = (result.stdout or "").strip() + match = re.search(r"(\d+)", stdout) + if not match: + return ( + None, + f"could not parse topic_partitions_per_shard from output: {stdout!r}", + ) + return int(match.group(1)), None + + +def _get_total_partitions( + broker_container: str, *, runner: object | None = None +) -> tuple[int | None, str | None]: + """Sum the PARTITIONS column of `rpk topic list` across every topic. + + `rpk topic list` has no documented `-f json` mode verified against this + lane's rpk version (mirrors the same absence already noted for `rpk group + describe` above) -- parse the fixed-width plain-text table instead, + locating the PARTITIONS column by its header label rather than a + hardcoded position, since column position/order in rpk table output is + not guaranteed stable across subcommands or rpk versions. + """ + try: + result = _run( + ["docker", "exec", broker_container, "rpk", "topic", "list"], + runner=runner, + ) + except subprocess.TimeoutExpired: + return None, f"timed out listing topics via {broker_container}" + except FileNotFoundError: + return None, "docker command not found" + if result.returncode != 0: + return ( + None, + f"rpk topic list failed (exit {result.returncode}): {(result.stderr or '').strip()}", + ) + lines = (result.stdout or "").splitlines() + if not lines: + return None, "rpk topic list produced no output" + header = lines[0].split() + try: + partitions_idx = header.index("PARTITIONS") + except ValueError: + return None, f"rpk topic list header missing PARTITIONS column: {lines[0]!r}" + total = 0 + for line in lines[1:]: + fields = line.split() + if len(fields) <= partitions_idx: + continue + try: + total += int(fields[partitions_idx]) + except ValueError: + continue + return total, None + + +def check_partition_headroom( + broker_container: str, + *, + runner: object | None = None, + warn_threshold: float = DEFAULT_PARTITION_WARN_THRESHOLD, +) -> PartitionHeadroomCheck: + """Live partition-allocation headroom vs. `topic_partitions_per_shard` (OMN-14013).""" + cap, cap_err = _get_partition_cap(broker_container, runner=runner) + total, total_err = _get_total_partitions(broker_container, runner=runner) + error = cap_err or total_err + if error is not None or cap is None or total is None or cap <= 0: + detail = error or ("cap reported as <= 0" if cap is not None else "unknown") + return PartitionHeadroomCheck( + cap=cap, + total_partitions=total, + usage_ratio=None, + warn_threshold=warn_threshold, + at_or_over_cap=False, + crossed_warn_threshold=False, + detail=f"could not determine partition headroom: {detail}", + error=error or detail, + ) + usage_ratio = total / cap + at_or_over_cap = total >= cap + crossed_warn_threshold = usage_ratio >= warn_threshold + if at_or_over_cap: + detail = ( + f"{total}/{cap} partitions ({usage_ratio:.1%}) -- AT OR OVER CAP: " + "new topic creates will fail" + ) + elif crossed_warn_threshold: + detail = ( + f"{total}/{cap} partitions ({usage_ratio:.1%}) -- crossed " + f"{warn_threshold:.0%} warn threshold" + ) + else: + detail = f"{total}/{cap} partitions ({usage_ratio:.1%})" + return PartitionHeadroomCheck( + cap=cap, + total_partitions=total, + usage_ratio=usage_ratio, + warn_threshold=warn_threshold, + at_or_over_cap=at_or_over_cap, + crossed_warn_threshold=crossed_warn_threshold, + detail=detail, + error=None, + ) + + def check_consumer_group( broker_container: str, group: str, *, runner: object | None = None ) -> ConsumerGroupCheck: @@ -484,6 +685,7 @@ def run_health_gate( opener: object | None = None, require_digest_change: bool = True, sleep_fn: object | None = None, + partition_warn_threshold: float = DEFAULT_PARTITION_WARN_THRESHOLD, ) -> HealthGateReport: report = HealthGateReport( lane=lane, @@ -519,6 +721,13 @@ def run_health_gate( report.cluster_healthy = cluster_healthy report.cluster_detail = cluster_detail + partition_headroom = check_partition_headroom( + broker_container, runner=runner, warn_threshold=partition_warn_threshold + ) + report.partition_headroom = partition_headroom + if partition_headroom.error is not None: + report.errors.append(partition_headroom.error) + for group in consumer_groups: report.consumer_groups.append( check_consumer_group_with_retry( @@ -603,6 +812,16 @@ def main(argv: list[str] | None = None) -> int: "--consumer-groups-file", default=str(Path(__file__).resolve().parent / "consumer_groups_stability.yaml"), ) + parser.add_argument( + "--partition-warn-threshold", + type=float, + default=DEFAULT_PARTITION_WARN_THRESHOLD, + help=( + "Fraction of topic_partitions_per_shard in use that triggers a " + "visible (non-blocking) partition-headroom WARN [OMN-14013]. " + f"Default {DEFAULT_PARTITION_WARN_THRESHOLD:.0%}." + ), + ) parser.add_argument("--json", action="store_true", dest="json_output") parser.add_argument( "--no-require-digest-change", @@ -635,6 +854,7 @@ def main(argv: list[str] | None = None) -> int: min_contracts=args.min_contracts, consumer_groups=consumer_groups, require_digest_change=args.require_digest_change, + partition_warn_threshold=args.partition_warn_threshold, ) if args.json_output: @@ -647,6 +867,12 @@ def main(argv: list[str] | None = None) -> int: ) print(f" health_ok={report.health_ok} ({report.health_detail})") print(f" cluster_healthy={report.cluster_healthy} ({report.cluster_detail})") + if report.partition_headroom is not None: + ph = report.partition_headroom + print( + f" partition_headroom: {ph.detail}" + + (f" error={ph.error}" if ph.error else "") + ) for g in report.consumer_groups: print( f" group {g.group}: state={g.state} stable={g.stable}" diff --git a/scripts/seed-infisical.py b/scripts/seed-infisical.py index ea305a5ddc..507501573d 100644 --- a/scripts/seed-infisical.py +++ b/scripts/seed-infisical.py @@ -211,7 +211,7 @@ def _load_infisical_credentials() -> tuple[str, str, str, str]: """ infisical_addr = os.environ.get( "INFISICAL_ADDR", - "http://localhost:8880", # fallback-ok: local seed default + "http://localhost:8880", # url-authority-ok: INFISICAL_ADDR is bootstrap-only by registry contract, and a resolver cannot reach Infisical before this seed tool establishes it. ) client_id = os.environ.get("INFISICAL_CLIENT_ID", "") client_secret = os.environ.get("INFISICAL_CLIENT_SECRET", "") diff --git a/scripts/sync-node-migrations.sh b/scripts/sync-node-migrations.sh index def3b6a531..d48b8cfe2a 100755 --- a/scripts/sync-node-migrations.sh +++ b/scripts/sync-node-migrations.sh @@ -51,6 +51,22 @@ # copy already exists here at dev tip — so a new drift-causing merge can no # longer land in omnimarket in the first place. # +# LEGACY-DECLARED EXEMPTION (OMN-15717): a vendored file that no longer has +# a source in omnimarket (the owning node was deleted/rebuilt upstream) is +# NOT "stale" if it carries a checked-in row in +# docker/migrations/forward/_ledger/application-migrations.tsv. Operator +# ruling 2026-08-04 (OMN-15695) requires applied migration history to be +# preserved permanently, never deleted or rewritten — a live database can +# still carry a legacy-applied ledger row for a since-deleted node migration, +# and bootstrap.sql's adoption path needs that file's checked-in bytes to +# verify the historical checksum. Removing the vendored file the moment +# omnimarket deletes its source would re-open exactly the gap OMN-15717 +# fixed: an applied-but-undeclared migration bootstrap.sql cannot resolve. +# A file counts as "legacy-declared" only if application-migrations.tsv +# carries a row for it; an UNDECLARED file with no current omnimarket source +# is still flagged stale (that combination is 6th-occurrence-class drift, +# not preserved history). +# # USAGE # scripts/sync-node-migrations.sh # vendor (writes files) # scripts/sync-node-migrations.sh --check # CI mode: fail if drift exists @@ -70,7 +86,19 @@ fi # Repo root = parent of this script's directory. SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -DEST_ROOT="${REPO_ROOT}/docker/migrations/forward/nodes" +DEST_ROOT="${SYNC_NODE_MIGRATIONS_DEST_ROOT:-${REPO_ROOT}/docker/migrations/forward/nodes}" +APPLICATION_MIGRATION_MANIFEST="${APPLICATION_MIGRATION_MANIFEST:-${REPO_ROOT}/docker/migrations/forward/_ledger/application-migrations.tsv}" + +# OMN-15717: a vendored file with a checked-in application-migrations.tsv row +# is preserved applied history, not drift — see the LEGACY-DECLARED EXEMPTION +# note above. Match on the manifest's artifact_path column (relative to the +# forward-migration root, i.e. "nodes//.sql"). +is_legacy_declared() { + node_and_filename="$1" # "/.sql" + [ -f "${APPLICATION_MIGRATION_MANIFEST}" ] || return 1 + awk -F '\t' -v path="nodes/${node_and_filename}" '$1 == path { found=1 } END { exit !found }' \ + "${APPLICATION_MIGRATION_MANIFEST}" +} resolve_omnimarket_src() { if [ -n "${OMNIMARKET_SRC:-}" ] && [ -d "${OMNIMARKET_SRC}/src/omnimarket/nodes" ]; then @@ -177,16 +205,24 @@ sort -o "${EXPECTED_LIST}" "${EXPECTED_LIST}" if [ "${CHECK_MODE}" -eq 1 ]; then while IFS= read -r extra_file; do if [ -n "${extra_file}" ] && ! grep -Fxq "${extra_file}" "${EXPECTED_LIST}"; then - echo "[sync-node-migrations] DRIFT: stale vendored migration ${extra_file}" >&2 - DRIFT=1 + if is_legacy_declared "${extra_file}"; then + echo "[sync-node-migrations] legacy-declared (OMN-15717), not stale: ${extra_file}" + else + echo "[sync-node-migrations] DRIFT: stale vendored migration ${extra_file}" >&2 + DRIFT=1 + fi fi done < "${ACTUAL_LIST}" else while IFS= read -r extra_file; do if [ -n "${extra_file}" ] && ! grep -Fxq "${extra_file}" "${EXPECTED_LIST}"; then - rm -f "${DEST_ROOT}/${extra_file}" - echo "[sync-node-migrations] removed stale ${extra_file}" - COPIED=$((COPIED + 1)) + if is_legacy_declared "${extra_file}"; then + echo "[sync-node-migrations] kept legacy-declared (OMN-15717) ${extra_file}" + else + rm -f "${DEST_ROOT}/${extra_file}" + echo "[sync-node-migrations] removed stale ${extra_file}" + COPIED=$((COPIED + 1)) + fi fi done < "${ACTUAL_LIST}" fi diff --git a/scripts/system_health_check.sh b/scripts/system_health_check.sh index 4a359b657b..104ea94432 100755 --- a/scripts/system_health_check.sh +++ b/scripts/system_health_check.sh @@ -14,6 +14,12 @@ # --json Output results as JSON # --ci Non-interactive mode (implies --json, sets exit codes for CI) # --cross-repo Enable cross-repo checks (env audit, cloud bus refs) +# --lane Lane-liveness subset ONLY (dev_lane_liveness, redpanda, +# runtime_containers). Runs from a containerized CI runner +# with only docker.sock + network access — deliberately +# skips the checks that need POSTGRES_HOST / VALKEY_HOST / +# INFISICAL_* credentials. This is the mode the scheduled +# enforcement surface runs (OMN-15190). # --verbose Show detailed output for each check # --help Show this help message # @@ -27,15 +33,34 @@ # 3. valkey - Valkey (Redis-compatible) connectivity # 4. infra_containers - Core infra containers running # 5. keycloak - Keycloak auth (yellow if not running) -# 6. runtime_containers - Runtime profile containers (yellow if not running) +# 6. runtime_containers - Runtime profile containers (RED under keep-alive) # 7. required_topics - Required Kafka topics exist # 8. migration_parity - Docker and src migration directories in sync # 9. env_audit - No rogue .env files (--cross-repo only) # 10. cloud_bus_refs - No unsuppressed 29092 references (--cross-repo only) # cloud-bus-ok OMN-4922 # 11. bus_endpoint - KAFKA_BOOTSTRAP_SERVERS must not contain 29092 # cloud-bus-ok OMN-4922 # 12. infisical_folders - /shared// folders exist in Infisical (when INFISICAL_ADDR is set) +# 13. dev_lane_liveness - The lab/dev compose lane is UP and reachable on the +# exact path CI publishers use (OMN-15190) # -# OMN-3772 OMN-3903 +# LANE KEEP-ALIVE (ONEX_LANE_KEEPALIVE, default 1 — operator ruling 2026-07-29) +# +# The lab/dev lane used to be documented as ephemeral-by-design (OMN-13414): +# GC/idle-reclaimed to zero containers between uses, rediscovered reactively +# by whichever PR happened to hit the resulting CI cascade. That posture is +# REVERSED by operator ruling (WS-4): the lab lane is KEEP-ALIVE, because +# testing things live is the entire point of the lab. Lane-down is therefore +# a DEFECT, not an expected state. +# +# This file is where that ruling changes behavior. Before it, a fully +# torn-down lane scored `runtime_containers: yellow ("none running — runtime +# profile not active")`, and yellow exits 0 — i.e. the canonical health gate +# reported SUCCESS on a lane that was stranding every repo's +# occ-autobind / occ-companion-effect publish with connection-refused. +# Under keep-alive those states are RED. Set ONEX_LANE_KEEPALIVE=0 to restore +# the advisory posture for a lane that genuinely is ephemeral. +# +# OMN-3772 OMN-3903 OMN-15190 set -euo pipefail @@ -50,6 +75,43 @@ FLAG_JSON=false FLAG_CI=false FLAG_CROSS_REPO=false FLAG_VERBOSE=false +FLAG_LANE=false + +# ----- Lane keep-alive posture (OMN-15190) ----- +# 1 = the lab/dev lane is expected to be UP at all times (operator ruling +# 2026-07-29). 0 = pre-ruling ephemeral posture, lane-absent is advisory only. +LANE_KEEPALIVE="${ONEX_LANE_KEEPALIVE:-1}" + +# Lane identity + probe targets come from the rendered service contract +# (docker/runtime-policy.env, generated from contracts/services/ +# runtime_policy.contract.yaml) — the same source refresh_dev_lane.sh reads, +# so the lane this check watches cannot drift from the lane the refresh +# script deploys. +# +# Read by targeted key extraction, NOT `source`: that file is a generated +# artifact whose key set changes when the contract is re-rendered, and +# sourcing it into this script's global scope would let a future render +# silently redefine POSTGRES_* / VALKEY_* / KAFKA_BOOTSTRAP_SERVERS and +# change what the OTHER checks in this file are asserting. Two keys are +# wanted; two keys are read. +policy_env_value() { + local key="$1" file="${REPO_ROOT}/docker/runtime-policy.env" + [[ -f "$file" ]] || return 0 + sed -n "s/^${key}=//p" "$file" | tail -n 1 | tr -d "\"'" +} +DEV_LANE_COMPOSE_PROJECT="${DEV_LANE_COMPOSE_PROJECT:-$(policy_env_value DEV_COMPOSE_PROJECT)}" +DEV_LANE_COMPOSE_PROJECT="${DEV_LANE_COMPOSE_PROJECT:-omnibase-infra}" +DEV_LANE_MAIN_PORT="${DEV_LANE_MAIN_PORT:-$(policy_env_value DEV_RUNTIME_MAIN_PORT)}" +DEV_LANE_MAIN_PORT="${DEV_LANE_MAIN_PORT:-8085}" +# The broker host-port CI publishers connect to. No contract var exists for it +# (the contract renders in-cluster addresses); this is the PUBLISHED host port +# that occ-autobind / occ-companion-effect dial. +DEV_LANE_BROKER_PORT="${DEV_LANE_BROKER_PORT:-19092}" +# fallback-ok: localhost IS the lane host in the documented primary context +# (this script runs ON the lane host); the containerized deploy runner +# overrides via compose env LANE_PROBE_HOST=host.docker.internal (OMN-14958), +# exactly as refresh_dev_lane.sh / refresh_stability_lane.sh do. +LANE_PROBE_HOST="${LANE_PROBE_HOST:-localhost}" # fallback-ok: localhost IS the lane host when this script runs ON the lane host (its documented primary context); the containerized deploy runner overrides via LANE_PROBE_HOST=host.docker.internal (OMN-14958), same as refresh_dev_lane.sh # ----- State ----- OVERALL_STATUS="green" # green | yellow | red @@ -105,6 +167,7 @@ while [[ $# -gt 0 ]]; do --json) FLAG_JSON=true; shift ;; --ci) FLAG_CI=true; FLAG_JSON=true; shift ;; --cross-repo) FLAG_CROSS_REPO=true; shift ;; + --lane) FLAG_LANE=true; shift ;; --verbose) FLAG_VERBOSE=true; shift ;; --help|-h) show_help ;; *) echo "Unknown option: $1" >&2; exit 2 ;; @@ -251,15 +314,164 @@ check_runtime_containers() { fi done + # OMN-15190: under the keep-alive ruling a missing runtime service is a + # DEFECT, not "profile not active". Yellow exits 0, so the pre-ruling + # severity made this gate report success on a lane that was down. + local absent_severity="yellow" + local absent_note=" (runtime profile not active)" + if [[ "$LANE_KEEPALIVE" == "1" ]]; then + absent_severity="red" + absent_note=" — lab lane is KEEP-ALIVE (ONEX_LANE_KEEPALIVE=1), a missing runtime service is a defect" + fi + if [[ ${#missing[@]} -eq 0 ]]; then log_check "$name" "green" "all runtime containers running (${found}/${#expected[@]})" elif [[ $found -gt 0 ]]; then - log_check "$name" "yellow" "partial: ${found}/${#expected[@]} running, missing: ${missing[*]}" + log_check "$name" "$absent_severity" "partial: ${found}/${#expected[@]} running, missing: ${missing[*]}${absent_note}" else - log_check "$name" "yellow" "none running (runtime profile not active)" + log_check "$name" "$absent_severity" "none running${absent_note}" fi } +# Bounded TCP connect probe. +# +# Deliberately NOT `nc -w`: that flag bounds READS, not the connect itself, so +# an `nc -w 5` against a black-holed host can hang far past the timeout +# (memory `reference_nc_w_flag_does_not_bound_connect`). bash's /dev/tcp does +# the connect; the bound comes from coreutils `timeout` where present and from +# an explicit watchdog otherwise, so this works on the BSD-userland gate host +# where `timeout` is absent. +# +# Returns 0 when the port accepts a connection, 1 otherwise. +tcp_probe() { + local host="$1" port="$2" secs="${3:-5}" + + if command -v timeout >/dev/null 2>&1; then + timeout "$secs" bash -c "exec 3<>/dev/tcp/${host}/${port}" 2>/dev/null + return $? + fi + + ( exec 3<>/dev/tcp/"${host}"/"${port}" ) 2>/dev/null & + local probe_pid=$! + local waited=0 + while kill -0 "$probe_pid" 2>/dev/null && [[ "$waited" -lt "$secs" ]]; do + sleep 1 + waited=$((waited + 1)) + done + if kill -0 "$probe_pid" 2>/dev/null; then + kill -9 "$probe_pid" 2>/dev/null || true + wait "$probe_pid" 2>/dev/null || true + return 1 + fi + wait "$probe_pid" +} + +check_dev_lane_liveness() { + local name="dev_lane_liveness" + local project="$DEV_LANE_COMPOSE_PROJECT" + + # Indeterminate is not health. A check that cannot see the lane must not + # report the lane as fine — that is the exact inversion OMN-13915 shipped. + if ! command -v docker >/dev/null 2>&1; then + log_check "$name" "red" "docker CLI unavailable — lane liveness is unprovable (fail-closed)" + return + fi + + local rows + if ! rows=$(docker ps -a --filter "label=com.docker.compose.project=${project}" \ + --format '{{.Names}}|{{.State}}|{{.Status}}' 2>&1); then + log_check "$name" "red" "docker daemon unreachable: $(json_escape "$rows") (fail-closed)" + return + fi + + # The compose PROJECT LABEL is the membership oracle, not a hardcoded + # container-name list: the dev lane's compose file leaves some services + # without an explicit container_name (compose-assigned), so a name map + # silently under-counts — the same trap verify_dev_refresh.py documents. + local total=0 + [[ -n "$rows" ]] && total=$(printf '%s\n' "$rows" | grep -c '|') + + if [[ "$total" -eq 0 ]]; then + if [[ "$LANE_KEEPALIVE" == "1" ]]; then + log_check "$name" "red" \ + "compose project '${project}' has ZERO containers — the lab lane is fully GC/idle-reclaimed (OMN-15190). Under the keep-alive ruling this is a defect: while it is down, every repo's occ-autobind / occ-companion-effect publish fails connection-refused and cascades into occ-preflight / Receipt Gate org-wide. Recovery: docs/runbooks/cold-lane-full-bringup.md" + else + log_check "$name" "yellow" \ + "compose project '${project}' has ZERO containers (ONEX_LANE_KEEPALIVE=0 — lane treated as ephemeral per OMN-13414)" + fi + return + fi + + local running=0 + local exited_nonzero=() not_running=() unhealthy=() + local cname cstate cstatus + while IFS='|' read -r cname cstate cstatus; do + [[ -z "$cname" ]] && continue + case "$cstate" in + running) + ((running++)) || true + # Docker health is used here ONLY as a secondary signal. It is + # never the sole verdict (OMN-13915: 37/48 runners sat + # "Up (healthy)" with a dead listener; OMN-15233: 59/64 read + # unhealthy while the registry said 64/64 online). + [[ "$cstatus" == *"(unhealthy)"* ]] && unhealthy+=("$cname") + ;; + exited) + # One-shots (migrations, provisioners) legitimately exit 0. A + # NONZERO exit is a real failure that leaves the lane serving + # with an unapplied schema — the OMN-15312 class, which is + # invisible to every "is it up?" probe. + if [[ "$cstatus" =~ Exited\ \(([0-9]+)\) ]]; then + [[ "${BASH_REMATCH[1]}" != "0" ]] && exited_nonzero+=("${cname}(exit ${BASH_REMATCH[1]})") + fi + ;; + *) + # created / restarting / dead / paused — a container stuck in + # any of these in a keep-alive lane is a defect, and + # `restarting` specifically is a crash loop. + not_running+=("${cname}(${cstate})") + ;; + esac + done <<< "$rows" + + # Reachability on the EXACT path the CI publishers use. This is the signal + # that actually decides whether the org's receipt path works; container + # state alone does not (a running broker with an unpublished/unroutable + # host port strands CI just as completely as a torn-down lane). + local red_reasons=() + if ! tcp_probe "$LANE_PROBE_HOST" "$DEV_LANE_BROKER_PORT" 5; then + red_reasons+=("broker ${LANE_PROBE_HOST}:${DEV_LANE_BROKER_PORT} refused connection (the occ-autobind / occ-companion-effect publish path)") + fi + local http_code + http_code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 8 \ + "http://${LANE_PROBE_HOST}:${DEV_LANE_MAIN_PORT}/health" 2>/dev/null) || http_code="000" + if [[ "$http_code" != "200" ]]; then + red_reasons+=("runtime /health on ${LANE_PROBE_HOST}:${DEV_LANE_MAIN_PORT} returned ${http_code}") + fi + [[ ${#exited_nonzero[@]} -gt 0 ]] && red_reasons+=("nonzero-exit container(s): ${exited_nonzero[*]}") + [[ ${#not_running[@]} -gt 0 ]] && red_reasons+=("not-running container(s): ${not_running[*]}") + + local census="${total} container(s), ${running} running, probe host ${LANE_PROBE_HOST}" + if [[ ${#red_reasons[@]} -gt 0 ]]; then + local joined="${red_reasons[0]}" + local i + for ((i = 1; i < ${#red_reasons[@]}; i++)); do + joined="${joined}; ${red_reasons[$i]}" + done + log_check "$name" "red" "lane '${project}' DEGRADED — ${joined} [${census}]" + return + fi + + if [[ ${#unhealthy[@]} -gt 0 ]]; then + # Advisory, not red: a running-but-unhealthy sidecar does not strand + # the CI publish path, and a permanently-red check is a disabled check. + log_check "$name" "yellow" "lane '${project}' serving, but docker-unhealthy: ${unhealthy[*]} [${census}]" + return + fi + + log_check "$name" "green" "lane '${project}' up and reachable — broker :${DEV_LANE_BROKER_PORT} open, /health 200 [${census}]" +} + check_required_topics() { local name="required_topics" @@ -477,18 +689,31 @@ if [[ "$FLAG_JSON" == "false" ]]; then echo "" fi -check_postgres -check_redpanda -check_valkey -check_infra_containers -check_keycloak -check_runtime_containers -check_required_topics -check_migration_parity -check_env_audit -check_cloud_bus_refs -check_bus_endpoint -check_infisical_folders +if [[ "$FLAG_LANE" == "true" ]]; then + # Lane-liveness subset (OMN-15190). Runs from the containerized deploy + # runner, which has docker.sock + host-gateway network reachability but + # none of the POSTGRES_/VALKEY_/INFISICAL_ credentials the full gate + # needs — so the full gate cannot BE the scheduled surface, and a + # credential-less full run would die on `${POSTGRES_HOST:?}` instead of + # reporting on the lane. + check_dev_lane_liveness + check_redpanda + check_runtime_containers +else + check_postgres + check_redpanda + check_valkey + check_infra_containers + check_keycloak + check_runtime_containers + check_required_topics + check_migration_parity + check_env_audit + check_cloud_bus_refs + check_bus_endpoint + check_infisical_folders + check_dev_lane_liveness +fi # ===================================================================== # Output @@ -517,7 +742,9 @@ if [[ "$FLAG_JSON" == "true" ]]; then "flags": { "cross_repo": ${FLAG_CROSS_REPO}, "ci": ${FLAG_CI}, - "verbose": ${FLAG_VERBOSE} + "lane": ${FLAG_LANE}, + "verbose": ${FLAG_VERBOSE}, + "lane_keepalive": "${LANE_KEEPALIVE}" } } EOF diff --git a/scripts/tests/test_keycloak_desired_clients_contract.py b/scripts/tests/test_keycloak_desired_clients_contract.py index 666f62a92e..474ea3f936 100644 --- a/scripts/tests/test_keycloak_desired_clients_contract.py +++ b/scripts/tests/test_keycloak_desired_clients_contract.py @@ -26,3 +26,23 @@ def test_omniweb_allows_the_managed_staging_callback() -> None: assert "https://dev.app.omninode.ai/*" in omniweb["redirectUris"] assert "https://dev.app.omninode.ai" in omniweb["webOrigins"] + + +def test_omniweb_user_identity_claim_contract() -> None: + config = json.loads(_CONFIG_PATH.read_text()) + omniweb = _client(config, "omniweb") + mappers = {mapper["name"]: mapper for mapper in omniweb["protocolMappers"]} + + principal = mappers["principal_id"] + assert principal["protocolMapper"] == "oidc-usermodel-attribute-mapper" + assert principal["config"]["user.attribute"] == "principal_id" + assert principal["config"]["claim.name"] == "principal_id" + assert principal["config"]["id.token.claim"] == "true" + assert principal["config"]["access.token.claim"] == "true" + assert principal["config"]["userinfo.token.claim"] == "true" + + assert "gateway-attach-audience" not in mappers + assert all( + mapper.get("config", {}).get("included.custom.audience") != "gateway-attach" + for mapper in mappers.values() + ) diff --git a/scripts/trigger_rebuild_on_merge.py b/scripts/trigger_rebuild_on_merge.py index 5e5b0f7735..d9ae4f81ab 100644 --- a/scripts/trigger_rebuild_on_merge.py +++ b/scripts/trigger_rebuild_on_merge.py @@ -16,7 +16,7 @@ # # Triggers when: # - PR had the "runtime_change" label, OR -# - Any changed file matches src/omnimarket/** or src/omnibase_infra/nodes/** +# - The canonical deploy-gate classifier identifies a changed runtime path # # Lane policy (the triggering ref decides the lane — no hardcoded origin/main): # - merge to dev -> runtime_lane=dev, source_branch=dev @@ -26,11 +26,15 @@ # # Tickets: OMN-8917 (original auto-trigger), OMN-12573 (re-point to node_redeploy) # -# Required environment variables (when not --dry-run): -# KAFKA_BOOTSTRAP_SERVERS -- broker address(es), e.g. host:9092 -# KAFKA_SASL_USERNAME -- SASL username / API key -# KAFKA_SASL_PASSWORD -- SASL password / API secret -# DEPLOY_AGENT_HMAC_SECRET -- HMAC secret for payload signing +# Required inputs (when not --dry-run): +# --bus-lane -- control-bus lane declared by the overlay +# --bus-overlay -- checked-in config/ci_bus_lanes.yaml path +# --consumer-model -- canonical strict consumer model source +# +# Optional environment variables: +# KAFKA_BOOTSTRAP_SERVERS -- drift guard / from-secret broker only +# KAFKA_SASL_USERNAME -- SASL username / API key (cloud broker only) +# KAFKA_SASL_PASSWORD -- SASL password / API secret (cloud broker only) # # Usage: # python scripts/trigger_rebuild_on_merge.py \ @@ -42,16 +46,20 @@ from __future__ import annotations -import fnmatch -import hashlib -import hmac +import ast +import importlib.util import json import os import sys import uuid -from datetime import UTC, datetime +from collections.abc import Callable +from pathlib import Path +from typing import Literal, cast +from uuid import UUID import click +import yaml +from pydantic import BaseModel, ConfigDict, ValidationError, field_validator from omnibase_infra.utils.util_producer_effect_assertion import ( ProducerZeroOutputError, @@ -64,13 +72,10 @@ # command downstream. TOPIC = "onex.cmd.omnimarket.redeploy-start.v1" -_RUNTIME_PATH_PATTERNS = [ - "src/omnimarket/*", - "src/omnibase_infra/nodes/*", -] - _RUNTIME_LABEL = "runtime_change" +RuntimePathClassifier = Callable[[list[str]], list[str]] + # Maps the merged PR's base branch to a runtime lane. Values match # deploy_agent.events.EnumRuntimeLane (dev | stability-test | prod). prod is not # triggerable from CI: production deploys the stability-proven digest through @@ -81,17 +86,335 @@ } -def should_trigger(changed_files: list[str], labels: list[str]) -> bool: - """Return True if a rebuild should be triggered.""" - if _RUNTIME_LABEL in labels: +class ModelCiBusLane(BaseModel): + """One checked-in CI control-bus lane declaration.""" + + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + + broker: str + + @field_validator("broker") + @classmethod + def validate_broker(cls, value: str) -> str: + """Reject empty broker declarations at the contract boundary.""" + if not value: + raise ValueError("broker must not be empty") + return value + + +class ModelCiBusOverlay(BaseModel): + """Typed contract for the authoritative lane-to-broker overlay.""" + + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + + default: str + lanes: dict[str, ModelCiBusLane] + + @field_validator("default") + @classmethod + def validate_default(cls, value: str) -> str: + """The current overlay contract has an explicit in-memory default.""" + if value != "inmemory": + raise ValueError("default must be 'inmemory'") + return value + + @field_validator("lanes") + @classmethod + def validate_lanes( + cls, value: dict[str, ModelCiBusLane] + ) -> dict[str, ModelCiBusLane]: + """A bus overlay without declared lanes cannot route a producer.""" + if not value: + raise ValueError("lanes must not be empty") + if any(not lane.strip() for lane in value): + raise ValueError("lane names must not be empty") + return value + + +# The script is also loaded directly from its file path by hermetic tests. Give +# Pydantic the explicit namespace so postponed annotations resolve without +# depending on the module having first been inserted into sys.modules. +ModelCiBusOverlay.model_rebuild(_types_namespace={"ModelCiBusLane": ModelCiBusLane}) + + +class ModelRedeployStartCommandWire(BaseModel): + """Strict producer-side subset of the redeploy orchestrator command.""" + + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + + correlation_id: UUID + scope: Literal["full"] = "full" + git_ref: str + runtime_lane: Literal["dev", "stability-test"] + build_source: Literal["workspace", "release"] + requested_by: str + + @field_validator("git_ref") + @classmethod + def validate_git_ref(cls, value: str) -> str: + """The live trigger carries an exact hexadecimal merge commit SHA.""" + is_hex = all(character in "0123456789abcdef" for character in value) + if not (7 <= len(value) <= 64) or not is_hex: + raise ValueError( + "git_ref must be a 7-64 character lowercase hex commit SHA" + ) + return value + + @field_validator("requested_by") + @classmethod + def validate_requested_by(cls, value: str) -> str: + if not value: + raise ValueError("requested_by must not be empty") + return value + + +ModelRedeployStartCommandWire.model_rebuild( + _types_namespace={"Literal": Literal, "UUID": UUID} +) + + +def load_ci_bus_overlay(path: Path) -> ModelCiBusOverlay: + """Load and strictly validate the checked-in CI bus overlay.""" + if not path.is_file(): + raise ValueError(f"CI bus overlay does not exist: {path}") + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + return ModelCiBusOverlay.model_validate(raw) + except (OSError, yaml.YAMLError, ValidationError) as exc: + raise ValueError(f"Invalid CI bus overlay {path}: {exc}") from exc + + +def resolve_ci_bus_broker( + *, + overlay: ModelCiBusOverlay, + lane: str, + injected_broker: str, +) -> str: + """Resolve one control-bus lane without allowing opaque target drift.""" + lane_key = lane.strip() + if not lane_key: + raise ValueError("CI bus lane must not be empty") + declaration = overlay.lanes.get(lane_key) + if declaration is None: + raise ValueError( + f"CI bus lane {lane_key!r} is not declared; " + f"declared lanes: {sorted(overlay.lanes)}" + ) + + broker = declaration.broker + if broker == "inmemory": + raise ValueError( + f"CI bus lane {lane_key!r} declares 'inmemory'; a post-merge " + "redeploy command requires a cross-process broker" + ) + if broker == "from-secret": + if not injected_broker: + raise ValueError( + f"CI bus lane {lane_key!r} declares 'from-secret', but " + "KAFKA_BOOTSTRAP_SERVERS is empty" + ) + return injected_broker + if injected_broker and injected_broker != broker: + raise ValueError( + "LANE BUS DRIFT: injected KAFKA_BOOTSTRAP_SERVERS does not match " + f"the checked-in broker for lane {lane_key!r} ({broker})" + ) + return broker + + +def build_kafka_producer_config( + bootstrap_servers: str, + username: str, + password: str, +) -> dict[str, str | int | float | bool]: + """Build plaintext local or SASL_SSL cloud transport deterministically.""" + if bool(username) != bool(password): + raise ValueError( + "KAFKA_SASL_USERNAME and KAFKA_SASL_PASSWORD must both be set " + "or both be empty" + ) + config: dict[str, str | int | float | bool] = { + "bootstrap.servers": bootstrap_servers, + } + if username and password: + config.update( + { + "security.protocol": "SASL_SSL", + "sasl.mechanisms": "PLAIN", + "sasl.username": username, + "sasl.password": password, + } + ) + return config + + +def _field_call_is_required(value: ast.expr | None) -> bool: + """Return whether one annotated consumer field has no default.""" + if value is None: return True - for f in changed_files: - for pattern in _RUNTIME_PATH_PATTERNS: - if fnmatch.fnmatch(f, pattern) or f.startswith(pattern.rstrip("*")): - return True + if not isinstance(value, ast.Call): + return False + if value.args and isinstance(value.args[0], ast.Constant): + if value.args[0].value is Ellipsis: + return True + for keyword in value.keywords: + if keyword.arg == "default" and isinstance(keyword.value, ast.Constant): + return keyword.value.value is Ellipsis return False +def _class_declares_extra_forbid(node: ast.ClassDef) -> bool: + for statement in node.body: + if not isinstance(statement, ast.Assign): + continue + if not any( + isinstance(target, ast.Name) and target.id == "model_config" + for target in statement.targets + ): + continue + if not isinstance(statement.value, ast.Call): + return False + return any( + keyword.arg == "extra" + and isinstance(keyword.value, ast.Constant) + and keyword.value.value == "forbid" + for keyword in statement.value.keywords + ) + return False + + +def load_consumer_model_contract( + model_path: Path, +) -> tuple[frozenset[str], frozenset[str]]: + """Read field and required-field truth from the canonical consumer model.""" + if not model_path.is_file(): + raise ValueError(f"consumer model does not exist: {model_path}") + try: + tree = ast.parse( + model_path.read_text(encoding="utf-8"), filename=str(model_path) + ) + except (OSError, SyntaxError) as exc: + raise ValueError(f"invalid consumer model {model_path}: {exc}") from exc + + model_node = next( + ( + node + for node in tree.body + if isinstance(node, ast.ClassDef) + and node.name == "ModelRedeployStartCommand" + ), + None, + ) + if model_node is None: + raise ValueError( + f"consumer model {model_path} does not define ModelRedeployStartCommand" + ) + if not _class_declares_extra_forbid(model_node): + raise ValueError( + "ModelRedeployStartCommand must declare ConfigDict(extra='forbid')" + ) + + fields: set[str] = set() + required: set[str] = set() + for statement in model_node.body: + if not isinstance(statement, ast.AnnAssign): + continue + if not isinstance(statement.target, ast.Name): + continue + name = statement.target.id + fields.add(name) + if _field_call_is_required(statement.value): + required.add(name) + if not fields: + raise ValueError("ModelRedeployStartCommand declares no annotated fields") + return frozenset(fields), frozenset(required) + + +def assert_consumer_model_accepts_payload( + *, payload: dict[str, object], model_path: Path +) -> None: + """Fail before publish if the strict consumer cannot accept these keys.""" + fields, required = load_consumer_model_contract(model_path) + payload_fields = frozenset(payload) + extras = sorted(payload_fields - fields) + missing = sorted(required - payload_fields) + if extras: + raise ValueError( + f"consumer rejects extra fields from producer payload: {extras}" + ) + if missing: + raise ValueError(f"producer payload misses required consumer fields: {missing}") + + +def build_redeploy_start_payload( + *, + runtime_lane: str, + build_source: str, + source_sha: str, + correlation_id: str, + requested_by: str, +) -> dict[str, object]: + """Build exactly the strict command shape consumed by node_redeploy.""" + command = ModelRedeployStartCommandWire( + correlation_id=correlation_id, + scope="full", + git_ref=source_sha, + runtime_lane=runtime_lane, + build_source=build_source, + requested_by=requested_by, + ) + return command.model_dump(mode="json") + + +def load_runtime_path_classifier(path: Path) -> RuntimePathClassifier: + """Load the exact deploy-gate runtime-path classifier used by hosted CI. + + Runtime deployment scope has one owner: omniclaude's deploy-gate validator. + Loading its ``find_runtime_paths`` callable keeps the post-merge publisher + aligned with the required deploy gate instead of maintaining a second path + allowlist that can silently drift. + """ + if not path.is_file(): + raise ValueError(f"runtime path validator does not exist: {path}") + + module_name = "_canonical_deploy_path_classifier" + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise ValueError(f"cannot load runtime path validator: {path}") + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception as exc: + sys.modules.pop(module_name, None) + raise ValueError(f"invalid runtime path validator {path}: {exc}") from exc + + classifier = getattr(module, "find_runtime_paths", None) + if not callable(classifier): + raise ValueError( + f"runtime path validator {path} does not define find_runtime_paths" + ) + return cast("RuntimePathClassifier", classifier) + + +def classify_runtime_paths( + changed_files: list[str], classifier: RuntimePathClassifier +) -> list[str]: + """Run and validate the canonical classifier's output fail-closed.""" + runtime_paths = classifier(changed_files) + if not isinstance(runtime_paths, list) or any( + not isinstance(path, str) or not path.strip() for path in runtime_paths + ): + raise ValueError("runtime path validator returned an invalid path list") + return runtime_paths + + +def should_trigger(runtime_paths: list[str], labels: list[str]) -> bool: + """Return True for a runtime label or canonical deploy-path hit.""" + return _RUNTIME_LABEL in labels or bool(runtime_paths) + + def lane_for_base_branch(base_branch: str) -> str: """Map a merged PR's base branch to a node_redeploy_orchestrator runtime lane. @@ -109,25 +432,26 @@ def lane_for_base_branch(base_branch: str) -> str: return lane -def _sign_envelope(envelope: dict[str, object], secret: str) -> dict[str, object]: - body_dict = {k: v for k, v in envelope.items() if k != "_signature"} - body = json.dumps(body_dict, sort_keys=True, separators=(",", ":")).encode() - signature = hmac.new(secret.encode(), body, hashlib.sha256).hexdigest() - return {**envelope, "_signature": signature} +def build_source_for_base_branch(base_branch: str) -> str: + """Map branch lineage to the deploy agent's provenance mode.""" + if base_branch == "dev": + return "workspace" + if base_branch == "main": + return "release" + raise ValueError(f"No build source mapping for base branch {base_branch!r}") def publish_redeploy_start_event( bootstrap_servers: str, username: str, password: str, - hmac_secret: str, runtime_lane: str, - source_branch: str, + build_source: str, source_sha: str, correlation_id: str, requested_by: str, ) -> int: - """Publish a signed redeploy-start command to node_redeploy_orchestrator via SASL_SSL. + """Publish a strict redeploy-start command to node_redeploy_orchestrator. Returns the number of commands delivered (``1`` on success). Raises on any delivery failure so the caller can assert a non-zero emit count — a producer @@ -135,27 +459,17 @@ def publish_redeploy_start_event( """ from confluent_kafka import Producer - envelope = { - "correlation_id": correlation_id, - "requested_by": requested_by, - "runtime_lane": runtime_lane, - "source_branch": source_branch, - "source_sha": source_sha, - # dev dogfoods OCC drafting; stability gates on readiness before prod. - "requires_occ": True, - "requires_readiness_gate": runtime_lane != "dev", - "requested_at": datetime.now(UTC).isoformat(), - } - signed = _sign_envelope(envelope, hmac_secret) + payload = build_redeploy_start_payload( + runtime_lane=runtime_lane, + build_source=build_source, + source_sha=source_sha, + correlation_id=correlation_id, + requested_by=requested_by, + ) - producer_config: dict[str, str | int | float | bool] = { - "bootstrap.servers": bootstrap_servers, - "security.protocol": "SASL_SSL", - "sasl.mechanisms": "PLAIN", - "sasl.username": username, - "sasl.password": password, - } - producer = Producer(producer_config) + producer = Producer( + build_kafka_producer_config(bootstrap_servers, username, password) + ) delivery_error: BaseException | None = None @@ -164,7 +478,7 @@ def _on_delivery(err: object, _msg: object) -> None: if err is not None: delivery_error = RuntimeError(str(err)) - message = json.dumps(signed, default=str).encode("utf-8") + message = json.dumps(payload, default=str).encode("utf-8") key = f"gha-redeploy/{correlation_id}".encode() producer.produce( @@ -173,10 +487,15 @@ def _on_delivery(err: object, _msg: object) -> None: value=message, on_delivery=_on_delivery, ) - producer.flush(timeout=30) + remaining = producer.flush(timeout=30) if delivery_error is not None: raise RuntimeError(f"Kafka delivery failed: {delivery_error}") from None + if remaining and remaining > 0: + raise RuntimeError( + f"Kafka delivery timed out: {remaining} message(s) remain " + "undelivered after the 30 second flush" + ) # Exactly one redeploy-start command was delivered; the caller asserts N>0. return 1 @@ -213,6 +532,29 @@ def _on_delivery(err: object, _msg: object) -> None: default="", help="Correlation ID (auto-generated if not provided)", ) +@click.option( + "--bus-lane", + default="", + help="Control-bus lane id declared by the CI bus overlay (normally 'dev')", +) +@click.option( + "--bus-overlay", + type=click.Path(path_type=Path), + default=None, + help="Path to the authoritative checked-in config/ci_bus_lanes.yaml", +) +@click.option( + "--consumer-model", + type=click.Path(path_type=Path), + default=None, + help="Path to omnimarket's canonical ModelRedeployStartCommand source", +) +@click.option( + "--runtime-path-validator", + type=click.Path(path_type=Path, exists=True, dir_okay=False), + required=True, + help="Path to omniclaude's canonical deploy-gate validator source", +) @click.option( "--dry-run", is_flag=True, @@ -226,12 +568,16 @@ def main( source_sha: str, requested_by: str, correlation_id: str, + bus_lane: str, + bus_overlay: Path | None, + consumer_model: Path | None, + runtime_path_validator: Path, dry_run: bool, ) -> None: """Publish a node_redeploy_orchestrator start command if a PR contains runtime changes. - Triggers when PR had the runtime_change label OR changed files match - src/omnimarket/** or src/omnibase_infra/nodes/**. The triggering base branch + Triggers when the PR had the runtime_change label or the canonical deploy + gate classifies a changed path as runtime-scoped. The triggering base branch decides the runtime lane; the merge SHA is the ref node_redeploy_orchestrator rebuilds. """ files: list[str] = ( @@ -246,8 +592,16 @@ def main( corr_id = correlation_id or str(uuid.uuid4()) runtime_lane = lane_for_base_branch(base_branch) + build_source = build_source_for_base_branch(base_branch) + + try: + classifier = load_runtime_path_classifier(runtime_path_validator) + runtime_paths = classify_runtime_paths(files, classifier) + except ValueError as exc: + click.echo(f"ERROR: {exc}", err=True) + sys.exit(1) - if not should_trigger(files, label_list): + if not should_trigger(runtime_paths, label_list): click.echo( "No rebuild trigger: no runtime_change label or runtime path changes detected." ) @@ -256,21 +610,52 @@ def main( click.echo( f"Redeploy triggered: runtime_lane={runtime_lane} source_branch={base_branch} " f"source_sha={source_sha} correlation_id={corr_id} labels={label_list} " - f"files_matched={[f for f in files if any(f.startswith(p.rstrip('*')) for p in _RUNTIME_PATH_PATTERNS)]}" + f"files_matched={runtime_paths}" ) if dry_run: click.echo("(dry-run: skipping Kafka publish)") sys.exit(0) - bootstrap_servers = os.environ.get("KAFKA_BOOTSTRAP_SERVERS", "") + injected_broker = os.environ.get("KAFKA_BOOTSTRAP_SERVERS", "").strip() username = os.environ.get("KAFKA_SASL_USERNAME", "") password = os.environ.get("KAFKA_SASL_PASSWORD", "") - hmac_secret = os.environ.get("DEPLOY_AGENT_HMAC_SECRET", "") + + if bus_overlay is None or not bus_lane.strip() or consumer_model is None: + click.echo( + "ERROR: --bus-lane, --bus-overlay, and --consumer-model are " + "required for a live redeploy publish", + err=True, + ) + sys.exit(1) + + try: + overlay = load_ci_bus_overlay(bus_overlay) + bootstrap_servers = resolve_ci_bus_broker( + overlay=overlay, + lane=bus_lane, + injected_broker=injected_broker, + ) + # Validate the transport pair before the producer is constructed. + build_kafka_producer_config(bootstrap_servers, username, password) + candidate_payload = build_redeploy_start_payload( + runtime_lane=runtime_lane, + build_source=build_source, + source_sha=source_sha, + correlation_id=corr_id, + requested_by=requested_by, + ) + assert_consumer_model_accepts_payload( + payload=candidate_payload, + model_path=consumer_model, + ) + except ValueError as exc: + click.echo(f"ERROR: {exc}", err=True) + sys.exit(1) # A runtime change was detected and this is not a dry run, so the job's # PURPOSE is now to publish exactly one redeploy-start command. If any - # precondition for publishing is absent (broker, SASL creds, HMAC secret), + # precondition for publishing is absent (resolved broker), # this producer CANNOT emit — that is zero output and MUST fail closed # (RT-5 / OMN-14467), never "skip publish" green. The dead-producer bug this # replaces printed "KAFKA_BOOTSTRAP_SERVERS is not set -- skipping publish" @@ -280,10 +665,7 @@ def main( require_producer_preconditions( artifact=TOPIC, preconditions={ - "KAFKA_BOOTSTRAP_SERVERS": bootstrap_servers, - "KAFKA_SASL_USERNAME": username, - "KAFKA_SASL_PASSWORD": password, - "DEPLOY_AGENT_HMAC_SECRET": hmac_secret, + "CI_BUS_BROKER": bootstrap_servers, }, ) except ProducerZeroOutputError as exc: @@ -295,9 +677,8 @@ def main( bootstrap_servers=bootstrap_servers, username=username, password=password, - hmac_secret=hmac_secret, runtime_lane=runtime_lane, - source_branch=base_branch, + build_source=build_source, source_sha=source_sha, correlation_id=corr_id, requested_by=requested_by, diff --git a/scripts/validation/check_no_infra_inmemory_import.sh b/scripts/validation/check_no_infra_inmemory_import.sh index efa2ea835d..56b22510b7 100755 --- a/scripts/validation/check_no_infra_inmemory_import.sh +++ b/scripts/validation/check_no_infra_inmemory_import.sh @@ -57,6 +57,45 @@ _is_allowlisted() { return 1 } +# OMN-14988: collapse repeated slashes so grep-reported paths compare equal to +# the single-slash ALLOWLIST entries. +# +# BSD grep (macOS) reports "src//omnibase_infra/foo.py" for the search root +# "src/"; GNU grep (Linux CI) reports "src/omnibase_infra/foo.py". Both must +# normalize to the single-slash form. +# +# TRAP (do not "simplify" this back): the obvious one-liner +# file="${file//\/\//\/}" +# is bash-version-dependent. Under bash >= 4.3 it collapses correctly, but +# under bash 3.2 — which IS `/usr/bin/env bash` on stock macOS, and therefore +# the interpreter this hook actually runs under locally — the backslash in the +# replacement word is retained literally, yielding +# src\/omnibase_infra/backends/auto_configure.py +# which matches no ALLOWLIST entry, so every allowlisted file is reported as a +# violation (8/8 false positives, zero true positives). Holding the pattern and +# replacement in variables removes the escaping ambiguity entirely and behaves +# identically on bash 3.2 and 5.x. Regression-tested in +# tests/unit/scripts/validation/test_check_no_infra_inmemory_import.py. +_DOUBLE_SLASH='//' +_SINGLE_SLASH='/' + +_normalize_path() { + local p="$1" + while [[ "$p" == *"$_DOUBLE_SLASH"* ]]; do + p="${p//$_DOUBLE_SLASH/$_SINGLE_SLASH}" + done + printf '%s' "$p" +} + +# Hidden entry point used by the normalization regression test so the pure +# path-normalization behavior can be asserted per bash interpreter without +# depending on the local grep flavor. +if [[ "${1:-}" == "--print-normalized-path" ]]; then + _normalize_path "${2:-}" + printf '\n' + exit 0 +fi + # Match real import statements (optionally indented for function-local # imports) of the infra in-memory adapter. Docstring/example lines (e.g. a # ">>> from ..." doctest) are excluded by anchoring on leading whitespace + @@ -66,7 +105,7 @@ PATTERN='^[[:space:]]*from omnibase_infra\.event_bus\.event_bus_inmemory import| while IFS= read -r line; do file="${line%%:*}" # Normalize any doubled slash from grep's "src/" prefix (src//foo -> src/foo). - file="${file//\/\//\/}" + file="$(_normalize_path "$file")" if _is_allowlisted "$file"; then continue fi diff --git a/scripts/validation/validate_application_migration_manifest.py b/scripts/validation/validate_application_migration_manifest.py new file mode 100644 index 0000000000..d822fc6a5b --- /dev/null +++ b/scripts/validation/validate_application_migration_manifest.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Validate the deterministic application-migration declaration set. + +The forward runner also performs a portable POSIX-shell validation before it +touches PostgreSQL. This typed gate is the richer CI/operator proof: every +vendored node SQL artifact is either bound to one stream, owner, domain, +version, and content SHA-256, or is explicitly blocked by a ticket. There are +no root-name defaults and no environment-provided exceptions. +""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import sys +from collections.abc import Hashable, Sequence +from dataclasses import dataclass +from pathlib import Path + +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_TICKET = re.compile(r"^OMN-[0-9]+$") +_NODE_NAME = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*$") +_CLOUD_NAME = re.compile(r"^[A-Za-z0-9_.-]+$") +_LEGACY_SOURCE_CHECKSUM = re.compile(r"^[A-Za-z0-9_.:-]+$") +_ALLOWED_DOMAINS = frozenset({"tenant", "omninode_internal"}) + + +class ManifestError(ValueError): + """A checked-in declaration is incomplete, ambiguous, or contradictory.""" + + +@dataclass(frozen=True, slots=True) +class MigrationDeclaration: + artifact_path: str + migration_stream: str + owner: str + domain: str + version: str + checksum: str + + +@dataclass(frozen=True, slots=True) +class BlockedMigration: + artifact_path: str + version: str + checksum: str + ticket: str + reason: str + + +@dataclass(frozen=True, slots=True) +class LegacyNodeMigrationDeclaration: + migration_stream: str + owner: str + domain: str + version: str + source_checksum: str + ticket: str + + +@dataclass(frozen=True, slots=True) +class CloudAlias: + migration_name: str + runner_version: str + + +@dataclass(frozen=True, slots=True) +class ValidationResult: + declarations: tuple[MigrationDeclaration, ...] + blocked: tuple[BlockedMigration, ...] + legacy_node_declarations: tuple[LegacyNodeMigrationDeclaration, ...] + cloud_aliases: tuple[CloudAlias, ...] + + +def _read_tsv( + path: Path, field_count: int, *, allow_empty: bool = False +) -> list[tuple[int, list[str]]]: + if not path.is_file(): + raise ManifestError(f"required declaration file is missing: {path}") + rows: list[tuple[int, list[str]]] = [] + for line_number, raw_line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + fields = raw_line.split("\t") + if len(fields) != field_count or any(field == "" for field in fields): + raise ManifestError( + f"{path}:{line_number}: expected {field_count} non-empty TSV fields" + ) + rows.append((line_number, fields)) + if not rows and not allow_empty: + raise ManifestError(f"declaration file must not be empty: {path}") + return rows + + +def _node_identity(artifact_path: str) -> tuple[str, str, str]: + parts = artifact_path.split("/") + if len(parts) != 3 or parts[0] != "nodes" or not parts[2].endswith(".sql"): + raise ManifestError( + f"artifact must be nodes//.sql: {artifact_path!r}" + ) + _, node_name, filename = parts + if _NODE_NAME.fullmatch(node_name) is None: + raise ManifestError(f"invalid node name in artifact: {artifact_path!r}") + stream = f"node:{node_name}" + version = f"{stream}:{filename}" + return stream, version, filename + + +def _content_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _node_identity_from_version(version: str) -> tuple[str, str, str]: + parts = version.split(":") + if len(parts) != 3 or parts[0] != "node" or not parts[2].endswith(".sql"): + raise ManifestError(f"invalid historical node migration version: {version!r}") + _, node_name, filename = parts + if _NODE_NAME.fullmatch(node_name) is None: + raise ManifestError(f"invalid node name in historical version: {version!r}") + if _NODE_NAME.fullmatch(filename.removesuffix(".sql")) is None: + raise ManifestError(f"invalid filename in historical version: {version!r}") + return f"node:{node_name}", node_name, filename + + +def validate_manifests( + migrations_dir: Path, + declaration_path: Path, + blocked_path: Path, + legacy_node_declaration_path: Path, + cloud_alias_path: Path, + *, + require_complete: bool = False, +) -> ValidationResult: + """Parse and validate all application migration declarations.""" + + declarations: list[MigrationDeclaration] = [] + blocked: list[BlockedMigration] = [] + legacy_node_declarations: list[LegacyNodeMigrationDeclaration] = [] + aliases: list[CloudAlias] = [] + + for line_number, fields in _read_tsv(declaration_path, 6): + declaration = MigrationDeclaration(*fields) + expected_stream, expected_version, _ = _node_identity(declaration.artifact_path) + if declaration.migration_stream != expected_stream: + raise ManifestError( + f"{declaration_path}:{line_number}: unknown migration stream " + f"{declaration.migration_stream!r}; expected {expected_stream!r}" + ) + if declaration.owner != expected_stream: + raise ManifestError( + f"{declaration_path}:{line_number}: owner must equal the node stream" + ) + if declaration.domain not in _ALLOWED_DOMAINS: + raise ManifestError( + f"{declaration_path}:{line_number}: unknown domain " + f"{declaration.domain!r}" + ) + if declaration.version != expected_version: + raise ManifestError( + f"{declaration_path}:{line_number}: version must preserve exact " + f"runner identity {expected_version!r}" + ) + if _SHA256.fullmatch(declaration.checksum) is None: + raise ManifestError( + f"{declaration_path}:{line_number}: checksum is not lowercase SHA-256" + ) + artifact = migrations_dir / declaration.artifact_path + if not artifact.is_file(): + raise ManifestError(f"declared artifact is missing: {artifact}") + actual_checksum = _content_sha256(artifact) + if actual_checksum != declaration.checksum: + raise ManifestError( + f"conflicting checksum for {declaration.version}: " + f"declared={declaration.checksum}, actual={actual_checksum}" + ) + declarations.append(declaration) + + for line_number, fields in _read_tsv(blocked_path, 5, allow_empty=True): + item = BlockedMigration(*fields) + _, expected_version, _ = _node_identity(item.artifact_path) + if item.version != expected_version: + raise ManifestError( + f"{blocked_path}:{line_number}: blocked version must preserve exact " + f"runner identity {expected_version!r}" + ) + if _SHA256.fullmatch(item.checksum) is None: + raise ManifestError( + f"{blocked_path}:{line_number}: checksum is not lowercase SHA-256" + ) + if _TICKET.fullmatch(item.ticket) is None: + raise ManifestError( + f"{blocked_path}:{line_number}: invalid blocker ticket {item.ticket!r}" + ) + artifact = migrations_dir / item.artifact_path + if not artifact.is_file(): + raise ManifestError(f"blocked artifact is missing: {artifact}") + actual_checksum = _content_sha256(artifact) + if actual_checksum != item.checksum: + raise ManifestError( + f"conflicting checksum for blocked {item.version}: " + f"declared={item.checksum}, actual={actual_checksum}" + ) + blocked.append(item) + + for line_number, fields in _read_tsv( + legacy_node_declaration_path, 6, allow_empty=True + ): + legacy_item = LegacyNodeMigrationDeclaration(*fields) + expected_stream, node_name, filename = _node_identity_from_version( + legacy_item.version + ) + if legacy_item.migration_stream != expected_stream: + raise ManifestError( + f"{legacy_node_declaration_path}:{line_number}: unknown migration " + f"stream {legacy_item.migration_stream!r}; expected {expected_stream!r}" + ) + if legacy_item.owner != expected_stream: + raise ManifestError( + f"{legacy_node_declaration_path}:{line_number}: owner must equal " + "the node stream" + ) + if legacy_item.domain not in _ALLOWED_DOMAINS: + raise ManifestError( + f"{legacy_node_declaration_path}:{line_number}: unknown domain " + f"{legacy_item.domain!r}" + ) + if _LEGACY_SOURCE_CHECKSUM.fullmatch(legacy_item.source_checksum) is None: + raise ManifestError( + f"{legacy_node_declaration_path}:{line_number}: malformed legacy " + "source checksum" + ) + if _TICKET.fullmatch(legacy_item.ticket) is None: + raise ManifestError( + f"{legacy_node_declaration_path}:{line_number}: invalid blocker " + f"ticket {legacy_item.ticket!r}" + ) + artifact = migrations_dir / "nodes" / node_name / filename + if artifact.exists(): + raise ManifestError( + f"{legacy_node_declaration_path}:{line_number}: legacy declaration " + f"has vendored artifact: {artifact.relative_to(migrations_dir)}" + ) + legacy_node_declarations.append(legacy_item) + + for line_number, fields in _read_tsv(cloud_alias_path, 2): + alias = CloudAlias(*fields) + if ( + _CLOUD_NAME.fullmatch(alias.migration_name) is None + or _CLOUD_NAME.fullmatch(alias.runner_version) is None + or not alias.runner_version.endswith(".sql") + ): + raise ManifestError( + f"{cloud_alias_path}:{line_number}: malformed cloud migration alias" + ) + aliases.append(alias) + + declared_paths = [item.artifact_path for item in declarations] + blocked_paths = [item.artifact_path for item in blocked] + legacy_versions = [item.version for item in legacy_node_declarations] + declared_identities = [ + (item.migration_stream, item.domain, item.version) for item in declarations + ] + _reject_duplicates(declared_paths, "double migration declaration") + _reject_duplicates(blocked_paths, "duplicate blocked migration") + _reject_duplicates(legacy_versions, "duplicate historical node migration") + _reject_duplicates(declared_identities, "duplicate migration version") + _reject_duplicates( + [item.migration_name for item in aliases], "duplicate cloud migration alias" + ) + _reject_duplicates( + [item.runner_version for item in aliases], "duplicate cloud runner version" + ) + + overlap = sorted(set(declared_paths) & set(blocked_paths)) + if overlap: + raise ManifestError( + f"double migration declaration across active/blocked sets: {overlap!r}" + ) + active_versions = {item.version for item in declarations} + blocked_versions = {item.version for item in blocked} + legacy_overlap = sorted((active_versions | blocked_versions) & set(legacy_versions)) + if legacy_overlap: + raise ManifestError( + "historical node migration conflicts with an active or blocked " + f"declaration: {legacy_overlap!r}" + ) + + filesystem_paths = { + str(path.relative_to(migrations_dir)) + for path in (migrations_dir / "nodes").glob("*/*.sql") + if path.is_file() + } + manifest_paths = set(declared_paths) | set(blocked_paths) + missing = sorted(filesystem_paths - manifest_paths) + extra = sorted(manifest_paths - filesystem_paths) + if missing or extra: + raise ManifestError( + "migration declaration set differs from the vendored node tree: " + f"missing={missing!r}, extra={extra!r}" + ) + if require_complete and blocked: + blockers = sorted({item.ticket for item in blocked}) + raise ManifestError( + f"migration declaration set is incomplete: {len(blocked)} blocked " + f"artifact(s), blockers={blockers!r}" + ) + + return ValidationResult( + tuple(declarations), + tuple(blocked), + tuple(legacy_node_declarations), + tuple(aliases), + ) + + +def _reject_duplicates(values: Sequence[Hashable], label: str) -> None: + seen: set[Hashable] = set() + duplicates: set[Hashable] = set() + for value in values: + if value in seen: + duplicates.add(value) + seen.add(value) + if duplicates: + raise ManifestError(f"{label}: {sorted(map(str, duplicates))!r}") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + repo_root = Path(__file__).resolve().parents[2] + default_migrations = repo_root / "docker" / "migrations" / "forward" + default_ledger = default_migrations / "_ledger" + parser.add_argument("--migrations-dir", type=Path, default=default_migrations) + parser.add_argument( + "--declarations", + type=Path, + default=default_ledger / "application-migrations.tsv", + ) + parser.add_argument( + "--blocked", + type=Path, + default=default_ledger / "application-migration-blocks.tsv", + ) + parser.add_argument( + "--legacy-node-declarations", + type=Path, + default=default_ledger / "legacy-node-migrations.tsv", + ) + parser.add_argument( + "--cloud-aliases", + type=Path, + default=default_ledger / "cloud-migration-aliases.tsv", + ) + parser.add_argument( + "--require-complete", + action="store_true", + help="Fail when any checked-in blocker remains.", + ) + args = parser.parse_args(argv) + + try: + result = validate_manifests( + args.migrations_dir, + args.declarations, + args.blocked, + args.legacy_node_declarations, + args.cloud_aliases, + require_complete=args.require_complete, + ) + except ManifestError as exc: + print(f"FAIL: {exc}", file=sys.stderr) + return 1 + + print( + "PASS: deterministic application migration declarations validated " + f"({len(result.declarations)} active, {len(result.blocked)} blocked, " + f"{len(result.legacy_node_declarations)} historical, " + f"{len(result.cloud_aliases)} cloud aliases)." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation/validate_test_root_collection.py b/scripts/validation/validate_test_root_collection.py new file mode 100644 index 0000000000..5552102f2d --- /dev/null +++ b/scripts/validation/validate_test_root_collection.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""OMN-15378 / OMN-15410: fail closed on a `tests/` directory CI can never run. + +Why this exists +---------------- +``scripts/deploy-agent/tests/`` sat completely uncollected for ~5 weeks: no +entry in ``pyproject.toml`` ``testpaths``, no CI job, no governed selector +mapping reached it. A test asserting a literal superseded by OMN-12990 was RED +that entire time with zero signal, found only by hand in an unrelated PR +(OMN-14968 / #2536). This is a false-green class, not a stale-test nit — any +regression covered by an uncollected root is unenforced. + +OMN-15410 collected the four remaining roots of that class +(``scripts/ci/tests``, ``scripts/tests``, ``scripts/runtime_build/tests``, +``src/omnibase_infra/services/observability/agent_actions/tests`` — 366 tests, +4 of them RED since 2026-04-02) and hardened this guard against the two seams +that let the class exist in the first place. + +What this checks +----------------- +1. **Reachability** (:func:`find_violations`). Every directory containing at + least one ``test_*.py`` file must sit under a root pytest actually + collects, i.e. under an entry of ``[tool.pytest.ini_options] testpaths`` in + ``pyproject.toml``; or be a registered :data:`STANDALONE_PROJECT_ROOTS` + entry — a directory that owns its own ``pyproject.toml`` (a genuinely + separate uv (sub-)project, e.g. ``scripts/deploy-agent``) AND has a live + GitHub Actions workflow that runs its tests. All three legs are verified + (OMN-15378 / #2553): the ``pyproject.toml`` exists, the workflow file + exists and references the root, and the workflow is reachable on a pull + request (its own ``pull_request`` trigger, or a PR-reachable workflow calls + it via ``uses:``). An allowlist entry with no wiring behind it IS the + OMN-15378 defect; or be named in :data:`KNOWN_UNCOLLECTED_DEBT`, which is + EMPTY as of OMN-15410. Do NOT add a root there to make a violation pass — + wire its collection (path 1 or 2 above) instead; every entry would be a + live defect. + +2. **The ci.yml seam** (:func:`check_full_suite_invocation`). ``testpaths`` + only governs if CI lets it. Before OMN-15410 the full-suite step ran + ``uv run pytest tests/``, a hardcoded positional path that made the + *workflow*, not ``pyproject.toml``, the real definition of the suite — a + root added to ``testpaths`` would still never have run. The step now passes + no positional path at all, and this check fails closed if one reappears. + +3. **The selector seam** (:func:`check_collocated_selector_coverage`). The + full suite is only one of two pytest steps; a NARROWED smart-selection run + reaches only what ``scripts/ci/detect_test_paths.py`` maps. A collocated + root present in ``testpaths`` but absent from that mapping would run only + on unrelated full-suite escalations. Parity is asserted in both directions. + +Why the four OMN-15410 roots are ``testpaths`` entries rather than a move into +``tests/``: they are collocated with the code they cover, two of them declare +their own packages, and once ``testpaths`` is the single source of truth a +move buys no collection guarantee this list does not already provide. + +Deliberately excluded: ``scripts/deploy-agent/tests/`` cannot be added to +``testpaths`` or folded into ``tests/`` — both trees declare a top-level +``tests`` package (``tests/__init__.py`` vs +``scripts/deploy-agent/tests/__init__.py``), so pytest raises +``ImportPathMismatchError`` when both are collected in one session (verified +locally 2026-07-29). That is why it is a registered standalone project. The +four OMN-15410 roots have no such conflict — a combined collection of all five +``testpaths`` entries was verified clean (27,436 tests, 2026-07-30). +""" + +from __future__ import annotations + +import re +import shlex +import sys +import tomllib +from pathlib import Path +from typing import Any + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] + +# Directories that own a private pyproject.toml (a genuine separate uv +# (sub-)project) and are exercised by a DEDICATED CI workflow rather than +# folded into the main `tests/` tree. The pyproject.toml and the workflow file +# must exist on disk, the workflow must REFERENCE the root, and the workflow +# must be reachable on a pull request -- either it triggers on `pull_request` +# itself, or a workflow that does calls it via `uses:` (OMN-15378 AC3: the +# deploy-agent workflow is `workflow_call`-only and is invoked by ci.yml's +# `deploy-agent-tests` job so its result lands under the required "CI Summary" +# context). An unwired allowlist entry -- a workflow file that exists but never +# runs on a PR, or runs but does not touch the root -- is exactly the class of +# defect this guard exists to catch. +STANDALONE_PROJECT_ROOTS: dict[str, str] = { + "scripts/deploy-agent": ".github/workflows/deploy-agent-tests.yml", +} + +# Uncollected roots explicitly tolerated. EMPTY as of OMN-15410: the four +# entries OMN-15378 grandfathered in (scripts/ci/tests, scripts/tests, +# scripts/runtime_build/tests, and the agent_actions root) are now collected +# via pyproject.toml `testpaths`, so the guard has converged to zero known +# uncollected roots -- its stated OMN-15378 goal. +# +# Do NOT add an entry here to make a violation pass; that is the exact move +# this guard exists to prevent. Wire the root's collection instead (add it to +# `testpaths` plus a COLLOCATED_TEST_ROOTS mapping, or register a standalone +# project with real CI wiring). An entry here is only ever legitimate as a +# same-PR, ticket-bearing record of debt that genuinely cannot be wired yet. +KNOWN_UNCOLLECTED_DEBT: frozenset[str] = frozenset() + +_IGNORED_DIR_PARTS = (".git", "__pycache__", ".venv", "node_modules") + +# The ci.yml step whose pytest invocation defines the full suite. +FULL_SUITE_STEP_NAME = "Run pytest (full suite)" +CI_WORKFLOW = ".github/workflows/ci.yml" + +# GitHub Actions expressions are substituted out before shell tokenization: +# `--junitxml=junit-${{ matrix.split }}.xml` would otherwise tokenize into +# three words, one of which looks like a positional argument. +_GH_EXPRESSION = re.compile(r"\$\{\{.*?\}\}", re.DOTALL) +_GH_EXPRESSION_PLACEHOLDER = "GH_EXPR" + +# pytest options that take their value as a SEPARATE token. Needed so +# `--splits 15` is not misread as the positional path `15`. Options written +# `--opt=value` need no entry here. +_PYTEST_VALUE_OPTIONS = frozenset( + { + "-c", + "-k", + "-m", + "-n", + "-o", + "-p", + "--deselect", + "--dist", + "--group", + "--ignore", + "--junitxml", + "--maxfail", + "--rootdir", + "--splits", + "--timeout", + "--timeout-method", + } +) + + +def collected_roots(repo_root: Path = REPO_ROOT) -> tuple[str, ...]: + """Return ``testpaths`` from pyproject.toml, POSIX with trailing slash. + + Fails closed: a missing pyproject.toml, a missing ``testpaths`` key, or an + empty list all raise. An empty ``testpaths`` is especially dangerous — bare + ``pytest`` would then collect from the rootdir, i.e. the entire repository + including ``.venv`` — and the full-suite CI step now relies on this list. + """ + pyproject = repo_root / "pyproject.toml" + if not pyproject.is_file(): + raise FileNotFoundError( + f"{pyproject} does not exist; cannot determine collected test roots" + ) + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + ini_options = data.get("tool", {}).get("pytest", {}).get("ini_options", {}) + paths = ini_options.get("testpaths") + if not paths: + raise ValueError( + f"{pyproject} declares no [tool.pytest.ini_options] testpaths; bare " + "`pytest` would collect the whole repository (OMN-15410)" + ) + return tuple(str(p).rstrip("/") + "/" for p in paths) + + +def find_test_dirs(repo_root: Path) -> list[str]: + """Return every repo-relative directory literally named ``tests`` (POSIX, + trailing slash) that directly contains at least one ``test_*.py`` file. + + Scoped to directories named ``tests`` -- not "any file matching + ``test_*.py`` anywhere" -- because several modules in this repo are named + ``test_selection_*.py`` as PRODUCT CODE (the change-aware selector, + ``scripts/ci/test_selection_models.py`` / ``test_selection_loader.py``), + not pytest tests; a bare ``test_*.py`` glob false-positives on those. + """ + found: set[str] = set() + for tests_dir in repo_root.rglob("tests"): + if not tests_dir.is_dir(): + continue + rel = tests_dir.relative_to(repo_root) + if any(part in _IGNORED_DIR_PARTS for part in rel.parts): + continue + # Recursive: scripts/deploy-agent/tests/unit/test_*.py must count as + # content of the scripts/deploy-agent/tests/ root, not be missed + # because the test files sit one level below the `tests` dir itself. + if any(tests_dir.rglob("test_*.py")): + found.add(rel.as_posix() + "/") + return sorted(found) + + +def _is_collected(test_dir: str, roots: tuple[str, ...]) -> bool: + return any(test_dir.startswith(root) for root in roots) + + +def _load_workflow(path: Path) -> dict[str, Any]: + """Parse a workflow file; an unparseable file yields an empty mapping.""" + try: + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return {} + return loaded if isinstance(loaded, dict) else {} + + +def _workflow_triggers(workflow: dict[str, Any]) -> set[str]: + """Return the workflow's trigger names. + + PyYAML resolves the bare ``on:`` key to the boolean ``True`` (YAML 1.1), so + both spellings are checked. ``on: [push, pull_request]`` (list form) and + ``on: push`` (scalar form) are normalized alongside the mapping form. + """ + raw = workflow.get(True, workflow.get("on")) + if isinstance(raw, dict): + return {str(key) for key in raw} + if isinstance(raw, list): + return {str(item) for item in raw} + if isinstance(raw, str): + return {raw} + return set() + + +def _workflow_runs_on_pull_request( + workflow_rel: str, repo_root: Path, _seen: frozenset[str] = frozenset() +) -> bool: + """True when ``workflow_rel`` actually executes on a pull request. + + Either it declares a ``pull_request`` trigger itself, or it is a reusable + (``workflow_call``) workflow invoked via ``uses:`` by another workflow that + is itself PR-reachable. A registered wiring workflow that no event and no + caller can ever reach runs zero tests -- the OMN-15378 defect wearing a + workflow file as a disguise. + """ + if workflow_rel in _seen: # cycle guard + return False + workflow = _load_workflow(repo_root / workflow_rel) + triggers = _workflow_triggers(workflow) + if "pull_request" in triggers or "pull_request_target" in triggers: + return True + if "workflow_call" not in triggers: + return False + reference = f"./{workflow_rel}" + workflows_dir = repo_root / ".github" / "workflows" + candidates = sorted(workflows_dir.glob("*.yml")) + sorted( + workflows_dir.glob("*.yaml") + ) + for candidate in candidates: + candidate_rel = candidate.relative_to(repo_root).as_posix() + if candidate_rel == workflow_rel: + continue + jobs = _load_workflow(candidate).get("jobs") + if not isinstance(jobs, dict): + continue + calls_it = any( + str((body or {}).get("uses") or "") == reference + for body in jobs.values() + if isinstance(body, dict) + ) + if calls_it and _workflow_runs_on_pull_request( + candidate_rel, repo_root, _seen | {workflow_rel} + ): + return True + return False + + +def _standalone_problem(test_dir: str, repo_root: Path) -> str | None: + """Return None if legitimately wired, 'unregistered' if no match, else a + description of what is wrong with the registered entry.""" + for root, workflow in STANDALONE_PROJECT_ROOTS.items(): + root_prefix = root.rstrip("/") + "/" + if test_dir == root_prefix or test_dir.startswith(root_prefix): + pyproject = repo_root / root / "pyproject.toml" + workflow_path = repo_root / workflow + if not pyproject.is_file(): + return ( + f"registered as a STANDALONE_PROJECT_ROOTS entry but " + f"{root}/pyproject.toml does not exist" + ) + if not workflow_path.is_file(): + return ( + f"registered as a STANDALONE_PROJECT_ROOTS entry but " + f"the wiring workflow {workflow} does not exist" + ) + if root.rstrip("/") not in workflow_path.read_text(encoding="utf-8"): + return ( + f"registered as a STANDALONE_PROJECT_ROOTS entry but the " + f"wiring workflow {workflow} never references {root} — it " + f"cannot be running those tests" + ) + if not _workflow_runs_on_pull_request(workflow, repo_root): + return ( + f"registered as a STANDALONE_PROJECT_ROOTS entry but the " + f"wiring workflow {workflow} never runs on a pull request " + f"(no pull_request trigger, and no PR-reachable workflow " + f"calls it via uses:) — the tests are invisible again" + ) + return None + return "unregistered" + + +def find_violations(repo_root: Path = REPO_ROOT) -> list[str]: + """Return a human-readable violation string per uncollected test root.""" + roots = collected_roots(repo_root) + violations: list[str] = [] + + # A testpaths entry that does not exist on disk aborts collection with + # pytest exit 5 ("no tests ran"), reddening the whole suite for a + # bookkeeping error. Catch it here, not in a 40-minute CI job. + for root in roots: + if not (repo_root / root).is_dir(): + violations.append( + f"{root}: listed in pyproject.toml testpaths but is not a " + "directory on disk — pytest would abort collection with exit 5. " + "Remove the entry or restore the directory." + ) + + for test_dir in find_test_dirs(repo_root): + if _is_collected(test_dir, roots): + continue + if test_dir.rstrip("/") in KNOWN_UNCOLLECTED_DEBT: + continue + problem = _standalone_problem(test_dir, repo_root) + if problem is None: + continue + if problem == "unregistered": + violations.append( + f"{test_dir}: not under any pyproject.toml testpaths root " + f"({', '.join(roots)}), not a registered " + "STANDALONE_PROJECT_ROOTS entry, and not in " + "KNOWN_UNCOLLECTED_DEBT — no pytest invocation in CI can ever " + "run these tests (OMN-15378 class). Add it to testpaths (plus a " + "COLLOCATED_TEST_ROOTS mapping in " + "scripts/ci/detect_test_paths.py if it lives outside tests/), or " + "register it in STANDALONE_PROJECT_ROOTS with real CI wiring." + ) + else: + violations.append(f"{test_dir}: {problem}") + return violations + + +def _full_suite_run_block(repo_root: Path) -> str: + """Return the shell body of ci.yml's full-suite pytest step.""" + workflow_path = repo_root / CI_WORKFLOW + if not workflow_path.is_file(): + raise FileNotFoundError(f"{CI_WORKFLOW} does not exist") + workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) + for job in (workflow.get("jobs") or {}).values(): + for step in job.get("steps") or []: + if isinstance(step, dict) and step.get("name") == FULL_SUITE_STEP_NAME: + return str(step.get("run", "")) + raise LookupError( + f"{CI_WORKFLOW} has no step named {FULL_SUITE_STEP_NAME!r}; the " + "full-suite invocation cannot be verified (OMN-15410)" + ) + + +def positional_pytest_args(run_block: str) -> list[str]: + """Return the positional (non-option) arguments a run block passes pytest. + + GitHub expressions are replaced with a placeholder first, then the block is + tokenized as shell so a quoted marker expression stays one token. + """ + text = _GH_EXPRESSION.sub(_GH_EXPRESSION_PLACEHOLDER, run_block) + text = text.replace("\\\n", " ") + tokens = shlex.split(text) + if "pytest" not in tokens: + return [] + positionals: list[str] = [] + skip_next = False + for token in tokens[tokens.index("pytest") + 1 :]: + if skip_next: + skip_next = False + continue + if token.startswith("-"): + skip_next = token in _PYTEST_VALUE_OPTIONS + continue + positionals.append(token) + return positionals + + +def check_full_suite_invocation(repo_root: Path = REPO_ROOT) -> list[str]: + """Fail closed when ci.yml's full suite names its own paths (OMN-15410). + + A positional path there silently overrides ``testpaths``, which is how four + collectable roots went unrun for months while ``pyproject.toml`` looked + correct. + """ + positionals = positional_pytest_args(_full_suite_run_block(repo_root)) + if not positionals: + return [] + return [ + f"{CI_WORKFLOW} step {FULL_SUITE_STEP_NAME!r} passes positional path(s) " + f"{positionals} to pytest, overriding pyproject.toml testpaths. The " + "full suite must pass NO positional path so it inherits every collected " + "root (OMN-15410); use --ignore to exclude, never a positional include." + ] + + +def check_collocated_selector_coverage(repo_root: Path = REPO_ROOT) -> list[str]: + """Assert testpaths <-> COLLOCATED_TEST_ROOTS parity (OMN-15410). + + A collocated root collected only by the full suite is reachable solely via + unrelated escalations; the change-aware selector must be able to select it + from a diff that touches the code it covers. + """ + sys.path.insert(0, str(repo_root)) + try: + from scripts.ci.detect_test_paths import ( + COLLOCATED_TEST_ROOTS, + TESTS_PREFIX, + ) + finally: + sys.path.remove(str(repo_root)) + + violations: list[str] = [] + mapped = set(COLLOCATED_TEST_ROOTS.values()) + declared = {root for root in collected_roots(repo_root) if root != TESTS_PREFIX} + + for root in sorted(declared - mapped): + violations.append( + f"{root}: collected via pyproject.toml testpaths but no " + "COLLOCATED_TEST_ROOTS entry in scripts/ci/detect_test_paths.py maps " + "any source prefix to it — a narrowed smart-selection run can never " + "select it (OMN-15410)." + ) + for root in sorted(mapped - declared): + violations.append( + f"{root}: mapped by COLLOCATED_TEST_ROOTS in " + "scripts/ci/detect_test_paths.py but absent from pyproject.toml " + "testpaths — the selector would hand pytest a path the full suite " + "never collects (OMN-15410)." + ) + return violations + + +def main() -> int: + violations = ( + find_violations() + + check_full_suite_invocation() + + check_collocated_selector_coverage() + ) + if violations: + print("FAIL: test-collection defect(s) detected (OMN-15378/OMN-15410 class):") + for violation in violations: + print(f" - {violation}") + return 1 + print( + "OK: every tests/ directory is reachable by a wired pytest invocation, the " + "full suite inherits pyproject testpaths, and every collocated root is " + "selector-reachable." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/omnibase_infra/adapters/llm/adapter_llm_provider_openai.py b/src/omnibase_infra/adapters/llm/adapter_llm_provider_openai.py index 98b3998bd2..9e6fa0d66e 100644 --- a/src/omnibase_infra/adapters/llm/adapter_llm_provider_openai.py +++ b/src/omnibase_infra/adapters/llm/adapter_llm_provider_openai.py @@ -167,8 +167,8 @@ class AdapterLlmProviderOpenai: This adapter supports all OpenAI-compatible inference servers including vLLM, text-generation-inference, and the OpenAI API itself. - Falls back to the ``LLM_CODER_URL`` environment variable (default: - ``http://localhost:8000``) if ``base_url`` is not provided at construction. + Falls back to the contract-provided ``LLM_CODER_URL`` when ``base_url`` is + not provided at construction. Attributes: _provider_name: Provider identifier. @@ -182,8 +182,9 @@ class AdapterLlmProviderOpenai: _capabilities_cache: Cached model capabilities. Example: + >>> resolved_endpoint = "routing-authority-provided endpoint" >>> adapter = AdapterLlmProviderOpenai( - ... base_url="http://localhost:8000", + ... base_url=resolved_endpoint, ... default_model="qwen2.5-coder-14b", ... ) >>> response = await adapter.generate_async(request) diff --git a/src/omnibase_infra/cli/cli_auth.py b/src/omnibase_infra/cli/cli_auth.py new file mode 100644 index 0000000000..1564c37f94 --- /dev/null +++ b/src/omnibase_infra/cli/cli_auth.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""``onex auth`` -- credential in, gateway JWT out (OMN-15922). + +Four commands, one credential, no per-harness code. Claude Code, Codex, cursor +and a bare terminal all shell out to the same binary, which is the property +that makes ``onex auth login`` + ``onex delegate`` work identically from all +four: the auth logic lives here, and a marketplace skill stays a thin shim over +it (zero auth logic in skill markdown). + + onex auth login --tenant-slug S --client-id C --client-secret-stdin + onex auth status + onex auth token + onex auth logout + +SECRET HANDLING + The secret is read from stdin, never from an argv flag. A ``--client-secret + `` option would put the credential in the process table, in shell + history, and in any exec log -- three durable copies that outlive the + session. It is written only to ``~/.onex/credentials.json`` at mode 0600 and + referenced from ``config.yaml`` by name; ``status`` prints tenant, principal + and expiry, and never prints secret material. + +WHERE THE TRANSPORT COMES FROM + ``token`` mints over the network, so it needs a concrete + ``ProtocolGatewayTransport``. That adapter ships in this same distribution + (``omnibase_infra.gateway.client.gateway_transport_httpx``) and is + constructed directly. ``login``/``status``/``logout`` touch no network at + all -- they are pure local file operations over ``~/.onex`` -- so they work + regardless. +""" + +from __future__ import annotations + +import asyncio +import socket +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import NoReturn + +import click + +from omnibase_core.errors.model_onex_error import ModelOnexError +from omnibase_infra.gateway.client.gateway_token_minter import ( + GatewayTokenMinter, +) +from omnibase_infra.gateway.client.gateway_transport_httpx import ( + GatewayTransportHttpx, +) +from omnibase_infra.gateway.client.store_gateway_credential import ( + StoreGatewayCredential, +) +from omnibase_infra.gateway.models.model_gateway_credential import ( + ModelGatewayCredential, +) + +__all__ = ["auth_group"] + + +def _store() -> StoreGatewayCredential: + return StoreGatewayCredential(onex_home=Path.home() / ".onex") + + +def _fail(message: str) -> NoReturn: + """Report on stderr and exit non-zero. + + Typed ``NoReturn`` deliberately: every fail-closed branch below relies on + control not continuing past it, and ``NoReturn`` makes the type checker + enforce that rather than leaving it to reviewer attention. + """ + click.echo(f"Error: {message}", err=True) + sys.exit(1) + + +def _load_credential() -> ModelGatewayCredential: + try: + return _store().load() + except ModelOnexError as exc: + _fail(str(exc)) + + +@click.group("auth") +def auth_group() -> None: # stub-ok + """Manage the gateway credential and the tokens minted from it.""" + + +@auth_group.command("login") +@click.option( + "--tenant-slug", required=True, help="Tenant slug the credential belongs to." +) +@click.option( + "--client-id", + required=True, + help="Keycloak clientId of the per-tenant confidential client (this IS the principal_id).", +) +@click.option( + "--token-endpoint", + required=True, + help="Realm token endpoint, e.g. https:///realms//protocol/openid-connect/token", +) +@click.option( + "--base-url", required=True, help="Gateway origin, e.g. https://api.omninode.ai" +) +@click.option( + "--client-secret-stdin", + is_flag=True, + required=True, + help="Read the client secret from stdin. The only accepted form -- a flag value would leak into the process table and shell history.", +) +@click.option( + "--edge-instance-id", + default="", + help="Host label for session bookkeeping. Defaults to this machine's hostname.", +) +def auth_login( + tenant_slug: str, + client_id: str, + token_endpoint: str, + base_url: str, + client_secret_stdin: bool, + edge_instance_id: str, +) -> None: + """Store a gateway credential by reference under ~/.onex. + + Writes the secret to ~/.onex/credentials.json (mode 0600) and a + reference-only block to ~/.onex/config.yaml. Nothing else in config.yaml + is disturbed. + """ + if not client_secret_stdin: # pragma: no cover - click marks the flag required + _fail("--client-secret-stdin is required; the secret is never taken from argv.") + + secret = sys.stdin.read().strip() + if not secret: + _fail( + "no client secret on stdin. Pipe it, e.g.: pbpaste | onex auth login ... --client-secret-stdin" + ) + + try: + _store().save( + tenant_slug=tenant_slug, + client_id=client_id, + client_secret=secret, + token_endpoint=token_endpoint, + base_url=base_url, + edge_instance_id=edge_instance_id or socket.gethostname(), + ) + except ModelOnexError as exc: + _fail(str(exc)) + + click.echo( + f"Stored gateway credential for tenant '{tenant_slug}' (client_id {client_id})." + ) + click.echo("Secret written by reference to ~/.onex/credentials.json (mode 0600).") + + +@auth_group.command("status") +def auth_status() -> None: + """Print the stored credential's identity and endpoints. + + Never prints secret material -- not the client secret, not a token. This + command is what an operator pastes into an issue. + """ + credential = _load_credential() + click.echo(f"tenant_slug: {credential.tenant_slug}") + click.echo(f"principal_id: {credential.client_id}") + click.echo(f"token_endpoint: {credential.token_endpoint}") + click.echo(f"gateway base_url: {credential.base_url}") + click.echo(f"edge_instance_id: {credential.edge_instance_id}") + click.echo("client_secret: stored by reference (not shown)") + + +@auth_group.command("token") +def auth_token() -> None: + """Mint and print a currently-valid gateway access token. + + The escape hatch any harness can shell out to. Emits the raw token on + stdout and nothing else, so it composes; every diagnostic goes to stderr. + Exits non-zero if the credential is missing, the grant is refused, or the + token's audience is not exactly the gateway-attach set. + """ + credential = _load_credential() + minter = GatewayTokenMinter( + transport=GatewayTransportHttpx(), + credential=credential, + ) + try: + token = asyncio.run(minter.token_for(now=datetime.now(UTC))) + except ModelOnexError as exc: + _fail(str(exc)) + click.echo(token.access_token.get_secret_value()) + + +@auth_group.command("logout") +def auth_logout() -> None: + """Remove the stored credential and the secret it references.""" + try: + _store().clear() + except ModelOnexError as exc: + _fail(str(exc)) + click.echo("Removed the gateway credential from ~/.onex.") diff --git a/src/omnibase_infra/cli/cli_delegate.py b/src/omnibase_infra/cli/cli_delegate.py index 531956f98a..e434487769 100644 --- a/src/omnibase_infra/cli/cli_delegate.py +++ b/src/omnibase_infra/cli/cli_delegate.py @@ -86,6 +86,11 @@ from omnibase_infra.backends.backend_probe import probe_kafka from omnibase_infra.backends.enum_probe_state import EnumProbeState from omnibase_infra.cli.cli_node import _resolve_packaged_contract +from omnibase_infra.cli.omnimarket_drift_guard import ( + DRIFT_OVERRIDE_ENV, + OmnimarketDriftError, + check_omnimarket_drift, +) from omnibase_infra.cli.receipt_mode import ( default_emit_socket_path, run_receipt_mode, @@ -487,6 +492,33 @@ def _on_alarm(signum: int, frame: object) -> None: "/emit_spool/ for later replay." ), ) +@click.option( + "--omni-home", + type=click.Path(path_type=Path), + envvar="OMNI_HOME", + default=None, + help=( + "Canonical omni_home workspace root for the local omnimarket drift " + "check (OMN-13930). Defaults to the $OMNI_HOME environment variable " + "-- the envvar binding is load-bearing: without it the guard " + "silently receives omni_home=None and never fires, because callers " + "never pass this flag explicitly." + ), +) +@click.option( + "--allow-omnimarket-drift", + "allow_omnimarket_drift", + is_flag=True, + envvar=DRIFT_OVERRIDE_ENV, + default=False, + help=( + "Dispatch even when the omnimarket co-install has drifted from the " + "canonical clone (OMN-13930). Refusal is the DEFAULT; this is the " + "only supported way past it, and it is named in the refusal message. " + f"Bound to ${DRIFT_OVERRIDE_ENV}. Results produced under an override " + "come from an UNVERIFIED build and are not evidence." + ), +) def delegate_command( prompt: str, task_type: str | None, @@ -498,6 +530,8 @@ def delegate_command( timeout: int, verbose: bool, emit_socket: Path | None, + omni_home: Path | None, + allow_omnimarket_drift: bool, ) -> None: """Delegate PROMPT to a local LLM and print exactly one typed result. @@ -527,6 +561,8 @@ def delegate_command( timeout=timeout, verbose=verbose, emit_socket=emit_socket, + omni_home=omni_home, + allow_drift=allow_omnimarket_drift, ) except ValueError as exc: raise click.UsageError(str(exc)) from exc @@ -545,6 +581,8 @@ def run_delegate( timeout: int, verbose: bool, emit_socket: Path | None, + omni_home: Path | None = None, + allow_drift: bool = False, ) -> int: """Build the payload, resolve the contract, and dispatch in receipt mode. @@ -579,6 +617,21 @@ def run_delegate( backstop trip returns exit code 1 with a clear stderr message instead of hanging indefinitely. """ + # OMN-13930: ``DELEGATE_NODE_NAME`` is an omnimarket-provided node, so + # this surface carries the same stale/absent co-install exposure as + # ``onex skill`` and ``onex node`` -- it was simply the one of the three + # never wired to the guard, and a drifted venv surfaced here as a bare + # contract-resolution failure with no pointer to the repair command. + # Runs FIRST, before any bus probe or payload write, so a drifted venv + # never produces a receipt that could be mistaken for evidence. + try: + check_omnimarket_drift( + omni_home=str(omni_home) if omni_home else None, + allow_drift=allow_drift, + ) + except OmnimarketDriftError as exc: + raise click.ClickException(str(exc)) from exc + resolved_task_type = task_type or classify_task_type(prompt) resolved_source = source or DELEGATE_SOURCE if bus is None: diff --git a/src/omnibase_infra/cli/cli_node.py b/src/omnibase_infra/cli/cli_node.py index a11ed1818b..080a21e764 100644 --- a/src/omnibase_infra/cli/cli_node.py +++ b/src/omnibase_infra/cli/cli_node.py @@ -27,6 +27,7 @@ from omnibase_core.models.errors.model_onex_error import ModelOnexError from omnibase_core.runtime.runtime_local import RuntimeLocal, parse_backend_overrides from omnibase_infra.cli.omnimarket_drift_guard import ( + DRIFT_OVERRIDE_ENV, OmnimarketDriftError, check_omnimarket_drift, ) @@ -197,6 +198,20 @@ def _entry_point_module(value: str) -> str: "drift guard silently receives omni_home=None and never fires." ), ) +@click.option( + "--allow-omnimarket-drift", + "allow_omnimarket_drift", + is_flag=True, + envvar=DRIFT_OVERRIDE_ENV, + default=False, + help=( + "Dispatch even when the omnimarket co-install has drifted from the " + "canonical clone (OMN-13930). Refusal is the DEFAULT; this is the " + "only supported way past it, and it is named in the refusal message. " + f"Bound to ${DRIFT_OVERRIDE_ENV}. Results produced under an override " + "come from an UNVERIFIED build and are not evidence." + ), +) def run_node_by_name( node_name: str, contract_path: Path | None, @@ -208,6 +223,7 @@ def run_node_by_name( output_mode: str, emit_socket: Path | None, omni_home: Path | None, + allow_omnimarket_drift: bool, ) -> None: """Run a packaged ONEX node on the local runtime, resolved by NAME. @@ -228,7 +244,10 @@ def run_node_by_name( onex node merge_sweep --output receipt # one typed result JSON on stdout """ try: - check_omnimarket_drift(omni_home=str(omni_home) if omni_home else None) + check_omnimarket_drift( + omni_home=str(omni_home) if omni_home else None, + allow_drift=allow_omnimarket_drift, + ) except OmnimarketDriftError as exc: raise click.ClickException(str(exc)) from exc diff --git a/src/omnibase_infra/cli/cli_occ.py b/src/omnibase_infra/cli/cli_occ.py index b0afb923df..331fe4596d 100644 --- a/src/omnibase_infra/cli/cli_occ.py +++ b/src/omnibase_infra/cli/cli_occ.py @@ -33,16 +33,31 @@ the renderer-owned Evidence section and re-appends exactly one canonical block, so ``stamp(stamp(x)) == stamp(x)`` is a fixpoint. +``onex occ compute-contract-hash CONTRACT_PATH [--evidence-item-id ID]`` (OMN-15711) + Print the ``contract_sha256`` (and, with ``--evidence-item-id``, the + per-entry ``contract_entry_sha256``) that a DoD receipt for CONTRACT_PATH + must carry to pass ``validator_receipt_gate.check_receipt_contract_binding``. + This command owns zero hashing logic — it imports + :func:`omnibase_core.validation.validator_receipt_gate.compute_contract_sha256` + and :func:`...compute_contract_entry_sha256` verbatim, so a receipt authored + via this CLI and the gate's own hash check can never diverge. Packages the + hash-chaining the gate previously kept internal, so neither an agent + hand-authoring a receipt nor an automation authoring one on its behalf has + to reimplement SHA-256 canonicalization to stay bound to the gate. + .. versionadded:: OMN-14190 +.. versionadded:: OMN-15711 ``compute-contract-hash`` """ from __future__ import annotations +import json import sys from collections.abc import Sequence from pathlib import Path import click +import yaml # Canonical OCC stamp schema — single import block. The models + parser/renderer # were relocated from omnibase_core to omnibase_compat (the lowest shared layer) @@ -54,13 +69,25 @@ render_pr_occ_metadata_stamp, ) +# Receipt-Gate's own hash-chaining functions (OMN-15711 / OMN-13888). Imported +# verbatim -- never reimplemented -- so this CLI's output and the gate's own +# `check_receipt_contract_binding` check can never compute a different hash for +# the same contract bytes/entry. +from omnibase_core.validation.validator_receipt_gate import ( + ContractEntryNotFoundError, + compute_contract_entry_sha256, + compute_contract_sha256, +) + __all__ = [ "occ", "occ_validate", "occ_stamp", + "occ_compute_contract_hash", "parse_evidence_source_token", "validate_pr_body", "stamp_pr_body", + "compute_receipt_contract_hashes", ] @@ -246,3 +273,76 @@ def occ_stamp( file.write_text(stamped, encoding="utf-8") else: click.echo(stamped, nl=False) + + +# --------------------------------------------------------------------------- +# compute-contract-hash (OMN-15711 / FM5) — package validator_receipt_gate's +# hash-chaining as a supported CLI so authoring a DoD receipt never requires +# reimplementing SHA-256 canonicalization inline. +# --------------------------------------------------------------------------- + + +def compute_receipt_contract_hashes( + contract_path: Path, *, evidence_item_id: str | None = None +) -> dict[str, str]: + """Return the receipt-binding hash(es) for ``contract_path``. + + Always includes ``contract_sha256`` (the whole-file hash, prefixed + ``sha256:`` to match the canonical receipt field format). When + ``evidence_item_id`` is given, also includes ``contract_entry_sha256`` (the + per-entry hash for that ``dod_evidence`` item id). + + Delegates entirely to + :func:`omnibase_core.validation.validator_receipt_gate.compute_contract_sha256` + and ``compute_contract_entry_sha256`` — this function performs no hashing of + its own, only the "sha256:" prefix formatting the gate's receipt models + expect and YAML parsing for the entry-hash path. + + Raises: + ContractEntryNotFoundError: when ``evidence_item_id`` is given but no + ``dod_evidence`` item in the contract carries that id. + """ + result: dict[str, str] = { + "contract_sha256": f"sha256:{compute_contract_sha256(contract_path)}" + } + if evidence_item_id is not None: + contract_data = yaml.safe_load(contract_path.read_text(encoding="utf-8")) + result["contract_entry_sha256"] = compute_contract_entry_sha256( + contract_data, evidence_item_id + ) + return result + + +@occ.command("compute-contract-hash") +@click.argument( + "contract_path", + type=click.Path(exists=True, dir_okay=False, path_type=Path), +) +@click.option( + "--evidence-item-id", + "evidence_item_id", + default=None, + metavar="ID", + help="dod_evidence item id to also compute the per-entry " + "contract_entry_sha256 for (OMN-13888 per-entry hash scheme).", +) +def occ_compute_contract_hash( + contract_path: Path, evidence_item_id: str | None +) -> None: + """Print the contract_sha256 (+ optional contract_entry_sha256) a DoD + receipt for CONTRACT_PATH must carry to bind against Receipt-Gate. + + Prints a single-line JSON object to stdout, e.g.: + ``{"contract_sha256": "sha256:...."}`` or, with --evidence-item-id, + ``{"contract_entry_sha256": "sha256:....", "contract_sha256": "sha256:...."}``. + Exits non-zero with an actionable message when --evidence-item-id names a + dod_evidence item absent from the contract. + """ + try: + result = compute_receipt_contract_hashes( + contract_path, evidence_item_id=evidence_item_id + ) + except ContractEntryNotFoundError as exc: + raise click.UsageError(str(exc)) from exc + + click.echo(json.dumps(result, sort_keys=True)) diff --git a/src/omnibase_infra/cli/cli_skill.py b/src/omnibase_infra/cli/cli_skill.py index 4f6a767d3f..cfcdfe81bd 100644 --- a/src/omnibase_infra/cli/cli_skill.py +++ b/src/omnibase_infra/cli/cli_skill.py @@ -40,6 +40,7 @@ from omnibase_infra.cli.model_skill_mapping import ModelSkillMapping from omnibase_infra.cli.model_skill_mapping_registry import ModelSkillMappingRegistry from omnibase_infra.cli.omnimarket_drift_guard import ( + DRIFT_OVERRIDE_ENV, OmnimarketDriftError, check_omnimarket_drift, ) @@ -257,6 +258,20 @@ def _write_payload( "explicitly." ), ) +@click.option( + "--allow-omnimarket-drift", + "allow_omnimarket_drift", + is_flag=True, + envvar=DRIFT_OVERRIDE_ENV, + default=False, + help=( + "Dispatch even when the omnimarket co-install has drifted from the " + "canonical clone (OMN-13930). Refusal is the DEFAULT; this is the " + "only supported way past it, and it is named in the refusal message. " + "Bound to $" + DRIFT_OVERRIDE_ENV + ". Results produced under an " + "override come from an UNVERIFIED build and are not evidence." + ), +) @click.argument("skill_args", nargs=-1, type=click.UNPROCESSED) def run_skill_by_name( skill_name: str, @@ -265,6 +280,7 @@ def run_skill_by_name( verbose: bool, emit_socket: Path | None, omni_home: Path | None, + allow_omnimarket_drift: bool, skill_args: tuple[str, ...], ) -> None: """Dispatch a skill to its backing node and print one typed result. @@ -283,7 +299,10 @@ def run_skill_by_name( onex skill delegate "summarize this paragraph" --task-type document """ try: - check_omnimarket_drift(omni_home=str(omni_home) if omni_home else None) + check_omnimarket_drift( + omni_home=str(omni_home) if omni_home else None, + allow_drift=allow_omnimarket_drift, + ) except OmnimarketDriftError as exc: raise click.ClickException(str(exc)) from exc diff --git a/src/omnibase_infra/cli/commands.py b/src/omnibase_infra/cli/commands.py index 0bcf6bc9ed..c7f0e730d9 100644 --- a/src/omnibase_infra/cli/commands.py +++ b/src/omnibase_infra/cli/commands.py @@ -757,5 +757,14 @@ def _print_result(name: str, result: object) -> None: cli.add_command(env_group) +# ============================================================================= +# Gateway Auth Commands (login, status, token, logout) [OMN-15922] +# ============================================================================= + +from omnibase_infra.cli.cli_auth import auth_group + +cli.add_command(auth_group) + + if __name__ == "__main__": cli() diff --git a/src/omnibase_infra/cli/omnimarket_drift_guard.py b/src/omnibase_infra/cli/omnimarket_drift_guard.py index ef51fd972f..b8602b3698 100644 --- a/src/omnibase_infra/cli/omnimarket_drift_guard.py +++ b/src/omnibase_infra/cli/omnimarket_drift_guard.py @@ -57,6 +57,7 @@ from pathlib import Path __all__ = [ + "DRIFT_OVERRIDE_ENV", "OmnimarketDriftError", "canonical_local_omnimarket_commit", "check_omnimarket_drift", @@ -65,6 +66,19 @@ logger = logging.getLogger(__name__) +# The single supported way past a drift refusal (OMN-13930). Named in every +# refusal message so the escape hatch is discoverable from the failure alone. +# +# This module NEVER reads it itself: the value arrives as the ``allow_drift`` +# argument, bound at the CLI boundary by click's ``envvar=`` (the same +# mechanism ``--omni-home`` uses). That keeps this module a pure function of +# its arguments and keeps the read out of ``src/`` where the +# ``check-env-reads`` hook (correctly) forbids raw ``os.environ`` access. +# Click's BOOL conversion is what makes the override fail closed: ``0`` / +# ``false`` parse as False, and an unparseable value is a hard usage error, +# so neither one silently disables the guard. +DRIFT_OVERRIDE_ENV = "ONEX_ALLOW_OMNIMARKET_DRIFT" + # Local `git rev-parse HEAD` only -- this never touches the network, so a # generous timeout still keeps the hot path fast. _GIT_TIMEOUT_SECONDS = 2 @@ -126,42 +140,83 @@ def canonical_local_omnimarket_commit(omni_home: str | None = None) -> str | Non return sha if len(sha) == 40 else None -def check_omnimarket_drift(omni_home: str | None = None) -> None: +def check_omnimarket_drift( + omni_home: str | None = None, *, allow_drift: bool = False +) -> None: """Fail fast if the current venv's omnimarket is missing or has drifted from the canonical local clone. - Fails OPEN (returns silently) only when the canonical local clone cannot - be determined -- see the module docstring for why. Never performs - network I/O. + Refusal is the DEFAULT and is never silently skipped. Two ways past it, + both deliberate: + + * Fails OPEN (returns silently) when the canonical local clone cannot be + determined -- see the module docstring for why. + * Downgrades to a loud WARNING when ``allow_drift`` is True -- the + operator's explicit opt-out, bound at the CLI boundary to + ``ONEX_ALLOW_OMNIMARKET_DRIFT`` (:data:`DRIFT_OVERRIDE_ENV`, + OMN-13930). Every refusal message names that variable, so the escape + hatch is discoverable from the failure itself rather than requiring a + source read. Before it existed the only workaround was unsetting + ``$OMNI_HOME``, which disables the guard globally and SILENTLY -- + strictly worse than a named, logged override. + + Never performs network I/O. + + Args: + omni_home: Canonical workspace root to resolve the reference clone + from. ``None`` (no ``$OMNI_HOME``) means "cannot determine" and + fails open. + allow_drift: Explicit operator opt-out. Keyword-only and defaulting + to False so refusal stays the default at EVERY call site, + including ones added later -- a forgotten argument fails closed. Raises: - OmnimarketDriftError: a canonical clone IS present locally, and - either (a) omnimarket is not installed from git in the current - interpreter at all (absent, or a non-VCS/PyPI install), or (b) - its installed commit does not match the canonical local clone's - HEAD commit. + OmnimarketDriftError: a canonical clone IS present locally, + ``allow_drift`` is False, and either (a) omnimarket is not + installed from git in the current interpreter at all (absent, or + a non-VCS/PyPI install), or (b) its installed commit does not + match the canonical local clone's HEAD commit. """ canonical = canonical_local_omnimarket_commit(omni_home=omni_home) if canonical is None: return installed = installed_omnimarket_commit() + if installed == canonical: + return + if installed is None: - raise OmnimarketDriftError( + detail = ( "omnimarket is NOT INSTALLED from git in this interpreter " "(absent, or installed from PyPI/a non-VCS source), but a " f"canonical clone exists at $OMNI_HOME/omnimarket (HEAD " - f"{canonical[:12]}). 'onex skill'/'onex run' dispatch for " - "market-provided nodes (e.g. node_aislop_sweep) will fail with " - "'Unknown node'. Repair with: " + f"{canonical[:12]}). 'onex skill'/'onex run'/'onex delegate' " + "dispatch for market-provided nodes (e.g. node_aislop_sweep) " + "will fail with 'Unknown node'. Repair with: " "scripts/install-node-skill-package.sh --execute (or " - "scripts/check-omnimarket-venv-drift.sh --repair). " - "See docs/runbooks/node-skill-package-install.md." + "scripts/check-omnimarket-venv-drift.sh --repair)." ) - if installed != canonical: - raise OmnimarketDriftError( + else: + detail = ( f"omnimarket venv is STALE: installed commit {installed[:12]} != " f"canonical $OMNI_HOME/omnimarket HEAD {canonical[:12]}. Repair with: " "scripts/check-omnimarket-venv-drift.sh --repair (or re-run " - "scripts/install-node-skill-package.sh --execute directly). " - "See docs/runbooks/node-skill-package-install.md." + "scripts/install-node-skill-package.sh --execute directly)." ) + + if allow_drift: + # Loud on every dispatch, by design: a silent bypass would recreate + # the invisible-drift failure this guard exists to end. + logger.warning( + "%s DISPATCHING ANYWAY because %s is set -- results from " + "market-provided nodes come from an UNVERIFIED omnimarket build " + "and must not be treated as evidence.", + detail, + DRIFT_OVERRIDE_ENV, + ) + return + + raise OmnimarketDriftError( + f"{detail} To dispatch anyway despite the drift (results are NOT " + f"evidence), set {DRIFT_OVERRIDE_ENV}=1. " + "See docs/runbooks/node-skill-package-install.md." + ) diff --git a/src/omnibase_infra/configs/routing_tiers.yaml b/src/omnibase_infra/configs/routing_tiers.yaml deleted file mode 100644 index 52517b3888..0000000000 --- a/src/omnibase_infra/configs/routing_tiers.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. -# SPDX-License-Identifier: MIT -# Copyright (c) 2026 OmniNode Team -# -# routing_tiers.yaml — Declarative delegation escalation ladder. -# -# Loaded by node_delegation_routing_reducer at startup. -# To change tier order, model assignments, or retry counts, edit this file only. -# No handler changes are needed for operational tuning. -# -# Tier evaluation order: local → cheap_cloud → claude -# The handler iterates tiers top-to-bottom, stopping at the first accepted result. -# -# Endpoint URLs are resolved from the deployed bifrost contract at -# ~/.omninode/delegation/bifrost_delegation.yaml. The backend_id field maps -# each model to a backend entry in the bifrost contract. See OMN-10657. -# -# Related Tickets: -# - OMN-8029: Delegation pipeline — local→cheap-cloud→claude routing -# - OMN-10657: Endpoint resolution from deployed contract, not env vars -tiers: - - name: local - models: - - id: qwen3-coder-30b - backend_id: local-qwen-coder-30b - max_context_tokens: 65536 - use_for: - - code_generation - - code_review - - refactor - - test - - research - - id: deepseek-r1-14b - backend_id: local-deepseek-r1-14b - max_context_tokens: 24576 - use_for: - - test - - research - - reasoning - - planning - - review - - document - fast_path_threshold_tokens: 24576 - eval_before_accept: true - eval_model: deepseek-r1-14b - max_retries: 2 - - name: cheap_cloud - models: - - id: glm-z-ai - backend_id: cloud-glm - max_context_tokens: 128000 - use_for: - - code_generation - - reasoning - - research - - id: gemini-flash - backend_id: cloud-gemini-flash - max_context_tokens: 1000000 - use_for: - - summarization - - simple_tasks - - document - eval_before_accept: true - eval_model: deepseek-r1-14b - max_retries: 1 - - name: claude - models: - - id: claude-sonnet-4-6 - backend_id: cloud-sonnet - max_context_tokens: 200000 - use_for: - - escalation - - complex_reasoning - - agent_orchestration - - code_generation - - reasoning - - planning - - test - - document - - research - eval_before_accept: false - max_retries: 0 - # CLI agent delegation tier (OMN-10137) - # Models dispatched via subprocess rather than HTTP API. - # Availability confirmed at dispatch time via shutil.which inside HandlerLlmCliSubprocess. - # - # OMN-13215: the shelled ``codex-cli`` model was REMOVED from this tier. The - # delegation ceiling executes over the canonical HTTP path (claude tier -> - # cloud-sonnet) — there is no codex subprocess inference path anywhere. - - name: cli_agents - models: - - id: claude-cli - backend_id: cli-claude - max_context_tokens: 200000 - use_for: - - agent_delegation - - complex_reasoning - - id: opencode-cli - backend_id: cli-opencode - max_context_tokens: 200000 - use_for: - - agent_delegation - - code_generation - eval_before_accept: false - max_retries: 0 diff --git a/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py b/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py index acd87f25da..ff9ca4d5aa 100644 --- a/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py +++ b/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py @@ -25,8 +25,9 @@ class EnumOmnibaseInfraTopic(str, Enum): CMD_CODING_AGENT_INVOKE_V1 = "onex.cmd.omnibase-infra.coding-agent-invoke.v1" # onex.cmd.omnibase-infra.coding-agent-invoke.v1 CMD_CODING_AGENT_WORKSPACE_VALIDATE_V1 = "onex.cmd.omnibase-infra.coding-agent-workspace-validate.v1" # onex.cmd.omnibase-infra.coding-agent-workspace-validate.v1 CMD_CONSUMER_RESTART_V1 = "onex.cmd.omnibase-infra.consumer-restart.v1" # onex.cmd.omnibase-infra.consumer-restart.v1 - CMD_DELEGATION_INFERENCE_REQUEST_V1 = "onex.cmd.omnibase-infra.delegation-inference-request.v1" # onex.cmd.omnibase-infra.delegation-inference-request.v1 - CMD_DELEGATION_REQUEST_V1 = "onex.cmd.omnibase-infra.delegation-request.v1" # onex.cmd.omnibase-infra.delegation-request.v1 + CMD_GATEWAY_ATTACH_REQUEST_V1 = "onex.cmd.omnibase-infra.gateway-attach-request.v1" # onex.cmd.omnibase-infra.gateway-attach-request.v1 + CMD_GATEWAY_DETACH_REQUEST_V1 = "onex.cmd.omnibase-infra.gateway-detach-request.v1" # onex.cmd.omnibase-infra.gateway-detach-request.v1 + CMD_GATEWAY_HEARTBEAT_REQUEST_V1 = "onex.cmd.omnibase-infra.gateway-heartbeat-request.v1" # onex.cmd.omnibase-infra.gateway-heartbeat-request.v1 CMD_LLM_COMPLETION_REQUEST_V1 = "onex.cmd.omnibase-infra.llm-completion-request.v1" # onex.cmd.omnibase-infra.llm-completion-request.v1 CMD_LLM_EMBEDDING_REQUEST_V1 = "onex.cmd.omnibase-infra.llm-embedding-request.v1" # onex.cmd.omnibase-infra.llm-embedding-request.v1 CMD_LLM_INFERENCE_REQUEST_V1 = "onex.cmd.omnibase-infra.llm-inference-request.v1" # onex.cmd.omnibase-infra.llm-inference-request.v1 @@ -68,7 +69,7 @@ class EnumOmnibaseInfraTopic(str, Enum): EVT_CONSUMER_HEALTH_V1 = "onex.evt.omnibase-infra.consumer-health.v1" # onex.evt.omnibase-infra.consumer-health.v1 EVT_DB_ERROR_V1 = "onex.evt.omnibase-infra.db-error.v1" # onex.evt.omnibase-infra.db-error.v1 EVT_EVENT_FORWARDED_V1 = "onex.evt.omnibase-infra.event-forwarded.v1" # onex.evt.omnibase-infra.event-forwarded.v1 - EVT_GATEWAY_HEARTBEAT_V1 = "onex.evt.omnibase-infra.gateway-heartbeat.v1" # onex.evt.omnibase-infra.gateway-heartbeat.v1 + EVT_GATEWAY_SESSION_V1 = "onex.evt.omnibase-infra.gateway-session.v1" # onex.evt.omnibase-infra.gateway-session.v1 EVT_GMAIL_INTENT_RECEIVED_V1 = "onex.evt.omnibase-infra.gmail-intent-received.v1" # onex.evt.omnibase-infra.gmail-intent-received.v1 EVT_INFERENCE_RESPONSE_V1 = "onex.evt.omnibase-infra.inference-response.v1" # onex.evt.omnibase-infra.inference-response.v1 EVT_LLM_CALL_COMPLETED_V1 = "onex.evt.omnibase-infra.llm-call-completed.v1" # onex.evt.omnibase-infra.llm-call-completed.v1 diff --git a/src/omnibase_infra/errors/__init__.py b/src/omnibase_infra/errors/__init__.py index bf90e00f02..6b52e53f88 100644 --- a/src/omnibase_infra/errors/__init__.py +++ b/src/omnibase_infra/errors/__init__.py @@ -40,6 +40,7 @@ EventRegistryFingerprintMismatchError: Live event registry fingerprint != expected EventRegistryFingerprintMissingError: Event registry artifact file not found ProjectionError: Raised by NodeProjectionEffect when a synchronous projection write fails + ProjectionTenantContextError: Projection write refused — no tenant resolved under enforcement Correlation ID Assignment: All infrastructure errors support correlation_id for distributed tracing. @@ -147,11 +148,17 @@ from omnibase_infra.errors.error_message_type_registry import MessageTypeRegistryError from omnibase_infra.errors.error_payload_registry import PayloadRegistryError from omnibase_infra.errors.error_policy_registry import PolicyRegistryError -from omnibase_infra.errors.error_projection import ProjectionError +from omnibase_infra.errors.error_projection import ( + ProjectionError, + ProjectionTenantContextError, +) from omnibase_infra.errors.error_schema_fingerprint import ( SchemaFingerprintMismatchError, SchemaFingerprintMissingError, ) +from omnibase_infra.errors.error_topic_provisioning import ( + TopicReplicationPolicyError, +) from omnibase_infra.errors.repository import ( RepositoryContractError, RepositoryError, @@ -171,6 +178,7 @@ "ArchitectureViolationError", # Projection errors (OMN-2510) "ProjectionError", + "ProjectionTenantContextError", # Binding resolution errors "BindingResolutionError", "ChainPropagationError", @@ -225,6 +233,8 @@ "ServiceRegistrationError", "ServiceRegistryUnavailableError", "ServiceResolutionError", + # Topic provisioning policy errors (OMN-15395) + "TopicReplicationPolicyError", "UnknownHandlerTypeError", # Error catalog lookup (OMN-518) "get_resolution", diff --git a/src/omnibase_infra/errors/error_projection.py b/src/omnibase_infra/errors/error_projection.py index 2a064bed59..fc17e0b685 100644 --- a/src/omnibase_infra/errors/error_projection.py +++ b/src/omnibase_infra/errors/error_projection.py @@ -98,4 +98,20 @@ def __init__( self.projection_type = projection_type -__all__ = ["ProjectionError"] +class ProjectionTenantContextError(ProjectionError): + """Raised when a tenant projection has no valid authenticated authority. + + OMN-15421. Tenant-scoped projection tables compare their UUID tenant key + with the transaction-local ``app.tenant_id`` setting. The adapter accepts + only an opaque capability minted after canonical signed-envelope verification + and an authoritative signer-to-tenant binding check. Ordinary security-context + fields, gateway metadata, request/payload values, environment values, empty + strings, slugs, and shared sentinels are never authority or fallbacks. + + Distinct from the generic :class:`ProjectionError` so callers and operators + can tell a tenant-attribution refusal apart from a connection or schema + failure; the two have completely different remediations. + """ + + +__all__ = ["ProjectionError", "ProjectionTenantContextError"] diff --git a/src/omnibase_infra/errors/error_topic_provisioning.py b/src/omnibase_infra/errors/error_topic_provisioning.py new file mode 100644 index 0000000000..247c547801 --- /dev/null +++ b/src/omnibase_infra/errors/error_topic_provisioning.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Topic-provisioning policy errors (OMN-15395). + +``TopicReplicationPolicyError`` is raised by +:class:`~omnibase_infra.topics.model_topic_provisioning_policy.ModelTopicProvisioningPolicy` +when a topic spec cannot be provisioned under the resolved environment policy — +either because the owning contract declares no replication factor in an +environment that forbids implicit defaults, or because the resolved replication +factor is below the environment's durability floor (RF1 against managed +staging / MSK). + +It is a distinct class (not a bare ``ProtocolConfigurationError``) so the +provisioning call sites can re-raise it past their best-effort ``except +Exception`` boundaries without also re-raising unrelated configuration errors. +A durability violation must never degrade into a warning-and-continue. +""" + +from __future__ import annotations + +from omnibase_core.enums import EnumCoreErrorCode +from omnibase_infra.errors.error_infra import RuntimeHostError +from omnibase_infra.models.errors.model_infra_error_context import ( + ModelInfraErrorContext, +) + + +class TopicReplicationPolicyError(RuntimeHostError): + """Raised when a topic spec violates the environment's replication policy. + + Example: + >>> from omnibase_infra.enums import EnumInfraTransportType + >>> context = ModelInfraErrorContext.with_correlation( + ... transport_type=EnumInfraTransportType.KAFKA, + ... operation="resolve_topic_spec", + ... ) + >>> raise TopicReplicationPolicyError( # doctest: +SKIP + ... "replication_factor=1 is rejected in managed staging", + ... context=context, + ... ) + """ + + def __init__( + self, + message: str, + context: ModelInfraErrorContext | None = None, + **extra_context: object, + ) -> None: + """Initialize with the INVALID_CONFIGURATION error code.""" + super().__init__( + message=message, + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + context=context, + **extra_context, + ) + + +__all__: list[str] = ["TopicReplicationPolicyError"] diff --git a/src/omnibase_infra/event_bus/event_bus_kafka.py b/src/omnibase_infra/event_bus/event_bus_kafka.py index f3432e959d..fca6902a3c 100644 --- a/src/omnibase_infra/event_bus/event_bus_kafka.py +++ b/src/omnibase_infra/event_bus/event_bus_kafka.py @@ -206,6 +206,7 @@ async def handler(msg): KafkaError, UnknownTopicOrPartitionError, ) +from aiokafka.structs import TopicPartition from omnibase_infra.enums import EnumConsumerGroupPurpose, EnumInfraTransportType from omnibase_infra.errors import ( @@ -223,7 +224,11 @@ async def handler(msg): build_aiokafka_auth_kwargs, ) from omnibase_infra.event_bus.mixin_kafka_broadcast import MixinKafkaBroadcast -from omnibase_infra.event_bus.mixin_kafka_dlq import MixinKafkaDlq +from omnibase_infra.event_bus.mixin_kafka_dlq import ( + _REPLAY_COUNT_HEADER, + _REPLAY_COUNT_PARSE_FAILURE_SENTINEL, + MixinKafkaDlq, +) from omnibase_infra.event_bus.models import ( ModelEventBusReadiness, ModelEventHeaders, @@ -253,6 +258,11 @@ async def handler(msg): logger = logging.getLogger(__name__) +# OMN-15232: pause between fail-closed rewinds so a persistently unreachable DLQ +# does not turn the consume loop into a hot spin over the same offset. The +# rewind itself is what keeps the data safe; this only bounds the retry rate. +DLQ_UNPERSISTED_REWIND_BACKOFF_SECONDS: float = 1.0 + class EventBusKafka( MixinKafkaBroadcast, @@ -1309,6 +1319,7 @@ async def subscribe( group_id: str | None = None, purpose: EnumConsumerGroupPurpose = EnumConsumerGroupPurpose.CONSUME, required_for_readiness: bool = False, + auto_offset_reset: str | None = None, ) -> Callable[[], Awaitable[None]]: """Subscribe to topic with callback handler. @@ -1338,6 +1349,21 @@ async def subscribe( required_for_readiness: Whether this subscription must have active partition assignments for the runtime to report as ready via ``/ready``. Defaults to False (does not block readiness). + auto_offset_reset: Optional per-subscription override of this + consumer's ``auto_offset_reset`` policy ("earliest"/"latest"). + When ``None`` (the default, unchanged behavior), the bus-level + ``self._config.auto_offset_reset`` applies, as before this + parameter existed. Each ``(topic, group_id)`` pair already + gets its own dedicated ``AIOKafkaConsumer`` + (``_start_consumer_for_topic_unlocked``), so this override is + scoped to exactly the one consumer this call creates -- it + does not affect any other subscription's offset-reset + behavior. Added for OMN-15789 so the + ``event_bus_substrate`` fixture's ``real_broker`` leg can + exercise the same per-call ``auto_offset_reset`` surface + ``EventBusSemanticFake`` (omnibase_core) already has, instead + of needing a separate ``EventBusKafka`` instance per offset + policy under test. Returns: Async unsubscribe function to remove this subscription @@ -1420,7 +1446,9 @@ async def handler(msg): # calls during cold start. _pending_consumer_keys (set above under # the lock) guards against duplicate starts across concurrent callers. if need_consumer_start: - await self._start_consumer_for_topic_unlocked(topic, effective_group_id) + await self._start_consumer_for_topic_unlocked( + topic, effective_group_id, auto_offset_reset_override=auto_offset_reset + ) async with self._lock: logger.debug( @@ -1664,7 +1692,11 @@ def _resolve_group_instance_id(self, effective_group_id: str) -> str: return f"{effective_group_id[:prefix_budget]}-{host_hash}" async def _start_consumer_for_topic_unlocked( - self, topic: str, group_id: str + self, + topic: str, + group_id: str, + *, + auto_offset_reset_override: str | None = None, ) -> None: """Start a Kafka consumer without holding the shared lock. @@ -1690,6 +1722,11 @@ async def _start_consumer_for_topic_unlocked( group ID (e.g. ``"my-group.__t.events"`` with ``topic="events"``). + auto_offset_reset_override: Optional per-consumer override of + ``auto_offset_reset``. ``None`` (the default) preserves the + pre-OMN-15789 behavior of always using + ``self._config.auto_offset_reset``. + Raises: ProtocolConfigurationError: If group_id is empty or contains only whitespace (must be derived from compute_consumer_group_id or @@ -1706,6 +1743,11 @@ async def _start_consumer_for_topic_unlocked( group_id, topic, correlation_id, consumer_key ) resolved_group_instance_id = self._resolve_group_instance_id(effective_group_id) + resolved_auto_offset_reset = ( + auto_offset_reset_override + if auto_offset_reset_override is not None + else self._config.auto_offset_reset + ) # Apply consumer configuration from config model consumer = AIOKafkaConsumer( @@ -1713,7 +1755,7 @@ async def _start_consumer_for_topic_unlocked( bootstrap_servers=self._bootstrap_servers, group_id=effective_group_id, group_instance_id=resolved_group_instance_id, - auto_offset_reset=self._config.auto_offset_reset, + auto_offset_reset=resolved_auto_offset_reset, enable_auto_commit=self._config.enable_auto_commit, session_timeout_ms=self._config.session_timeout_ms, heartbeat_interval_ms=self._config.heartbeat_interval_ms, @@ -1840,7 +1882,7 @@ async def _start_consumer_for_topic_unlocked( bootstrap_servers=self._bootstrap_servers, group_id=effective_group_id, group_instance_id=resolved_group_instance_id, - auto_offset_reset=self._config.auto_offset_reset, + auto_offset_reset=resolved_auto_offset_reset, enable_auto_commit=self._config.enable_auto_commit, session_timeout_ms=self._config.session_timeout_ms, heartbeat_interval_ms=self._config.heartbeat_interval_ms, @@ -2047,8 +2089,19 @@ async def _dispatch_to_subscriber( topic: str, group_id: str, correlation_id: UUID, - ) -> None: - """Invoke a single subscriber callback, routing to DLQ on exhausted retries.""" + ) -> bool: + """Invoke a single subscriber callback, routing to DLQ on exhausted retries. + + Returns: + ``True`` when it is safe for the partition offset to advance past + this message — the callback succeeded, retries remain (so the + message is still live), or the DLQ write for an exhausted message + was confirmed durable. ``False`` when retries were exhausted AND + the DLQ write was NOT confirmed: the message then exists nowhere + durable, so the caller must rewind rather than let the offset move + (OMN-15232, same discipline as the OMN-14936 gate in + ``runtime/event_bus_subcontract_wiring.py``). + """ try: await callback(event_message) except Exception as e: @@ -2072,13 +2125,42 @@ async def _dispatch_to_subscriber( ) if retries_exhausted: - await self._publish_to_dlq( + dlq_result = await self._publish_to_dlq( original_topic=topic, failed_message=event_message, error=e, correlation_id=correlation_id, consumer_group=group_id, ) + # OMN-15232: only an explicit ``False`` counts as a confirmed + # non-persist -- duck-typed hosts/test doubles that still + # return ``None`` keep their prior behavior, matching the + # allowance the OMN-14936 gate makes in + # runtime/event_bus_subcontract_wiring.py. + dlq_persisted = dlq_result is not False + if not dlq_persisted: + # TRY400 suppressed below: the handler traceback was + # already emitted by the logger.exception above; this line + # reports the DLQ persistence outcome, not a second copy of + # that stack. + logger.error( # noqa: TRY400 + "dlq_publish_not_persisted topic=%s subscription_id=%s " + "correlation_id=%s error_type=%s -- retries exhausted and " + "the DLQ write was NOT confirmed; offset will be rewound " + "instead of advancing (OMN-15232)", + topic, + subscription_id, + str(correlation_id), + type(e).__name__, + extra={ + "topic": topic, + "group_id": group_id, + "subscription_id": subscription_id, + "correlation_id": str(correlation_id), + "error_type": type(e).__name__, + }, + ) + return dlq_persisted else: logger.warning( f"Handler failed but retries available ({retry_count}/{max_retries})", @@ -2090,6 +2172,110 @@ async def _dispatch_to_subscriber( }, ) + return True + + async def _rewind_after_unpersisted_dlq( + self, + consumer: AIOKafkaConsumer, + msg: object, + topic: str, + group_id: str, + correlation_id: UUID, + failure_stage: str, + ) -> None: + """Withhold offset advancement after an unconfirmed DLQ write (OMN-15232). + + The consumers this class builds run with + ``enable_auto_commit=self._config.enable_auto_commit``, which defaults + to ``True``: the client commits the *fetch position* on its own cadence, + and this loop never calls ``commit`` itself. So — unlike the manual-commit + path in ``runtime/event_bus_subcontract_wiring.py``, where the OMN-14936 + gate simply returns without committing — merely declining to commit here + does nothing. The offset advances anyway and the message is lost. + + The fail-closed action that works under BOTH commit models is a rewind of + the fetch position to the failed message's own offset + (``consumer.seek(tp, msg.offset)``) — the identical "does NOT advance the + committed offset" mechanism ``KafkaTransport.nack`` uses in + ``event_bus/kafka_transport.py``. Under auto-commit the committer then + commits the rewound position, which cannot be past the message; under + manual commit the message is simply refetched. Either way Kafka + redelivers and the DLQ write is retried instead of the record being + silently dropped. + + Only the failed message's own partition is rewound; sibling partitions + are untouched (same per-partition discipline as OMN-14757). + """ + msg_topic = getattr(msg, "topic", None) or topic + partition = getattr(msg, "partition", None) + offset = getattr(msg, "offset", None) + + if partition is None or offset is None: + logger.error( + "dlq_unpersisted_rewind_impossible topic=%s stage=%s " + "correlation_id=%s -- message carries no partition/offset " + "coordinate, so offset advancement cannot be withheld; this " + "record may be lost (OMN-15232)", + topic, + failure_stage, + str(correlation_id), + extra={ + "topic": topic, + "group_id": group_id, + "correlation_id": str(correlation_id), + "failure_stage": failure_stage, + }, + ) + return + + try: + consumer.seek(TopicPartition(msg_topic, int(partition)), int(offset)) + except Exception as seek_error: + logger.exception( + "dlq_unpersisted_rewind_failed topic=%s partition=%s offset=%s " + "stage=%s correlation_id=%s error=%s -- could NOT withhold offset " + "advancement after an unconfirmed DLQ write; this record may be " + "lost (OMN-15232)", + msg_topic, + partition, + offset, + failure_stage, + str(correlation_id), + str(seek_error), + extra={ + "topic": msg_topic, + "group_id": group_id, + "partition": partition, + "offset": offset, + "failure_stage": failure_stage, + "correlation_id": str(correlation_id), + }, + ) + return + + logger.error( + "dlq_unpersisted_offset_rewound topic=%s partition=%s offset=%s " + "stage=%s correlation_id=%s -- DLQ persistence was NOT confirmed; " + "fetch position rewound so the offset cannot advance past an " + "undelivered DLQ write (OMN-15232)", + msg_topic, + partition, + offset, + failure_stage, + str(correlation_id), + extra={ + "topic": msg_topic, + "group_id": group_id, + "partition": partition, + "offset": offset, + "failure_stage": failure_stage, + "correlation_id": str(correlation_id), + }, + ) + + if DLQ_UNPERSISTED_REWIND_BACKOFF_SECONDS > 0: + await asyncio.sleep(DLQ_UNPERSISTED_REWIND_BACKOFF_SECONDS) + async def _emit_consume_loop_error_health_event( self, topic: str, @@ -2230,7 +2416,7 @@ async def _consume_loop( ) # Deserialization errors are permanent failures - route to DLQ # Create minimal message from raw Kafka data for DLQ context - await self._publish_raw_to_dlq( + dlq_result = await self._publish_raw_to_dlq( original_topic=topic, raw_msg=msg, error=e, @@ -2238,11 +2424,29 @@ async def _consume_loop( failure_type="deserialization_error", consumer_group=effective_consumer_group, ) + # OMN-15232: an undeserializable message that did NOT reach + # the DLQ exists nowhere durable. Skipping it here while the + # client auto-commits the position is a silent, committed, + # unrecoverable drop -- the OMN-14936 failure mode at a call + # site that fix did not cover. Rewind instead so Kafka + # redelivers and the DLQ write is retried. Only an explicit + # ``False`` counts as a confirmed non-persist (duck-typed + # hosts returning ``None`` keep prior behavior). + if dlq_result is False: + await self._rewind_after_unpersisted_dlq( + consumer, + msg, + topic, + group_id, + correlation_id, + "deserialization_error", + ) continue # Skip this message but continue consuming # Dispatch to all subscribers + offset_may_advance = True for _sub_group_id, subscription_id, callback in subscribers: - await self._dispatch_to_subscriber( + dispatch_offset_safe = await self._dispatch_to_subscriber( callback, subscription_id, event_message, @@ -2250,6 +2454,25 @@ async def _consume_loop( group_id, correlation_id, ) + # OMN-15232: same gate on the dispatch path. Every subscriber + # still gets the message (one failing subscriber must not + # starve the others), but if ANY of them exhausted retries + # without a confirmed DLQ record, the offset must not move. + # Redelivery may duplicate for the subscribers that + # succeeded -- at-least-once, which is the delivery contract + # here, and strictly preferable to losing the record. + if dispatch_offset_safe is False: + offset_may_advance = False + + if not offset_may_advance: + await self._rewind_after_unpersisted_dlq( + consumer, + msg, + topic, + group_id, + correlation_id, + "handler_retries_exhausted", + ) except asyncio.CancelledError: # Graceful cancellation - this is expected during shutdown @@ -2597,6 +2820,7 @@ async def _enforce_onex_topic_format( result, reason = validate_onex_topic_format(topic) if result in ( TopicValidationResult.VALID, + TopicValidationResult.VALID_TENANT_WIRE, TopicValidationResult.VALID_LEGACY_DLQ, TopicValidationResult.SKIPPED_INTERNAL, ): @@ -2751,6 +2975,42 @@ def _kafka_headers_to_model( retry_count = self._parse_int_header( headers_dict.get("retry_count"), 0, "retry_count" ) + + # OMN-14551: a message republished by node_dlq_replay_effect carries + # x-replay-count instead of retry_count (see mixin_kafka_dlq.py's + # _REPLAY_COUNT_HEADER doc for the producer/reader contract). When + # present it is the authoritative replay lineage counter and wins + # over any stale "retry_count" header -- the replay engine never + # stamps "retry_count", so its presence here is unrelated/stale data. + # Every ModelEventMessage built from this method flows into + # _publish_raw_to_dlq (handler_exception / subcontract-wiring call + # sites) and _publish_to_dlq (typed path, reads + # failed_message.headers.retry_count directly) -- if this conversion + # drops the header, both paths reset replay lineage to 0 on every + # republish and should_replay's guard never trips (the 2026-08-05 + # dev DLQ amplification incident). + replay_count_str = headers_dict.get(_REPLAY_COUNT_HEADER) + if replay_count_str is not None: + try: + parsed_replay_count = int(replay_count_str) + except (ValueError, TypeError): + logger.warning( + "Malformed %s header %r, failing closed to stop replay", + _REPLAY_COUNT_HEADER, + replay_count_str, + ) + retry_count = _REPLAY_COUNT_PARSE_FAILURE_SENTINEL + else: + if parsed_replay_count < 0: + logger.warning( + "Negative %s header value %d, failing closed to stop replay", + _REPLAY_COUNT_HEADER, + parsed_replay_count, + ) + retry_count = _REPLAY_COUNT_PARSE_FAILURE_SENTINEL + else: + retry_count = parsed_replay_count + max_retries = self._parse_int_header( headers_dict.get("max_retries"), 3, "max_retries" ) diff --git a/src/omnibase_infra/event_bus/kafka_auth.py b/src/omnibase_infra/event_bus/kafka_auth.py index 56027a060d..5e9059f5f3 100644 --- a/src/omnibase_infra/event_bus/kafka_auth.py +++ b/src/omnibase_infra/event_bus/kafka_auth.py @@ -135,7 +135,7 @@ def build_aiokafka_auth_kwargs(config: ModelKafkaEventBusConfig) -> dict[str, ob kwargs["sasl_mechanism"] = "OAUTHBEARER" kwargs["sasl_oauth_token_provider"] = MSKTokenProvider(region=config.msk_region) - if config.ssl_ca_file is not None: + if config.security_protocol in ("SSL", "SASL_SSL"): kwargs["ssl_context"] = ssl.create_default_context(cafile=config.ssl_ca_file) return kwargs diff --git a/src/omnibase_infra/event_bus/kafka_transport.py b/src/omnibase_infra/event_bus/kafka_transport.py index fda22c4075..89235ce61a 100644 --- a/src/omnibase_infra/event_bus/kafka_transport.py +++ b/src/omnibase_infra/event_bus/kafka_transport.py @@ -3,10 +3,11 @@ """Kafka face for the unified-runtime transport protocols (epic OMN-14717, S3). -Net-new, currently UNUSED in production (ticket OMN-14756; see +Introduced for the unified-runtime transport seam (ticket OMN-14756; see ``docs/plans/2026-07-17-single-runtime-transport-di-unification-plan.md`` sections -(c)/(d), step S3). This is the concrete Kafka implementation of the two core -transport protocols +(c)/(d), step S3) and used by the production hybrid gateway runtime for explicit +cross-broker acknowledgement. This is the concrete Kafka implementation of the two +core transport protocols * ``omnibase_core.protocols.runtime.protocol_transport_consumer.ProtocolTransportConsumer`` * ``omnibase_core.protocols.runtime.protocol_transport_producer.ProtocolTransportProducer`` @@ -169,43 +170,91 @@ async def start(self) -> None: await self._producer.start() if self._topics: - # Group ``subscribe`` (topics passed positionally): the consumer joins - # the group and, on join, NATIVELY resumes each partition from its - # group-committed offset (a fresh group starts at ``auto_offset_reset``). - # This is what makes "restart resumes from the committed offset" and - # "uncommitted offsets redeliver on restart" hold, with no manual - # ``seek`` dance. A single transport instance is the sole group member, - # so it is assigned every partition of its topics — matching the unified - # runtime's single-poll-loop-per-topic-set model (single-owner-per-topic - # is the S6 boot invariant, R1). The one-time join latency is absorbed by - # ``_prime`` below so the runtime's first ``poll`` still returns promptly. - self._consumer = AIOKafkaConsumer( - *self._topics, - bootstrap_servers=self._config.bootstrap_servers, - group_id=self._group, - # FORCED for the new transport consumers (plan S3): the runtime, not - # the client, decides when an offset is durable. Legacy push - # consumers keep their per-consumer config setting untouched. - enable_auto_commit=False, - auto_offset_reset=self._auto_offset_reset, - session_timeout_ms=self._config.session_timeout_ms, - heartbeat_interval_ms=self._config.heartbeat_interval_ms, - max_poll_interval_ms=self._config.max_poll_interval_ms, - retry_backoff_ms=self._config.reconnect_backoff_ms, - **self._client_version_kwargs(AIOKafkaConsumer), - **self._auth_kwargs(), - ) - await self._consumer.start() - # Trigger the group join + first fetch now, buffering the first batch, so - # the runtime's first poll() returns the available records instead of - # racing the lazy rebalance. - await self._prime(self._consumer) + await self._start_consumer() except BaseException: await self.close() raise self._started = True + async def _start_consumer(self) -> None: + """Construct, start, and prime a fresh consumer bound to ``self._topics``. + + Factored out of ``start()`` so ``restart_consumer`` can rejoin the group + from zero (a brand-new client instance -- the only way to recover from a + silent ``max_poll_interval_ms`` idle-eviction, see + ``has_group_membership``) without touching ``self._producer``. A single + ``KafkaTransport`` instance can back both one gateway direction's + consumer AND the other direction's (plus status/heartbeat) producer, so + a watchdog-triggered consumer recreate (``NodeGatewayDelivery``, + OMN-15748/OMN-15690) must never go through ``close()`` + ``start()``, + which stops both clients. + """ + # Group ``subscribe`` (topics passed positionally): the consumer joins + # the group and, on join, NATIVELY resumes each partition from its + # group-committed offset (a fresh group starts at ``auto_offset_reset``). + # This is what makes "restart resumes from the committed offset" and + # "uncommitted offsets redeliver on restart" hold, with no manual + # ``seek`` dance. A single transport instance is the sole group member, + # so it is assigned every partition of its topics — matching the unified + # runtime's single-poll-loop-per-topic-set model (single-owner-per-topic + # is the S6 boot invariant, R1). The one-time join latency is absorbed by + # ``_prime`` below so the runtime's first ``poll`` still returns promptly. + self._consumer = AIOKafkaConsumer( + *self._topics, + bootstrap_servers=self._config.bootstrap_servers, + group_id=self._group, + # FORCED for the new transport consumers (plan S3): the runtime, not + # the client, decides when an offset is durable. Legacy push + # consumers keep their per-consumer config setting untouched. + enable_auto_commit=False, + auto_offset_reset=self._auto_offset_reset, + session_timeout_ms=self._config.session_timeout_ms, + heartbeat_interval_ms=self._config.heartbeat_interval_ms, + max_poll_interval_ms=self._config.max_poll_interval_ms, + retry_backoff_ms=self._config.reconnect_backoff_ms, + **self._client_version_kwargs(AIOKafkaConsumer), + **self._auth_kwargs(), + ) + await self._consumer.start() + # Trigger the group join + first fetch now, buffering the first batch, so + # the runtime's first poll() returns the available records instead of + # racing the lazy rebalance. + await self._prime(self._consumer) + + async def restart_consumer(self) -> None: + """Recreate ONLY the consumer-side client; the shared producer stays up. + + A gateway direction's membership-loss watchdog recovery + (``NodeGatewayDelivery._recover_stalled_direction``, + OMN-15748/OMN-15690) must rejoin the group from a fresh client without + killing the producer this same instance also serves for the OTHER + direction's forward-publish and every status/heartbeat publish -- + ``close()`` + ``start()`` stops both clients and would silently break + those for the duration of the recreate. Requires an assigned + ``topics`` set (a producer-only transport has nothing to restart). + """ + if not self._topics: + context = ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.KAFKA, + operation="restart_consumer", + target_name="kafka_transport", + ) + raise ProtocolConfigurationError( + "KafkaTransport.restart_consumer() requires topics=[...]; " + "this instance is producer-only.", + context=context, + parameter="topics", + value=list(self._topics), + ) + self._buffer.clear() + if self._consumer is not None: + try: + await self._consumer.stop() + finally: + self._consumer = None + await self._start_consumer() + async def _prime(self, consumer: AIOKafkaConsumer) -> None: """Eagerly fetch ONE batch into ``self._buffer`` after a position change. @@ -263,6 +312,25 @@ async def close(self) -> None: if first_error is not None: raise first_error + def has_group_membership(self) -> bool: + """Best-effort local check: does the consumer still hold a partition assignment? + + Not authoritative on its own -- ``assignment()`` reflects the client's + last-known state and can be momentarily stale across a rebalance. Used + as a corroborating signal for the delivery-loop staleness watchdog + (``NodeGatewayDelivery``, OMN-15748/OMN-15690): a silent + ``max_poll_interval_ms`` idle-eviction (aiokafka's client-side + ``GroupCoordinator._heartbeat_routine``) drops group membership with + zero exception raised to the caller, so the watchdog cannot rely on + task failure alone to detect it. + """ + if self._consumer is None: + return False + try: + return bool(self._consumer.assignment()) + except Exception: # noqa: BLE001 — boundary: best-effort probe, never raises + return False + async def caught_up_topics(self, topics: frozenset[str]) -> frozenset[str]: """Return the subset of ``topics`` whose consumer has caught up (LAG=0). diff --git a/src/omnibase_infra/event_bus/mixin_kafka_dlq.py b/src/omnibase_infra/event_bus/mixin_kafka_dlq.py index 7ec57c7d37..154d175039 100644 --- a/src/omnibase_infra/event_bus/mixin_kafka_dlq.py +++ b/src/omnibase_infra/event_bus/mixin_kafka_dlq.py @@ -104,6 +104,111 @@ def _resolve_dlq_topic(self, dlq_topic: str | None = None) -> str: logger = logging.getLogger(__name__) +# Header ``node_dlq_replay_effect.engine_dlq_replay.DLQProducer.replay_message`` +# stamps onto a message it republishes to its original topic. Must match that +# producer's header name verbatim -- this is the sole read side of that +# contract for the raw Kafka header shape. ``EventBusKafka._kafka_headers_to_model`` +# reads the same constant to populate ``ModelEventHeaders.retry_count`` for the +# parsed-message shape (see the isinstance branch below). +_REPLAY_COUNT_HEADER: str = "x-replay-count" + +# OMN-14551 defect 4: a malformed/negative/absent-value replay-count header +# must NOT reset replay lineage to 0 -- that would silently reopen the +# unbounded-replay amplification the header exists to bound. Any +# ``max_replay_count`` in practice (default 5) is tiny; this sentinel +# guarantees ``retry_count >= max_replay_count`` trips on the very next +# replay-eligibility check, forcing quarantine instead of blind replay. +_REPLAY_COUNT_PARSE_FAILURE_SENTINEL: int = 2**31 - 1 + + +def _extract_replay_count_from_raw_headers(raw_msg: object) -> int: + """Read the replay lineage count off a DLQ-bound message, if present. + + OMN-14551: a message replayed by ``node_dlq_replay_effect`` carries an + ``x-replay-count`` header when it lands back on its original topic. If it + fails again there, ``_publish_raw_to_dlq``/``_publish_to_dlq`` must carry + that count forward into the new DLQ record's ``retry_count`` field so + ``should_replay``'s ``retry_count >= max_replay_count`` guard is + reachable -- otherwise every re-published copy resets to 0 and the guard + can never trip (the live DLQ amplification incident this fixes). + + Two ``raw_msg.headers`` shapes are recognized, matching the two shapes + the four production ``_publish_raw_to_dlq`` call sites actually pass: + + 1. Raw Kafka ``list[tuple[str, bytes]]`` header shape (a real + ``ConsumerRecord`` or a test double of one) -- the deserialization- + error call site (``EventBusKafka._consume_loop``). Read directly off + the ``x-replay-count`` entry. + 2. A parsed ``ModelEventHeaders`` instance -- the ``handler_exception`` + call site (``handler_wiring.py``) and the five + ``event_bus_subcontract_wiring.py`` call sites, all of which pass a + ``ModelEventMessage``/``ProtocolEventMessage`` as ``raw_msg`` once the + consume loop has already deserialized it. Its ``.headers.retry_count`` + was itself already populated from ``x-replay-count`` by + ``EventBusKafka._kafka_headers_to_model`` at consume time, so it is + read directly here rather than re-parsed. + + Any other ``raw_msg.headers`` shape (missing, unrecognized type) has no + replay lineage to read and defaults to 0 -- the pre-existing + first-failure behavior. + """ + headers = getattr(raw_msg, "headers", None) + + if isinstance(headers, ModelEventHeaders): + retry_count = headers.retry_count + if isinstance(retry_count, int) and not isinstance(retry_count, bool): + if retry_count >= 0: + return retry_count + logger.warning( + "Negative ModelEventHeaders.retry_count %d on raw DLQ " + "message, failing closed to stop replay", + retry_count, + ) + return _REPLAY_COUNT_PARSE_FAILURE_SENTINEL + logger.warning( + "Invalid ModelEventHeaders.retry_count %r on raw DLQ message, " + "failing closed to stop replay", + retry_count, + ) + return _REPLAY_COUNT_PARSE_FAILURE_SENTINEL + + if not isinstance(headers, list | tuple): + return 0 + for entry in headers: + if not isinstance(entry, list | tuple) or len(entry) != 2: + continue + key, value = entry + if key != _REPLAY_COUNT_HEADER: + continue + if value is None: + logger.warning( + "Empty %s header value on raw DLQ message, failing closed " + "to stop replay", + _REPLAY_COUNT_HEADER, + ) + return _REPLAY_COUNT_PARSE_FAILURE_SENTINEL + try: + decoded = value.decode("utf-8") if isinstance(value, bytes) else str(value) + parsed = int(decoded) + except (ValueError, TypeError, UnicodeDecodeError): + logger.warning( + "Malformed %s header %r on raw DLQ message, failing closed " + "to stop replay", + _REPLAY_COUNT_HEADER, + value, + ) + return _REPLAY_COUNT_PARSE_FAILURE_SENTINEL + if parsed < 0: + logger.warning( + "Negative %s header value %d on raw DLQ message, failing " + "closed to stop replay", + _REPLAY_COUNT_HEADER, + parsed, + ) + return _REPLAY_COUNT_PARSE_FAILURE_SENTINEL + return parsed + return 0 + class MixinKafkaDlq: """Mixin providing Dead Letter Queue functionality for Kafka event bus. @@ -232,7 +337,7 @@ async def _publish_to_dlq( dlq_topic: str | None = None, failure_class: str | None = None, validation_detail: str | None = None, - ) -> None: + ) -> bool: """Publish failed message to dead letter queue with metrics and alerting. This method publishes messages that failed processing to the configured @@ -261,9 +366,21 @@ async def _publish_to_dlq( validation_detail: Optional real pydantic ``ValidationError`` detail (OMN-14492) for the ``publisher_malformed`` case. + Returns: + ``True`` if the DLQ message was actually published (producer send + acked within timeout, either on the resolved topic or the category + fallback), ``False`` otherwise (rejected input, producer + unavailable, or every send failed/timed out). Mirrors the + ``_publish_raw_to_dlq`` contract added under OMN-14936: callers + that gate offset advancement on durable DLQ persistence + (OMN-15232) MUST check this return value instead of assuming the + write succeeded just because no exception escaped this method. + Note: This method logs errors if DLQ publishing fails but does not raise - exceptions to prevent cascading failures in the consumer loop. + exceptions to prevent cascading failures in the consumer loop. The + boolean return value is the only signal of actual persistence + success. """ # Validate original_topic - reject whitespace-only values if not original_topic or not original_topic.strip(): @@ -274,7 +391,7 @@ async def _publish_to_dlq( "error_type": type(error).__name__, }, ) - return + return False # Track timing for metrics start_time = datetime.now(UTC) @@ -625,6 +742,12 @@ async def _publish_to_dlq( # Invoke DLQ callbacks for custom alerting await self._invoke_dlq_callbacks(dlq_event) + # OMN-15232: surface the real persistence outcome so callers can gate + # offset advancement on it. ``success`` is True only when a + # send_and_wait ack came back within the timeout, on either the + # resolved topic or the category fallback. + return success + async def _invoke_dlq_callbacks(self, event: ModelDlqEvent) -> None: """Invoke registered DLQ callbacks with error isolation. @@ -727,6 +850,11 @@ async def _publish_raw_to_dlq( start_time = datetime.now(UTC) error_type = type(error).__name__ + # OMN-14551: propagate the replay lineage instead of resetting it -- + # see _extract_replay_count_from_raw_headers for why this must match + # the replay engine's header name exactly. + replay_count = _extract_replay_count_from_raw_headers(raw_msg) + # Explicit bus configuration wins; otherwise use caller-provided # category routing before falling back to the default intents DLQ. resolved_dlq_topic = self._resolve_dlq_topic(dlq_topic) @@ -774,7 +902,7 @@ async def _publish_raw_to_dlq( "failure_type": failure_type, "failure_timestamp": start_time.isoformat(), "correlation_id": str(correlation_id), - "retry_count": 0, + "retry_count": replay_count, "error_type": error_type, } # OMN-14492: structured classification fields — only present when the @@ -822,7 +950,7 @@ async def _publish_raw_to_dlq( "failure_type": failure_type, "failure_timestamp": start_time.isoformat(), "correlation_id": str(correlation_id), - "retry_count": 0, + "retry_count": replay_count, "error_type": error_type, "serialization_fallback": True, } @@ -1024,7 +1152,7 @@ async def _publish_raw_to_dlq( correlation_id=correlation_id, error_type=error_type, error_message=sanitized_failure_reason, - retry_count=0, + retry_count=replay_count, message_offset=message_offset_str, message_partition=raw_partition, success=success, diff --git a/src/omnibase_infra/event_bus/model_contract_attach_result.py b/src/omnibase_infra/event_bus/model_contract_attach_result.py index 7f31d065f4..ce9ed38835 100644 --- a/src/omnibase_infra/event_bus/model_contract_attach_result.py +++ b/src/omnibase_infra/event_bus/model_contract_attach_result.py @@ -25,6 +25,9 @@ class ModelContractAttachResult(BaseModel): contract_name: The wired contract's node name. status: Whether the contract's consumer attached, was skipped as not-ready, or failed during attach. + dispatcher_ids: Exact dispatcher scope owned by the contract's + subscription callbacks. Preserved on NOT_READY results so a later + reconciliation attempt cannot fall back to process-global fan-out. topics_subscribed: Topics whose consumers were actually attached. readiness: The readiness confirm outcome for the contract's topics. detail: Human-readable detail for non-attached outcomes (no secrets). @@ -34,6 +37,7 @@ class ModelContractAttachResult(BaseModel): contract_name: str status: EnumContractAttachStatus + dispatcher_ids: tuple[str, ...] = Field(default_factory=tuple) topics_subscribed: tuple[str, ...] = Field(default_factory=tuple) readiness: ModelTopicSetReadiness | None = Field(default=None) detail: str = Field(default="") diff --git a/src/omnibase_infra/event_bus/model_runtime_attach_readiness.py b/src/omnibase_infra/event_bus/model_runtime_attach_readiness.py index fb484b032e..cd123a9bf1 100644 --- a/src/omnibase_infra/event_bus/model_runtime_attach_readiness.py +++ b/src/omnibase_infra/event_bus/model_runtime_attach_readiness.py @@ -7,6 +7,8 @@ Related Tickets: - OMN-13237: Per-contract scoped topic provisioning at runtime boot. + - OMN-15512: Ride the existing runtime-manifest event/projection so the + NOT-READY blocker set is durably queryable instead of log-only. """ from __future__ import annotations @@ -76,5 +78,31 @@ def from_results( results=results, ) + @property + def not_ready_results(self) -> tuple[ModelContractAttachResult, ...]: + """The blocker set: every contract whose consumer did NOT attach. + + Covers both ``NOT_READY`` (readiness confirm failed, attach skipped) + and ``FAILED`` (attach raised after readiness passed). Order follows + ``results``, which is boot-walk order. + """ + return tuple( + r for r in self.results if r.status is not EnumContractAttachStatus.ATTACHED + ) + + def blockers_only(self) -> ModelRuntimeAttachReadiness: + """Return a copy whose ``results`` hold ONLY the blocker set. + + Used for the published/persisted copy (OMN-15512). The counts are + preserved, so ``required_contracts - attached_contracts`` still equals + ``len(results)`` on the narrowed copy and no information is lost: + contracts that DID attach are already enumerated on the same + runtime-manifest payload (``contracts`` / ``handlers`` / the + subscribed-topic set). Re-emitting several hundred attached results + would roughly double the envelope for zero added signal, and boot walks + 475+ contracts today. + """ + return self.model_copy(update={"results": self.not_ready_results}) + __all__: list[str] = ["ModelRuntimeAttachReadiness"] diff --git a/src/omnibase_infra/event_bus/service_topic_manager.py b/src/omnibase_infra/event_bus/service_topic_manager.py index 17c5f1a554..f03c66a22a 100644 --- a/src/omnibase_infra/event_bus/service_topic_manager.py +++ b/src/omnibase_infra/event_bus/service_topic_manager.py @@ -28,6 +28,8 @@ from typing import TYPE_CHECKING from uuid import UUID, uuid4 +from omnibase_infra.enums import EnumInfraTransportType +from omnibase_infra.errors import TopicReplicationPolicyError from omnibase_infra.event_bus.enum_topic_readiness_failure_reason import ( EnumTopicReadinessFailureReason, ) @@ -44,6 +46,21 @@ from omnibase_infra.event_bus.model_topic_set_readiness import ( ModelTopicSetReadiness, ) +from omnibase_infra.models.errors.model_infra_error_context import ( + ModelInfraErrorContext, +) +from omnibase_infra.topics.broker_capacity_probe import ( + bind_policy_to_broker_capacity, + is_invalid_replication_factor_error, +) +from omnibase_infra.topics.model_topic_provisioning_diff import ( + ModelTopicProvisioningDiff, + build_provisioning_diff, +) +from omnibase_infra.topics.model_topic_provisioning_policy import ( + ModelTopicProvisioningPolicy, + resolve_specs_for_creation, +) from omnibase_infra.topics.model_topic_spec import ModelTopicSpec from omnibase_infra.utils import sanitize_error_message @@ -58,12 +75,24 @@ ENV_BOOTSTRAP_SERVERS = "KAFKA_BOOTSTRAP_SERVERS" ENV_TOPIC_PARTITION_CAP = "ONEX_TOPIC_PROVISIONER_MAX_PARTITIONS" -# Default partition and replication settings for standard event topics +# Default partition count for standard event topics. +# +# OMN-15395: the companion ``DEFAULT_EVENT_TOPIC_REPLICATION_FACTOR = 1`` is +# GONE. It was the mechanism that overrode the MSK broker's own RF2 default down +# to RF1 on every topic whose contract declared nothing — 519 RF1 topics. +# Replication is now resolved by ``ModelTopicProvisioningPolicy`` on the +# creation path, which fails closed against a managed cluster. DEFAULT_EVENT_TOPIC_PARTITIONS = 6 -DEFAULT_EVENT_TOPIC_REPLICATION_FACTOR = 1 -def _topic_partition_cap_from_env() -> int | None: +def topic_partition_cap_from_env() -> int | None: + """Return the lane's partition cap, or ``None`` when uncapped. + + Public because the operator CLI (``scripts/create_kafka_topics.py``) must + apply the SAME cap the runtime provisioner applies. A CLI that creates a + topic at its contract-declared 6 partitions on a lane the runtime caps to 1 + manufactures permanent partition drift between the two live creation paths. + """ raw_value = os.environ.get(ENV_TOPIC_PARTITION_CAP) if raw_value is None or raw_value.strip() == "": return None @@ -97,6 +126,44 @@ def _topic_provisioning_sort_key(spec: ModelTopicSpec) -> tuple[int, str]: return (spec.provisioning_priority, spec.suffix) +def unhostable_replication_error( + *, + topic: str, + requested_replication_factor: int | None, + policy: ModelTopicProvisioningPolicy, + cause: BaseException, +) -> TopicReplicationPolicyError: + """Build the typed, receipted error for a broker-rejected replica count.""" + context = ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.KAFKA, + operation="create_topic", + target_name=topic, + ) + measured = ( + f"the cluster measured {policy.broker_count} broker(s)" + if policy.broker_count is not None + else ( + "the cluster's broker count could NOT be measured, so no capacity " + "ceiling was installed and the declared value was passed through " + "unreduced" + ) + ) + return TopicReplicationPolicyError( + f"Broker refused to create topic {topic!r} with " + f"replication_factor={requested_replication_factor}: " + f"INVALID_REPLICATION_FACTOR. {measured}. This is a hard provisioning " + "failure, not a best-effort miss — the topic does NOT exist. Fix the " + f"owning contract's topic_config.replication_factor, or run against a " + "cluster whose describe_cluster is reachable so the measured capacity " + f"ceiling can reduce it (OMN-15395). Broker error: {cause!r}", + context=context, + topic=topic, + declared_replication_factor=requested_replication_factor, + profile=policy.profile.value, + broker_count=policy.broker_count, + ) + + class TopicProvisioner: """Provisions Kafka topics automatically on startup. @@ -125,6 +192,7 @@ def __init__( contracts_root: Path, skill_manifests_root: Path | None = None, skill_manifests_roots: list[Path] | None = None, + policy: ModelTopicProvisioningPolicy | None = None, ) -> None: """Initialize the topic provisioner. @@ -143,12 +211,22 @@ def __init__( topics.yaml manifests (supports multiple roots: skills, CLI relays, services). When both singular and plural are set, the singular root is prepended to the list. + policy: Replication policy for the target broker (OMN-15395). + Defaults to the policy derived from the live Kafka client + configuration, so a managed (MSK) target rejects a declared RF1 + fail-closed and resolves an undeclared replication factor to the + managed durability floor rather than to 1. Whatever is supplied + here is UNMEASURED — it carries no capacity ceiling until the + first admin connection binds a live ``describe_cluster`` broker + count to it (see :meth:`_measured_policy`). The measurement may + only install a ceiling and raise an undeclared default; it never + weakens the durability floor. Raises: FileNotFoundError: If *contracts_root* does not point to an existing directory. - Ticket: OMN-4594, OMN-4622, OMN-5132 + Ticket: OMN-4594, OMN-4622, OMN-5132, OMN-15395 """ if not contracts_root.is_dir(): raise FileNotFoundError( @@ -160,14 +238,325 @@ def __init__( self._contracts_root = contracts_root self._skill_manifests_root = skill_manifests_root self._skill_manifests_roots = skill_manifests_roots - self._topic_partition_cap = _topic_partition_cap_from_env() + self._topic_partition_cap = topic_partition_cap_from_env() + self._policy = policy or ModelTopicProvisioningPolicy.from_env() + # OMN-15395 (D4): memoize the capacity probe ATTEMPT, not merely a + # successful one. Keying the "already probed?" test on + # ``policy.broker_count is None`` re-probed an UNMEASURABLE cluster on + # every entrypoint — three entrypoints, three describe_cluster round + # trips, none of which could ever succeed — which is the per-call + # fan-out (d) exists to eliminate, reintroduced on the failure path. A + # policy supplied already-measured counts as probed. + self._capacity_probed = self._policy.broker_count is not None self._topic_specs = self._build_topic_specs() + # OMN-15395 (c): the contract-derived spec registry every path resolves + # against, so a caller that knows only a topic NAME still creates that + # topic to its owning contract's declared partitions/replication/config + # instead of falling back to bare module defaults. + self._spec_by_name: dict[str, ModelTopicSpec] = { + spec.suffix: spec for spec in self._topic_specs + } + # OMN-15395 (d): live broker snapshot, so a pass over an + # already-provisioned cluster issues zero CreateTopics instead of + # ~1,280 blind authorizations. Lifecycle: re-fetched at the top of every + # ``ensure_provisioned_topics_exist`` pass, fetched lazily once for the + # ``ensure_topic_exists`` path, and folded forward on each create. It is + # deliberately not invalidated on a timer — the only staleness a + # provisioner instance can observe is a topic deleted out-of-band, and + # the cost there is a skipped create that the next full pass repairs. + self._existing_topics: frozenset[str] | None = None + # OMN-15395 (c): resolved specs of topics THIS provisioner created, used + # as the readiness expectation so a freshly created topic is confirmed + # against the spec it was created with. + self._created_specs: dict[str, ModelTopicSpec] = {} + + @property + def policy(self) -> ModelTopicProvisioningPolicy: + """The replication policy this provisioner resolves specs against. + + Unmeasured until the first admin connection; thereafter bound to the + cluster's live broker count (OMN-15395). + """ + return self._policy + + async def _measured_policy(self, admin: object) -> ModelTopicProvisioningPolicy: + """Bind the policy to the cluster's live broker count, once. + + The capacity ceiling that reduces a contract-declared replication + factor MUST come from a measurement of the target broker, never from an + inference off the SASL mechanism: ``ModelKafkaEventBusConfig`` accepts + PLAIN / SCRAM / OAUTHBEARER as well as MSK IAM, so "not IAM" says + nothing at all about node count. Measuring here — on the same admin + client that is about to issue the ``CreateTopics`` — is the only place + the ceiling can be honest. + + The ATTEMPT is memoized, not the success (OMN-15395 D4). A cluster whose + ``describe_cluster`` is absent or failing leaves ``broker_count`` at + ``None`` forever, so testing that field re-ran the probe on every + entrypoint; the sentinel bounds it to one attempt per instance. + """ + if self._capacity_probed: + return self._policy + self._capacity_probed = True + self._policy = await bind_policy_to_broker_capacity(admin, self._policy) + return self._policy def _creation_partitions(self, spec: ModelTopicSpec) -> int: if self._topic_partition_cap is None: return spec.partitions return min(spec.partitions, self._topic_partition_cap) + def _creation_spec(self, spec: ModelTopicSpec) -> ModelTopicSpec: + """Return the spec as this provisioner would actually CREATE it. + + Replication resolved through the policy, partitions clamped by the + lane's env cap. This is the single "effective spec" every site must + compare against — the creation site, and (OMN-15395 D3) the drift site. + + Raises: + TopicReplicationPolicyError: The spec violates the policy. + """ + resolved = self._resolve_spec(spec) + partitions = self._creation_partitions(resolved) + if partitions == resolved.partitions: + return resolved + return resolved.model_copy(update={"partitions": partitions}) + + def _resolve_spec(self, spec: ModelTopicSpec) -> ModelTopicSpec: + """Resolve one spec's replication factor through the environment policy. + + Raises: + TopicReplicationPolicyError: RF below the environment floor, or + undeclared RF where the policy has no default. + """ + return self._policy.resolve_spec(spec) + + def _resolve_specs_for_creation( + self, + specs: Sequence[ModelTopicSpec], + correlation_id: UUID, + ) -> tuple[ModelTopicSpec, ...]: + """Resolve every spec BEFORE any ``CreateTopics`` is issued (OMN-15395 a/b). + + Fail-closed and batch-scoped: a single spec that violates the + environment replication policy — the RF1-on-MSK case — aborts the whole + pass with ZERO creates issued. Not a warning, not a clamp-and-continue, + and not a per-topic skip that lets the rest of the pass proceed while a + durability defect sits unfixed in a contract we own. Every violation is + collected first so one boot surfaces every offending contract instead of + one per redeploy. + + The batch resolution itself lives in + :func:`~omnibase_infra.topics.model_topic_provisioning_policy.resolve_specs_for_creation` + so the operator CLI (``scripts/create_kafka_topics.py``) enforces the + identical fail-closed rule; this wrapper only adds the + correlation-scoped log line. + + Returns: + The resolved specs, each carrying an explicit replication factor. + + Raises: + TopicReplicationPolicyError: Any spec violates the policy. + """ + try: + return resolve_specs_for_creation(self._policy, specs) + except TopicReplicationPolicyError: + logger.exception( + "Refusing to provision topic(s) under the %s replication " + "policy; no CreateTopics issued (correlation_id=%s)", + self._policy.profile.value, + correlation_id, + ) + raise + + async def _fetch_broker_topic_metadata( + self, + admin: object, + ) -> tuple[dict[str, Mapping[str, object]], frozenset[str]]: + """Snapshot the broker's live topics in a single metadata request. + + Returns ``(metadata_by_topic, existing_names)``. One metadata request + replaces the previous "issue CreateTopics for every known topic and use + ``TopicAlreadyExistsError`` as flow control" pattern. + """ + describe = getattr(admin, "describe_topics", None) + entries: object = [] + if describe is not None: + entries = await describe() + metadata: dict[str, Mapping[str, object]] = {} + if isinstance(entries, Sequence) and not isinstance(entries, (str, bytes)): + for entry in entries: + if not isinstance(entry, Mapping): + continue + name = entry.get("topic") + if not isinstance(name, str): + continue + error_code = entry.get("error_code") + if isinstance(error_code, int) and error_code != 0: + # Broker knows the name but cannot serve it (e.g. unknown + # topic) — treat as absent so it is created, not skipped. + continue + metadata[name] = entry + return metadata, frozenset(metadata) + + async def _existing_topic_names(self, admin: object) -> frozenset[str]: + """Return the cached live topic snapshot, fetching it once if needed.""" + if self._existing_topics is None: + _, names = await self._fetch_broker_topic_metadata(admin) + self._existing_topics = names + return self._existing_topics + + def _note_topic_created( + self, + topic_name: str, + spec: ModelTopicSpec | None = None, + ) -> None: + """Fold a freshly created topic into the cached snapshot + readiness specs.""" + if self._existing_topics is not None: + # ``.union`` rather than ``|``: the repo's union-count ratchet parses + # a bare ``|`` here as a type union and counts it against the budget. + self._existing_topics = self._existing_topics.union({topic_name}) + if spec is not None: + self._created_specs[topic_name] = spec + + def _report_spec_drift( + self, + present_topics: Sequence[str], + metadata: Mapping[str, Mapping[str, object]], + correlation_id: UUID, + ) -> list[str]: + """Report partition/replication drift on already-existing topics. + + OMN-15395 (d): drift on a live topic is REPORTED, never silently + re-created or mutated — repairing the 519 pre-existing RF1 topics is the + operator-gated WS-M reassignment lane, not this provisioner's call. + Reuses ``evaluate_topic_readiness`` rather than re-implementing the + comparison. + + The expectation is the spec this provisioner would actually CREATE — + :meth:`_creation_spec`, i.e. replication resolved through the policy AND + partitions clamped by the lane's env cap. "There is one resolver" only + holds if every site uses what it returns, and that includes the site + that decides what counts as drift: + + * comparing a broker against an unresolved RF2 on a cluster measured at + one node reports every RF2 topic as drifted even though the + provisioner deliberately and correctly created it at RF1 there; and + * comparing against the UNCAPPED ``spec.partitions`` reports every + contract-declared 6-partition topic as ``partition_mismatch`` on every + dev/stability lane, where ``ONEX_TOPIC_PROVISIONER_MAX_PARTITIONS=1`` + means the provisioner itself created them with one partition + (OMN-15395 D3). + + Both seed the operator-gated WS-M reassignment queue that consumes this + feed with targets the cluster cannot host — a provisioner reporting + drift against topics it just created, correctly. + + Partition divergence the cap *explains* is not silently dropped either: + a pre-existing topic carrying more partitions than the current cap + allows (created before the cap was lowered) is reported under a distinct + ``partition_cap_suppressed`` label. Kafka cannot reduce a partition + count, so emitting that as ``partition_mismatch`` would feed the repair + queue an impossible instruction; emitting nothing at all would hide a + real difference between contract and broker. + + A spec the policy REFUSES (a contract declaring RF1 against managed) is + reported here rather than raised: the fail-closed abort is scoped to + topics being created, and a pre-existing topic is already on the broker. + """ + expected: dict[str, ModelTopicSpec] = {} + refusals: list[str] = [] + for name in present_topics: + declared = self._spec_by_name.get(name) + if declared is None: + continue + try: + expected[name] = self._creation_spec(declared) + except TopicReplicationPolicyError as exc: + refusals.append(f"{name}: replication_policy_violation: {exc}") + if refusals: + logger.warning( + "%d existing topic(s) have a contract spec the %s replication " + "policy would refuse to create: %s (correlation_id=%s)", + len(refusals), + self._policy.profile.value, + refusals, + correlation_id, + ) + if not expected: + return refusals + evaluation = evaluate_topic_readiness( + tuple(expected), + [metadata[name] for name in expected if name in metadata], + expected_specs=expected, + ) + drift: list[str] = [] + cap_suppressed: list[str] = [] + for failure in evaluation.failures: + if failure.reason not in ( + EnumTopicReadinessFailureReason.PARTITION_MISMATCH, + EnumTopicReadinessFailureReason.REPLICATION_MISMATCH, + ): + continue + entry = f"{failure.topic}: {failure.reason.value}: {failure.detail}" + if ( + failure.reason is EnumTopicReadinessFailureReason.PARTITION_MISMATCH + and self._partition_gap_is_cap_explained( + failure.topic, metadata.get(failure.topic) + ) + ): + cap_suppressed.append( + f"{failure.topic}: partition_cap_suppressed: {failure.detail} " + f"(explained by {ENV_TOPIC_PARTITION_CAP}=" + f"{self._topic_partition_cap}; partitions cannot be reduced, " + "so this is NOT a reassignment target)" + ) + continue + drift.append(entry) + if cap_suppressed: + logger.info( + "Partition divergence on %d existing topic(s) is explained by " + "the lane partition cap — NOT reported as drift (OMN-15395): " + "%s (correlation_id=%s)", + len(cap_suppressed), + cap_suppressed, + correlation_id, + ) + if drift: + logger.warning( + "Topic spec drift on %d existing topic(s) — reported, NOT " + "re-created or mutated (OMN-15395): %s (correlation_id=%s)", + len(drift), + drift, + correlation_id, + ) + return refusals + drift + + def _partition_gap_is_cap_explained( + self, + topic: str, + entry: Mapping[str, object] | None, + ) -> bool: + """True when the broker's extra partitions are explained by the cap. + + The lane cap can only ever be *lowered* against topics that already + exist — Kafka has no partition-reduction operation — so a topic whose + broker partition count sits between the capped expectation + (exclusive) and the contract-declared count (inclusive) is a topic + created before the current cap, not a contract/broker disagreement the + reassignment lane can act on. + """ + if self._topic_partition_cap is None or entry is None: + return False + declared = self._spec_by_name.get(topic) + if declared is None: + return False + raw = entry.get("partitions") + if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)): + return False + observed = len([p for p in raw if isinstance(p, Mapping)]) + return self._topic_partition_cap < observed <= declared.partitions + def _build_topic_specs(self) -> tuple[ModelTopicSpec, ...]: """Build topic specs from contract YAML extraction. @@ -196,8 +585,12 @@ def _build_topic_specs(self) -> tuple[ModelTopicSpec, ...]: for entry in contract_entries: # Per-topic config (OMN-13238): when a contract declares a # ``topic_config`` block the extractor carries partitions / - # replication_factor / kafka_config; otherwise these stay None and - # ModelTopicSpec applies its canonical defaults. + # replication_factor / kafka_config. + # + # OMN-15395: an undeclared replication_factor is carried through as + # None — "the contract declared nothing" — and is resolved (or + # refused) by the environment policy on the creation path. It is NOT + # silently coerced to 1 here any more. result_specs.append( ModelTopicSpec( suffix=entry.topic, @@ -207,11 +600,7 @@ def _build_topic_specs(self) -> tuple[ModelTopicSpec, ...]: if entry.partitions is not None else DEFAULT_EVENT_TOPIC_PARTITIONS ), - replication_factor=( - entry.replication_factor - if entry.replication_factor is not None - else DEFAULT_EVENT_TOPIC_REPLICATION_FACTOR - ), + replication_factor=entry.replication_factor, kafka_config=( dict(entry.kafka_config) if entry.kafka_config is not None @@ -238,14 +627,18 @@ async def ensure_provisioned_topics_exist( ) -> dict[str, list[str] | str]: """Ensure all ONEX provisioned topics exist. - Creates any missing topics discovered from contract YAML extraction. - The snapshot topic gets special compaction configuration via - ModelSnapshotTopicConfig. + Lists the broker's live topics FIRST and creates only the genuinely + missing ones (OMN-15395 d) — a pass over an already-provisioned cluster + issues zero ``CreateTopics``. Every created topic's replication factor is + resolved through the environment policy before any create is issued + (OMN-15395 a/b), so an RF1 spec against managed staging aborts the whole + pass instead of creating a topic that cannot survive a broker loss. - This method is best-effort: individual topic creation failures are - logged as warnings but do not prevent other topics from being created. - Unrecoverable failures (connection, authentication, etc.) are also - logged as warnings and never block startup. + Individual topic creation failures are best-effort: they log warnings and + do not prevent other topics from being created. Unrecoverable failures + (connection, authentication) also degrade to a warning and never block + startup. A replication-policy violation is NOT best-effort — it + propagates to the caller with nothing created. Args: correlation_id: Optional correlation ID for tracing. @@ -255,12 +648,23 @@ async def ensure_provisioned_topics_exist( - created: List of newly created topic names - existing: List of topics that already existed - failed: List of topics that failed to create + - drift: Partition/replication drift found on existing topics + (reported only — never re-created or mutated) - status: "success", "partial", or "unavailable" + + Raises: + TopicReplicationPolicyError: A missing topic's spec violates the + environment replication policy (raised before any + ``CreateTopics`` — the pass creates nothing), or the broker + refused a ``CreateTopics`` with ``INVALID_REPLICATION_FACTOR`` + (OMN-15395 D5 — an unhostable replica count leaves the topic + absent, so it is never degraded to a warning + ``failed``). """ correlation_id = correlation_id or uuid4() created: list[str] = [] existing: list[str] = [] failed: list[str] = [] + drift: list[str] = [] try: from aiokafka.admin import AIOKafkaAdminClient, NewTopic @@ -277,6 +681,7 @@ async def ensure_provisioned_topics_exist( "created": created, "existing": existing, "failed": [s.suffix for s in self._topic_specs], + "drift": drift, "status": "unavailable", } @@ -293,7 +698,45 @@ async def ensure_provisioned_topics_exist( ) await admin.start() - for spec in self._topic_specs: + # Measure the cluster's node count BEFORE resolving anything: the + # capacity ceiling that may reduce a contract-declared replication + # factor has to be a measurement of this broker, not an inference + # from its auth mechanism (OMN-15395). + await self._measured_policy(admin) + + # (d) One metadata request replaces the blind create-everything + # sweep. Only names the broker does not already have are candidates. + metadata, existing_names = await self._fetch_broker_topic_metadata(admin) + self._existing_topics = existing_names + diff: ModelTopicProvisioningDiff = build_provisioning_diff( + (spec.suffix for spec in self._topic_specs), existing_names + ) + existing.extend(diff.present_topics) + drift.extend( + self._report_spec_drift(diff.present_topics, metadata, correlation_id) + ) + + missing = set(diff.missing_topics) + missing_specs = [ + spec for spec in self._topic_specs if spec.suffix in missing + ] + logger.info( + "Topic provisioning diff: desired=%d present=%d missing=%d " + "(correlation_id=%s)", + len(diff.desired_topics), + len(diff.present_topics), + len(diff.missing_topics), + correlation_id, + ) + + # (a)/(b) Resolve every missing spec's replication factor BEFORE the + # first CreateTopics. A floor violation raises out of this method — + # it is deliberately outside the best-effort boundary below. + resolved_specs = self._resolve_specs_for_creation( + missing_specs, correlation_id + ) + + for spec in resolved_specs: try: partitions = self._creation_partitions(spec) new_topic = NewTopic( @@ -307,22 +750,39 @@ async def ensure_provisioned_topics_exist( await admin.create_topics([new_topic]) created.append(spec.suffix) + self._note_topic_created(spec.suffix, spec) logger.info( - "Created topic: %s (partitions=%d)", + "Created topic: %s (partitions=%d, replication_factor=%s)", spec.suffix, partitions, + spec.replication_factor, extra={"correlation_id": str(correlation_id)}, ) except TopicAlreadyExistsError: existing.append(spec.suffix) + self._note_topic_created(spec.suffix) logger.debug( "Topic already exists: %s", spec.suffix, extra={"correlation_id": str(correlation_id)}, ) - except Exception as e: # noqa: BLE001 — boundary: logs warning and degrades + except Exception as e: + # Boundary: an unhostable replica count is re-raised + # fail-closed; everything else degrades to a warning. + if is_invalid_replication_factor_error(e): + # (D5) A replica count the broker cannot host is a + # durability failure, not a best-effort miss. It leaves + # the topic ABSENT, so degrading it to a warning + + # status="partial" is the silent-uncreated-topic bug the + # capacity measurement was supposed to make impossible. + raise unhostable_replication_error( + topic=spec.suffix, + requested_replication_factor=spec.replication_factor, + policy=self._policy, + cause=e, + ) from e failed.append(spec.suffix) logger.warning( "Failed to create topic %s: %s", @@ -334,6 +794,10 @@ async def ensure_provisioned_topics_exist( }, ) + except TopicReplicationPolicyError: + # Durability violations are fail-closed: never degraded to a warning. + raise + except Exception as e: # noqa: BLE001 — boundary: logs warning and degrades logger.warning( "Topic auto-creation interrupted by %s. " @@ -365,6 +829,7 @@ async def ensure_provisioned_topics_exist( "created": created, "existing": existing, "failed": failed + not_attempted, + "drift": drift, "status": interrupted_status, } @@ -387,6 +852,7 @@ async def ensure_provisioned_topics_exist( "created_count": len(created), "existing_count": len(existing), "failed_count": len(failed), + "drift_count": len(drift), "status": status, "correlation_id": str(correlation_id), }, @@ -396,6 +862,7 @@ async def ensure_provisioned_topics_exist( "created": created, "existing": existing, "failed": failed, + "drift": drift, "status": status, } @@ -413,10 +880,16 @@ async def ensure_topic_exists( multiple topics, prefer :meth:`ensure_provisioned_topics_exist` which reuses a single admin connection for all topics. + Spec resolution (OMN-15395 c): when the caller supplies neither *config* + nor *spec*, the topic's OWN contract-derived spec is looked up from this + provisioner's registry. The per-contract boot interleave calls this + method with a bare topic name, and it used to land on a hardcoded RF1 — + that bare-default branch no longer exists. + Args: topic_name: The topic name to create. config: Optional snapshot-topic configuration (compaction etc.). If - None, falls back to *spec* or default event topic settings. + None, falls back to *spec*, then to the contract-derived spec. correlation_id: Optional correlation ID for tracing. spec: Optional contract-derived ``ModelTopicSpec`` (partitions, replication, kafka_config). Used by the per-contract boot @@ -426,6 +899,12 @@ async def ensure_topic_exists( Returns: True if topic was created or already exists, False on failure. + + Raises: + TopicReplicationPolicyError: The resolved spec violates the + environment replication policy (raised before + ``CreateTopics``), or the broker refused the ``CreateTopics`` + with ``INVALID_REPLICATION_FACTOR`` (OMN-15395 D5). """ correlation_id = correlation_id or uuid4() @@ -446,6 +925,9 @@ async def ensure_topic_exists( TopicAlreadyExistsError = _TopicAlreadyExistsError admin: AIOKafkaAdminClient | None = None + # Recorded so a broker INVALID_REPLICATION_FACTOR rejection can name the + # value it refused (OMN-15395 D5). + requested_replication_factor: int | None = None try: auth_kwargs = build_aiokafka_auth_kwargs_from_env() admin = AIOKafkaAdminClient( @@ -455,31 +937,78 @@ async def ensure_topic_exists( ) await admin.start() + # Same measured-capacity binding as the full pass: the ceiling is + # read off this cluster, never inferred (OMN-15395). + await self._measured_policy(admin) + + # (d) Skip creation entirely when the broker already has the topic. + # The snapshot is fetched once per provisioner instance, so the + # per-contract boot interleave costs ONE metadata request instead of + # one blind CreateTopics authorization per contract topic. + if topic_name in await self._existing_topic_names(admin): + logger.debug( + "Topic already exists (broker snapshot), skipping create: %s", + topic_name, + extra={"correlation_id": str(correlation_id)}, + ) + return True + + created_spec: ModelTopicSpec | None = None if config is not None: + # Snapshot-topic config carries its own replication factor; it + # is still resolved through the SAME policy as every other + # creation site, and it is the RESOLVER'S OUTPUT that reaches + # NewTopic — not the raw declared value. "There is one resolver" + # only holds if every creation site uses what it returns. + resolved_rf = self._policy.resolve_replication_factor( + topic=topic_name, declared=config.replication_factor + ) + requested_replication_factor = resolved_rf new_topic = NewTopic( name=topic_name, num_partitions=config.partition_count, - replication_factor=config.replication_factor, + replication_factor=resolved_rf, topic_configs=config.to_kafka_config(), ) - elif spec is not None: - # Contract-derived spec (OMN-13237 per-contract interleave): - # honor declared partitions/replication/kafka_config. - new_topic = NewTopic( - name=topic_name, - num_partitions=self._creation_partitions(spec), - replication_factor=spec.replication_factor, - topic_configs=dict(spec.kafka_config) if spec.kafka_config else {}, + created_spec = ModelTopicSpec( + suffix=topic_name, + partitions=config.partition_count, + replication_factor=resolved_rf, + kafka_config=config.to_kafka_config(), ) else: - default_spec = ModelTopicSpec(suffix=topic_name) + # (c) Caller-supplied spec wins; otherwise use the topic's own + # contract-derived spec. Only a topic this provisioner knows + # nothing about falls back to a bare spec, whose undeclared + # replication factor the policy resolves or refuses. + effective_spec = ( + spec + if spec is not None + else self._spec_by_name.get( + topic_name, ModelTopicSpec(suffix=topic_name) + ) + ) + # Record the exact effective spec handed to the broker. The + # dev/stability partition cap is part of creation semantics; + # retaining the uncapped spec here makes the subsequent + # readiness check expect (for example) 6 partitions after we + # deliberately created 1, so a cold-start consumer can never + # attach until the process restarts and treats the topic as + # pre-existing (OMN-15978 live finding). + resolved = self._creation_spec(effective_spec) + created_spec = resolved + requested_replication_factor = resolved.replication_factor new_topic = NewTopic( name=topic_name, - num_partitions=self._creation_partitions(default_spec), - replication_factor=DEFAULT_EVENT_TOPIC_REPLICATION_FACTOR, + num_partitions=resolved.partitions, + replication_factor=resolved.replication_factor, + topic_configs=dict(resolved.kafka_config) + if resolved.kafka_config + else {}, ) await admin.create_topics([new_topic]) + self._note_topic_created(topic_name, created_spec) logger.info( "Created topic: %s", topic_name, @@ -488,6 +1017,7 @@ async def ensure_topic_exists( return True except TopicAlreadyExistsError: + self._note_topic_created(topic_name) logger.debug( "Topic already exists: %s", topic_name, @@ -495,7 +1025,24 @@ async def ensure_topic_exists( ) return True - except Exception as e: # noqa: BLE001 — boundary: logs warning and degrades + except TopicReplicationPolicyError: + # Durability violations are fail-closed: never degraded to a + # warning-and-False, which the caller would read as "best effort". + raise + + except Exception as e: + # Boundary: an unhostable replica count is re-raised fail-closed; + # everything else degrades to a warning-and-False. + if is_invalid_replication_factor_error(e): + # (D5) Same rule as the batch path: a broker-rejected replica + # count is fail-closed, never a warning-and-False the caller + # reads as "best effort". + raise unhostable_replication_error( + topic=topic_name, + requested_replication_factor=requested_replication_factor, + policy=self._policy, + cause=e, + ) from e logger.warning( "Failed to create topic %s: %s", topic_name, @@ -531,6 +1078,17 @@ async def confirm_topics_ready( cadence / max-attempts; on exhaustion each unready topic carries a classified failure reason. + Spec pass-through (OMN-15395 c): when the caller supplies no + *expected_specs*, the RESOLVED specs of the topics this provisioner + actually created in this process are used, so a freshly created topic is + verified against the partitions/replication it was created with. Specs + are deliberately NOT injected for pre-existing topics: on a cluster + carrying the 519 legacy RF1 topics, asserting the contract's RF against + them would flip healthy topics to NOT-READY and block consumer attach. + Drift on pre-existing topics is reported by + :meth:`ensure_provisioned_topics_exist` instead (OMN-15395 d) and + repaired by the operator-gated WS-M reassignment lane. + Args: topics: The topic names to confirm. expected_specs: Optional per-topic expected spec (partitions/RF/ @@ -546,7 +1104,15 @@ async def confirm_topics_ready( if not requested: return ModelTopicSetReadiness(status=EnumTopicReadinessStatus.SKIPPED) knobs = config or ModelTopicReadinessConfig() - specs = dict(expected_specs or {}) + specs = ( + dict(expected_specs) + if expected_specs is not None + else { + name: spec + for name in requested + if (spec := self._created_specs.get(name)) is not None + } + ) try: from aiokafka.admin import AIOKafkaAdminClient @@ -743,12 +1309,16 @@ def _partition_leader(partition: Mapping[str, object]) -> int | None: def _replication_mismatch( partitions: Sequence[Mapping[str, object]], - expected_rf: int, + expected_rf: int | None, ) -> str | None: """Return a detail string when replica counts disagree with the spec. - Skipped (returns None) where the broker does not expose a replica list. + Skipped (returns None) where the broker does not expose a replica list, or + where the owning contract declared no replication factor (OMN-15395: an + undeclared RF is not an expectation to assert against). """ + if expected_rf is None: + return None for partition in partitions: replicas = partition.get("replicas") if not ( @@ -803,4 +1373,9 @@ async def _run() -> None: _cli_main() -__all__ = ["TopicProvisioner", "evaluate_topic_readiness"] +__all__ = [ + "TopicProvisioner", + "evaluate_topic_readiness", + "topic_partition_cap_from_env", + "unhostable_replication_error", +] diff --git a/src/omnibase_infra/gate/executor.py b/src/omnibase_infra/gate/executor.py index 9e1f06ee85..b9cda6760a 100644 --- a/src/omnibase_infra/gate/executor.py +++ b/src/omnibase_infra/gate/executor.py @@ -20,7 +20,9 @@ from omnibase_core.models.gate.model_omnigate_config import ModelOmniGateConfig from omnibase_infra.gate.validator_registry import execute_validator -_BASE_ENV_ALLOWLIST = frozenset({"PATH", "HOME", "USER", "LANG", "LC_ALL"}) +# ``LC_ALL`` remains available through a check's explicit ``allowed_env``. +# Ambient overrides are not trusted because invalid values alter receipt evidence. +_BASE_ENV_ALLOWLIST = frozenset({"PATH", "HOME", "USER", "LANG"}) _SECRET_PATTERNS = ( re.compile(r"(?i)(api[_-]?key|token|password|secret)\s*[:=]\s*\S+"), re.compile(r"(?i)(bearer|basic)\s+[A-Za-z0-9._~+/=-]{16,}"), diff --git a/src/omnibase_infra/gateway/client/__init__.py b/src/omnibase_infra/gateway/client/__init__.py new file mode 100644 index 0000000000..23bf3556ad --- /dev/null +++ b/src/omnibase_infra/gateway/client/__init__.py @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Client half of the gateway attach cycle -- what ``onex auth`` drives (OMN-15922). + +The server half of this contract lives in ``nodes/node_gateway_attach_effect``; +this package is the caller that holds a credential, mints a token against it, +attaches, and re-attaches before its session ceiling. The two halves share the +session and renewal models rather than mirroring them, so the contract has one +definition and cannot drift. +""" diff --git a/src/omnibase_infra/gateway/client/gateway_renewal_planner.py b/src/omnibase_infra/gateway/client/gateway_renewal_planner.py new file mode 100644 index 0000000000..0a5a850f10 --- /dev/null +++ b/src/omnibase_infra/gateway/client/gateway_renewal_planner.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""GatewayRenewalPlanner — the client half of the RE_ATTACH cycle (OMN-15922). + +The server computes the window and hands it over on the attach response +(OMN-15952, ``service_gateway_renewal_policy`` on the node). This class does +not recompute it -- recomputing would make the client a second authority over +a term the contract already settled, and the two would drift the first time the +node's ``renewal_margin_seconds`` changed. It reads the directive and answers +three questions about it: + + * ``plan_instant`` -- which moment inside the jitter window is mine? + * ``is_renewal_due`` -- has that window opened? + * ``assert_window_is_honourable`` -- can I still make it? + +Every method is pure: ``now`` and the random generator are both injected. That +is what makes the boundary cases (a token whose whole life is shorter than the +margin, an instant exactly at ``renew_not_before``) driveable by tests rather +than sleepable-through. + +WHY THE JITTER DRAW IS AN INJECTED GENERATOR + A fleet provisioned in one bootstrap batch shares an attach instant, so + without spreading it shares a renewal instant -- and the synchronisation is + self-sustaining, because a batch that renews together stays together. + Taking a ``random.Random`` rather than calling the module-level ``random`` + lets a test prove the draw actually spreads, instead of asserting only that + each individual result is inside the window (which a constant would satisfy). +""" + +from __future__ import annotations + +import random +from datetime import datetime, timedelta + +from omnibase_core.enums.enum_core_error_code import EnumCoreErrorCode +from omnibase_core.errors.model_onex_error import ModelOnexError +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_renewal_directive import ( + ModelGatewayRenewalDirective, +) + +__all__ = ["GatewayRenewalPlanner"] + + +class GatewayRenewalPlanner: + """Reads a server-issued renewal directive and schedules against it.""" + + def plan_instant( + self, + directive: ModelGatewayRenewalDirective, + *, + rng: random.Random, + ) -> datetime: + """Pick this client's own renewal moment, uniform inside the window. + + Args: + directive: The cycle the gateway declared at attach. + rng: Generator for the decorrelation draw. + + Returns: + An instant in ``[renew_not_before, renew_at]``. A zero-width window + (``jitter_seconds`` 0, or a floor collision on a very short-lived + session) collapses to that single instant rather than erroring -- + no spreading is a legitimate configuration, not a failure. + """ + window = (directive.renew_at - directive.renew_not_before).total_seconds() + if window <= 0: + return directive.renew_at + return directive.renew_not_before + timedelta(seconds=rng.random() * window) + + def is_renewal_due( + self, + directive: ModelGatewayRenewalDirective, + *, + now: datetime, + ) -> bool: + """True once ``now`` has reached the window's opening edge. + + Inclusive at ``renew_not_before``: at the instant the window opens, + renewal is due. Erring early costs one extra token; erring late costs + the session. + """ + return now >= directive.renew_not_before + + def assert_window_is_honourable( + self, + directive: ModelGatewayRenewalDirective, + *, + now: datetime, + minimum_lead_seconds: int, + ) -> None: + """Refuse a window this client can no longer complete inside. + + A margin shorter than the client's own round trip is a real deployment + (a short token lifespan, a margin tuned down, a machine that slept). + Running the cycle anyway means re-granting at an instant that has + already passed by the time the grant returns, which presents as an + intermittent 401 rather than as the configuration problem it is. + + Args: + directive: The cycle the gateway declared. + now: Current instant. + minimum_lead_seconds: Time this client needs for grant + attach. + + Raises: + ModelOnexError: When less than ``minimum_lead_seconds`` remains + before ``renew_at``. + """ + remaining = (directive.renew_at - now).total_seconds() + if remaining < minimum_lead_seconds: + raise ModelOnexError( + "gateway renewal window cannot be honoured: renew_at is " + f"{remaining:.0f}s away but this client needs " + f"{minimum_lead_seconds}s for a re-grant plus re-attach " + f"(session_expires_at {directive.session_expires_at.isoformat()}, " + f"margin {directive.margin_seconds}s). Re-attach now, or raise " + "the gateway's renewal_margin_seconds.", + error_code=EnumCoreErrorCode.TIMEOUT_EXCEEDED, + ) diff --git a/src/omnibase_infra/gateway/client/gateway_session_keeper.py b/src/omnibase_infra/gateway/client/gateway_session_keeper.py new file mode 100644 index 0000000000..c6725fc814 --- /dev/null +++ b/src/omnibase_infra/gateway/client/gateway_session_keeper.py @@ -0,0 +1,310 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""GatewaySessionKeeper — Bearer on every gateway call (OMN-15922). + +The one place a gateway request is constructed, so that a token is never +hand-passed and no code path can produce an anonymous call. Every request here +goes through ``_post``, which takes the Authorization header from the minter +rather than from an argument -- an anonymous call is not something a caller can +express, which is a stronger property than every caller remembering to pass one. + +THE RENEWAL CYCLE THIS IMPLEMENTS (OMN-15952, contract 0.3.0) + ``expires_at`` is stamped once at attach, from + ``min(token exp, max_session_ttl_seconds)``, and nothing moves it. A + heartbeat proves liveness and non-revocation; it buys no time. So + ``ensure_attached`` does not "renew" -- inside the jitter window it performs + a fresh ``client_credentials`` grant and a fresh attach, minting a NEW + ``session_id``. Continuity across the boundary is this object's property, + never the session record's. + +FAIL-CLOSED + An expired session is refused locally, before a request is sent: the client + already holds ``expires_at``, so spending a round trip to be told what it + knows is both slower and, if the gateway is unreachable, indistinguishable + from an outage. A rejected call raises. There is no branch on which a dead + session is used and no branch on which failure degrades to a local path -- + a locally-successful delegation while the operator believes it ran in cloud + is the exact failure this refusal exists to prevent. +""" + +from __future__ import annotations + +import json +import random +from datetime import datetime +from typing import Final + +from omnibase_core.enums.enum_core_error_code import EnumCoreErrorCode +from omnibase_core.errors.model_onex_error import ModelOnexError +from omnibase_core.protocols.http.protocol_http_client import ProtocolHttpResponse +from omnibase_infra.gateway.client.gateway_renewal_planner import ( + GatewayRenewalPlanner, +) +from omnibase_infra.gateway.client.gateway_token_minter import ( + GatewayTokenMinter, +) +from omnibase_infra.gateway.models.model_gateway_attachment import ( + ModelGatewayAttachment, +) +from omnibase_infra.gateway.models.model_gateway_credential import ( + ModelGatewayCredential, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_renewal_directive import ( + ModelGatewayRenewalDirective, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session import ( + ModelGatewaySession, +) +from omnibase_infra.protocols.protocol_gateway_transport import ( + ProtocolGatewayTransport, +) + +__all__ = ["GatewaySessionKeeper"] + +_ATTACH_PATH: Final[str] = "/v1/gateway/attach" +_HEARTBEAT_PATH: Final[str] = "/v1/gateway/heartbeat" + +# Seconds this client needs for a grant plus an attach plus one backoff-retry. +# Checked against the directive's remaining lead before a renewal is attempted. +_RENEWAL_LEAD_SECONDS: Final[int] = 30 + + +class GatewaySessionKeeper: + """Attaches to the gateway and keeps the session alive by re-attaching.""" + + def __init__( + self, + *, + transport: ProtocolGatewayTransport, + credential: ModelGatewayCredential, + minter: GatewayTokenMinter, + rng: random.Random, + planner: GatewayRenewalPlanner | None = None, + ) -> None: + self._transport = transport + self._credential = credential + self._minter = minter + self._rng = rng + self._planner = planner if planner is not None else GatewayRenewalPlanner() + self._attachment: ModelGatewayAttachment | None = None + self._renew_at: datetime | None = None + + @property + def attachment(self) -> ModelGatewayAttachment | None: + """The session currently held, or None before the first attach.""" + return self._attachment + + async def attach(self, *, now: datetime) -> ModelGatewayAttachment: + """Open a new session, replacing any this client already held. + + Raises: + ModelOnexError: If the grant fails, the gateway rejects the token, + or the response omits the required renewal directive. + """ + # force_fresh_token: the session ceiling is stamped from the token's + # own exp, so attaching with a cached token that is merely still-valid + # buys only the token's remaining life. On the renewal path that would + # mint a successor already inside its own renewal window. + response = await self._post( + _ATTACH_PATH, + payload={"edge_instance_id": self._credential.edge_instance_id}, + now=now, + force_fresh_token=True, + ) + body = await self._require_ok(response, operation="attach") + attachment = self._parse_attachment(body) + + # Draw this client's own moment inside the declared jitter window, once + # per session -- redrawing per call would let a fleet re-synchronise on + # whichever draw happened to fire first. + self._attachment = attachment + self._renew_at = self._planner.plan_instant(attachment.renewal, rng=self._rng) + return attachment + + async def heartbeat(self, *, now: datetime) -> ModelGatewaySession: + """Prove liveness on the held session. Never extends ``expires_at``. + + Raises: + ModelOnexError: If no session is held, if the held session has + passed its ceiling (refused locally -- see module docstring), + or if the gateway rejects the call. + """ + attachment = self._require_attachment() + if now >= attachment.session.expires_at: + raise ModelOnexError( + "gateway session " + f"{attachment.session.session_id} expired at " + f"{attachment.session.expires_at.isoformat()}; a heartbeat " + "cannot extend it (renewal mode is " + f"{attachment.renewal.mode.value}). Call ensure_attached() to " + "re-attach with a fresh grant.", + error_code=EnumCoreErrorCode.INVALID_STATE, + ) + + response = await self._post( + _HEARTBEAT_PATH, + payload={"session_id": str(attachment.session.session_id)}, + now=now, + ) + body = await self._require_ok(response, operation="heartbeat") + document = self._decode_object(body, source="gateway heartbeat response") + session = ModelGatewaySession.model_validate( + self._require_mapping(document, "session") + ) + + # The contract's central negative, checked on the receiving side: if a + # heartbeat ever comes back with a moved ceiling, that is a server-side + # regression, and a client that silently adopted the new value would + # hide it. + if session.expires_at != attachment.session.expires_at: + raise ModelOnexError( + "gateway heartbeat returned a session whose expires_at moved " + f"({attachment.session.expires_at.isoformat()} -> " + f"{session.expires_at.isoformat()}). expires_at is stamped once " + "at attach; a heartbeat must never extend it.", + error_code=EnumCoreErrorCode.VALIDATION_FAILED, + ) + return session + + async def ensure_attached(self, *, now: datetime) -> ModelGatewayAttachment: + """Return a session valid at ``now``, re-attaching when the cycle says to. + + Re-attaches when the jitter window has opened (or the ceiling has + already passed). Re-attaching is a fresh grant plus a fresh attach -- + a NEW ``session_id`` -- never an extension of the incumbent. + """ + attachment = self._attachment + renew_at = self._renew_at + if attachment is None or renew_at is None: + return await self.attach(now=now) + + if now < renew_at and now < attachment.session.expires_at: + return attachment + return await self.attach(now=now) + + def assert_renewal_is_reachable(self, *, now: datetime) -> None: + """Raise if the held session's renewal window is too tight to complete. + + Deliberately NOT called from ``ensure_attached``: at or past + ``renew_at`` the honest action is to re-attach immediately, and a guard + there would turn every on-deadline renewal into a hard error. This is a + diagnostic for ``onex auth status`` and for a supervisor deciding + whether a deployment's ``renewal_margin_seconds`` is survivable, which + is a question about the configuration rather than about this call. + """ + attachment = self._require_attachment() + self._planner.assert_window_is_honourable( + attachment.renewal, + now=now, + minimum_lead_seconds=_RENEWAL_LEAD_SECONDS, + ) + + # -- transport --------------------------------------------------------- + + async def _post( + self, + path: str, + *, + payload: dict[str, str], + now: datetime, + force_fresh_token: bool = False, + ) -> ProtocolHttpResponse: + """The single gateway request constructor. Always Bearer-authenticated. + + The token goes in the Authorization header and nowhere else -- the + gateway's own seam table binds ``Authorization: Bearer `` to the + bus command's ``access_token`` field, so a client that also placed it + in the body would be duplicating a credential into a payload that gets + logged and forwarded. + """ + token = await self._minter.token_for(now=now, force_refresh=force_fresh_token) + return await self._transport.post_json( + self._credential.base_url.rstrip("/") + path, + body=json.dumps(payload), + headers={ + "Authorization": f"Bearer {token.access_token.get_secret_value()}", + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) + + async def _require_ok( + self, response: ProtocolHttpResponse, *, operation: str + ) -> str: + if response.status != 200: + # Status only -- the remote body is not echoed into a local error + # (see GatewayTokenMinter._grant for the same reasoning). + raise ModelOnexError( + f"gateway {operation} rejected (HTTP {response.status}) for " + f"tenant '{self._credential.tenant_slug}'. If the credential was " + "rotated or disabled, re-run 'onex auth login'.", + error_code=EnumCoreErrorCode.AUTHENTICATION_ERROR, + ) + return await response.text() + + # -- parsing ----------------------------------------------------------- + + def _parse_attachment(self, body: str) -> ModelGatewayAttachment: + document = self._decode_object(body, source="gateway attach response") + if "renewal" not in document or document["renewal"] is None: + raise ModelOnexError( + "gateway attach response carries no 'renewal' directive. It is " + "REQUIRED on node_gateway_attach_effect (contract 0.3.0) and " + "optional only at the onex-api edge for rollout ordering -- a " + "client without it has no defined behaviour at session expiry. " + "The gateway or the edge is running a pre-OMN-15952 build.", + error_code=EnumCoreErrorCode.VALIDATION_FAILED, + ) + return ModelGatewayAttachment( + session=ModelGatewaySession.model_validate( + self._require_mapping(document, "session") + ), + heartbeat_interval_seconds=self._require_int( + document, "heartbeat_interval_seconds" + ), + renewal=ModelGatewayRenewalDirective.model_validate( + self._require_mapping(document, "renewal") + ), + ) + + def _require_attachment(self) -> ModelGatewayAttachment: + attachment = self._attachment + if attachment is None: + raise ModelOnexError( + "no gateway session is attached; call attach() or " + "ensure_attached() first.", + error_code=EnumCoreErrorCode.INVALID_STATE, + ) + return attachment + + def _decode_object(self, raw: str, *, source: str) -> dict[str, object]: + try: + document = json.loads(raw) + except json.JSONDecodeError as exc: + raise ModelOnexError( + f"{source} is not valid JSON.", + error_code=EnumCoreErrorCode.PARSING_ERROR, + ) from exc + if not isinstance(document, dict): + raise ModelOnexError( + f"{source} is {type(document).__name__}, expected a JSON object.", + error_code=EnumCoreErrorCode.PARSING_ERROR, + ) + return {str(key): value for key, value in document.items()} + + def _require_mapping(self, document: dict[str, object], key: str) -> object: + if key not in document: + raise ModelOnexError( + f"gateway response has no '{key}' field.", + error_code=EnumCoreErrorCode.VALIDATION_FAILED, + ) + return document[key] + + def _require_int(self, document: dict[str, object], key: str) -> int: + value = document.get(key) + if isinstance(value, bool) or not isinstance(value, int): + raise ModelOnexError( + f"gateway response has no usable integer '{key}' field.", + error_code=EnumCoreErrorCode.VALIDATION_FAILED, + ) + return value diff --git a/src/omnibase_infra/gateway/client/gateway_token_minter.py b/src/omnibase_infra/gateway/client/gateway_token_minter.py new file mode 100644 index 0000000000..a28f052592 --- /dev/null +++ b/src/omnibase_infra/gateway/client/gateway_token_minter.py @@ -0,0 +1,253 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""GatewayTokenMinter — credential -> gateway JWT (OMN-15922). + +One ``client_credentials`` grant against the tenant realm's token endpoint, +one audience assertion, one in-memory cache. Deliberately no ``refresh_token`` +path: RFC 6749 §4.4.3 says the client-credentials grant SHOULD NOT issue one, +and Keycloak does not -- so "refresh" here means re-grant, which is also +exactly what the gateway's RE_ATTACH renewal mode assumes its clients do. + +WHAT THE AUDIENCE CHECK IS, AND IS NOT + It is NOT a security control. This client verifies no signature and holds + no JWKS; the gateway does that (``service_keycloak_token_validator`` on the + node, ``gateway_auth.py`` at the edge). A forged token would sail past this + check and die at the gateway, which is the correct place for it to die. + + It IS a fail-fast against a live, specific, already-observed defect: the + P0B per-tenant provisioner stamps only ``aud=redpanda-events`` + (``keycloak_client_manager.py`` BROKER_TOKEN_AUDIENCE), while attach + requires exact set equality with ``{"gateway-attach"}``. Without this + check, a correctly-configured operator with a genuinely broken credential + sees an opaque 401 from a remote service several calls downstream. With + it, they see which audience they got and which was needed, at the mint. + + The comparison is SET EQUALITY, mirroring the gateway rather than being + merely compatible with it. A superset (a dual-audience broker+attach token) + is rejected there and so is rejected here -- a client that accepted more + than the gateway does would report success and then fail on the wire. + +CACHING + In memory, per instance, re-granted once ``now`` reaches ``exp - skew``. + Not written to disk: a cached bearer on disk is a credential at rest with + none of the protections the actual credential file gets, in exchange for + saving one sub-second grant per process. +""" + +from __future__ import annotations + +import base64 +import binascii +import json +from datetime import UTC, datetime, timedelta +from typing import Final + +from pydantic import SecretStr + +from omnibase_core.enums.enum_core_error_code import EnumCoreErrorCode +from omnibase_core.errors.model_onex_error import ModelOnexError +from omnibase_infra.gateway.models.model_gateway_access_token import ( + ModelGatewayAccessToken, +) +from omnibase_infra.gateway.models.model_gateway_credential import ( + ModelGatewayCredential, +) +from omnibase_infra.protocols.protocol_gateway_transport import ( + ProtocolGatewayTransport, +) + +__all__ = ["GATEWAY_ATTACH_AUDIENCES", "GatewayTokenMinter"] + +# Exact audience set the gateway requires, mirrored from +# node_gateway_attach_effect/contract.yaml (required_audience: gateway-attach) +# and onex-api gateway_auth.GATEWAY_EXPECTED_AUDIENCES. A contract term, not a +# deployment knob -- which is why it is a constant here rather than config. +GATEWAY_ATTACH_AUDIENCES: Final[frozenset[str]] = frozenset({"gateway-attach"}) + +# Re-grant this far ahead of exp. Sized to cover clock skew between this +# machine and Keycloak plus one in-flight request: a token that is valid when +# the request leaves and expired when the gateway validates it is the classic +# expiry-boundary defect, and it presents as an intermittent 401. +_DEFAULT_SKEW_SECONDS: Final[int] = 30 + + +class GatewayTokenMinter: + """Mints and caches gateway access tokens for one credential.""" + + def __init__( + self, + *, + transport: ProtocolGatewayTransport, + credential: ModelGatewayCredential, + skew_seconds: int = _DEFAULT_SKEW_SECONDS, + ) -> None: + self._transport = transport + self._credential = credential + self._skew = timedelta(seconds=skew_seconds) + self._cached: ModelGatewayAccessToken | None = None + + async def token_for( + self, *, now: datetime, force_refresh: bool = False + ) -> ModelGatewayAccessToken: + """Return a token valid at ``now``, re-granting if inside the skew window. + + Args: + now: Caller-supplied instant. Injected rather than read from the + clock so the boundary cases are driven directly by tests. + force_refresh: Skip the cache and grant unconditionally. Used by the + attach path, where a still-valid-but-short token is not good + enough: the gateway stamps a session's ``expires_at`` from + ``min(token exp, max_session_ttl_seconds)``, so attaching with a + token that has two minutes left buys a two-minute session. That + is exactly the state the renewal cycle is trying to leave, so + renewing into it would loop. + + Returns: + A token whose ``expires_at`` is at least ``skew`` beyond ``now``. + + Raises: + ModelOnexError: If the grant fails, the response is malformed, or + the audience is not exactly the gateway-attach set. + """ + cached = self._cached + if ( + not force_refresh + and cached is not None + and now < cached.expires_at - self._skew + ): + return cached + minted = await self._grant(now=now) + self._cached = minted + return minted + + async def _grant(self, *, now: datetime) -> ModelGatewayAccessToken: + response = await self._transport.post_form( + self._credential.token_endpoint, + form={ + "grant_type": "client_credentials", + "client_id": self._credential.client_id, + "client_secret": self._credential.client_secret.get_secret_value(), + }, + headers={"Accept": "application/json"}, + ) + if response.status != 200: + # The body is NOT echoed: an OAuth2 error response is attacker- and + # proxy-influenced, and an error path that pastes a remote body into + # a local message is how secrets and tokens reach logs. + raise ModelOnexError( + "gateway token grant rejected by " + f"{self._credential.token_endpoint} (HTTP {response.status}) for " + f"client_id '{self._credential.client_id}'. Check the credential " + "with 'onex auth status', or re-run 'onex auth login' with a " + "freshly rotated secret.", + error_code=EnumCoreErrorCode.AUTHENTICATION_ERROR, + ) + + payload = self._decode_json_object( + await response.text(), source=self._credential.token_endpoint + ) + access_token = self._require_string(payload, "access_token") + expires_in = self._require_int(payload, "expires_in") + audiences = self._audiences_of(access_token) + + if audiences != GATEWAY_ATTACH_AUDIENCES: + raise ModelOnexError( + "minted token carries audience " + f"{sorted(audiences)} but the gateway requires exactly " + f"{sorted(GATEWAY_ATTACH_AUDIENCES)} (set equality, not " + f"membership). The Keycloak client '{self._credential.client_id}' " + "needs an audience mapper stamping 'gateway-attach'.", + error_code=EnumCoreErrorCode.AUTHENTICATION_ERROR, + ) + + return ModelGatewayAccessToken( + access_token=SecretStr(access_token), + expires_at=now + timedelta(seconds=expires_in), + audiences=audiences, + ) + + # -- parsing ----------------------------------------------------------- + + def _audiences_of(self, access_token: str) -> frozenset[str]: + """Normalise the ``aud`` claim to a set, mirroring the gateway. + + RFC 7519 §4.1.3 permits ``aud`` to be a single case-sensitive string OR + an array of them, and Keycloak switches to the array form the moment a + client carries more than one audience mapper. Multiplicity and order + must therefore not be observable -- ``aud`` is a set. + """ + segments = access_token.split(".") + if len(segments) != 3: + raise ModelOnexError( + "the token endpoint returned an access_token that is not a JWT " + f"(expected 3 dot-separated segments, found {len(segments)}).", + error_code=EnumCoreErrorCode.PARSING_ERROR, + ) + padding = "=" * (-len(segments[1]) % 4) + try: + claims_bytes = base64.urlsafe_b64decode(segments[1] + padding) + except (binascii.Error, ValueError) as exc: + raise ModelOnexError( + "the access_token's claims segment is not valid base64url.", + error_code=EnumCoreErrorCode.PARSING_ERROR, + ) from exc + + claims = self._decode_json_object( + claims_bytes.decode("utf-8", errors="replace"), + source="access_token claims", + ) + raw = claims.get("aud") + if isinstance(raw, str): + return frozenset({raw}) + if isinstance(raw, list): + if not all(isinstance(entry, str) for entry in raw): + raise ModelOnexError( + "the access_token's 'aud' claim is a list containing " + "non-string entries.", + error_code=EnumCoreErrorCode.PARSING_ERROR, + ) + return frozenset(str(entry) for entry in raw) + raise ModelOnexError( + "the access_token carries no usable 'aud' claim; the gateway " + "requires exactly " + f"{sorted(GATEWAY_ATTACH_AUDIENCES)}.", + error_code=EnumCoreErrorCode.AUTHENTICATION_ERROR, + ) + + def _decode_json_object(self, raw: str, *, source: str) -> dict[str, object]: + try: + document = json.loads(raw) + except json.JSONDecodeError as exc: + raise ModelOnexError( + f"{source} returned a body that is not valid JSON.", + error_code=EnumCoreErrorCode.PARSING_ERROR, + ) from exc + if not isinstance(document, dict): + raise ModelOnexError( + f"{source} returned {type(document).__name__}, expected a JSON object.", + error_code=EnumCoreErrorCode.PARSING_ERROR, + ) + return {str(key): value for key, value in document.items()} + + def _require_string(self, payload: dict[str, object], key: str) -> str: + value = payload.get(key) + if not isinstance(value, str) or not value: + raise ModelOnexError( + f"the token endpoint response has no usable '{key}' field.", + error_code=EnumCoreErrorCode.PARSING_ERROR, + ) + return value + + def _require_int(self, payload: dict[str, object], key: str) -> int: + value = payload.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ModelOnexError( + f"the token endpoint response has no usable '{key}' field.", + error_code=EnumCoreErrorCode.PARSING_ERROR, + ) + return value + + @staticmethod + def utc_now() -> datetime: + """The clock, in one named place, so callers do not each pick one.""" + return datetime.now(UTC) diff --git a/src/omnibase_infra/gateway/client/gateway_transport_httpx.py b/src/omnibase_infra/gateway/client/gateway_transport_httpx.py new file mode 100644 index 0000000000..1fa67bad33 --- /dev/null +++ b/src/omnibase_infra/gateway/client/gateway_transport_httpx.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""The one component in the ``onex auth`` slice that opens a socket (OMN-15922). + +EFFECT layer: this is external I/O and nothing else. Every decision the gateway +client makes -- whether a status is acceptable, whether an audience matches, +whether a renewal window has opened -- lives above this line in the services, +which is what lets the whole cycle be driven in tests by an in-memory fake with +no network and no sleeping. The rule that keeps that true is the one below. + +WHAT THIS DOES NOT DO: CLASSIFY + Neither method raises on a non-2xx. That is the ``ProtocolGatewayTransport`` + contract, and it is load-bearing rather than lax: a 401 from Keycloak and a + 401 from the gateway need *different* operator-facing messages naming + different remediations, and only the caller knows which call it made. An + adapter that raised here would flatten both into one transport error and + throw away the status the caller needs to say anything useful. So a reached + server always comes back as a response. + + A server that was NOT reached is the opposite case and does raise: there is + no status to hand back, and returning a synthetic one (a fake 503) would be + indistinguishable from a real server that answered 503 -- which is the + difference between "your gateway is refusing you" and "your gateway is not + there". ``InfraUnavailableError`` keeps them apart. + +SECRET DISCIPLINE + ``post_form`` carries ``client_secret`` and ``post_json`` carries a Bearer. + Nothing here logs a request, a body, a header, or an exception payload, and + the raised errors name only the URL's operation -- never the form, never the + headers. httpx's own exception reprs do not carry request bodies, but the + error path deliberately does not interpolate ``exc`` for that reason. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping + +import httpx + +from omnibase_infra.enums import EnumInfraTransportType +from omnibase_infra.errors import InfraUnavailableError, ModelInfraErrorContext + +__all__ = ["GatewayTransportHttpx", "GatewayHttpResponse"] + +# One grant or one attach against a control-plane endpoint. Long enough to +# absorb a cold Keycloak realm, short enough that an unattended runtime's +# renewal cycle still fits inside the margin the directive declares. +_DEFAULT_TIMEOUT_SECONDS: float = 10.0 + + +class GatewayHttpResponse: + """A reached server's answer, satisfying ``ProtocolHttpResponse``. + + Holds the body as already-read text rather than a live stream: the caller + reads it at most once, and a streamed body would outlive the + ``httpx.AsyncClient`` context that produced it. + """ + + def __init__(self, status: int, body: str) -> None: + self._status = status + self._body = body + + @property + def status(self) -> int: + return self._status + + async def text(self) -> str: + return self._body + + async def json(self) -> object: + """Present for protocol conformance; the gateway client never calls it. + + The client parses through typed Pydantic models instead, so that the + wire shape has exactly one authority. Kept because + ``ProtocolHttpResponse`` declares it and a partial implementation would + fail the structural check for the next caller, not this one. + """ + return json.loads(self._body) + + +class GatewayTransportHttpx: + """``ProtocolGatewayTransport`` over ``httpx.AsyncClient``.""" + + def __init__(self, *, timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS) -> None: + self._timeout = timeout_seconds + + async def post_form( + self, + url: str, + *, + form: Mapping[str, str], + headers: Mapping[str, str], + ) -> GatewayHttpResponse: + """POST ``form`` url-encoded. Carries ``client_secret`` -- never logged.""" + return await self._post( + url, + operation="gateway_token_grant", + data=dict(form), + content=None, + headers=dict(headers), + ) + + async def post_json( + self, + url: str, + *, + body: str, + headers: Mapping[str, str], + ) -> GatewayHttpResponse: + """POST an already-serialized JSON body. Carries a Bearer -- never logged.""" + return await self._post( + url, + operation="gateway_request", + data=None, + content=body.encode("utf-8"), + headers=dict(headers), + ) + + async def _post( + self, + url: str, + *, + operation: str, + data: dict[str, str] | None, + content: bytes | None, + headers: dict[str, str], + ) -> GatewayHttpResponse: + try: + # The freestanding-IO guard exists to catch imperative IO that + # BYPASSES a transport contract. Here the raw call is what BACKS + # one: this module is the sole ProtocolGatewayTransport + # implementation, and confining the socket to this single line is + # precisely what keeps the credential store, token minter, renewal + # planner and session keeper transport-free and driveable by an + # in-memory fake. An outbound OAuth2 client_credentials grant plus + # gateway attach from a CLI has no bus-mediated transport to route + # through -- it is a client calling out, not a node emitting. + timeout = self._timeout + client = httpx.AsyncClient(timeout=timeout) # no-contract-check: the seam + async with client: + response = await client.post( + url, + data=data, + content=content, + headers=headers, + ) + except httpx.HTTPError as exc: + # The URL, not the payload: the form carries a client secret and the + # headers carry a bearer, so neither is interpolated here. + raise InfraUnavailableError( + f"gateway transport could not reach {url}", + context=ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation=operation, + ), + ) from exc + + return GatewayHttpResponse(response.status_code, response.text) diff --git a/src/omnibase_infra/gateway/client/store_gateway_credential.py b/src/omnibase_infra/gateway/client/store_gateway_credential.py new file mode 100644 index 0000000000..6b2491a724 --- /dev/null +++ b/src/omnibase_infra/gateway/client/store_gateway_credential.py @@ -0,0 +1,327 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""StoreGatewayCredential — the ``~/.onex`` gateway credential (OMN-15922). + +Two files, split on exactly one axis: whether the content is a secret. + +``~/.onex/config.yaml`` (the surface ``onex config get`` already reads, same +reader shape -- ``yaml.safe_load`` over the same path) carries a ``gateway:`` +block of references and endpoints. It never carries a secret VALUE. That is not +a style preference: config.yaml is world-readable by default and is the file +operators paste into issues and screen-shares. A literal ``client_secret`` here +is refused outright rather than accepted-with-a-warning, because a convenience +fallback would make the by-reference rule advisory and every future credential +would take the easy path. + +``~/.onex/credentials.json`` (mode 0600, enforced on read as well as on write) +holds ``{: }``. Enforcing the mode on READ matters more than on +write: the file survives ``chmod``, backup/restore, and ``scp``, so a write-time +check alone proves nothing about the file actually being loaded. + +Every failure path raises ``ModelOnexError``. There is no "return None and let +the caller decide" branch, because the caller that decides wrong makes an +anonymous call the operator believes is authenticated -- the single failure +this store exists to prevent (OMN-15680 AC-e restated at the auth layer). + +Schema note (OMN-16037): ``config.yaml`` currently has two divergent writers, +``cli_config.py`` (``mode``/``kafka``/``logging``) and ``cli_init.py`` +(``version``/``credentials``/``paths``). This store deliberately does not pick +a side and does not widen the drift: it reads and writes ONLY its own +``gateway:`` block, preserves every other top-level key byte-for-byte through a +load/mutate/dump round trip, and follows ``cli_config.py``'s reader shape. +""" + +from __future__ import annotations + +import json +import stat +from pathlib import Path +from typing import Final + +import yaml +from pydantic import SecretStr + +from omnibase_core.enums.enum_core_error_code import EnumCoreErrorCode +from omnibase_core.errors.model_onex_error import ModelOnexError +from omnibase_infra.gateway.models.model_gateway_credential import ( + ModelGatewayCredential, +) + +__all__ = ["StoreGatewayCredential"] + +_GATEWAY_BLOCK: Final[str] = "gateway" +_SECRET_REF_KEY: Final[str] = "client_secret_ref" +_REMEDIATION: Final[str] = ( + "run 'onex auth login --tenant-slug --client-id " + "--client-secret-stdin'" +) +# Field name -> config key. tenant_slug/client_id/token_endpoint/base_url are +# the four the mint and the attach both need; edge_instance_id is bookkeeping +# only and so is the one key with a defensible default (the hostname is not +# available to a transport-free package, so login always writes it). +_REQUIRED_KEYS: Final[tuple[str, ...]] = ( + "tenant_slug", + "client_id", + _SECRET_REF_KEY, + "token_endpoint", + "base_url", +) + + +class StoreGatewayCredential: + """Reads and writes the gateway credential under an ``~/.onex`` root.""" + + def __init__(self, *, onex_home: Path) -> None: + """Bind the store to a directory. + + Args: + onex_home: Directory holding ``config.yaml`` and + ``credentials.json``. Injected rather than derived from + ``Path.home()`` inside the class so tests drive a real + directory instead of patching the home lookup. + """ + self._onex_home = onex_home + + @property + def config_path(self) -> Path: + return self._onex_home / "config.yaml" + + @property + def credentials_path(self) -> Path: + return self._onex_home / "credentials.json" + + # -- read -------------------------------------------------------------- + + def load(self) -> ModelGatewayCredential: + """Resolve the credential, or raise naming what to do about it. + + Raises: + ModelOnexError: On any missing, blank, malformed, mis-permissioned + or secret-carrying configuration. Never returns a partially + resolved credential. + """ + block = self._load_gateway_block() + + if "client_secret" in block: + raise ModelOnexError( + f"{self.config_path} carries an inline 'client_secret'. The " + "secret value must live only in " + f"{self.credentials_path} (mode 0600), referenced from config " + f"by '{_SECRET_REF_KEY}'. Remove it and {_REMEDIATION}.", + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + + values: dict[str, str] = {} + for key in _REQUIRED_KEYS: + values[key] = self._require_text(block, key) + + edge_instance_id = self._require_text(block, "edge_instance_id") + secret = self._read_secret(values[_SECRET_REF_KEY]) + + return ModelGatewayCredential( + tenant_slug=values["tenant_slug"], + client_id=values["client_id"], + client_secret=SecretStr(secret), + token_endpoint=values["token_endpoint"], + base_url=values["base_url"], + edge_instance_id=edge_instance_id, + ) + + def _load_gateway_block(self) -> dict[str, object]: + document = self._load_config_document(must_exist=True) + block = document.get(_GATEWAY_BLOCK) + if block is None: + raise ModelOnexError( + f"{self.config_path} has no '{_GATEWAY_BLOCK}:' block -- this " + f"machine holds no gateway credential. To create one, {_REMEDIATION}.", + error_code=EnumCoreErrorCode.CONFIGURATION_NOT_FOUND, + ) + if not isinstance(block, dict): + raise ModelOnexError( + f"{self.config_path}: '{_GATEWAY_BLOCK}' must be a mapping, " + f"found {type(block).__name__}.", + error_code=EnumCoreErrorCode.CONFIGURATION_PARSE_ERROR, + ) + return {str(key): value for key, value in block.items()} + + def _load_config_document(self, *, must_exist: bool) -> dict[str, object]: + if not self.config_path.exists(): + if not must_exist: + return {} + raise ModelOnexError( + f"no ONEX config at {self.config_path} -- this machine holds no " + f"gateway credential. To create one, {_REMEDIATION}.", + error_code=EnumCoreErrorCode.CONFIGURATION_NOT_FOUND, + ) + # yaml-ok: user-authored config file with two divergent writers + # (OMN-16037); a Pydantic model here would either reject the other + # writer's keys or silently drop them on the round trip. + document = yaml.safe_load(self.config_path.read_text()) + if document is None: + return {} + if not isinstance(document, dict): + raise ModelOnexError( + f"{self.config_path} must be a YAML mapping, found " + f"{type(document).__name__}.", + error_code=EnumCoreErrorCode.CONFIGURATION_PARSE_ERROR, + ) + return {str(key): value for key, value in document.items()} + + def _require_text(self, block: dict[str, object], key: str) -> str: + """Read one non-blank string, treating blank as absent-and-wrong.""" + if key not in block: + raise ModelOnexError( + f"{self.config_path}: '{_GATEWAY_BLOCK}.{key}' is missing. " + f"To rewrite the block, {_REMEDIATION}.", + error_code=EnumCoreErrorCode.MISSING_REQUIRED_PARAMETER, + ) + value = block[key] + if not isinstance(value, str) or not value.strip(): + raise ModelOnexError( + f"{self.config_path}: '{_GATEWAY_BLOCK}.{key}' must be a " + f"non-empty string. To rewrite the block, {_REMEDIATION}.", + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + return value + + def _read_secret(self, secret_ref: str) -> str: + """Resolve the referenced secret from the 0600 credentials file.""" + if not self.credentials_path.exists(): + raise ModelOnexError( + f"no credentials.json at {self.credentials_path}, but config " + f"references secret '{secret_ref}'. To restore it, {_REMEDIATION}.", + error_code=EnumCoreErrorCode.CONFIGURATION_NOT_FOUND, + ) + + mode = stat.S_IMODE(self.credentials_path.stat().st_mode) + if mode & 0o077: + raise ModelOnexError( + f"{self.credentials_path} is mode {mode:04o}; it must be 0600 " + "(owner-only). Refusing to read a group- or world-readable " + "credential file. Fix with: chmod 600 " + f"{self.credentials_path}", + error_code=EnumCoreErrorCode.PERMISSION_DENIED, + ) + + raw = self.credentials_path.read_text() + try: + document = json.loads(raw) + except json.JSONDecodeError as exc: + raise ModelOnexError( + f"{self.credentials_path} is not valid JSON. To rewrite it, " + f"{_REMEDIATION}.", + error_code=EnumCoreErrorCode.CONFIGURATION_PARSE_ERROR, + ) from exc + + if not isinstance(document, dict): + raise ModelOnexError( + f"{self.credentials_path} must be a JSON object mapping " + "secret refs to secret values.", + error_code=EnumCoreErrorCode.CONFIGURATION_PARSE_ERROR, + ) + if secret_ref not in document: + raise ModelOnexError( + f"{self.credentials_path} has no entry for secret ref " + f"'{secret_ref}' named by {self.config_path}. To restore it, " + f"{_REMEDIATION}.", + error_code=EnumCoreErrorCode.CONFIGURATION_NOT_FOUND, + ) + secret = document[secret_ref] + if not isinstance(secret, str) or not secret: + raise ModelOnexError( + f"{self.credentials_path}: entry '{secret_ref}' must be a " + "non-empty string.", + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + return secret + + # -- write ------------------------------------------------------------- + + def save( + self, + *, + tenant_slug: str, + client_id: str, + client_secret: str, + token_endpoint: str, + base_url: str, + edge_instance_id: str, + ) -> None: + """Write the reference-only config block and the 0600 secret file. + + Every other top-level key in ``config.yaml`` survives the round trip -- + ``onex auth login`` must not be a way to lose someone's ``kafka:`` + settings (OMN-16037: two writers already disagree about this file). + """ + secret_ref = f"{tenant_slug}-gateway" + self._onex_home.mkdir(parents=True, exist_ok=True) + + document = self._load_config_document(must_exist=False) + document[_GATEWAY_BLOCK] = { + "tenant_slug": tenant_slug, + "client_id": client_id, + _SECRET_REF_KEY: secret_ref, + "token_endpoint": token_endpoint, + "base_url": base_url, + "edge_instance_id": edge_instance_id, + } + self.config_path.write_text(yaml.safe_dump(document, sort_keys=False)) + + secrets = self._load_secret_document() + secrets[secret_ref] = client_secret + self._write_secret_document(secrets) + + def clear(self) -> None: + """Remove both the config block and the referenced secret. + + Order matters: the secret goes first. If the process dies between the + two writes, what survives is a config that names a missing secret -- + which ``load`` refuses loudly -- rather than an orphaned secret sitting + on disk with nothing pointing at it. + """ + document = self._load_config_document(must_exist=False) + block = document.get(_GATEWAY_BLOCK) + secret_ref = "" + if isinstance(block, dict): + candidate = block.get(_SECRET_REF_KEY) + if isinstance(candidate, str): + secret_ref = candidate + + if secret_ref: + secrets = self._load_secret_document() + if secret_ref in secrets: + del secrets[secret_ref] + self._write_secret_document(secrets) + + if _GATEWAY_BLOCK in document: + del document[_GATEWAY_BLOCK] + self.config_path.write_text(yaml.safe_dump(document, sort_keys=False)) + + def _load_secret_document(self) -> dict[str, str]: + if not self.credentials_path.exists(): + return {} + try: + document = json.loads(self.credentials_path.read_text()) + except json.JSONDecodeError as exc: + raise ModelOnexError( + f"{self.credentials_path} is not valid JSON; refusing to " + "overwrite it and lose the credentials it may hold.", + error_code=EnumCoreErrorCode.CONFIGURATION_PARSE_ERROR, + ) from exc + if not isinstance(document, dict): + raise ModelOnexError( + f"{self.credentials_path} must be a JSON object.", + error_code=EnumCoreErrorCode.CONFIGURATION_PARSE_ERROR, + ) + return {str(key): str(value) for key, value in document.items()} + + def _write_secret_document(self, secrets: dict[str, str]) -> None: + """Write the secret file so it is never briefly world-readable. + + ``touch`` + ``chmod`` before ``write_text``: creating the file at the + umask default and tightening it afterwards leaves a window in which the + secret is on disk at 0644. + """ + self.credentials_path.touch(mode=0o600, exist_ok=True) + self.credentials_path.chmod(0o600) + self.credentials_path.write_text(json.dumps(secrets, indent=2, sort_keys=True)) diff --git a/src/omnibase_infra/gateway/models/model_gateway_access_token.py b/src/omnibase_infra/gateway/models/model_gateway_access_token.py new file mode 100644 index 0000000000..3f50c50f9a --- /dev/null +++ b/src/omnibase_infra/gateway/models/model_gateway_access_token.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""ModelGatewayAccessToken -- one minted client-credentials token (OMN-15922). + +``expires_at`` is absolute, derived once from the grant's ``expires_in`` at the +instant of the grant. Carrying the relative form instead would push the "when +does this die" arithmetic to every call site, and each of those would have to +remember which instant it was relative to. + +``audiences`` is the normalised ``aud`` claim as a SET. RFC 7519 4.1.3 permits +``aud`` to be a single string or an array, and Keycloak emits the array form as +soon as a client carries more than one audience mapper -- so multiplicity and +order must not be observable to anything downstream. The gateway compares the +set for exact equality against ``{"gateway-attach"}``; this field is what lets +the client apply the identical comparison before the token is ever used. +""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, SecretStr + +__all__ = ["ModelGatewayAccessToken"] + + +class ModelGatewayAccessToken(BaseModel): + """A minted access token with its absolute expiry and audience set.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + # SecretStr for the same reason ModelGatewayCredential.client_secret is: + # a bearer token in a traceback is a credential in a log aggregator. + access_token: SecretStr + expires_at: datetime + audiences: frozenset[str] diff --git a/src/omnibase_infra/gateway/models/model_gateway_attachment.py b/src/omnibase_infra/gateway/models/model_gateway_attachment.py new file mode 100644 index 0000000000..94e52ee2d2 --- /dev/null +++ b/src/omnibase_infra/gateway/models/model_gateway_attachment.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""ModelGatewayAttachment -- everything one successful attach hands the client. + +The client-side reading of ``ModelGatewayAttachResponse`` +(``node_gateway_attach_effect`` 0.3.0) reduced to the three things a client +actually acts on: the session it now holds, how often to prove liveness, and +the renewal cycle it must run to survive its own ceiling. The response's +``session_event`` is deliberately not carried -- it is the node's thin-publish +payload for the bus, not instruction to the caller. + +``session`` and ``renewal`` are the node's OWN models, imported rather than +mirrored. The client and the server are the same package here, so there is +exactly one definition of the attach contract's shape and no way for the two +sides to drift: a field added to ``ModelGatewaySession`` is immediately a field +this client parses, and a rename breaks the import rather than silently +producing a client that ignores what it was told. + +``renewal`` is REQUIRED here, matching the node and NOT the edge. The edge +(``onex-api``) accepts it as optional purely so the two repos may deploy in +either order; a *client* that accepted its absence would run an unattended +runtime with no renewal policy at all, which is the exact gap OMN-15952 was +filed against. Absence is refused at parse time. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_renewal_directive import ( + ModelGatewayRenewalDirective, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session import ( + ModelGatewaySession, +) + +__all__ = ["ModelGatewayAttachment"] + + +class ModelGatewayAttachment(BaseModel): + """A live gateway session plus the terms for keeping it.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + session: ModelGatewaySession + heartbeat_interval_seconds: int = Field(gt=0) + renewal: ModelGatewayRenewalDirective diff --git a/src/omnibase_infra/gateway/models/model_gateway_credential.py b/src/omnibase_infra/gateway/models/model_gateway_credential.py new file mode 100644 index 0000000000..3e4697816f --- /dev/null +++ b/src/omnibase_infra/gateway/models/model_gateway_credential.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""ModelGatewayCredential -- one tenant's machine identity for the gateway (OMN-15922). + +The resolved form of what ``~/.onex`` holds: the per-tenant confidential +Keycloak client (``client_credentials`` grant, ``clientId`` == the immutable +``principal_id``) plus the two contract-supplied URLs it is used against. + +``client_secret`` is a ``SecretStr`` deliberately. This model is constructed on +the CLI path, passed into services, and therefore ends up inside tracebacks, +``repr()`` output and structured log records whenever anything downstream +raises. ``SecretStr`` renders as ``**********`` in every one of those, which is +the only reason a plain-string field would ever have been "fine until it +wasn't". Read the value only where it is about to go on the wire, via +``get_secret_value()``. + +No field here is optional. A half-configured credential is exactly the state +that produces an anonymous call the operator believes is authenticated, so +absence is resolved (and refused) at the store boundary, never represented. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, SecretStr + +__all__ = ["ModelGatewayCredential"] + + +class ModelGatewayCredential(BaseModel): + """A tenant's client-credentials identity plus its resolved endpoints.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + tenant_slug: str = Field(min_length=1) + # Keycloak clientId of the per-tenant confidential client. This IS the + # principal_id the gateway resolves authority from -- it is not a label. + client_id: str = Field(min_length=1) + client_secret: SecretStr + # Realm token endpoint, supplied by configuration rather than assembled + # from a hardcoded issuer -- the realm differs per tenant. + token_endpoint: str = Field(min_length=1) + # Gateway origin the attach/heartbeat paths are appended to. + base_url: str = Field(min_length=1) + # Caller-declared host label, used by the gateway for session bookkeeping + # only -- never for authorization, which comes from the token claims. + edge_instance_id: str = Field(min_length=1, max_length=255) diff --git a/src/omnibase_infra/handlers/registration_storage/models/model_update_registration_request.py b/src/omnibase_infra/handlers/registration_storage/models/model_update_registration_request.py index 68610c2d72..8455eb512b 100644 --- a/src/omnibase_infra/handlers/registration_storage/models/model_update_registration_request.py +++ b/src/omnibase_infra/handlers/registration_storage/models/model_update_registration_request.py @@ -47,10 +47,11 @@ class ModelUpdateRegistrationRequest(BaseModel): >>> from omnibase_infra.nodes.node_registration_storage_effect.models import ( ... ModelRegistrationUpdate, ... ) + >>> resolved_health_endpoint = "" >>> request = ModelUpdateRegistrationRequest( ... node_id=uuid4(), ... updates=ModelRegistrationUpdate( - ... endpoints={"health": "http://localhost:8080/health"}, + ... endpoints={"health": resolved_health_endpoint}, ... ), ... correlation_id=uuid4(), ... ) diff --git a/src/omnibase_infra/idempotency/__init__.py b/src/omnibase_infra/idempotency/__init__.py index 5ede314da1..6df15b74e6 100644 --- a/src/omnibase_infra/idempotency/__init__.py +++ b/src/omnibase_infra/idempotency/__init__.py @@ -20,6 +20,7 @@ Stores: - StoreIdempotencyInmemory: In-memory store for testing (OMN-945) - StoreIdempotencyPostgres: Production PostgreSQL store (OMN-945) + - StoreIdempotencySqlite: Durable single-edge SQLite store Example - InMemory (Testing): >>> from omnibase_infra.idempotency import StoreIdempotencyInmemory @@ -78,6 +79,7 @@ ) from omnibase_infra.idempotency.store_inmemory import StoreIdempotencyInmemory from omnibase_infra.idempotency.store_postgres import StoreIdempotencyPostgres +from omnibase_infra.idempotency.store_sqlite import StoreIdempotencySqlite __all__: list[str] = [ # Stores @@ -89,6 +91,7 @@ "ModelIdempotencyStoreMetrics", "ModelPostgresIdempotencyStoreConfig", "StoreIdempotencyPostgres", + "StoreIdempotencySqlite", # Protocol "ProtocolIdempotencyStore", ] diff --git a/src/omnibase_infra/idempotency/store_sqlite.py b/src/omnibase_infra/idempotency/store_sqlite.py new file mode 100644 index 0000000000..4cd0c50b92 --- /dev/null +++ b/src/omnibase_infra/idempotency/store_sqlite.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Durable, edge-local SQLite idempotency store. + +The gateway runs as a single process on an operator edge and cannot depend on +the cloud database to decide whether a cross-broker delivery already +completed. This store provides that local durable decision surface while +implementing the same ``ProtocolIdempotencyStore`` contract used by the other +runtime stores. + +Each operation uses its own SQLite connection. That keeps connection/thread +ownership explicit when the blocking standard-library driver is moved through +``asyncio.to_thread``. WAL plus ``synchronous=FULL`` makes a successful +``mark_processed`` durable before the caller is allowed to commit its source +Kafka offset. +""" + +from __future__ import annotations + +import asyncio +import sqlite3 +from datetime import UTC, datetime +from pathlib import Path +from uuid import UUID + +from omnibase_infra.idempotency.protocol_idempotency_store import ( + ProtocolIdempotencyStore, +) + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS idempotency_records ( + domain TEXT NOT NULL, + message_id TEXT NOT NULL, + correlation_id TEXT, + processed_at REAL NOT NULL, + PRIMARY KEY (domain, message_id) +) +""" + + +class StoreIdempotencySqlite(ProtocolIdempotencyStore): + """Coroutine-safe durable idempotency store backed by one SQLite file.""" + + def __init__(self, path: Path) -> None: + self._path = path + self._lock = asyncio.Lock() + self._started = False + + async def start(self) -> None: + """Create and verify the store, failing boot when it is unavailable.""" + async with self._lock: + await asyncio.to_thread(self._initialize_sync) + self._started = True + + async def close(self) -> None: + """Close the lifecycle boundary (operations use short-lived connections).""" + async with self._lock: + self._started = False + + async def check_and_record( + self, + message_id: UUID, + domain: str | None = None, + correlation_id: UUID | None = None, + ) -> bool: + """Atomically insert a completion marker if one does not exist.""" + async with self._lock: + self._require_started() + return await asyncio.to_thread( + self._check_and_record_sync, + message_id, + domain, + correlation_id, + ) + + async def is_processed( + self, + message_id: UUID, + domain: str | None = None, + ) -> bool: + """Return whether the durable completion marker exists.""" + async with self._lock: + self._require_started() + return await asyncio.to_thread( + self._is_processed_sync, + message_id, + domain, + ) + + async def mark_processed( + self, + message_id: UUID, + domain: str | None = None, + correlation_id: UUID | None = None, + processed_at: datetime | None = None, + ) -> None: + """Durably upsert a completion marker before source acknowledgement.""" + timestamp = processed_at or datetime.now(UTC) + async with self._lock: + self._require_started() + await asyncio.to_thread( + self._mark_processed_sync, + message_id, + domain, + correlation_id, + timestamp, + ) + + async def cleanup_expired(self, ttl_seconds: int) -> int: + """Remove completion markers older than the contract-declared TTL.""" + if ttl_seconds < 1: + raise ValueError("ttl_seconds must be positive") + cutoff = datetime.now(UTC).timestamp() - ttl_seconds + async with self._lock: + self._require_started() + return await asyncio.to_thread(self._cleanup_expired_sync, cutoff) + + def _initialize_sync(self) -> None: + self._path.parent.mkdir(parents=True, exist_ok=True) + with self._connect() as connection: + connection.execute(_SCHEMA) + result = connection.execute("PRAGMA quick_check").fetchone() + if result != ("ok",): + raise sqlite3.DatabaseError( + f"gateway idempotency store quick_check failed: {result!r}" + ) + + def _connect(self) -> sqlite3.Connection: + # This class is the injected ProtocolIdempotencyStore implementation; + # opening its edge-local database here is the service boundary itself. + connection = sqlite3.connect( # no-contract-check: injected store boundary + self._path, + timeout=5.0, + ) + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("PRAGMA synchronous=FULL") + return connection + + def _check_and_record_sync( + self, + message_id: UUID, + domain: str | None, + correlation_id: UUID | None, + ) -> bool: + with self._connect() as connection: + cursor = connection.execute( + """ + INSERT OR IGNORE INTO idempotency_records + (domain, message_id, correlation_id, processed_at) + VALUES (?, ?, ?, ?) + """, + ( + self._domain(domain), + str(message_id), + str(correlation_id) if correlation_id is not None else None, + datetime.now(UTC).timestamp(), + ), + ) + return cursor.rowcount == 1 + + def _is_processed_sync(self, message_id: UUID, domain: str | None) -> bool: + with self._connect() as connection: + row = connection.execute( + """ + SELECT 1 FROM idempotency_records + WHERE domain = ? AND message_id = ? + """, + (self._domain(domain), str(message_id)), + ).fetchone() + return row is not None + + def _mark_processed_sync( + self, + message_id: UUID, + domain: str | None, + correlation_id: UUID | None, + processed_at: datetime, + ) -> None: + if processed_at.tzinfo is None: + raise ValueError("processed_at must be timezone-aware") + with self._connect() as connection: + connection.execute( + """ + INSERT INTO idempotency_records + (domain, message_id, correlation_id, processed_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(domain, message_id) DO UPDATE SET + correlation_id = excluded.correlation_id, + processed_at = excluded.processed_at + """, + ( + self._domain(domain), + str(message_id), + str(correlation_id) if correlation_id is not None else None, + processed_at.timestamp(), + ), + ) + + def _cleanup_expired_sync(self, cutoff: float) -> int: + with self._connect() as connection: + cursor = connection.execute( + "DELETE FROM idempotency_records WHERE processed_at < ?", + (cutoff,), + ) + return cursor.rowcount + + def _require_started(self) -> None: + if not self._started: + raise RuntimeError("SQLite idempotency store is not started") + + @staticmethod + def _domain(domain: str | None) -> str: + return domain or "" + + +__all__ = ["StoreIdempotencySqlite"] diff --git a/src/omnibase_infra/migration/cutover/__init__.py b/src/omnibase_infra/migration/cutover/__init__.py new file mode 100644 index 0000000000..2ddfdff75d --- /dev/null +++ b/src/omnibase_infra/migration/cutover/__init__.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Transformation receipts and durable per-family database cutover proof.""" + +from omnibase_infra.migration.cutover.cutover_coordinator import ( + CutoverCoordinator, +) +from omnibase_infra.migration.cutover.models import ( + ModelControlPlaneDeltaEvidence, + ModelCutoverContinuityEvidence, + ModelCutoverFamilyContract, + ModelCutoverFamilyState, + ModelCutoverJournalEvent, + ModelCutoverJournalRequest, + ModelPostgresEvidenceQuerySet, + ModelProjectionReplayEvidence, + ModelReceiptCheck, + ModelReverseDeltaEntry, + ModelReverseDeltaProof, + ModelRollbackDecision, + ModelTransformationEvidence, + ModelTransformationReceipt, +) +from omnibase_infra.migration.cutover.postgres_transformation_evidence_collector import ( + PostgresTransformationEvidenceCollector, +) +from omnibase_infra.migration.cutover.repository_postgres_cutover_journal import ( + RepositoryPostgresCutoverJournal, +) +from omnibase_infra.migration.cutover.transformation_receipt_builder import ( + TransformationReceiptBuilder, +) + +__all__ = [ + "CutoverCoordinator", + "ModelControlPlaneDeltaEvidence", + "ModelCutoverContinuityEvidence", + "ModelCutoverFamilyContract", + "ModelCutoverFamilyState", + "ModelCutoverJournalEvent", + "ModelCutoverJournalRequest", + "ModelPostgresEvidenceQuerySet", + "ModelProjectionReplayEvidence", + "ModelReceiptCheck", + "ModelReverseDeltaEntry", + "ModelReverseDeltaProof", + "ModelRollbackDecision", + "ModelTransformationEvidence", + "ModelTransformationReceipt", + "PostgresTransformationEvidenceCollector", + "RepositoryPostgresCutoverJournal", + "TransformationReceiptBuilder", +] diff --git a/src/omnibase_infra/migration/cutover/cutover_coordinator.py b/src/omnibase_infra/migration/cutover/cutover_coordinator.py new file mode 100644 index 0000000000..4184a90691 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/cutover_coordinator.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Application service joining pure receipts to the durable family journal.""" + +from __future__ import annotations + +from uuid import UUID + +from omnibase_infra.migration.cutover.models import ( + ModelCutoverContinuityEvidence, + ModelCutoverFamilyContract, + ModelCutoverJournalEvent, + ModelCutoverJournalRequest, + ModelReverseDeltaProof, + ModelRollbackDecision, + ModelTransformationEvidence, + ModelTransformationReceipt, +) +from omnibase_infra.migration.cutover.protocols import ( + ProtocolCutoverJournalRepository, +) +from omnibase_infra.migration.cutover.transformation_receipt_builder import ( + TransformationReceiptBuilder, +) + + +class CutoverCoordinator: + """Coordinate receipt persistence and guarded journal transitions.""" + + def __init__( + self, + repository: ProtocolCutoverJournalRepository, + receipt_service: TransformationReceiptBuilder | None = None, + ) -> None: + self._repository = repository + self._receipt_service = receipt_service or TransformationReceiptBuilder() + + async def register_family(self, contract: ModelCutoverFamilyContract) -> None: + """Persist a family's immutable cutover and rollback contract.""" + await self._repository.register_family(contract) + + async def reconcile( + self, + contract: ModelCutoverFamilyContract, + source: ModelTransformationEvidence, + target: ModelTransformationEvidence, + continuity: ModelCutoverContinuityEvidence, + ) -> ModelTransformationReceipt: + """Build and durably persist a complete PASS-or-FAIL receipt.""" + receipt = self._receipt_service.build(contract, source, target, continuity) + await self._repository.record_receipt(receipt) + return receipt + + async def append( + self, + family_id: UUID, + request: ModelCutoverJournalRequest, + ) -> ModelCutoverJournalEvent: + """Append a family event through the repository's state machine.""" + return await self._repository.append_event(family_id, request) + + async def record_reverse_delta( + self, + proof: ModelReverseDeltaProof, + ) -> None: + """Persist complete reverse-delta coverage before attesting it.""" + await self._repository.record_reverse_delta_proof(proof) + + async def evaluate_direct_rollback( + self, + family_id: UUID, + ) -> ModelRollbackDecision: + """Return the fail-closed rollback decision for one family.""" + return await self._repository.evaluate_direct_rollback(family_id) + + +__all__ = ["CutoverCoordinator"] diff --git a/src/omnibase_infra/migration/cutover/enums/__init__.py b/src/omnibase_infra/migration/cutover/enums/__init__.py new file mode 100644 index 0000000000..3e45609090 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/enums/__init__.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed cutover and receipt enums.""" + +from omnibase_infra.migration.cutover.enums.enum_cutover_event_kind import ( + EnumCutoverEventKind, +) +from omnibase_infra.migration.cutover.enums.enum_cutover_family_kind import ( + EnumCutoverFamilyKind, +) +from omnibase_infra.migration.cutover.enums.enum_cutover_family_status import ( + EnumCutoverFamilyStatus, +) +from omnibase_infra.migration.cutover.enums.enum_post_checkpoint_mode import ( + EnumPostCheckpointMode, +) +from omnibase_infra.migration.cutover.enums.enum_receipt_dimension import ( + EnumReceiptDimension, +) +from omnibase_infra.migration.cutover.enums.enum_receipt_status import ( + EnumReceiptStatus, +) +from omnibase_infra.migration.cutover.enums.enum_reverse_delta_operation import ( + EnumReverseDeltaOperation, +) + +__all__ = [ + "EnumCutoverEventKind", + "EnumCutoverFamilyKind", + "EnumCutoverFamilyStatus", + "EnumPostCheckpointMode", + "EnumReceiptDimension", + "EnumReceiptStatus", + "EnumReverseDeltaOperation", +] diff --git a/src/omnibase_infra/migration/cutover/enums/enum_cutover_event_kind.py b/src/omnibase_infra/migration/cutover/enums/enum_cutover_event_kind.py new file mode 100644 index 0000000000..f5f3c2ccc1 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/enums/enum_cutover_event_kind.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Append-only cutover journal event kinds.""" + +from enum import StrEnum + + +class EnumCutoverEventKind(StrEnum): + """Mechanically ordered events in one coherent family cutover.""" + + BACKFILL_STARTED = "backfill_started" + BACKFILL_COMPLETED = "backfill_completed" + DUAL_WRITE_STARTED = "dual_write_started" + DUAL_WRITE_ENDED = "dual_write_ended" + FINAL_DELTA_APPLIED = "final_delta_applied" + WRITER_CHECKPOINT = "writer_checkpoint" + APPLICATION_PATH_WRITE_PROVEN = "application_path_write_proven" + READER_CUTOVER = "reader_cutover" + OBSERVATION_WINDOW_STARTED = "observation_window_started" + OBSERVATION_WINDOW_COMPLETED = "observation_window_completed" + WRITER_QUIESCED = "writer_quiesced" + REVERSE_DELTA_PROVEN = "reverse_delta_proven" + FORWARD_FIX_RECORDED = "forward_fix_recorded" + PRE_CHECKPOINT_ROLLBACK = "pre_checkpoint_rollback" + MISMATCH_RESOLVED = "mismatch_resolved" + + +__all__ = ["EnumCutoverEventKind"] diff --git a/src/omnibase_infra/migration/cutover/enums/enum_cutover_family_kind.py b/src/omnibase_infra/migration/cutover/enums/enum_cutover_family_kind.py new file mode 100644 index 0000000000..b8b667a7e6 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/enums/enum_cutover_family_kind.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Kinds of coherent database families participating in a cutover.""" + +from enum import StrEnum + + +class EnumCutoverFamilyKind(StrEnum): + """Determines which continuity proof a family must carry.""" + + PROJECTION = "projection" + CONTROL_PLANE = "control_plane" + + +__all__ = ["EnumCutoverFamilyKind"] diff --git a/src/omnibase_infra/migration/cutover/enums/enum_cutover_family_status.py b/src/omnibase_infra/migration/cutover/enums/enum_cutover_family_status.py new file mode 100644 index 0000000000..76c130b255 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/enums/enum_cutover_family_status.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Durable state of an independently stoppable cutover family.""" + +from enum import StrEnum + + +class EnumCutoverFamilyStatus(StrEnum): + """Family-local state; one blocked family never blocks an unrelated family.""" + + READY = "ready" + BLOCKED = "blocked" + CHECKPOINTED = "checkpointed" + OBSERVING = "observing" + COMPLETE = "complete" + + +__all__ = ["EnumCutoverFamilyStatus"] diff --git a/src/omnibase_infra/migration/cutover/enums/enum_post_checkpoint_mode.py b/src/omnibase_infra/migration/cutover/enums/enum_post_checkpoint_mode.py new file mode 100644 index 0000000000..2815c92ee5 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/enums/enum_post_checkpoint_mode.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Allowed behavior after the first target-only authoritative write.""" + +from enum import StrEnum + + +class EnumPostCheckpointMode(StrEnum): + """Per-family rollback posture after source authority becomes stale.""" + + REVERSE_DELTA = "reverse_delta" + FORWARD_FIX_ONLY = "forward_fix_only" + + +__all__ = ["EnumPostCheckpointMode"] diff --git a/src/omnibase_infra/migration/cutover/enums/enum_receipt_dimension.py b/src/omnibase_infra/migration/cutover/enums/enum_receipt_dimension.py new file mode 100644 index 0000000000..fb387200cd --- /dev/null +++ b/src/omnibase_infra/migration/cutover/enums/enum_receipt_dimension.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Required dimensions of a transformation-aware receipt.""" + +from enum import StrEnum + + +class EnumReceiptDimension(StrEnum): + """Every family receipt evaluates every dimension explicitly.""" + + EVIDENCE_CONTRACTS = "evidence_contracts" + KEY_SET = "key_set" + ROW_COUNT = "row_count" + TRANSFORMATION_HASH = "transformation_hash" + FOREIGN_KEYS = "foreign_keys" + SEQUENCES = "sequences" + OWNERS = "owners" + GRANTS = "grants" + POLICIES = "policies" + VIEWS_FUNCTIONS = "views_functions" + EVENT_OFFSETS = "event_offsets" + CONTROL_PLANE_DELTA = "control_plane_delta" + COLLISIONS = "collisions" + DEPENDENCIES = "dependencies" + + +__all__ = ["EnumReceiptDimension"] diff --git a/src/omnibase_infra/migration/cutover/enums/enum_receipt_status.py b/src/omnibase_infra/migration/cutover/enums/enum_receipt_status.py new file mode 100644 index 0000000000..4d92cc5065 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/enums/enum_receipt_status.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Transformation receipt outcome.""" + +from enum import StrEnum + + +class EnumReceiptStatus(StrEnum): + """A receipt is either wholly proven or fail-closed.""" + + PASS = "pass" + FAIL = "fail" + + +__all__ = ["EnumReceiptStatus"] diff --git a/src/omnibase_infra/migration/cutover/enums/enum_reverse_delta_operation.py b/src/omnibase_infra/migration/cutover/enums/enum_reverse_delta_operation.py new file mode 100644 index 0000000000..b44db49b09 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/enums/enum_reverse_delta_operation.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Target mutations captured by a reverse-delta proof.""" + +from enum import StrEnum + + +class EnumReverseDeltaOperation(StrEnum): + """Mutation kind whose inverse is durably attested.""" + + INSERT = "insert" + UPDATE = "update" + DELETE = "delete" + + +__all__ = ["EnumReverseDeltaOperation"] diff --git a/src/omnibase_infra/migration/cutover/models/__init__.py b/src/omnibase_infra/migration/cutover/models/__init__.py new file mode 100644 index 0000000000..a9d2b54c0a --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/__init__.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed transformation receipt and cutover journal models.""" + +from omnibase_infra.migration.cutover.models.model_control_plane_delta_evidence import ( + ModelControlPlaneDeltaEvidence, +) +from omnibase_infra.migration.cutover.models.model_cutover_continuity_evidence import ( + ModelCutoverContinuityEvidence, +) +from omnibase_infra.migration.cutover.models.model_cutover_family_contract import ( + ModelCutoverFamilyContract, +) +from omnibase_infra.migration.cutover.models.model_cutover_family_state import ( + ModelCutoverFamilyState, +) +from omnibase_infra.migration.cutover.models.model_cutover_journal_event import ( + ModelCutoverJournalEvent, +) +from omnibase_infra.migration.cutover.models.model_cutover_journal_request import ( + ModelCutoverJournalRequest, +) +from omnibase_infra.migration.cutover.models.model_postgres_evidence_query_set import ( + ModelPostgresEvidenceQuerySet, +) +from omnibase_infra.migration.cutover.models.model_projection_replay_evidence import ( + ModelProjectionReplayEvidence, +) +from omnibase_infra.migration.cutover.models.model_receipt_check import ( + ModelReceiptCheck, +) +from omnibase_infra.migration.cutover.models.model_reverse_delta_entry import ( + ModelReverseDeltaEntry, +) +from omnibase_infra.migration.cutover.models.model_reverse_delta_proof import ( + ModelReverseDeltaProof, +) +from omnibase_infra.migration.cutover.models.model_rollback_decision import ( + ModelRollbackDecision, +) +from omnibase_infra.migration.cutover.models.model_transformation_evidence import ( + ModelTransformationEvidence, +) +from omnibase_infra.migration.cutover.models.model_transformation_receipt import ( + ModelTransformationReceipt, + calculate_transformation_receipt_hash, +) + +__all__ = [ + "ModelControlPlaneDeltaEvidence", + "ModelCutoverContinuityEvidence", + "ModelCutoverFamilyContract", + "ModelCutoverFamilyState", + "ModelCutoverJournalEvent", + "ModelCutoverJournalRequest", + "ModelPostgresEvidenceQuerySet", + "ModelProjectionReplayEvidence", + "ModelReceiptCheck", + "ModelReverseDeltaEntry", + "ModelReverseDeltaProof", + "ModelRollbackDecision", + "ModelTransformationEvidence", + "ModelTransformationReceipt", + "calculate_transformation_receipt_hash", +] diff --git a/src/omnibase_infra/migration/cutover/models/model_control_plane_delta_evidence.py b/src/omnibase_infra/migration/cutover/models/model_control_plane_delta_evidence.py new file mode 100644 index 0000000000..96fea07f96 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_control_plane_delta_evidence.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Control-plane snapshot and final-delta parity evidence.""" + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class ModelControlPlaneDeltaEvidence(BaseModel): + """Snapshot and final-delta parity for non-replayable control-plane truth.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + snapshot_id: UUID + source_snapshot_hash: str = Field(..., pattern=_SHA256_PATTERN) + target_snapshot_hash: str = Field(..., pattern=_SHA256_PATTERN) + final_delta_id: UUID + source_final_delta_hash: str = Field(..., pattern=_SHA256_PATTERN) + target_final_delta_hash: str = Field(..., pattern=_SHA256_PATTERN) + source_watermark: str = Field(..., min_length=1) + target_watermark: str = Field(..., min_length=1) + + +__all__ = ["ModelControlPlaneDeltaEvidence"] diff --git a/src/omnibase_infra/migration/cutover/models/model_cutover_continuity_evidence.py b/src/omnibase_infra/migration/cutover/models/model_cutover_continuity_evidence.py new file mode 100644 index 0000000000..bfb585b8cd --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_cutover_continuity_evidence.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Per-family projection or control-plane continuity evidence.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, field_validator + +from omnibase_infra.migration.cutover.models.model_control_plane_delta_evidence import ( + ModelControlPlaneDeltaEvidence, +) +from omnibase_infra.migration.cutover.models.model_projection_replay_evidence import ( + ModelProjectionReplayEvidence, +) + + +class ModelCutoverContinuityEvidence(BaseModel): + """Exactly one continuity mode is consumed according to family kind.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + projection_replays: tuple[ModelProjectionReplayEvidence, ...] = () + control_plane_delta: ModelControlPlaneDeltaEvidence | None = None + + @field_validator("projection_replays") + @classmethod + def _unique_projection_offsets( + cls, + value: tuple[ModelProjectionReplayEvidence, ...], + ) -> tuple[ModelProjectionReplayEvidence, ...]: + identities = [ + (item.projection_id, item.topic, item.partition) for item in value + ] + if identities != sorted( + identities, + key=lambda identity: (str(identity[0]), identity[1], identity[2]), + ): + raise ValueError("projection replay evidence must be sorted") + if len(identities) != len(set(identities)): + raise ValueError("projection replay evidence must be unique") + return value + + +__all__ = ["ModelCutoverContinuityEvidence"] diff --git a/src/omnibase_infra/migration/cutover/models/model_cutover_family_contract.py b/src/omnibase_infra/migration/cutover/models/model_cutover_family_contract.py new file mode 100644 index 0000000000..fe662e7386 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_cutover_family_contract.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Contract for one independently stoppable database relation family.""" + +from __future__ import annotations + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_infra.migration.cutover.enums import ( + EnumCutoverFamilyKind, + EnumPostCheckpointMode, +) + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class ModelCutoverFamilyContract(BaseModel): + """Declare continuity and rollback semantics before cutover work begins. + + Binding fields are secret-free topology references, never DSNs. A family + must choose reverse-delta or forward-fix-only up front. A dual-write window + is disabled by default and, when explicitly enabled, is strictly bounded. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + family_id: UUID + family_key: str = Field( + ..., + min_length=3, + pattern=r"^[a-z0-9][a-z0-9._-]+$", + description="Stable semantic key for the coherent relation family", + ) + family_kind: EnumCutoverFamilyKind + source_binding_ref: str = Field(..., min_length=1, max_length=200) + target_binding_ref: str = Field(..., min_length=1, max_length=200) + source_evidence_contract_hash: str = Field(..., pattern=_SHA256_PATTERN) + target_evidence_contract_hash: str = Field(..., pattern=_SHA256_PATTERN) + post_checkpoint_mode: EnumPostCheckpointMode + reverse_delta_contract_ref: str = Field(default="", max_length=300) + forward_fix_runbook_ref: str = Field(default="", max_length=300) + dual_write_max_seconds: int = Field( + default=0, + ge=0, + le=3600, + description="Zero disables dual-write; non-zero is a hard bounded window", + ) + observation_window_seconds: int = Field(..., ge=1, le=604800) + + @model_validator(mode="after") + def _validate_semantics(self) -> ModelCutoverFamilyContract: + if self.source_binding_ref == self.target_binding_ref: + raise ValueError("source and target binding refs must be distinct") + + if "://" in self.source_binding_ref or "://" in self.target_binding_ref: + raise ValueError("binding refs must be secret-free names, not DSNs") + + if self.post_checkpoint_mode is EnumPostCheckpointMode.REVERSE_DELTA: + if not self.reverse_delta_contract_ref: + raise ValueError( + "reverse_delta mode requires reverse_delta_contract_ref" + ) + if self.forward_fix_runbook_ref: + raise ValueError( + "reverse_delta mode cannot declare a forward-fix runbook" + ) + else: + if not self.forward_fix_runbook_ref: + raise ValueError( + "forward_fix_only mode requires forward_fix_runbook_ref" + ) + if self.reverse_delta_contract_ref: + raise ValueError( + "forward_fix_only mode cannot advertise reverse-delta proof" + ) + return self + + +__all__ = ["ModelCutoverFamilyContract"] diff --git a/src/omnibase_infra/migration/cutover/models/model_cutover_family_state.py b/src/omnibase_infra/migration/cutover/models/model_cutover_family_state.py new file mode 100644 index 0000000000..5a7a056b54 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_cutover_family_state.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Durable state projection for one cutover family.""" + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.migration.cutover.enums import EnumCutoverFamilyStatus +from omnibase_infra.migration.cutover.models.model_cutover_family_contract import ( + ModelCutoverFamilyContract, +) + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class ModelCutoverFamilyState(BaseModel): + """Represent the materialized family-local journal state.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + contract: ModelCutoverFamilyContract + status: EnumCutoverFamilyStatus + last_known_good_receipt_id: UUID | None = None + blocked_receipt_id: UUID | None = None + checkpoint_event_id: UUID | None = None + first_target_write_event_id: UUID | None = None + first_target_sequence: int | None = Field(default=None, ge=1) + quiescence_event_id: UUID | None = None + quiesced_target_sequence: int | None = Field(default=None, ge=1) + verified_reverse_delta_proof_id: UUID | None = None + dual_write_expires_at: datetime | None = None + observation_ends_at: datetime | None = None + last_event_at: datetime | None = None + last_sequence: int = Field(..., ge=0) + last_event_hash: str = Field(..., pattern=_SHA256_PATTERN) + + +__all__ = ["ModelCutoverFamilyState"] diff --git a/src/omnibase_infra/migration/cutover/models/model_cutover_journal_event.py b/src/omnibase_infra/migration/cutover/models/model_cutover_journal_event.py new file mode 100644 index 0000000000..af1f803b08 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_cutover_journal_event.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Hash-chained durable cutover journal event.""" + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.migration.cutover.models.model_cutover_journal_request import ( + ModelCutoverJournalRequest, +) + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class ModelCutoverJournalEvent(BaseModel): + """Hash-chained durable event returned by the repository.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + event_id: UUID + family_id: UUID + sequence: int = Field(..., ge=1) + previous_event_hash: str = Field(..., pattern=_SHA256_PATTERN) + event_hash: str = Field(..., pattern=_SHA256_PATTERN) + request: ModelCutoverJournalRequest + + +__all__ = ["ModelCutoverJournalEvent"] diff --git a/src/omnibase_infra/migration/cutover/models/model_cutover_journal_request.py b/src/omnibase_infra/migration/cutover/models/model_cutover_journal_request.py new file mode 100644 index 0000000000..47d8dc492f --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_cutover_journal_request.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed request for one append-only cutover journal event.""" + +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from omnibase_infra.migration.cutover.enums import EnumCutoverEventKind + +_RECEIPT_EVENTS = { + EnumCutoverEventKind.BACKFILL_COMPLETED, + EnumCutoverEventKind.FINAL_DELTA_APPLIED, + EnumCutoverEventKind.WRITER_CHECKPOINT, + EnumCutoverEventKind.REVERSE_DELTA_PROVEN, + EnumCutoverEventKind.FORWARD_FIX_RECORDED, + EnumCutoverEventKind.MISMATCH_RESOLVED, +} + + +class ModelCutoverJournalRequest(BaseModel): + """Validate event-specific durable evidence before persistence.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + kind: EnumCutoverEventKind + occurred_at: datetime + evidence_ref: str = Field(..., min_length=1, max_length=500) + receipt_id: UUID | None = None + source_binding_ref: str = Field(default="", max_length=200) + target_binding_ref: str = Field(default="", max_length=200) + database_ref: str = Field(default="", max_length=200) + principal: str = Field(default="", max_length=160) + schema_ref: str = Field(default="", max_length=160) + target_sequence: int | None = Field(default=None, ge=1) + dual_write_expires_at: datetime | None = None + observation_ends_at: datetime | None = None + reverse_delta_proof_id: UUID | None = None + + @field_validator("occurred_at", "dual_write_expires_at", "observation_ends_at") + @classmethod + def _timezone_aware(cls, value: datetime | None) -> datetime | None: + if value is not None and value.tzinfo is None: + raise ValueError("journal timestamps must be timezone-aware") + return value + + @model_validator(mode="after") + def _validate_event_evidence(self) -> ModelCutoverJournalRequest: + if self.kind in _RECEIPT_EVENTS and self.receipt_id is None: + raise ValueError(f"{self.kind.value} requires a receipt_id") + + if self.kind is EnumCutoverEventKind.WRITER_CHECKPOINT: + if not self.source_binding_ref or not self.target_binding_ref: + raise ValueError("writer checkpoint requires source/target bindings") + + if self.kind is EnumCutoverEventKind.APPLICATION_PATH_WRITE_PROVEN: + if not all((self.database_ref, self.principal, self.schema_ref)): + raise ValueError( + "application-path write proof requires database, principal, and schema" + ) + if self.target_sequence is None: + raise ValueError( + "application-path write proof requires target_sequence" + ) + + if self.kind is EnumCutoverEventKind.DUAL_WRITE_STARTED: + if self.dual_write_expires_at is None: + raise ValueError("dual-write start requires a finite expiry") + + if self.kind is EnumCutoverEventKind.OBSERVATION_WINDOW_STARTED: + if self.observation_ends_at is None: + raise ValueError("observation start requires an explicit end") + + if self.kind is EnumCutoverEventKind.WRITER_QUIESCED: + if self.target_sequence is None: + raise ValueError("writer quiescence requires final target sequence") + + if self.kind is EnumCutoverEventKind.REVERSE_DELTA_PROVEN: + if self.reverse_delta_proof_id is None: + raise ValueError("reverse-delta event requires proof id") + + if self.receipt_id is not None and self.kind not in _RECEIPT_EVENTS: + raise ValueError(f"{self.kind.value} does not accept receipt_id") + if (self.source_binding_ref or self.target_binding_ref) and self.kind is not ( + EnumCutoverEventKind.WRITER_CHECKPOINT + ): + raise ValueError("binding refs are only valid on writer checkpoint") + if any( + (self.database_ref, self.principal, self.schema_ref) + ) and self.kind is not (EnumCutoverEventKind.APPLICATION_PATH_WRITE_PROVEN): + raise ValueError("database/principal/schema are only valid on write proof") + if self.target_sequence is not None and self.kind not in ( + EnumCutoverEventKind.APPLICATION_PATH_WRITE_PROVEN, + EnumCutoverEventKind.WRITER_QUIESCED, + ): + raise ValueError("target_sequence is not valid for this event") + if self.dual_write_expires_at is not None and self.kind is not ( + EnumCutoverEventKind.DUAL_WRITE_STARTED + ): + raise ValueError("dual-write expiry is only valid on dual-write start") + if self.observation_ends_at is not None and self.kind is not ( + EnumCutoverEventKind.OBSERVATION_WINDOW_STARTED + ): + raise ValueError("observation end is only valid on observation start") + if self.reverse_delta_proof_id is not None and self.kind is not ( + EnumCutoverEventKind.REVERSE_DELTA_PROVEN + ): + raise ValueError("reverse-delta proof id is only valid on its proof event") + return self + + +__all__ = ["ModelCutoverJournalRequest"] diff --git a/src/omnibase_infra/migration/cutover/models/model_postgres_evidence_query_set.py b/src/omnibase_infra/migration/cutover/models/model_postgres_evidence_query_set.py new file mode 100644 index 0000000000..d7bf51a8a5 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_postgres_evidence_query_set.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed read-only query contract for PostgreSQL transformation evidence.""" + +from __future__ import annotations + +import re + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +_READ_PREFIX = re.compile(r"^\s*(?:SELECT|WITH)\b", re.IGNORECASE) +_MUTATING_TOKEN = re.compile( + r"\b(?:INSERT|UPDATE|DELETE|DROP|ALTER|CREATE|GRANT|REVOKE|TRUNCATE|COPY|CALL|DO)\b", + re.IGNORECASE, +) + + +class ModelPostgresEvidenceQuerySet(BaseModel): + """One-column, read-only queries producing canonical evidence signatures. + + Source queries may explicitly transform legacy values into target semantics + (for example by joining a checked-in legacy-value-to-UUID mapping). Every + dimension is required; no omitted query silently resolves to an empty proof. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + label: str = Field(..., min_length=1, max_length=160) + keys_sql: str = Field(..., min_length=6) + rows_sql: str = Field(..., min_length=6) + foreign_keys_sql: str = Field(..., min_length=6) + sequences_sql: str = Field(..., min_length=6) + owners_sql: str = Field(..., min_length=6) + grants_sql: str = Field(..., min_length=6) + policies_sql: str = Field(..., min_length=6) + views_functions_sql: str = Field(..., min_length=6) + dependencies_sql: str = Field(..., min_length=6) + collisions_sql: str = Field(..., min_length=6) + + @field_validator( + "keys_sql", + "rows_sql", + "foreign_keys_sql", + "sequences_sql", + "owners_sql", + "grants_sql", + "policies_sql", + "views_functions_sql", + "dependencies_sql", + "collisions_sql", + ) + @classmethod + def _read_only_single_statement(cls, value: str) -> str: + if not _READ_PREFIX.match(value): + raise ValueError("evidence query must start with SELECT or WITH") + if ";" in value: + raise ValueError("evidence query must be exactly one statement") + if _MUTATING_TOKEN.search(value): + raise ValueError("evidence query must be read-only") + return value + + +__all__ = ["ModelPostgresEvidenceQuerySet"] diff --git a/src/omnibase_infra/migration/cutover/models/model_projection_replay_evidence.py b/src/omnibase_infra/migration/cutover/models/model_projection_replay_evidence.py new file mode 100644 index 0000000000..d4699b5ed9 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_projection_replay_evidence.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Projection version and authoritative event-offset proof.""" + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelProjectionReplayEvidence(BaseModel): + """Bind one projection identity/version to source and target offsets.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + projection_id: UUID + projection_label: str = Field(..., min_length=1) + projection_version: str = Field(..., min_length=1) + topic: str = Field(..., min_length=1) + partition: int = Field(..., ge=0) + source_offset: int = Field(..., ge=0) + target_offset: int = Field(..., ge=0) + + +__all__ = ["ModelProjectionReplayEvidence"] diff --git a/src/omnibase_infra/migration/cutover/models/model_receipt_check.py b/src/omnibase_infra/migration/cutover/models/model_receipt_check.py new file mode 100644 index 0000000000..7ca29ca711 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_receipt_check.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""One explicit comparison in a family transformation receipt.""" + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.migration.cutover.enums import EnumReceiptDimension + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class ModelReceiptCheck(BaseModel): + """Immutable outcome for one required receipt dimension.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + dimension: EnumReceiptDimension + passed: bool + source_digest: str = Field(..., pattern=_SHA256_PATTERN) + target_digest: str = Field(..., pattern=_SHA256_PATTERN) + detail: str = Field(..., min_length=1, max_length=500) + + +__all__ = ["ModelReceiptCheck"] diff --git a/src/omnibase_infra/migration/cutover/models/model_reverse_delta_entry.py b/src/omnibase_infra/migration/cutover/models/model_reverse_delta_entry.py new file mode 100644 index 0000000000..a8ee9b2504 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_reverse_delta_entry.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""One target mutation with an attested inverse artifact.""" + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.migration.cutover.enums import EnumReverseDeltaOperation + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class ModelReverseDeltaEntry(BaseModel): + """Bind one target sequence to a verifiable inverse artifact.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + entry_id: UUID + family_id: UUID + target_sequence: int = Field(..., ge=1) + relation: str = Field( + ..., + pattern=r"^[a-z_][a-z0-9_]*\.[a-z_][a-z0-9_]*$", + ) + operation: EnumReverseDeltaOperation + primary_key_hash: str = Field(..., pattern=_SHA256_PATTERN) + before_image_hash: str = Field(..., pattern=_SHA256_PATTERN) + after_image_hash: str = Field(..., pattern=_SHA256_PATTERN) + inverse_artifact_ref: str = Field(..., min_length=1, max_length=500) + + +__all__ = ["ModelReverseDeltaEntry"] diff --git a/src/omnibase_infra/migration/cutover/models/model_reverse_delta_proof.py b/src/omnibase_infra/migration/cutover/models/model_reverse_delta_proof.py new file mode 100644 index 0000000000..3b572d1d43 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_reverse_delta_proof.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Complete reverse-delta coverage for a quiesced writer.""" + +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_infra.migration.cutover.models.model_reverse_delta_entry import ( + ModelReverseDeltaEntry, +) + + +class ModelReverseDeltaProof(BaseModel): + """Require contiguous inverse coverage through the quiesced sequence.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + proof_id: UUID + family_id: UUID + start_sequence: int = Field(..., ge=1) + end_sequence: int = Field(..., ge=1) + entries: tuple[ModelReverseDeltaEntry, ...] + quiescence_event_id: UUID + reconciliation_receipt_id: UUID + behavioral_readback_ref: str = Field(..., min_length=1, max_length=500) + proven_at: datetime + + @model_validator(mode="after") + def _coverage_is_contiguous(self) -> ModelReverseDeltaProof: + if self.proven_at.tzinfo is None: + raise ValueError("reverse-delta proof timestamp must be timezone-aware") + if self.end_sequence < self.start_sequence: + raise ValueError("reverse-delta end precedes start") + expected = list(range(self.start_sequence, self.end_sequence + 1)) + actual = [entry.target_sequence for entry in self.entries] + if actual != expected: + raise ValueError("reverse-delta entries must cover every sequence in order") + if any(entry.family_id != self.family_id for entry in self.entries): + raise ValueError("reverse-delta entry belongs to another family") + return self + + +__all__ = ["ModelReverseDeltaProof"] diff --git a/src/omnibase_infra/migration/cutover/models/model_rollback_decision.py b/src/omnibase_infra/migration/cutover/models/model_rollback_decision.py new file mode 100644 index 0000000000..bf93462256 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_rollback_decision.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Mechanical direct-DSN rollback verdict.""" + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.migration.cutover.enums import EnumPostCheckpointMode + + +class ModelRollbackDecision(BaseModel): + """Explain why direct rollback is permitted or refused.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + family_id: UUID + allowed: bool + direct_dsn_rollback: bool + post_checkpoint_mode: EnumPostCheckpointMode + reason: str = Field(..., min_length=1) + reverse_delta_proof_id: UUID | None = None + + +__all__ = ["ModelRollbackDecision"] diff --git a/src/omnibase_infra/migration/cutover/models/model_transformation_evidence.py b/src/omnibase_infra/migration/cutover/models/model_transformation_evidence.py new file mode 100644 index 0000000000..351dc198a3 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_transformation_evidence.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Canonical source or target evidence for transformation receipts.""" + +from __future__ import annotations + +import re + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class ModelTransformationEvidence(BaseModel): + """Canonicalized evidence for one side of a family transformation. + + ``keys`` and ``transformed_row_hashes`` are already projected into target + semantics. A legacy slug-to-UUID mapping or internal-column omission is + therefore explicit in the hash-bound query contract. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + label: str = Field(..., min_length=1) + evidence_contract_hash: str = Field(..., pattern=_SHA256_PATTERN) + keys: tuple[str, ...] + row_count: int = Field(..., ge=0) + transformed_row_hashes: tuple[str, ...] + foreign_keys: tuple[str, ...] + sequences: tuple[str, ...] + owners: tuple[str, ...] + grants: tuple[str, ...] + policies: tuple[str, ...] + views_functions: tuple[str, ...] + dependencies: tuple[str, ...] + collision_keys: tuple[str, ...] + + @field_validator( + "keys", + "transformed_row_hashes", + "foreign_keys", + "sequences", + "owners", + "grants", + "policies", + "views_functions", + "dependencies", + "collision_keys", + ) + @classmethod + def _canonical_tuple(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if value != tuple(sorted(value)): + raise ValueError("evidence tuples must be sorted deterministically") + if len(value) != len(set(value)): + raise ValueError("evidence tuples must not contain duplicates") + return value + + @field_validator("transformed_row_hashes") + @classmethod + def _row_hashes_are_sha256(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if any(re.fullmatch(_SHA256_PATTERN, item) is None for item in value): + raise ValueError("transformed row hashes must be lowercase SHA-256") + return value + + @model_validator(mode="after") + def _counts_are_coherent(self) -> ModelTransformationEvidence: + if self.row_count != len(self.keys): + raise ValueError("row_count must equal the canonical key-set size") + if self.row_count != len(self.transformed_row_hashes): + raise ValueError("row_count must equal transformed row-hash count") + return self + + +__all__ = ["ModelTransformationEvidence"] diff --git a/src/omnibase_infra/migration/cutover/models/model_transformation_receipt.py b/src/omnibase_infra/migration/cutover/models/model_transformation_receipt.py new file mode 100644 index 0000000000..3ee2759bfc --- /dev/null +++ b/src/omnibase_infra/migration/cutover/models/model_transformation_receipt.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Immutable, transformation-aware family receipt.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_infra.migration.cutover.enums import ( + EnumReceiptDimension, + EnumReceiptStatus, +) +from omnibase_infra.migration.cutover.models.model_cutover_continuity_evidence import ( + ModelCutoverContinuityEvidence, +) +from omnibase_infra.migration.cutover.models.model_receipt_check import ( + ModelReceiptCheck, +) +from omnibase_infra.migration.cutover.models.model_transformation_evidence import ( + ModelTransformationEvidence, +) + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +def calculate_transformation_receipt_hash( + *, + receipt_id: UUID, + family_id: UUID, + family_contract_hash: str, + generated_at: datetime, + source: ModelTransformationEvidence, + target: ModelTransformationEvidence, + continuity: ModelCutoverContinuityEvidence, + checks: tuple[ModelReceiptCheck, ...], + status: EnumReceiptStatus, +) -> str: + """Return the canonical binding for every immutable receipt field.""" + body = { + "receipt_id": str(receipt_id), + "family_id": str(family_id), + "family_contract_hash": family_contract_hash, + "generated_at": generated_at.isoformat(), + "source": source.model_dump(mode="json"), + "target": target.model_dump(mode="json"), + "continuity": continuity.model_dump(mode="json"), + "checks": [check.model_dump(mode="json") for check in checks], + "status": status.value, + } + encoded = json.dumps( + body, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +class ModelTransformationReceipt(BaseModel): + """Durable receipt binding every comparison and continuity proof.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + receipt_id: UUID + family_id: UUID + family_contract_hash: str = Field(..., pattern=_SHA256_PATTERN) + generated_at: datetime + source: ModelTransformationEvidence + target: ModelTransformationEvidence + continuity: ModelCutoverContinuityEvidence + checks: tuple[ModelReceiptCheck, ...] + status: EnumReceiptStatus + receipt_hash: str = Field(..., pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def _checks_are_complete_and_truthful(self) -> ModelTransformationReceipt: + dimensions = [check.dimension for check in self.checks] + if dimensions != list(EnumReceiptDimension): + raise ValueError( + "receipt checks must contain every dimension exactly once in " + "canonical enum order" + ) + passed = all(check.passed for check in self.checks) + if passed != (self.status is EnumReceiptStatus.PASS): + raise ValueError("receipt status must match the complete check set") + if self.generated_at.tzinfo is None: + raise ValueError("receipt timestamp must be timezone-aware") + expected_hash = calculate_transformation_receipt_hash( + receipt_id=self.receipt_id, + family_id=self.family_id, + family_contract_hash=self.family_contract_hash, + generated_at=self.generated_at, + source=self.source, + target=self.target, + continuity=self.continuity, + checks=self.checks, + status=self.status, + ) + if self.receipt_hash != expected_hash: + raise ValueError("receipt hash does not bind the complete receipt") + return self + + +__all__ = ["ModelTransformationReceipt", "calculate_transformation_receipt_hash"] diff --git a/src/omnibase_infra/migration/cutover/postgres_transformation_evidence_collector.py b/src/omnibase_infra/migration/cutover/postgres_transformation_evidence_collector.py new file mode 100644 index 0000000000..9579567d9c --- /dev/null +++ b/src/omnibase_infra/migration/cutover/postgres_transformation_evidence_collector.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""PostgreSQL collector for transformation-aware evidence query contracts.""" + +from __future__ import annotations + +import hashlib +import json + +import asyncpg + +from omnibase_infra.migration.cutover.models import ( + ModelPostgresEvidenceQuerySet, + ModelTransformationEvidence, +) + + +class PostgresTransformationEvidenceCollector: + """Execute a complete read-only query set against one PostgreSQL database.""" + + def __init__(self, connection: asyncpg.Connection) -> None: + self._connection = connection + + async def collect( + self, + queries: ModelPostgresEvidenceQuerySet, + ) -> ModelTransformationEvidence: + """Collect all receipt dimensions without defaults or omitted scans.""" + keys = await self._fetch_strings(queries.keys_sql) + rows = await self._fetch_strings(queries.rows_sql) + return ModelTransformationEvidence( + label=queries.label, + evidence_contract_hash=self._query_contract_hash(queries), + keys=keys, + row_count=len(rows), + transformed_row_hashes=tuple( + sorted(hashlib.sha256(row.encode("utf-8")).hexdigest() for row in rows) + ), + foreign_keys=await self._fetch_strings(queries.foreign_keys_sql), + sequences=await self._fetch_strings(queries.sequences_sql), + owners=await self._fetch_strings(queries.owners_sql), + grants=await self._fetch_strings(queries.grants_sql), + policies=await self._fetch_strings(queries.policies_sql), + views_functions=await self._fetch_strings(queries.views_functions_sql), + dependencies=await self._fetch_strings(queries.dependencies_sql), + collision_keys=await self._fetch_strings(queries.collisions_sql), + ) + + async def collect_pair( + self, + source_queries: ModelPostgresEvidenceQuerySet, + target_queries: ModelPostgresEvidenceQuerySet, + ) -> tuple[ModelTransformationEvidence, ModelTransformationEvidence]: + """Collect source and target in one read-only repeatable-read snapshot.""" + async with self._connection.transaction( + isolation="repeatable_read", + readonly=True, + ): + source = await self.collect(source_queries) + target = await self.collect(target_queries) + return source, target + + @staticmethod + def _query_contract_hash(queries: ModelPostgresEvidenceQuerySet) -> str: + encoded = json.dumps( + queries.model_dump(mode="json"), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + async def _fetch_strings(self, query: str) -> tuple[str, ...]: + rows = await self._connection.fetch(query) + values: list[str] = [] + for row in rows: + if len(row) != 1: + raise ValueError("evidence queries must return exactly one column") + value = row[0] + if value is None: + raise ValueError("evidence queries must not return NULL signatures") + if not isinstance(value, str): + raise TypeError("evidence queries must return text signatures") + values.append(value) + return tuple(sorted(values)) + + +__all__ = ["PostgresTransformationEvidenceCollector"] diff --git a/src/omnibase_infra/migration/cutover/protocols/__init__.py b/src/omnibase_infra/migration/cutover/protocols/__init__.py new file mode 100644 index 0000000000..258b7050b0 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/protocols/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Cutover receipt and journal persistence ports.""" + +from omnibase_infra.migration.cutover.protocols.protocol_cutover_journal_repository import ( + ProtocolCutoverJournalRepository, +) + +__all__ = ["ProtocolCutoverJournalRepository"] diff --git a/src/omnibase_infra/migration/cutover/protocols/protocol_cutover_journal_repository.py b/src/omnibase_infra/migration/cutover/protocols/protocol_cutover_journal_repository.py new file mode 100644 index 0000000000..4ba33f6b28 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/protocols/protocol_cutover_journal_repository.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Persistence port for durable cutover receipts and journal state.""" + +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from omnibase_infra.migration.cutover.models import ( + ModelCutoverFamilyContract, + ModelCutoverFamilyState, + ModelCutoverJournalEvent, + ModelCutoverJournalRequest, + ModelReverseDeltaProof, + ModelRollbackDecision, + ModelTransformationReceipt, +) + + +class ProtocolCutoverJournalRepository(Protocol): + """Storage operations required by the cutover coordinator.""" + + async def initialize(self) -> None: + """Create the explicit proof/journal schema idempotently.""" + ... + + async def register_family(self, contract: ModelCutoverFamilyContract) -> None: + """Register an immutable family contract or reject drift.""" + ... + + async def record_receipt(self, receipt: ModelTransformationReceipt) -> None: + """Persist a receipt and block only its family on mismatch.""" + ... + + async def append_event( + self, + family_id: UUID, + request: ModelCutoverJournalRequest, + ) -> ModelCutoverJournalEvent: + """Append one validated, hash-chained family event.""" + ... + + async def record_reverse_delta_proof( + self, + proof: ModelReverseDeltaProof, + ) -> None: + """Persist complete reverse-delta coverage for later journal attestation.""" + ... + + async def get_state(self, family_id: UUID) -> ModelCutoverFamilyState: + """Read the durable family-local projection.""" + ... + + async def evaluate_direct_rollback( + self, + family_id: UUID, + ) -> ModelRollbackDecision: + """Return the mechanical direct-DSN rollback decision.""" + ... + + +__all__ = ["ProtocolCutoverJournalRepository"] diff --git a/src/omnibase_infra/migration/cutover/repository_postgres_cutover_journal.py b/src/omnibase_infra/migration/cutover/repository_postgres_cutover_journal.py new file mode 100644 index 0000000000..3798a4b42d --- /dev/null +++ b/src/omnibase_infra/migration/cutover/repository_postgres_cutover_journal.py @@ -0,0 +1,776 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""PostgreSQL implementation of the durable per-family cutover journal.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timedelta +from importlib.resources import files +from uuid import UUID, uuid4 + +import asyncpg + +from omnibase_infra.migration.cutover.enums import ( + EnumCutoverEventKind, + EnumCutoverFamilyStatus, + EnumPostCheckpointMode, + EnumReceiptStatus, +) +from omnibase_infra.migration.cutover.models import ( + ModelCutoverFamilyContract, + ModelCutoverFamilyState, + ModelCutoverJournalEvent, + ModelCutoverJournalRequest, + ModelReverseDeltaProof, + ModelRollbackDecision, + ModelTransformationReceipt, +) + +_ZERO_HASH = "0" * 64 + + +def _canonical_json(value: object) -> str: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + default=str, + ) + + +def _sha256(value: object) -> str: + return hashlib.sha256(_canonical_json(value).encode("utf-8")).hexdigest() + + +class RepositoryPostgresCutoverJournal: + """Persist immutable receipts and serialize family-local state transitions.""" + + def __init__(self, connection: asyncpg.Connection) -> None: + self._connection = connection + + async def initialize(self) -> None: + """Create the proof/journal schema only when explicitly requested.""" + bootstrap = ( + files("omnibase_infra.migration.cutover.sql") + .joinpath("bootstrap.sql") + .read_text(encoding="utf-8") + ) + await self._connection.execute(bootstrap) + + async def register_family(self, contract: ModelCutoverFamilyContract) -> None: + """Register an immutable contract; reject a same-id semantic rewrite.""" + contract_json = _canonical_json(contract.model_dump(mode="json")) + contract_hash = _sha256(contract.model_dump(mode="json")) + async with self._connection.transaction(): + await self._connection.execute( + """ +INSERT INTO omninode_internal.cutover_family_contracts + (family_id, contract_json, contract_hash) +VALUES ($1, $2::jsonb, $3) +ON CONFLICT (family_id) DO NOTHING +""", + contract.family_id, + contract_json, + contract_hash, + ) + stored = await self._connection.fetchval( + """ +SELECT contract_hash +FROM omninode_internal.cutover_family_contracts +WHERE family_id = $1 +FOR UPDATE +""", + contract.family_id, + ) + if stored != contract_hash: + raise ValueError( + f"family contract drift for {contract.family_id!r}; " + "register a new family/version instead of rewriting history" + ) + + async def record_receipt(self, receipt: ModelTransformationReceipt) -> None: + """Persist a receipt and block only the mismatching family.""" + receipt_json = _canonical_json(receipt.model_dump(mode="json")) + async with self._connection.transaction(): + family = await self._lock_family(receipt.family_id) + if receipt.family_contract_hash != family["contract_hash"]: + raise ValueError( + "receipt is not bound to the registered family contract" + ) + existing = await self._connection.fetchrow( + """ +SELECT family_id, receipt_hash +FROM omninode_internal.transformation_receipts +WHERE receipt_id = $1 +""", + receipt.receipt_id, + ) + if existing is not None: + if ( + existing["family_id"] != receipt.family_id + or existing["receipt_hash"] != receipt.receipt_hash + ): + raise ValueError( + "receipt UUID is already bound to different evidence" + ) + return + + latest_generated_at = await self._connection.fetchval( + """ +SELECT max(generated_at) +FROM omninode_internal.transformation_receipts +WHERE family_id = $1 +""", + receipt.family_id, + ) + if ( + latest_generated_at is not None + and receipt.generated_at < latest_generated_at + ): + raise ValueError("stale receipt replay is forbidden") + + await self._connection.execute( + """ +INSERT INTO omninode_internal.transformation_receipts + (receipt_id, family_id, status, receipt_hash, receipt_json, generated_at) +VALUES ($1, $2, $3, $4, $5::jsonb, $6) +""", + receipt.receipt_id, + receipt.family_id, + receipt.status.value, + receipt.receipt_hash, + receipt_json, + receipt.generated_at, + ) + if receipt.status is EnumReceiptStatus.PASS: + await self._connection.execute( + """ +UPDATE omninode_internal.cutover_family_contracts +SET last_known_good_receipt_id = $2 +WHERE family_id = $1 +""", + receipt.family_id, + receipt.receipt_id, + ) + else: + await self._connection.execute( + """ +UPDATE omninode_internal.cutover_family_contracts +SET status = 'blocked', blocked_receipt_id = $2 +WHERE family_id = $1 +""", + receipt.family_id, + receipt.receipt_id, + ) + + # Keep the row lock live until the receipt and state update commit. + if family["family_id"] != receipt.family_id: + raise AssertionError("locked family identity changed") + + async def append_event( + self, + family_id: UUID, + request: ModelCutoverJournalRequest, + ) -> ModelCutoverJournalEvent: + """Validate, hash-chain, and append one family-local event atomically.""" + async with self._connection.transaction(): + row = await self._lock_family(family_id) + contract = self._contract_from_row(row) + await self._validate_transition(row, contract, request) + + sequence = int(row["last_sequence"]) + 1 + previous_hash = str(row["last_event_hash"]) + event_id = uuid4() + event_body = { + "event_id": str(event_id), + "family_id": family_id, + "sequence": sequence, + "previous_event_hash": previous_hash, + "request": request.model_dump(mode="json"), + } + event_hash = _sha256(event_body) + event = ModelCutoverJournalEvent( + event_id=event_id, + family_id=family_id, + sequence=sequence, + previous_event_hash=previous_hash, + event_hash=event_hash, + request=request, + ) + await self._connection.execute( + """ +INSERT INTO omninode_internal.cutover_journal + (event_id, family_id, sequence, event_kind, request_json, receipt_id, + previous_event_hash, event_hash, occurred_at) +VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9) +""", + event.event_id, + family_id, + sequence, + request.kind.value, + _canonical_json(request.model_dump(mode="json")), + request.receipt_id, + previous_hash, + event_hash, + request.occurred_at, + ) + await self._project_transition(row, contract, event) + return event + + async def record_reverse_delta_proof( + self, + proof: ModelReverseDeltaProof, + ) -> None: + """Persist contiguous reverse-delta coverage after writer quiescence.""" + async with self._connection.transaction(): + row = await self._lock_family(proof.family_id) + contract = self._contract_from_row(row) + if EnumCutoverFamilyStatus(str(row["status"])) is ( + EnumCutoverFamilyStatus.BLOCKED + ): + raise ValueError("blocked family cannot advertise reverse-delta proof") + if ( + contract.post_checkpoint_mode + is not EnumPostCheckpointMode.REVERSE_DELTA + ): + raise ValueError("forward-fix-only family cannot record reverse delta") + if row["first_target_sequence"] is None: + raise ValueError("no target-only write has been proven") + if row["quiesced_target_sequence"] is None: + raise ValueError("writer must be durably quiesced before proof") + if proof.start_sequence != int(row["first_target_sequence"]): + raise ValueError( + "reverse delta does not start at first target-only write" + ) + if proof.end_sequence != int(row["quiesced_target_sequence"]): + raise ValueError( + "reverse delta does not reach quiesced target sequence" + ) + if proof.quiescence_event_id != row["quiescence_event_id"]: + raise ValueError("reverse delta cites the wrong quiescence event") + quiesced_at = await self._connection.fetchval( + """ +SELECT occurred_at +FROM omninode_internal.cutover_journal +WHERE event_id = $1 AND family_id = $2 AND event_kind = 'writer_quiesced' +""", + proof.quiescence_event_id, + proof.family_id, + ) + if not isinstance(quiesced_at, datetime): + raise ValueError("reverse delta cites no durable quiescence event") + reconciled_at = await self._require_pass_receipt_after( + proof.family_id, + proof.reconciliation_receipt_id, + quiesced_at, + ) + if proof.proven_at < reconciled_at: + raise ValueError("reverse-delta proof predates its reconciliation") + await self._connection.execute( + """ +INSERT INTO omninode_internal.reverse_delta_proofs + (proof_id, family_id, start_sequence, end_sequence, quiescence_event_id, + reconciliation_receipt_id, proof_json, proven_at) +VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8) +""", + proof.proof_id, + proof.family_id, + proof.start_sequence, + proof.end_sequence, + proof.quiescence_event_id, + proof.reconciliation_receipt_id, + _canonical_json(proof.model_dump(mode="json")), + proof.proven_at, + ) + for entry in proof.entries: + await self._connection.execute( + """ +INSERT INTO omninode_internal.reverse_delta_entries + (entry_id, proof_id, family_id, target_sequence, entry_json) +VALUES ($1, $2, $3, $4, $5::jsonb) +""", + entry.entry_id, + proof.proof_id, + proof.family_id, + entry.target_sequence, + _canonical_json(entry.model_dump(mode="json")), + ) + + async def get_state(self, family_id: UUID) -> ModelCutoverFamilyState: + """Read the family-local state projection.""" + row = await self._connection.fetchrow( + """ +SELECT * +FROM omninode_internal.cutover_family_contracts +WHERE family_id = $1 +""", + family_id, + ) + if row is None: + raise KeyError(f"unknown cutover family {family_id!r}") + return self._state_from_row(row) + + async def evaluate_direct_rollback( + self, + family_id: UUID, + ) -> ModelRollbackDecision: + """Refuse unsafe direct rollback after target-only authority exists.""" + state = await self.get_state(family_id) + mode = state.contract.post_checkpoint_mode + if state.status is EnumCutoverFamilyStatus.BLOCKED: + return ModelRollbackDecision( + family_id=family_id, + allowed=False, + direct_dsn_rollback=False, + post_checkpoint_mode=mode, + reason="family is blocked by a failed transformation receipt", + ) + if state.dual_write_expires_at is not None: + return ModelRollbackDecision( + family_id=family_id, + allowed=False, + direct_dsn_rollback=False, + post_checkpoint_mode=mode, + reason="bounded dual-write is still open and must be quiesced", + ) + if state.first_target_write_event_id is None: + return ModelRollbackDecision( + family_id=family_id, + allowed=True, + direct_dsn_rollback=True, + post_checkpoint_mode=mode, + reason=( + "no target-only authoritative write is proven; source remains current" + ), + ) + if mode is EnumPostCheckpointMode.FORWARD_FIX_ONLY: + return ModelRollbackDecision( + family_id=family_id, + allowed=False, + direct_dsn_rollback=False, + post_checkpoint_mode=mode, + reason=( + "target-only authority exists and the family is forward-fix-only" + ), + ) + if state.verified_reverse_delta_proof_id is None: + return ModelRollbackDecision( + family_id=family_id, + allowed=False, + direct_dsn_rollback=False, + post_checkpoint_mode=mode, + reason=( + "target-only authority exists; complete reverse delta, writer " + "quiescence, reconciliation, and behavioral readback are unproven" + ), + ) + return ModelRollbackDecision( + family_id=family_id, + allowed=True, + direct_dsn_rollback=True, + post_checkpoint_mode=mode, + reason=( + "writer is quiesced and complete reverse delta, reconciliation, " + "and behavioral readback are durably proven" + ), + reverse_delta_proof_id=state.verified_reverse_delta_proof_id, + ) + + async def _lock_family(self, family_id: UUID) -> asyncpg.Record: + row = await self._connection.fetchrow( + """ +SELECT * +FROM omninode_internal.cutover_family_contracts +WHERE family_id = $1 +FOR UPDATE +""", + family_id, + ) + if row is None: + raise KeyError(f"unknown cutover family {family_id!r}") + return row + + @staticmethod + def _contract_from_row(row: asyncpg.Record) -> ModelCutoverFamilyContract: + raw = row["contract_json"] + if isinstance(raw, str): + return ModelCutoverFamilyContract.model_validate_json(raw) + return ModelCutoverFamilyContract.model_validate_json(_canonical_json(raw)) + + def _state_from_row(self, row: asyncpg.Record) -> ModelCutoverFamilyState: + return ModelCutoverFamilyState( + contract=self._contract_from_row(row), + status=EnumCutoverFamilyStatus(str(row["status"])), + last_known_good_receipt_id=row["last_known_good_receipt_id"], + blocked_receipt_id=row["blocked_receipt_id"], + checkpoint_event_id=row["checkpoint_event_id"], + first_target_write_event_id=row["first_target_write_event_id"], + first_target_sequence=row["first_target_sequence"], + quiescence_event_id=row["quiescence_event_id"], + quiesced_target_sequence=row["quiesced_target_sequence"], + verified_reverse_delta_proof_id=row["verified_reverse_delta_proof_id"], + dual_write_expires_at=row["dual_write_expires_at"], + observation_ends_at=row["observation_ends_at"], + last_event_at=row["last_event_at"], + last_sequence=int(row["last_sequence"]), + last_event_hash=str(row["last_event_hash"]), + ) + + async def _require_pass_receipt( + self, + family_id: UUID, + receipt_id: UUID, + ) -> None: + status = await self._connection.fetchval( + """ +SELECT status +FROM omninode_internal.transformation_receipts +WHERE receipt_id = $1 AND family_id = $2 +""", + receipt_id, + family_id, + ) + if status != EnumReceiptStatus.PASS.value: + raise ValueError("journal transition requires a PASS family receipt") + + async def _require_resolution_receipt( + self, + family_id: UUID, + receipt_id: UUID, + blocked_receipt_id: UUID | None, + ) -> None: + row = await self._connection.fetchrow( + """ +SELECT repair.status AS repair_status, + repair.generated_at AS repair_generated_at, + blocked.generated_at AS blocked_generated_at +FROM omninode_internal.transformation_receipts repair +JOIN omninode_internal.transformation_receipts blocked + ON blocked.receipt_id = $3 AND blocked.family_id = repair.family_id +WHERE repair.receipt_id = $1 AND repair.family_id = $2 +""", + receipt_id, + family_id, + blocked_receipt_id, + ) + if row is None or row["repair_status"] != EnumReceiptStatus.PASS.value: + raise ValueError("mismatch resolution requires a PASS family receipt") + if row["repair_generated_at"] <= row["blocked_generated_at"]: + raise ValueError("mismatch resolution receipt must postdate the failure") + + async def _require_pass_receipt_after( + self, + family_id: UUID, + receipt_id: UUID, + not_before: datetime, + ) -> datetime: + generated_at = await self._connection.fetchval( + """ +SELECT generated_at +FROM omninode_internal.transformation_receipts +WHERE receipt_id = $1 AND family_id = $2 AND status = 'pass' +""", + receipt_id, + family_id, + ) + if not isinstance(generated_at, datetime) or generated_at <= not_before: + raise ValueError("post-write proof requires a fresh reconciliation receipt") + return generated_at + + async def _validate_transition( + self, + row: asyncpg.Record, + contract: ModelCutoverFamilyContract, + request: ModelCutoverJournalRequest, + ) -> None: + kind = request.kind + status = EnumCutoverFamilyStatus(str(row["status"])) + previous_raw = row["last_event_kind"] + previous = EnumCutoverEventKind(previous_raw) if previous_raw else None + if ( + row["last_event_at"] is not None + and request.occurred_at < row["last_event_at"] + ): + raise ValueError("journal event time precedes the durable prior event") + + if status is EnumCutoverFamilyStatus.BLOCKED: + if kind is not EnumCutoverEventKind.MISMATCH_RESOLVED: + raise ValueError("family is blocked; silent fallback is forbidden") + if request.receipt_id is None: + raise ValueError("mismatch resolution requires a PASS receipt") + await self._require_resolution_receipt( + contract.family_id, + request.receipt_id, + row["blocked_receipt_id"], + ) + return + + if kind is EnumCutoverEventKind.MISMATCH_RESOLVED: + raise ValueError("family is not blocked") + + dual_expiry = row["dual_write_expires_at"] + if ( + dual_expiry is not None + and kind is not EnumCutoverEventKind.DUAL_WRITE_ENDED + ): + raise ValueError("bounded dual-write must end before another transition") + + if request.receipt_id is not None: + await self._require_pass_receipt(contract.family_id, request.receipt_id) + + if kind is EnumCutoverEventKind.BACKFILL_STARTED: + if previous not in (None, EnumCutoverEventKind.PRE_CHECKPOINT_ROLLBACK): + raise ValueError("backfill start is out of order") + elif kind is EnumCutoverEventKind.BACKFILL_COMPLETED: + self._require_previous(previous, EnumCutoverEventKind.BACKFILL_STARTED) + elif kind is EnumCutoverEventKind.DUAL_WRITE_STARTED: + self._require_previous(previous, EnumCutoverEventKind.BACKFILL_COMPLETED) + self._validate_dual_write_window(contract, request) + elif kind is EnumCutoverEventKind.DUAL_WRITE_ENDED: + self._require_previous(previous, EnumCutoverEventKind.DUAL_WRITE_STARTED) + if dual_expiry is None: + raise ValueError("dual-write is not open") + if request.occurred_at > dual_expiry: + raise ValueError("dual-write exceeded its declared hard deadline") + elif kind is EnumCutoverEventKind.FINAL_DELTA_APPLIED: + if previous not in ( + EnumCutoverEventKind.BACKFILL_COMPLETED, + EnumCutoverEventKind.DUAL_WRITE_ENDED, + ): + raise ValueError("final delta requires completed backfill/dual-write") + elif kind is EnumCutoverEventKind.WRITER_CHECKPOINT: + self._require_previous(previous, EnumCutoverEventKind.FINAL_DELTA_APPLIED) + if request.source_binding_ref != contract.source_binding_ref: + raise ValueError("checkpoint source binding differs from contract") + if request.target_binding_ref != contract.target_binding_ref: + raise ValueError("checkpoint target binding differs from contract") + elif kind is EnumCutoverEventKind.APPLICATION_PATH_WRITE_PROVEN: + self._require_previous(previous, EnumCutoverEventKind.WRITER_CHECKPOINT) + elif kind is EnumCutoverEventKind.READER_CUTOVER: + self._require_previous( + previous, + EnumCutoverEventKind.APPLICATION_PATH_WRITE_PROVEN, + ) + elif kind is EnumCutoverEventKind.OBSERVATION_WINDOW_STARTED: + self._require_previous(previous, EnumCutoverEventKind.READER_CUTOVER) + minimum_end = request.occurred_at + timedelta( + seconds=contract.observation_window_seconds + ) + if request.observation_ends_at is None: + raise ValueError("observation end is missing") + if request.observation_ends_at < minimum_end: + raise ValueError("observation window is shorter than its contract") + elif kind is EnumCutoverEventKind.OBSERVATION_WINDOW_COMPLETED: + self._require_previous( + previous, + EnumCutoverEventKind.OBSERVATION_WINDOW_STARTED, + ) + observation_ends_at = row["observation_ends_at"] + if observation_ends_at is None: + raise ValueError("declared observation deadline is not durable") + if request.occurred_at < observation_ends_at: + raise ValueError("observation window has not reached its deadline") + elif kind is EnumCutoverEventKind.WRITER_QUIESCED: + if row["first_target_write_event_id"] is None: + raise ValueError("writer cannot quiesce before target-only write proof") + if request.target_sequence is None: + raise ValueError("writer quiescence sequence is missing") + if request.target_sequence < int(row["first_target_sequence"]): + raise ValueError("quiescence sequence precedes first target write") + elif kind is EnumCutoverEventKind.REVERSE_DELTA_PROVEN: + await self._validate_reverse_delta_event(row, contract, request) + elif kind is EnumCutoverEventKind.FORWARD_FIX_RECORDED: + if ( + contract.post_checkpoint_mode + is not EnumPostCheckpointMode.FORWARD_FIX_ONLY + ): + raise ValueError("reverse-delta family cannot claim forward-fix-only") + if row["first_target_write_event_id"] is None: + raise ValueError("forward fix is not post-checkpoint evidence") + first_write_at = await self._connection.fetchval( + """ +SELECT occurred_at +FROM omninode_internal.cutover_journal +WHERE event_id = $1 AND family_id = $2 + AND event_kind = 'application_path_write_proven' +""", + row["first_target_write_event_id"], + contract.family_id, + ) + if first_write_at is None or request.receipt_id is None: + raise ValueError("forward fix lacks durable target-write evidence") + await self._require_pass_receipt_after( + contract.family_id, + request.receipt_id, + first_write_at, + ) + elif kind is EnumCutoverEventKind.PRE_CHECKPOINT_ROLLBACK: + if row["first_target_write_event_id"] is not None: + raise ValueError( + "direct pre-checkpoint rollback after target write refused" + ) + + @staticmethod + def _require_previous( + actual: EnumCutoverEventKind | None, + expected: EnumCutoverEventKind, + ) -> None: + if actual is not expected: + raise ValueError( + f"cutover transition requires {expected.value}, found " + f"{actual.value if actual else 'no prior event'}" + ) + + @staticmethod + def _validate_dual_write_window( + contract: ModelCutoverFamilyContract, + request: ModelCutoverJournalRequest, + ) -> None: + if contract.dual_write_max_seconds == 0: + raise ValueError("dual-write is disabled by the family contract") + if request.dual_write_expires_at is None: + raise ValueError("dual-write deadline is missing") + if request.dual_write_expires_at <= request.occurred_at: + raise ValueError("dual-write deadline must be in the future") + maximum = request.occurred_at + timedelta( + seconds=contract.dual_write_max_seconds + ) + if request.dual_write_expires_at > maximum: + raise ValueError("dual-write deadline exceeds the contract maximum") + + async def _validate_reverse_delta_event( + self, + row: asyncpg.Record, + contract: ModelCutoverFamilyContract, + request: ModelCutoverJournalRequest, + ) -> None: + if contract.post_checkpoint_mode is not EnumPostCheckpointMode.REVERSE_DELTA: + raise ValueError("forward-fix-only family cannot prove reverse delta") + if row["quiescence_event_id"] is None: + raise ValueError("writer quiescence is unproven") + proof = await self._connection.fetchrow( + """ +SELECT reconciliation_receipt_id, quiescence_event_id, proven_at +FROM omninode_internal.reverse_delta_proofs +WHERE proof_id = $1 AND family_id = $2 +""", + request.reverse_delta_proof_id, + contract.family_id, + ) + if proof is None: + raise ValueError("reverse-delta proof is not durable") + if proof["quiescence_event_id"] != row["quiescence_event_id"]: + raise ValueError("reverse-delta proof predates current quiescence") + if proof["reconciliation_receipt_id"] != request.receipt_id: + raise ValueError("reverse-delta proof and journal receipt differ") + if request.occurred_at < proof["proven_at"]: + raise ValueError("reverse-delta journal event predates its durable proof") + + async def _project_transition( + self, + previous_row: asyncpg.Record, + contract: ModelCutoverFamilyContract, + event: ModelCutoverJournalEvent, + ) -> None: + request = event.request + kind = request.kind + projected: dict[str, object] = { + "status": previous_row["status"], + "blocked_receipt_id": previous_row["blocked_receipt_id"], + "checkpoint_event_id": previous_row["checkpoint_event_id"], + "first_target_write_event_id": previous_row["first_target_write_event_id"], + "first_target_sequence": previous_row["first_target_sequence"], + "quiescence_event_id": previous_row["quiescence_event_id"], + "quiesced_target_sequence": previous_row["quiesced_target_sequence"], + "verified_reverse_delta_proof_id": previous_row[ + "verified_reverse_delta_proof_id" + ], + "dual_write_expires_at": previous_row["dual_write_expires_at"], + "observation_ends_at": previous_row["observation_ends_at"], + "last_sequence": event.sequence, + "last_event_hash": event.event_hash, + "last_event_kind": previous_row["last_event_kind"], + "last_event_at": event.request.occurred_at, + } + preserve_phase = kind is EnumCutoverEventKind.MISMATCH_RESOLVED + if not preserve_phase: + projected["last_event_kind"] = kind.value + + if kind is EnumCutoverEventKind.DUAL_WRITE_STARTED: + projected["dual_write_expires_at"] = request.dual_write_expires_at + elif kind is EnumCutoverEventKind.DUAL_WRITE_ENDED: + projected["dual_write_expires_at"] = None + elif kind is EnumCutoverEventKind.WRITER_CHECKPOINT: + projected["checkpoint_event_id"] = event.event_id + projected["status"] = EnumCutoverFamilyStatus.CHECKPOINTED.value + elif kind is EnumCutoverEventKind.APPLICATION_PATH_WRITE_PROVEN: + projected["first_target_write_event_id"] = event.event_id + projected["first_target_sequence"] = request.target_sequence + elif kind is EnumCutoverEventKind.OBSERVATION_WINDOW_STARTED: + projected["status"] = EnumCutoverFamilyStatus.OBSERVING.value + projected["observation_ends_at"] = request.observation_ends_at + elif kind is EnumCutoverEventKind.OBSERVATION_WINDOW_COMPLETED: + projected["status"] = EnumCutoverFamilyStatus.COMPLETE.value + elif kind is EnumCutoverEventKind.WRITER_QUIESCED: + projected["quiescence_event_id"] = event.event_id + projected["quiesced_target_sequence"] = request.target_sequence + elif kind is EnumCutoverEventKind.REVERSE_DELTA_PROVEN: + projected["verified_reverse_delta_proof_id"] = ( + request.reverse_delta_proof_id + ) + elif kind is EnumCutoverEventKind.MISMATCH_RESOLVED: + projected["blocked_receipt_id"] = None + projected["status"] = self._status_after_resolution(previous_row).value + + await self._connection.execute( + """ +UPDATE omninode_internal.cutover_family_contracts +SET status = $2, + blocked_receipt_id = $3, + checkpoint_event_id = $4, + first_target_write_event_id = $5, + first_target_sequence = $6, + quiescence_event_id = $7, + quiesced_target_sequence = $8, + verified_reverse_delta_proof_id = $9, + dual_write_expires_at = $10, + observation_ends_at = $11, + last_sequence = $12, + last_event_hash = $13, + last_event_kind = $14, + last_event_at = $15 +WHERE family_id = $1 +""", + contract.family_id, + projected["status"], + projected["blocked_receipt_id"], + projected["checkpoint_event_id"], + projected["first_target_write_event_id"], + projected["first_target_sequence"], + projected["quiescence_event_id"], + projected["quiesced_target_sequence"], + projected["verified_reverse_delta_proof_id"], + projected["dual_write_expires_at"], + projected["observation_ends_at"], + projected["last_sequence"], + projected["last_event_hash"], + projected["last_event_kind"], + projected["last_event_at"], + ) + + @staticmethod + def _status_after_resolution(row: asyncpg.Record) -> EnumCutoverFamilyStatus: + previous = row["last_event_kind"] + if previous == EnumCutoverEventKind.OBSERVATION_WINDOW_COMPLETED.value: + return EnumCutoverFamilyStatus.COMPLETE + if previous == EnumCutoverEventKind.OBSERVATION_WINDOW_STARTED.value: + return EnumCutoverFamilyStatus.OBSERVING + if row["checkpoint_event_id"] is not None: + return EnumCutoverFamilyStatus.CHECKPOINTED + return EnumCutoverFamilyStatus.READY + + +__all__ = ["RepositoryPostgresCutoverJournal"] diff --git a/src/omnibase_infra/migration/cutover/sql/__init__.py b/src/omnibase_infra/migration/cutover/sql/__init__.py new file mode 100644 index 0000000000..778bd9e09f --- /dev/null +++ b/src/omnibase_infra/migration/cutover/sql/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Packaged PostgreSQL bootstrap for the explicit cutover proof repository.""" diff --git a/src/omnibase_infra/migration/cutover/sql/bootstrap.sql b/src/omnibase_infra/migration/cutover/sql/bootstrap.sql new file mode 100644 index 0000000000..2008051e9a --- /dev/null +++ b/src/omnibase_infra/migration/cutover/sql/bootstrap.sql @@ -0,0 +1,189 @@ +-- SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +-- SPDX-License-Identifier: MIT + +-- OMN-15420 proof schema. This file is loaded only by the explicit cutover +-- repository initializer; it is not part of the forward migration stream. +CREATE SCHEMA IF NOT EXISTS omninode_internal; + +CREATE TABLE IF NOT EXISTS omninode_internal.cutover_family_contracts ( + family_id UUID PRIMARY KEY, + contract_json JSONB NOT NULL CHECK (jsonb_typeof(contract_json) = 'object'), + contract_hash TEXT NOT NULL CHECK (contract_hash ~ '^[0-9a-f]{64}$'), + status TEXT NOT NULL DEFAULT 'ready' + CHECK (status IN ('ready', 'blocked', 'checkpointed', 'observing', 'complete')), + last_known_good_receipt_id UUID, + blocked_receipt_id UUID, + checkpoint_event_id UUID, + first_target_write_event_id UUID, + first_target_sequence BIGINT CHECK (first_target_sequence > 0), + quiescence_event_id UUID, + quiesced_target_sequence BIGINT CHECK (quiesced_target_sequence > 0), + verified_reverse_delta_proof_id UUID, + dual_write_expires_at TIMESTAMPTZ, + observation_ends_at TIMESTAMPTZ, + last_sequence BIGINT NOT NULL DEFAULT 0 CHECK (last_sequence >= 0), + last_event_hash TEXT NOT NULL DEFAULT repeat('0', 64) + CHECK (last_event_hash ~ '^[0-9a-f]{64}$'), + last_event_kind TEXT, + last_event_at TIMESTAMPTZ, + registered_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE IF NOT EXISTS omninode_internal.transformation_receipts ( + receipt_id UUID PRIMARY KEY, + family_id UUID NOT NULL REFERENCES omninode_internal.cutover_family_contracts(family_id), + status TEXT NOT NULL CHECK (status IN ('pass', 'fail')), + receipt_hash TEXT NOT NULL CHECK (receipt_hash ~ '^[0-9a-f]{64}$'), + receipt_json JSONB NOT NULL CHECK (jsonb_typeof(receipt_json) = 'object'), + generated_at TIMESTAMPTZ NOT NULL, + UNIQUE (family_id, receipt_hash), + UNIQUE (family_id, receipt_id) +); + +CREATE TABLE IF NOT EXISTS omninode_internal.cutover_journal ( + event_id UUID PRIMARY KEY, + family_id UUID NOT NULL REFERENCES omninode_internal.cutover_family_contracts(family_id), + sequence BIGINT NOT NULL CHECK (sequence > 0), + event_kind TEXT NOT NULL CHECK (event_kind IN ( + 'backfill_started', 'backfill_completed', 'dual_write_started', + 'dual_write_ended', 'final_delta_applied', 'writer_checkpoint', + 'application_path_write_proven', 'reader_cutover', + 'observation_window_started', 'observation_window_completed', + 'writer_quiesced', 'reverse_delta_proven', 'forward_fix_recorded', + 'pre_checkpoint_rollback', 'mismatch_resolved' + )), + request_json JSONB NOT NULL CHECK (jsonb_typeof(request_json) = 'object'), + receipt_id UUID REFERENCES omninode_internal.transformation_receipts(receipt_id), + previous_event_hash TEXT NOT NULL CHECK (previous_event_hash ~ '^[0-9a-f]{64}$'), + event_hash TEXT NOT NULL CHECK (event_hash ~ '^[0-9a-f]{64}$'), + occurred_at TIMESTAMPTZ NOT NULL, + UNIQUE (family_id, sequence), + UNIQUE (family_id, event_hash), + UNIQUE (family_id, event_id) +); + +CREATE TABLE IF NOT EXISTS omninode_internal.reverse_delta_proofs ( + proof_id UUID PRIMARY KEY, + family_id UUID NOT NULL REFERENCES omninode_internal.cutover_family_contracts(family_id), + start_sequence BIGINT NOT NULL CHECK (start_sequence > 0), + end_sequence BIGINT NOT NULL CHECK (end_sequence >= start_sequence), + quiescence_event_id UUID NOT NULL REFERENCES omninode_internal.cutover_journal(event_id), + reconciliation_receipt_id UUID NOT NULL REFERENCES omninode_internal.transformation_receipts(receipt_id), + proof_json JSONB NOT NULL CHECK (jsonb_typeof(proof_json) = 'object'), + proven_at TIMESTAMPTZ NOT NULL, + UNIQUE (family_id, proof_id) +); + +CREATE TABLE IF NOT EXISTS omninode_internal.reverse_delta_entries ( + entry_id UUID PRIMARY KEY, + proof_id UUID NOT NULL REFERENCES omninode_internal.reverse_delta_proofs(proof_id), + family_id UUID NOT NULL REFERENCES omninode_internal.cutover_family_contracts(family_id), + target_sequence BIGINT NOT NULL CHECK (target_sequence > 0), + entry_json JSONB NOT NULL CHECK (jsonb_typeof(entry_json) = 'object'), + UNIQUE (family_id, target_sequence) +); + +DO $constraints$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'cutover_family_last_receipt_fk' + AND conrelid = 'omninode_internal.cutover_family_contracts'::regclass + ) THEN + ALTER TABLE omninode_internal.cutover_family_contracts + ADD CONSTRAINT cutover_family_last_receipt_fk + FOREIGN KEY (family_id, last_known_good_receipt_id) + REFERENCES omninode_internal.transformation_receipts(family_id, receipt_id); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'cutover_family_blocked_receipt_fk' + AND conrelid = 'omninode_internal.cutover_family_contracts'::regclass + ) THEN + ALTER TABLE omninode_internal.cutover_family_contracts + ADD CONSTRAINT cutover_family_blocked_receipt_fk + FOREIGN KEY (family_id, blocked_receipt_id) + REFERENCES omninode_internal.transformation_receipts(family_id, receipt_id); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'cutover_family_checkpoint_event_fk' + AND conrelid = 'omninode_internal.cutover_family_contracts'::regclass + ) THEN + ALTER TABLE omninode_internal.cutover_family_contracts + ADD CONSTRAINT cutover_family_checkpoint_event_fk + FOREIGN KEY (family_id, checkpoint_event_id) + REFERENCES omninode_internal.cutover_journal(family_id, event_id); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'cutover_family_first_write_event_fk' + AND conrelid = 'omninode_internal.cutover_family_contracts'::regclass + ) THEN + ALTER TABLE omninode_internal.cutover_family_contracts + ADD CONSTRAINT cutover_family_first_write_event_fk + FOREIGN KEY (family_id, first_target_write_event_id) + REFERENCES omninode_internal.cutover_journal(family_id, event_id); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'cutover_family_quiescence_event_fk' + AND conrelid = 'omninode_internal.cutover_family_contracts'::regclass + ) THEN + ALTER TABLE omninode_internal.cutover_family_contracts + ADD CONSTRAINT cutover_family_quiescence_event_fk + FOREIGN KEY (family_id, quiescence_event_id) + REFERENCES omninode_internal.cutover_journal(family_id, event_id); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'cutover_family_reverse_proof_fk' + AND conrelid = 'omninode_internal.cutover_family_contracts'::regclass + ) THEN + ALTER TABLE omninode_internal.cutover_family_contracts + ADD CONSTRAINT cutover_family_reverse_proof_fk + FOREIGN KEY (family_id, verified_reverse_delta_proof_id) + REFERENCES omninode_internal.reverse_delta_proofs(family_id, proof_id); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'cutover_journal_family_receipt_fk' + AND conrelid = 'omninode_internal.cutover_journal'::regclass + ) THEN + ALTER TABLE omninode_internal.cutover_journal + ADD CONSTRAINT cutover_journal_family_receipt_fk + FOREIGN KEY (family_id, receipt_id) + REFERENCES omninode_internal.transformation_receipts(family_id, receipt_id); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'reverse_proof_family_receipt_fk' + AND conrelid = 'omninode_internal.reverse_delta_proofs'::regclass + ) THEN + ALTER TABLE omninode_internal.reverse_delta_proofs + ADD CONSTRAINT reverse_proof_family_receipt_fk + FOREIGN KEY (family_id, reconciliation_receipt_id) + REFERENCES omninode_internal.transformation_receipts(family_id, receipt_id); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'reverse_proof_family_event_fk' + AND conrelid = 'omninode_internal.reverse_delta_proofs'::regclass + ) THEN + ALTER TABLE omninode_internal.reverse_delta_proofs + ADD CONSTRAINT reverse_proof_family_event_fk + FOREIGN KEY (family_id, quiescence_event_id) + REFERENCES omninode_internal.cutover_journal(family_id, event_id); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'reverse_entry_family_proof_fk' + AND conrelid = 'omninode_internal.reverse_delta_entries'::regclass + ) THEN + ALTER TABLE omninode_internal.reverse_delta_entries + ADD CONSTRAINT reverse_entry_family_proof_fk + FOREIGN KEY (family_id, proof_id) + REFERENCES omninode_internal.reverse_delta_proofs(family_id, proof_id); + END IF; +END +$constraints$; diff --git a/src/omnibase_infra/migration/cutover/transformation_receipt_builder.py b/src/omnibase_infra/migration/cutover/transformation_receipt_builder.py new file mode 100644 index 0000000000..de1c98a483 --- /dev/null +++ b/src/omnibase_infra/migration/cutover/transformation_receipt_builder.py @@ -0,0 +1,281 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Pure builder for complete transformation-aware database receipts.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import UTC, datetime +from uuid import uuid4 + +from omnibase_infra.migration.cutover.enums import ( + EnumCutoverFamilyKind, + EnumReceiptDimension, + EnumReceiptStatus, +) +from omnibase_infra.migration.cutover.models import ( + ModelCutoverContinuityEvidence, + ModelCutoverFamilyContract, + ModelReceiptCheck, + ModelTransformationEvidence, + ModelTransformationReceipt, + calculate_transformation_receipt_hash, +) + + +def _digest(value: object) -> str: + encoded = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + default=str, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +class TransformationReceiptBuilder: + """Compare canonical evidence and emit an immutable all-dimension receipt.""" + + def build( + self, + contract: ModelCutoverFamilyContract, + source: ModelTransformationEvidence, + target: ModelTransformationEvidence, + continuity: ModelCutoverContinuityEvidence, + ) -> ModelTransformationReceipt: + """Build a PASS only when every required invariant is proven.""" + comparisons = self._comparisons(contract, source, target, continuity) + checks = tuple( + self._check(dimension, *comparisons[dimension]) + for dimension in EnumReceiptDimension + ) + status = ( + EnumReceiptStatus.PASS + if all(check.passed for check in checks) + else EnumReceiptStatus.FAIL + ) + receipt_id = uuid4() + generated_at = datetime.now(UTC) + family_contract_hash = _digest(contract.model_dump(mode="json")) + receipt_hash = calculate_transformation_receipt_hash( + receipt_id=receipt_id, + family_id=contract.family_id, + family_contract_hash=family_contract_hash, + generated_at=generated_at, + source=source, + target=target, + continuity=continuity, + checks=checks, + status=status, + ) + return ModelTransformationReceipt( + receipt_id=receipt_id, + family_id=contract.family_id, + family_contract_hash=family_contract_hash, + generated_at=generated_at, + source=source, + target=target, + continuity=continuity, + checks=checks, + status=status, + receipt_hash=receipt_hash, + ) + + def _comparisons( + self, + contract: ModelCutoverFamilyContract, + source: ModelTransformationEvidence, + target: ModelTransformationEvidence, + continuity: ModelCutoverContinuityEvidence, + ) -> dict[EnumReceiptDimension, tuple[object, object, bool, str]]: + source_row_digest = _digest(source.transformed_row_hashes) + target_row_digest = _digest(target.transformed_row_hashes) + event_source, event_target, event_passed, event_detail = ( + self._event_offset_comparison(contract, continuity) + ) + delta_source, delta_target, delta_passed, delta_detail = ( + self._control_plane_comparison(contract, continuity) + ) + return { + EnumReceiptDimension.EVIDENCE_CONTRACTS: ( + ( + contract.source_evidence_contract_hash, + source.evidence_contract_hash, + ), + ( + contract.target_evidence_contract_hash, + target.evidence_contract_hash, + ), + ( + source.evidence_contract_hash + == contract.source_evidence_contract_hash + and target.evidence_contract_hash + == contract.target_evidence_contract_hash + ), + "source and target evidence queries match the registered contracts", + ), + EnumReceiptDimension.KEY_SET: ( + source.keys, + target.keys, + source.keys == target.keys, + "canonical source and target key sets are equal", + ), + EnumReceiptDimension.ROW_COUNT: ( + source.row_count, + target.row_count, + source.row_count == target.row_count, + "source and target row counts are equal", + ), + EnumReceiptDimension.TRANSFORMATION_HASH: ( + source_row_digest, + target_row_digest, + source_row_digest == target_row_digest, + "transformation-aware row hashes are equal", + ), + EnumReceiptDimension.FOREIGN_KEYS: self._equal( + source.foreign_keys, target.foreign_keys, "foreign keys" + ), + EnumReceiptDimension.SEQUENCES: self._equal( + source.sequences, target.sequences, "sequence values and ownership" + ), + EnumReceiptDimension.OWNERS: self._equal( + source.owners, target.owners, "object owners" + ), + EnumReceiptDimension.GRANTS: self._equal( + source.grants, target.grants, "explicit grants" + ), + EnumReceiptDimension.POLICIES: self._equal( + source.policies, target.policies, "RLS policy signatures" + ), + EnumReceiptDimension.VIEWS_FUNCTIONS: self._equal( + source.views_functions, + target.views_functions, + "dependent view/function signatures", + ), + EnumReceiptDimension.EVENT_OFFSETS: ( + event_source, + event_target, + event_passed, + event_detail, + ), + EnumReceiptDimension.CONTROL_PLANE_DELTA: ( + delta_source, + delta_target, + delta_passed, + delta_detail, + ), + EnumReceiptDimension.COLLISIONS: ( + source.collision_keys, + target.collision_keys, + not source.collision_keys and not target.collision_keys, + "transformed source and target collision scans are empty", + ), + EnumReceiptDimension.DEPENDENCIES: self._equal( + source.dependencies, + target.dependencies, + "dependency signatures", + ), + } + + @staticmethod + def _equal( + source: object, + target: object, + label: str, + ) -> tuple[object, object, bool, str]: + return source, target, source == target, f"source and target {label} are equal" + + @staticmethod + def _event_offset_comparison( + contract: ModelCutoverFamilyContract, + continuity: ModelCutoverContinuityEvidence, + ) -> tuple[object, object, bool, str]: + replays = continuity.projection_replays + source = tuple( + ( + item.projection_id, + item.projection_label, + item.projection_version, + item.topic, + item.partition, + item.source_offset, + ) + for item in replays + ) + target = tuple( + ( + item.projection_id, + item.projection_label, + item.projection_version, + item.topic, + item.partition, + item.target_offset, + ) + for item in replays + ) + if contract.family_kind is EnumCutoverFamilyKind.PROJECTION: + return ( + source, + target, + bool(replays) and source == target, + "projection versions and authoritative event offsets are equal", + ) + return ( + source, + target, + not replays, + "control-plane families must not advertise projection replay evidence", + ) + + @staticmethod + def _control_plane_comparison( + contract: ModelCutoverFamilyContract, + continuity: ModelCutoverContinuityEvidence, + ) -> tuple[object, object, bool, str]: + delta = continuity.control_plane_delta + if delta is None: + passed = contract.family_kind is EnumCutoverFamilyKind.PROJECTION + return (), (), passed, "control-plane snapshot/final-delta evidence" + source = ( + delta.source_snapshot_hash, + delta.source_final_delta_hash, + delta.source_watermark, + ) + target = ( + delta.target_snapshot_hash, + delta.target_final_delta_hash, + delta.target_watermark, + ) + passed = ( + contract.family_kind is EnumCutoverFamilyKind.CONTROL_PLANE + and not continuity.projection_replays + and source == target + ) + return ( + source, + target, + passed, + "control-plane snapshot, final delta, and watermarks are equal", + ) + + @staticmethod + def _check( + dimension: EnumReceiptDimension, + source: object, + target: object, + passed: bool, + success_detail: str, + ) -> ModelReceiptCheck: + detail = success_detail if passed else f"MISMATCH: {success_detail}" + return ModelReceiptCheck( + dimension=dimension, + passed=passed, + source_digest=_digest(source), + target_digest=_digest(target), + detail=detail, + ) + + +__all__ = ["TransformationReceiptBuilder"] diff --git a/src/omnibase_infra/models/registration/model_node_registration.py b/src/omnibase_infra/models/registration/model_node_registration.py index 74b659a7a9..4e8dc3b9a5 100644 --- a/src/omnibase_infra/models/registration/model_node_registration.py +++ b/src/omnibase_infra/models/registration/model_node_registration.py @@ -62,12 +62,13 @@ class ModelNodeRegistration(BaseModel): >>> from datetime import datetime, UTC >>> from uuid import uuid4 >>> now = datetime.now(UTC) + >>> resolved_health_endpoint = "https://routing-authority.resolved.example/health" # url-authority-ok: docstring example value, not a runtime literal >>> registration = ModelNodeRegistration( ... node_id=uuid4(), ... node_type="effect", ... capabilities={"postgres": True}, - ... endpoints={"health": "http://localhost:8080/health"}, - ... health_endpoint="http://localhost:8080/health", + ... endpoints={"health": resolved_health_endpoint}, + ... health_endpoint=resolved_health_endpoint, ... registered_at=now, ... updated_at=now, ... ) diff --git a/src/omnibase_infra/models/registration/model_node_registration_record.py b/src/omnibase_infra/models/registration/model_node_registration_record.py index cd6fd108ae..4d91f9ad4e 100644 --- a/src/omnibase_infra/models/registration/model_node_registration_record.py +++ b/src/omnibase_infra/models/registration/model_node_registration_record.py @@ -90,14 +90,15 @@ class ModelNodeRegistrationRecord(ModelRegistrationRecordBase): ... ModelNodeMetadata, ... ) >>> now = datetime.now(UTC) + >>> resolved_health_endpoint = "" >>> record = ModelNodeRegistrationRecord( ... node_id=uuid4(), ... node_type=EnumNodeKind.EFFECT, ... node_version=ModelSemVer(major=1, minor=0, patch=0), ... capabilities=ModelNodeCapabilities(postgres=True), - ... endpoints={"health": "http://localhost:8080/health"}, + ... endpoints={"health": resolved_health_endpoint}, ... metadata=ModelNodeMetadata(environment="production"), - ... health_endpoint="http://localhost:8080/health", + ... health_endpoint=resolved_health_endpoint, ... registered_at=now, ... updated_at=now, ... ) diff --git a/src/omnibase_infra/models/routing/model_routing_subcontract.py b/src/omnibase_infra/models/routing/model_routing_subcontract.py index 7615313ad3..f07b2cb1dd 100644 --- a/src/omnibase_infra/models/routing/model_routing_subcontract.py +++ b/src/omnibase_infra/models/routing/model_routing_subcontract.py @@ -27,7 +27,17 @@ class ModelRoutingSubcontract(BaseModel): Attributes: version: Semantic version of this routing configuration. routing_strategy: Strategy for matching events to handlers. - Currently only "payload_type_match" is supported. + "payload_type_match" and "topic_match" are both contract-legal + values (OMN-15215) — this model's ``handlers`` shape only carries + ``routing_key``/``handler_key`` and has no per-entry ``topic`` + field, so it cannot represent topic_match's per-topic + disambiguation. That is expected: this model backs + ``handler_routing_loader``'s informational + ``RuntimeContractConfigLoader`` boot-summary pass, not the live + consumer-attach/dispatch decision — the real wiring path uses + ``omnibase_infra.runtime.auto_wiring.models.ModelHandlerRouting`` + (an untyped ``routing_strategy: str`` with a per-entry ``topic`` + field, OMN-14580/OMN-13825). handlers: List of routing entries mapping event models to handlers. default_handler: Optional fallback handler key for unmatched events. @@ -51,7 +61,7 @@ class ModelRoutingSubcontract(BaseModel): default_factory=lambda: ModelSemVer(major=1, minor=0, patch=0), description="Semantic version of this routing configuration", ) - routing_strategy: Literal["payload_type_match"] = Field( + routing_strategy: Literal["payload_type_match", "topic_match"] = Field( default="payload_type_match", description="Strategy for matching events to handlers", ) diff --git a/src/omnibase_infra/models/validation_ledger/model_payload_validation_ledger_append.py b/src/omnibase_infra/models/validation_ledger/model_payload_validation_ledger_append.py index 93c5d9e044..9fa39972c6 100644 --- a/src/omnibase_infra/models/validation_ledger/model_payload_validation_ledger_append.py +++ b/src/omnibase_infra/models/validation_ledger/model_payload_validation_ledger_append.py @@ -37,9 +37,10 @@ from pydantic import BaseModel, ConfigDict, Field -# NOTE: ModelIntentPayloadBase was removed in omnibase_core 0.6.2 -# Using pydantic.BaseModel directly as the base class, mirroring -# ModelPayloadLedgerAppend. +# NOTE: This infra-local payload DTO extends pydantic.BaseModel directly (repo +# convention; see docs/standards/ONEX_TERMINOLOGY.md), mirroring +# ModelPayloadLedgerAppend. omnibase_core's ModelIntentPayloadBase was never +# removed — it still exists and bases core's closed-set intent payloads. class ModelPayloadValidationLedgerAppend(BaseModel): diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/contract.yaml b/src/omnibase_infra/nodes/node_bus_forwarder_effect/contract.yaml index e5aff564f2..80999500d4 100644 --- a/src/omnibase_infra/nodes/node_bus_forwarder_effect/contract.yaml +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/contract.yaml @@ -7,16 +7,16 @@ node_type: "EFFECT_GENERIC" contract_version: major: 0 minor: 1 - patch: 0 + patch: 1 node_version: major: 0 minor: 1 - patch: 0 + patch: 2 description: > - Trust-boundary effect node that mirrors contract-declared tenant gateway topics between a local runtime bus and the hosted cloud Kafka edge. The node owns all publishes; handlers only validate and return transformed envelopes. + Trust-boundary effect node that mirrors contract-declared tenant gateway topics between a local runtime bus and the hosted cloud Kafka edge. The node owns all publishes; handlers only validate and return transformed envelopes. OMN-15792: the outbound handler resolves its physical wire topic through the single runtime topic resolver (resolve_physical_topic) instead of calling the prefix_topic primitive directly -- internal implementation change, wire format and behavior unchanged. runtime_profiles: - - effects + - canary input_model: name: "ModelGatewayEnvelope" module: "omnibase_infra.nodes.node_bus_forwarder_effect.models.model_gateway_envelope" @@ -36,10 +36,11 @@ config: transport: "kafka" broker_provider_id: "22222222-2222-2222-2222-222222222222" cloud_broker_ref: "gateway.cloud.kafka.broker" - cloud_auth_ref: "gateway.cloud.kafka.oauth" + cloud_auth_ref: "gateway.cloud.kafka.msk_iam" acl_provisioner_ref: "gateway.cloud.kafka.authorization" - client_id_ref: "gateway.cloud.kafka.oauth.client_id" - client_secret_api_key_ref: "gateway.cloud.kafka.oauth.client_secret" + msk_region_ref: "gateway.cloud.kafka.msk_region" + security_protocol: "SASL_SSL" + sasl_mechanism: "AWS_MSK_IAM" local_leg: supported_flavors: - "containerized" @@ -51,26 +52,43 @@ config: - "onex.evt.omnibase-infra.gateway-heartbeat.v1" outbound: - "onex.evt.omnibase-infra.inference-response.v1" + - "onex.evt.omnibase-infra.delegation-completed.v1" + - "onex.evt.omnibase-infra.delegation-failed.v1" + - "onex.evt.omniintelligence.llm-call-completed.v1" + - "onex.evt.omnibase-infra.llm-call-completed.v1" - "onex.evt.omnibase-infra.gateway-heartbeat.v1" + canary: + # OMN-15741 (G1): dedicated topic for the path-verifying healthcheck probe. + # Never mirrored, never carries tenant traffic -- the probe produces a tiny + # record here and reads it back across each leg using the same transport + # and credentials as real traffic, so the healthcheck can tell "connected + # and forwarding" apart from "started once, now silently broken" (the + # 2026-08-04 failure: ready-file present, forwarding 0% for 4 days). + topic: "onex.evt.omnibase-infra.gateway-canary.v1" + cadence_seconds: 30 + produce_deadline_seconds: 8 + readback_deadline_seconds: 12 liveness: heartbeat_interval_seconds: 15 max_silence_window_seconds: 60 lag_threshold_messages: 500 lag_threshold_seconds: 120 drain_deadline_seconds: 30 + reconnect_backoff_initial_seconds: 1 + reconnect_backoff_max_seconds: 30 + reconnect_backoff_jitter_seconds: 0.5 + degraded_after_seconds: 60 dedupe: + key: "envelope_id" + store: "sqlite" + store_path_required: true retention_hours: 24 -event_bus: - subscribe_topics: - - "onex.cmd.omnibase-infra.delegation-inference-request.v1" - - "onex.cmd.omnibase-infra.delegation-request.v1" - - "onex.evt.omnibase-infra.inference-response.v1" - - "onex.evt.omnibase-infra.gateway-heartbeat.v1" - publish_topics: - - "onex.cmd.omnibase-infra.delegation-inference-request.v1" - - "onex.cmd.omnibase-infra.delegation-request.v1" - - "onex.evt.omnibase-infra.inference-response.v1" - - "onex.evt.omnibase-infra.gateway-heartbeat.v1" + delivery: + semantics: "at_least_once" + source_offset_acknowledgement: "after_destination_ack_and_durable_marker" + auto_commit: false + retry_initial_seconds: 1 + retry_max_seconds: 30 handler_routing: routing_strategy: "operation_match" handlers: @@ -80,7 +98,7 @@ handler_routing: module: "omnibase_infra.nodes.node_bus_forwarder_effect.handlers.handler_forward_outbound" handler_type: "COMPUTE" description: > - Canonical handle() dispatch entrypoint accepts a typed or serialized outer event envelope, normalizes it, validates the local bare topic, and delegates to the tenant cloud wire-prefix transform. + Pure transform entrypoint for validating a local canonical envelope and attaching the configured tenant wire prefix. The dedicated gateway process owns transport subscriptions and publishing. - operation: "gateway.consume_inbound" handler: @@ -88,7 +106,7 @@ handler_routing: module: "omnibase_infra.nodes.node_bus_forwarder_effect.handlers.handler_consume_inbound" handler_type: "COMPUTE" description: > - Canonical handle() dispatch entrypoint accepts a typed or serialized outer event envelope, normalizes it, validates the tenant-prefixed cloud topic, strips it to the bare local topic, and stamps the verified gateway tenant slug into payload["tenant_id"] (via the shared omnibase_infra.shared.tenant_stamp.stamp_verified_tenant_slug helper) before republish. + Pure transform entrypoint for validating an attached cloud tenant, stripping the wire prefix, and stamping the verified tenant slug. The dedicated gateway process owns transport subscriptions and publishing. capabilities: - name: "tenant_prefix_transform" @@ -97,6 +115,8 @@ capabilities: description: "Rejects any canonical topic not declared in mirror topic sets." - name: "node_owned_publish" description: "Handlers never publish; the effect node owns both bus legs." + - name: "durable_source_acknowledgement" + description: "Source offsets commit only after destination broker acknowledgement and a durable envelope_id marker; acknowledged redelivery is suppressed across restarts." - name: "raw_dispatch_envelope_normalization" description: "Handlers deserialize serialized outer event envelopes before reading payload." - name: "verified_inbound_payload_tenant_stamp" @@ -105,7 +125,7 @@ metadata: description: "Hybrid gateway local runtime bus forwarder" author: "OmniNode Team" created: "2026-06-10" - updated: "2026-07-14" + updated: "2026-08-08" tags: - gateway - event-bus diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_forward_outbound.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_forward_outbound.py index 44b97a3e0a..5738179650 100644 --- a/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_forward_outbound.py +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_forward_outbound.py @@ -15,7 +15,7 @@ ModelGatewayForwarderConfig, ) from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( - prefix_topic, + resolve_physical_topic, ) @@ -64,9 +64,15 @@ def forward_outbound(self, envelope: ModelGatewayEnvelope) -> ModelGatewayEnvelo if envelope.canonical_topic not in config.mirror_topics.outbound: raise ValueError("canonical_topic is not declared for outbound mirroring") - expected_wire_topic = prefix_topic( - identity.tenant_slug, + # OMN-15792: routes through the single runtime topic resolver + # (``resolve_physical_topic``) instead of calling ``prefix_topic`` + # directly -- ``identity.tenant_slug`` is a required field (never + # None) on this forwarder's config, so this is the identical + # transform ``prefix_topic`` already performed, now via the sole + # resolver every publish/subscribe call site consults. + expected_wire_topic = resolve_physical_topic( envelope.canonical_topic, + tenant_slug=identity.tenant_slug, ) return envelope.model_copy( update={ diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/__init__.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/__init__.py index 1f449ca325..d125e5d14a 100644 --- a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/__init__.py +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/__init__.py @@ -2,16 +2,22 @@ # SPDX-License-Identifier: MIT """Models for the tenant gateway bus forwarder.""" +from .model_gateway_canary_config import ModelGatewayCanaryConfig from .model_gateway_cloud_bus_config import ModelGatewayCloudBusConfig from .model_gateway_envelope import ModelGatewayEnvelope from .model_gateway_forwarder_config import ModelGatewayForwarderConfig +from .model_gateway_forwarder_runtime_config import ModelGatewayForwarderRuntimeConfig +from .model_gateway_heartbeat import ModelGatewayHeartbeat from .model_gateway_mirror_topics import ModelGatewayMirrorTopics from .model_gateway_tenant_identity import ModelGatewayTenantIdentity __all__ = [ + "ModelGatewayCanaryConfig", "ModelGatewayCloudBusConfig", "ModelGatewayEnvelope", "ModelGatewayForwarderConfig", + "ModelGatewayForwarderRuntimeConfig", + "ModelGatewayHeartbeat", "ModelGatewayMirrorTopics", "ModelGatewayTenantIdentity", ] diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_canary_config.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_canary_config.py new file mode 100644 index 0000000000..00c23c980c --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_canary_config.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Contract-declared canary probe config for the gateway path-verifying healthcheck. + +OMN-15741 (G1): the container healthcheck must exercise the same transport and +credentials as real traffic — not a sentinel file — and must be able to tell +"local leg healthy, cloud leg dead" apart from "both legs healthy" (the exact +failure mode of the 2026-08-04 outage: the ready-file was present and the +process was up for four days while forwarding 0%). This model is the sole +authority for the canary topic, cadence, and per-leg deadlines; the probe +process consults it instead of hardcoding any of those values. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( + validate_canonical_topic, +) + + +class ModelGatewayCanaryConfig(BaseModel): + """Dedicated canary topic + cadence + deadlines for the path-verifying probe. + + The canary topic is deliberately separate from ``mirror_topics`` — it never + carries tenant traffic, so probing it can never be mistaken for (or + interfere with) the real forwarding path. ``cadence_seconds`` bounds how + often the probe is allowed to perform a real produce+readback round trip + against the live brokers; between real checks the probe process may serve + a cached result so the healthcheck does not spam either leg. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + topic: str = Field(..., min_length=1) + cadence_seconds: int = Field(..., ge=1) + produce_deadline_seconds: float = Field(..., gt=0) + readback_deadline_seconds: float = Field(..., gt=0) + + @field_validator("topic") + @classmethod + def _validate_topic(cls, value: str) -> str: + return validate_canonical_topic(value) + + @property + def total_deadline_seconds(self) -> float: + """Upper bound on one full real check across a single leg.""" + return self.produce_deadline_seconds + self.readback_deadline_seconds + + +__all__ = ["ModelGatewayCanaryConfig"] diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_cloud_bus_config.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_cloud_bus_config.py index 5dee5c06fd..7c5b5705ce 100644 --- a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_cloud_bus_config.py +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_cloud_bus_config.py @@ -7,7 +7,7 @@ from typing import Literal from uuid import UUID -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class ModelGatewayCloudBusConfig(BaseModel): @@ -19,17 +19,16 @@ class ModelGatewayCloudBusConfig(BaseModel): cloud_broker_ref: str = Field(..., min_length=1) cloud_auth_ref: str = Field(..., min_length=1) acl_provisioner_ref: str = Field(..., min_length=1) - client_id_ref: str = Field(..., min_length=1) - client_secret_api_key_ref: str = Field(..., min_length=1) + client_id_ref: str | None = Field(default=None, min_length=1) + client_secret_api_key_ref: str | None = Field(default=None, min_length=1) + msk_region_ref: str | None = Field(default=None, min_length=1) security_protocol: Literal["SASL_SSL"] = "SASL_SSL" - sasl_mechanism: Literal["OAUTHBEARER"] = "OAUTHBEARER" + sasl_mechanism: Literal["OAUTHBEARER", "AWS_MSK_IAM"] = "OAUTHBEARER" @field_validator( "cloud_broker_ref", "cloud_auth_ref", "acl_provisioner_ref", - "client_id_ref", - "client_secret_api_key_ref", ) @classmethod def _validate_contract_ref(cls, value: str) -> str: @@ -41,3 +40,41 @@ def _validate_contract_ref(cls, value: str) -> str: "gateway cloud bus config must use contract refs, not KAFKA_* env" ) return ref + + @field_validator( + "client_id_ref", + "client_secret_api_key_ref", + "msk_region_ref", + ) + @classmethod + def _validate_optional_contract_ref(cls, value: str | None) -> str | None: + if value is None: + return None + return cls._validate_contract_ref(value) + + @model_validator(mode="after") + def _validate_auth_refs(self) -> ModelGatewayCloudBusConfig: + if self.sasl_mechanism == "OAUTHBEARER": + if self.client_id_ref is None or self.client_secret_api_key_ref is None: + raise ValueError( + "OAUTHBEARER gateway cloud bus requires client_id_ref and " + "client_secret_api_key_ref" + ) + if self.msk_region_ref is not None: + raise ValueError( + "OAUTHBEARER gateway cloud bus must not declare msk_region_ref" + ) + else: + if self.msk_region_ref is None: + raise ValueError( + "AWS_MSK_IAM gateway cloud bus requires msk_region_ref" + ) + if ( + self.client_id_ref is not None + or self.client_secret_api_key_ref is not None + ): + raise ValueError( + "AWS_MSK_IAM gateway cloud bus uses the AWS credential chain, " + "not OAuth client refs" + ) + return self diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_forwarder_config.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_forwarder_config.py index dee016d4db..19a5085da9 100644 --- a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_forwarder_config.py +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_forwarder_config.py @@ -4,10 +4,14 @@ from __future__ import annotations +from pathlib import Path from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from omnibase_infra.nodes.node_bus_forwarder_effect.models.model_gateway_canary_config import ( + ModelGatewayCanaryConfig, +) from omnibase_infra.nodes.node_bus_forwarder_effect.models.model_gateway_cloud_bus_config import ( ModelGatewayCloudBusConfig, ) @@ -28,12 +32,30 @@ class ModelGatewayForwarderConfig(BaseModel): cloud_bus: ModelGatewayCloudBusConfig local_transport_flavor: Literal["containerized", "lightweight"] mirror_topics: ModelGatewayMirrorTopics + canary: ModelGatewayCanaryConfig heartbeat_interval_seconds: int = Field(default=15, ge=1) max_silence_window_seconds: int = Field(default=60, ge=1) lag_threshold_messages: int = Field(default=500, ge=1) lag_threshold_seconds: int = Field(default=120, ge=1) drain_deadline_seconds: int = Field(default=30, ge=1) - dedupe_retention_hours: int = Field(default=24, ge=1) + dedupe_store_path: Path + dedupe_retention_hours: int = Field(default=24, ge=24) + forward_retry_initial_seconds: float = Field(default=1.0, gt=0) + forward_retry_max_seconds: float = Field(default=30.0, gt=0) + reconnect_backoff_initial_seconds: float = Field(default=1.0, gt=0) + reconnect_backoff_max_seconds: float = Field(default=30.0, gt=0) + reconnect_backoff_jitter_seconds: float = Field(default=0.5, ge=0) + degraded_after_seconds: int = Field(default=60, ge=1) + + @field_validator("dedupe_store_path") + @classmethod + def _validate_dedupe_store_path(cls, value: Path) -> Path: + if not value.is_absolute(): + raise ValueError( + "dedupe_store_path must be absolute so deployment persistence " + "cannot depend on the container working directory" + ) + return value @model_validator(mode="after") def _validate_liveness_windows(self) -> ModelGatewayForwarderConfig: @@ -41,4 +63,21 @@ def _validate_liveness_windows(self) -> ModelGatewayForwarderConfig: raise ValueError( "max_silence_window_seconds must exceed heartbeat_interval_seconds" ) + if self.forward_retry_max_seconds < self.forward_retry_initial_seconds: + raise ValueError( + "forward_retry_max_seconds must be greater than or equal to " + "forward_retry_initial_seconds" + ) + if self.canary.topic in self.mirror_topics.inbound or ( + self.canary.topic in self.mirror_topics.outbound + ): + raise ValueError( + "canary.topic must be dedicated and must not appear in " + "mirror_topics.inbound or mirror_topics.outbound" + ) + if self.reconnect_backoff_max_seconds < self.reconnect_backoff_initial_seconds: + raise ValueError( + "reconnect_backoff_max_seconds must be greater than or equal to " + "reconnect_backoff_initial_seconds" + ) return self diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_forwarder_runtime_config.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_forwarder_runtime_config.py new file mode 100644 index 0000000000..41c88fbe32 --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_forwarder_runtime_config.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Resolved process configuration for the gateway bus forwarder.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, model_validator + +from omnibase_infra.event_bus.models.config import ModelKafkaEventBusConfig +from omnibase_infra.nodes.node_bus_forwarder_effect.models.model_gateway_forwarder_config import ( + ModelGatewayForwarderConfig, +) + + +class ModelGatewayForwarderRuntimeConfig(BaseModel): + """Contract plus the two resolved broker legs used by one edge process. + + ``forwarder.cloud_bus`` remains the provider-neutral declaration of the + required capabilities and secret references. ``cloud_bus`` and + ``local_bus`` are the resolved, typed materialization consumed by + ``KafkaTransport``. Keeping the layers separate prevents a process-wide + ``KAFKA_*`` environment from silently making both legs point at one broker. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + forwarder: ModelGatewayForwarderConfig + local_bus: ModelKafkaEventBusConfig + cloud_bus: ModelKafkaEventBusConfig + + @model_validator(mode="after") + def _validate_resolved_legs(self) -> ModelGatewayForwarderRuntimeConfig: + if self.forwarder.local_transport_flavor != "containerized": + raise ValueError( + "the production gateway process currently requires the " + "containerized local transport flavor" + ) + if self.local_bus.bootstrap_servers == self.cloud_bus.bootstrap_servers: + raise ValueError("gateway local_bus and cloud_bus must be distinct") + if self.local_bus.enable_auto_commit or self.cloud_bus.enable_auto_commit: + raise ValueError( + "gateway transport legs require enable_auto_commit=false; source " + "offsets are committed only after durable destination delivery" + ) + if ( + self.local_bus.auto_offset_reset != "earliest" + or self.cloud_bus.auto_offset_reset != "earliest" + ): + raise ValueError( + "gateway transport legs require auto_offset_reset=earliest on " + "both legs; enable_auto_commit=false only preserves offsets " + "already inside the consumer's read window -- a 'latest' leg " + "silently drops any backlog produced while the consumer group " + "was unjoined (crash, LeaveGroup, cold restart before rejoin), " + "and auto_offset_reset also fires mid-session on " + "OffsetOutOfRangeError, not only on first boot (OMN-15781)" + ) + if not any( + topic.endswith(".gateway-heartbeat.v1") + for topic in self.forwarder.mirror_topics.outbound + ): + raise ValueError( + "production gateway forwarder requires an outbound heartbeat topic" + ) + + declared_cloud = self.forwarder.cloud_bus + if self.cloud_bus.security_protocol != declared_cloud.security_protocol: + raise ValueError( + "resolved cloud security_protocol does not match the gateway contract" + ) + if self.cloud_bus.sasl_mechanism != declared_cloud.sasl_mechanism: + raise ValueError( + "resolved cloud sasl_mechanism does not match the gateway contract" + ) + if ( + self.cloud_bus.sasl_mechanism == "AWS_MSK_IAM" + and not self.cloud_bus.msk_region + ): + raise ValueError("resolved AWS_MSK_IAM cloud bus requires msk_region") + return self + + +__all__ = ["ModelGatewayForwarderRuntimeConfig"] diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_heartbeat.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_heartbeat.py new file mode 100644 index 0000000000..10dd7f27cc --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_heartbeat.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed tenant-edge liveness event emitted by the gateway forwarder.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelGatewayHeartbeat(BaseModel): + """Observable proof of one tenant-scoped edge's liveness/reconnect state. + + ``status="degraded"`` is emitted by the runtime reconnect-supervision + loop (``runtime/gateway_forwarder.py``) once a cloud-leg delivery + failure has persisted past the contract-declared + ``degraded_after_seconds`` window; ``consecutive_failures``/``detail`` + are only populated on that path. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + # Heartbeat tenant_id is the config-bound DNS-safe slug. + tenant_id: str + # Canonical MSK principal is t-, not a UUID field. + principal_id: str + status: Literal["active", "degraded"] = "active" + emitted_at: datetime + local_transport_flavor: Literal["containerized", "lightweight"] + consecutive_failures: int = Field(default=0, ge=0) + detail: str = "" + + +__all__ = ["ModelGatewayHeartbeat"] diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_tenant_identity.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_tenant_identity.py index 1dde68f326..58c042b437 100644 --- a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_tenant_identity.py +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_tenant_identity.py @@ -14,6 +14,7 @@ ) _TENANT_SLUG_RE = re.compile(r"^[a-z][a-z0-9-]{1,61}[a-z0-9]$") +_PRINCIPAL_ID_RE = re.compile(r"^t-[0-9a-f]{32}$") class ModelGatewayTenantIdentity(BaseModel): @@ -23,7 +24,8 @@ class ModelGatewayTenantIdentity(BaseModel): tenant_id: UUID tenant_slug: str - principal_id: UUID + # Canonical MSK principal is t-, not a UUID field. + principal_id: str @field_validator("tenant_slug") @classmethod @@ -34,3 +36,13 @@ def _validate_tenant_slug(cls, value: str) -> str: if not _TENANT_SLUG_RE.match(slug) or "--" in slug: raise ValueError("tenant_slug must be DNS-compatible lowercase slug") return slug + + @field_validator("principal_id") + @classmethod + def _validate_principal_id(cls, value: str) -> str: + principal_id = value.strip() + if not _PRINCIPAL_ID_RE.fullmatch(principal_id): + raise ValueError( + "principal_id must use the canonical t-<32 lowercase hex> form" + ) + return principal_id diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_delivery.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_delivery.py new file mode 100644 index 0000000000..87eb7bb188 --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_delivery.py @@ -0,0 +1,560 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Runtime-owned delivery and source acknowledgement for the gateway edge.""" + +from __future__ import annotations + +import asyncio +import json +import logging +import time +from collections.abc import Sequence +from typing import Literal, Protocol + +from omnibase_core.models.runtime.model_transport_message import ModelTransportMessage +from omnibase_infra.event_bus.topic_constants import get_dlq_topic_for_original +from omnibase_infra.idempotency import ProtocolIdempotencyStore +from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( + ModelGatewayForwarderConfig, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_forwarder import ( + ServiceGatewayForwarder, +) +from omnibase_infra.utils import sanitize_error_message + +logger = logging.getLogger(__name__) + +_DIRECTIONS: tuple[Literal["outbound", "inbound"], ...] = ("outbound", "inbound") + + +class ProtocolGatewayConsumer(Protocol): + """Concrete pull surface used by the Kafka-backed gateway runtime.""" + + async def poll( + self, + *, + max_messages: int, + timeout_ms: int, + ) -> Sequence[ModelTransportMessage]: ... + + async def commit(self, message: object) -> None: ... + + async def nack(self, message: object) -> None: ... + + +# The delivery loop's real sources (``KafkaTransport``) carry three optional +# capabilities beyond the ``ProtocolGatewayConsumer`` pull surface above: +# ``restart_consumer`` (recreate ONLY the consumer-side client -- a fresh +# group join -- without touching the shared producer another direction and +# status/heartbeat publishing may also depend on; see +# ``KafkaTransport.restart_consumer``), ``has_group_membership`` (a +# best-effort local assignment probe), and ``send`` (the same object is also +# a producer, for best-effort dead-lettering). None of these are declared as +# their own ``Protocol`` -- the architecture validator enforces one +# ``Protocol`` class per file, and ``ProtocolGatewayConsumer`` above already +# occupies this file's slot -- so they are duck-typed via +# ``getattr``/``callable`` instead. Absence of any of them (e.g. the +# unit-test fakes that only implement poll/commit/nack) degrades gracefully: +# no dead-letter, no forced recreate, no membership signal, never an error. +# +# NOTE: watchdog recovery deliberately does NOT use ``close()``/``start()`` +# (OMN-15748). Those stop/start BOTH the consumer and the producer, and a +# single ``KafkaTransport`` instance backs one direction's consumer AND the +# other direction's outbound publish (plus status/heartbeat) -- see +# ``runtime/gateway_forwarder.py``'s ``local_bus``/``cloud_bus`` wiring. +# ``restart_consumer`` is the consumer-scoped alternative. + + +def _build_quarantine_payload( + *, + direction: Literal["outbound", "inbound"], + message: ModelTransportMessage, + error: Exception, +) -> bytes: + """Serialize an undecodable record's forensic context for the DLQ.""" + payload: dict[str, object] = { + "original_topic": message.topic, + "original_partition": message.partition, + "original_offset": message.offset, + "direction": direction, + "failure_class": "gateway_undecodable_record", + "error_type": type(error).__name__, + "error_message": sanitize_error_message(error), + } + return json.dumps(payload).encode("utf-8") + + +class NodeGatewayDelivery: + """Poll both legs and acknowledge only after durable destination delivery. + + Cross-cluster Kafka cannot make the destination publish and source offset + commit one atomic transaction. The explicit order is therefore: + + 1. reject or transform the source envelope at the gateway trust boundary; + 2. await the destination broker acknowledgement; + 3. durably record the envelope ID in the edge-local store; + 4. commit the source offset. + + If step 4 fails, a restarted consumer sees the durable marker, skips the + destination publish, and commits the redelivered source record. The + irreducible publish-to-marker crash window remains at-least-once and is + observable through the stable envelope ID; downstream execution must use + that same ID as its idempotency key. + """ + + def __init__( + self, + *, + config: ModelGatewayForwarderConfig, + forwarder: ServiceGatewayForwarder, + local_consumer: ProtocolGatewayConsumer, + cloud_consumer: ProtocolGatewayConsumer, + idempotency_store: ProtocolIdempotencyStore, + poll_timeout_ms: int = 1_000, + watchdog_stale_seconds: float | None = None, + ) -> None: + self._config = config + self._forwarder = forwarder + self._local_consumer = local_consumer + self._cloud_consumer = cloud_consumer + self._idempotency_store = idempotency_store + self._poll_timeout_ms = poll_timeout_ms + # The two directional loops share one process and one durable marker + # namespace. Serialize the check -> publish -> marker -> commit sequence + # so the same envelope cannot race through both directions before either + # loop records it. + self._delivery_lock = asyncio.Lock() + self._tasks: list[asyncio.Task[None]] = [] + self._direction_sources: dict[ + Literal["outbound", "inbound"], ProtocolGatewayConsumer + ] = {"outbound": local_consumer, "inbound": cloud_consumer} + self._direction_tasks: dict[ + Literal["outbound", "inbound"], asyncio.Task[None] + ] = {} + # Membership-loss watchdog state (OMN-15748/OMN-15690). A silent + # aiokafka max_poll_interval_ms idle-eviction raises zero exception, + # so the exception-triggered reconnect supervision in + # runtime/gateway_forwarder.py structurally cannot observe it -- this + # watchdog is the independent detection path. Reuses the + # already-contract-declared but previously unwired + # ``max_silence_window_seconds`` as its staleness threshold. + self._last_progress_monotonic: dict[Literal["outbound", "inbound"], float] = {} + self._membership_lost_streak: dict[Literal["outbound", "inbound"], int] = {} + self._watchdog_degraded: set[Literal["outbound", "inbound"]] = set() + # Directions currently mid-recovery: ``_run_direction`` checks this to + # tell a watchdog-initiated ``stale_task.cancel()`` apart from a real + # shutdown/failure cancel, so it can return cleanly instead of + # re-raising CancelledError. Without this, the cancelled task's + # CancelledError would surface through ``wait()`` (a BaseException, + # escaping ``except Exception`` in the composition root's supervisor) + # and kill the whole process on every watchdog recovery (OMN-15748). + self._watchdog_recovering: set[Literal["outbound", "inbound"]] = set() + self._watchdog_stale_seconds = ( + watchdog_stale_seconds + if watchdog_stale_seconds is not None + else float(config.max_silence_window_seconds) + ) + self._watchdog_tick_seconds = min(self._watchdog_stale_seconds / 4, 30.0) + + async def start(self) -> None: + """Start one bounded pull loop per direction.""" + if self._tasks: + return + await self._idempotency_store.cleanup_expired(self._retention_seconds) + now = time.monotonic() + self._last_progress_monotonic = {"outbound": now, "inbound": now} + self._membership_lost_streak = {"outbound": 0, "inbound": 0} + self._watchdog_degraded.clear() + self._tasks = [ + self._spawn_direction_task("outbound"), + self._spawn_direction_task("inbound"), + asyncio.create_task( + self._run_cleanup_loop(), + name="gateway-delivery-dedupe-cleanup", + ), + asyncio.create_task( + self._run_watchdog_loop(), + name="gateway-delivery-watchdog", + ), + ] + + def _spawn_direction_task( + self, direction: Literal["outbound", "inbound"] + ) -> asyncio.Task[None]: + source = self._direction_sources[direction] + task = asyncio.create_task( + self._run_direction(direction, source), + name=f"gateway-delivery-{direction}", + ) + self._direction_tasks[direction] = task + return task + + async def wait(self) -> None: + """Propagate a delivery-loop failure to the composition root. + + Awaits the LIVE task set, not a one-time snapshot: a watchdog-initiated + direction recovery (``_recover_stalled_direction``) replaces that + direction's entry in ``self._tasks`` in place with a fresh task, and + this loop re-reads ``self._tasks`` after every wake so it picks up the + replacement and keeps supervising it. The superseded task's own + completion is a clean return (``_run_direction``'s + ``_watchdog_recovering`` check), never a propagated CancelledError, so + a watchdog recovery is invisible here -- it neither raises nor ends + ``wait()`` (OMN-15748; a plain ``asyncio.gather(*self._tasks)`` over a + frozen snapshot previously re-raised the cancelled task's + CancelledError from this call, which the composition root's + exception-triggered supervisor cannot distinguish from a real fault). + + The ``asyncio.wait`` below is bounded by ``_watchdog_tick_seconds`` + rather than waiting unboundedly for the next completion (CodeRabbit + finding on this PR). ``_recover_stalled_direction`` splices the + replacement task into ``self._tasks`` only AFTER the stale task has + already finished and this loop has already woken and rebuilt + ``watched`` from the (still stale-only) task list -- an unbounded + wait would then block on the remaining old tasks with no further + wakeup until one of THEM completes, so the replacement would never + be picked up and a real exception on the recovered direction would + go unsupervised. The bound guarantees this loop revisits + ``self._tasks`` at least once per watchdog tick even with zero new + completions. + """ + if not self._tasks: + raise RuntimeError("gateway delivery node is not started") + watched: set[asyncio.Task[None]] = set(self._tasks) + while watched: + done, watched = await asyncio.wait( + watched, + timeout=self._watchdog_tick_seconds, + return_when=asyncio.FIRST_COMPLETED, + ) + for task in done: + if task.cancelled(): + # Only a watchdog-initiated recovery (or a stop() call + # racing this loop) cancels a tracked task; either way it + # is not a supervised failure. + continue + exc = task.exception() + if exc is not None: + raise exc + # Pick up any watchdog-spawned replacement task not yet tracked -- + # including one that raced ahead and already finished (with an + # exception) before this rescan, which the bounded timeout above + # guarantees we reach even without a completion to wake us. + for task in self._tasks: + if task in watched: + continue + if not task.done() or ( + not task.cancelled() and task.exception() is not None + ): + watched.add(task) + + async def stop(self) -> None: + """Cancel pull loops without acknowledging any in-flight source record.""" + tasks = list(self._tasks) + self._tasks.clear() + self._direction_tasks.clear() + self._watchdog_degraded.clear() + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + async def deliver_message( + self, + direction: Literal["outbound", "inbound"], + source: ProtocolGatewayConsumer, + message: ModelTransportMessage, + ) -> None: + """Serialize and deliver one record in the durable acknowledgement order.""" + async with self._delivery_lock: + await self._deliver_message_locked(direction, source, message) + + async def _deliver_message_locked( + self, + direction: Literal["outbound", "inbound"], + source: ProtocolGatewayConsumer, + message: ModelTransportMessage, + ) -> None: + """Deliver one record while holding the process-wide delivery lock.""" + try: + envelope = self._forwarder.decode_message(message) + except asyncio.CancelledError: + raise + except Exception as decode_error: # noqa: BLE001 — boundary: any decode failure is quarantined, never a bare swallow + # A permanently malformed record can never decode no matter how + # many times it is redelivered -- routing it through the nack + # path below would seek back to the same offset and re-crash + # forever (OMN-15748 poison-pill DoS). Quarantine instead: log, + # best-effort dead-letter, commit past it, keep the loop alive. + await self._quarantine_undecodable_message( + direction, source, message, decode_error + ) + return + domain = f"gateway:{self._config.tenant_identity.tenant_slug}" + try: + if direction == "outbound": + self._forwarder.validate_outbound_message(message) + else: + self._forwarder.validate_inbound_message(message) + + if await self._idempotency_store.is_processed( + envelope.envelope_id, + domain=domain, + ): + logger.info( + "Gateway duplicate suppressed envelope_id=%s direction=%s " + "source_topic=%s source_partition=%s source_offset=%s", + envelope.envelope_id, + direction, + message.topic, + message.partition, + message.offset, + ) + await source.commit(message) + return + + if direction == "outbound": + await self._forwarder.forward_outbound_message(message) + else: + await self._forwarder.consume_inbound_message(message) + + await self._idempotency_store.mark_processed( + envelope.envelope_id, + domain=domain, + correlation_id=envelope.correlation_id, + ) + await source.commit(message) + logger.info( + "Gateway delivery acknowledged envelope_id=%s direction=%s " + "source_topic=%s source_partition=%s source_offset=%s", + envelope.envelope_id, + direction, + message.topic, + message.partition, + message.offset, + ) + except asyncio.CancelledError: + raise + except Exception: + try: + await source.nack(message) + except Exception: + logger.exception( + "Gateway source nack failed direction=%s source_topic=%s " + "source_partition=%s source_offset=%s", + direction, + message.topic, + message.partition, + message.offset, + ) + raise + + async def _quarantine_undecodable_message( + self, + direction: Literal["outbound", "inbound"], + source: ProtocolGatewayConsumer, + message: ModelTransportMessage, + error: Exception, + ) -> None: + """Dead-letter an undecodable record and commit past it (never redeliver). + + Commit failure still propagates (uncaught): that is a broker-level + fault, a different failure class already handled by the existing + reconnect-supervision loop in ``runtime/gateway_forwarder.py``. + """ + logger.error( + "Gateway undecodable record quarantined direction=%s source_topic=%s " + "source_partition=%s source_offset=%s error_type=%s error=%s", + direction, + message.topic, + message.partition, + message.offset, + type(error).__name__, + sanitize_error_message(error), + ) + sender = getattr(source, "send", None) + if callable(sender): + dlq_topic = get_dlq_topic_for_original(message.topic) + if dlq_topic is not None: + try: + await sender( + dlq_topic, + message.key, + _build_quarantine_payload( + direction=direction, message=message, error=error + ), + {"original_topic": message.topic.encode("utf-8")}, + ) + except Exception: + logger.exception( + "Gateway quarantine DLQ publish failed direction=%s " + "source_topic=%s source_partition=%s source_offset=%s", + direction, + message.topic, + message.partition, + message.offset, + ) + await source.commit(message) + + async def _run_direction( + self, + direction: Literal["outbound", "inbound"], + source: ProtocolGatewayConsumer, + ) -> None: + try: + while True: + messages = await source.poll( + max_messages=1, + timeout_ms=self._poll_timeout_ms, + ) + self._last_progress_monotonic[direction] = time.monotonic() + for message in messages: + await self.deliver_message(direction, source, message) + self._last_progress_monotonic[direction] = time.monotonic() + except asyncio.CancelledError: + if direction in self._watchdog_recovering: + # Watchdog-initiated: ``_recover_stalled_direction`` cancelled + # this task on purpose to swap in a fresh consumer. Return + # cleanly rather than re-raise so this completion registers + # as ordinary bookkeeping to ``wait()``, never a propagated + # CancelledError (OMN-15748). + return + raise + + async def _run_watchdog_loop(self) -> None: + """Detect a stalled direction independent of task exceptions. + + Two triggers, either sufficient: (1) no poll/deliver progress for + ``_watchdog_stale_seconds`` -- catches a task alive but hung inside a + stuck ``poll()`` (the live-observed 2026-08-09T10:03:07Z mechanism: + aiokafka's client-side idle-eviction fires a clean LeaveGroup with no + exception, then ``ensure_active_group`` refuses to rejoin while the + idle clock stays past threshold -- which it does forever once the + underlying poll never returns); (2) two consecutive lost-membership + probes -- catches membership silently lost even while poll keeps + returning. Recovery force-recreates just the affected direction's + transport (a fresh client instance re-joins the group from zero) and + publishes a DEGRADED status; a later tick with progress republishes + recovery. + """ + while True: + await asyncio.sleep(self._watchdog_tick_seconds) + now = time.monotonic() + for direction in _DIRECTIONS: + source = self._direction_sources[direction] + elapsed = now - self._last_progress_monotonic.get(direction, now) + membership_lost = self._probe_membership_lost(source) + self._membership_lost_streak[direction] = ( + self._membership_lost_streak.get(direction, 0) + 1 + if membership_lost + else 0 + ) + stale_by_time = elapsed >= self._watchdog_stale_seconds + stale_by_membership = self._membership_lost_streak[direction] >= 2 + if stale_by_time or stale_by_membership: + await self._recover_stalled_direction( + direction, + elapsed_seconds=elapsed, + membership_lost=membership_lost, + ) + elif direction in self._watchdog_degraded: + await self._mark_direction_recovered(direction) + + @staticmethod + def _probe_membership_lost(source: ProtocolGatewayConsumer) -> bool: + prober = getattr(source, "has_group_membership", None) + if not callable(prober): + return False + try: + return not bool(prober()) + except Exception: # noqa: BLE001 — boundary: probe failure counts as lost, fail closed + return True + + async def _recover_stalled_direction( + self, + direction: Literal["outbound", "inbound"], + *, + elapsed_seconds: float, + membership_lost: bool, + ) -> None: + logger.warning( + "Gateway %s delivery loop stalled elapsed_seconds=%.1f " + "membership_lost=%s; forcing recreate + rejoin", + direction, + elapsed_seconds, + membership_lost, + ) + stale_task = self._direction_tasks.get(direction) + if stale_task is not None and not stale_task.done(): + self._watchdog_recovering.add(direction) + try: + stale_task.cancel() + await asyncio.gather(stale_task, return_exceptions=True) + finally: + self._watchdog_recovering.discard(direction) + + source = self._direction_sources[direction] + # Consumer-scoped recreate ONLY -- never close()/start(), which would + # also stop the producer this same transport instance may serve for + # the OTHER direction's forward-publish and status/heartbeat publish + # (OMN-15748). + restarter = getattr(source, "restart_consumer", None) + if callable(restarter): + try: + await restarter() + except Exception: + logger.exception( + "Gateway %s transport recreate failed; will retry next " + "watchdog tick", + direction, + ) + return + + self._last_progress_monotonic[direction] = time.monotonic() + self._membership_lost_streak[direction] = 0 + new_task = self._spawn_direction_task(direction) + self._tasks = [new_task if t is stale_task else t for t in self._tasks] + if direction not in self._watchdog_degraded: + self._watchdog_degraded.add(direction) + await self._publish_watchdog_status( + "degraded", + detail=( + f"{direction} delivery loop membership-loss recovery " + f"elapsed_seconds={elapsed_seconds:.1f} " + f"membership_lost={membership_lost}" + ), + ) + + async def _mark_direction_recovered( + self, direction: Literal["outbound", "inbound"] + ) -> None: + self._watchdog_degraded.discard(direction) + await self._publish_watchdog_status( + "active", detail=f"{direction} delivery loop recovered" + ) + + async def _publish_watchdog_status( + self, status: Literal["active", "degraded"], *, detail: str + ) -> None: + """Best-effort status publish -- must never itself take down the watchdog.""" + try: + await self._forwarder.publish_status(status, detail=detail) + except Exception: + logger.exception("Gateway watchdog %s status publish failed", status) + + async def _run_cleanup_loop(self) -> None: + cleanup_interval_seconds = min(self._retention_seconds / 4, 3_600) + while True: + await asyncio.sleep(cleanup_interval_seconds) + removed = await self._idempotency_store.cleanup_expired( + self._retention_seconds + ) + if removed: + logger.info("Gateway dedupe cleanup removed=%d", removed) + + @property + def _retention_seconds(self) -> int: + return self._config.dedupe_retention_hours * 60 * 60 + + +__all__ = ["NodeGatewayDelivery"] diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_forwarder.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_forwarder.py index 5e4167d79a..357dbae5ee 100644 --- a/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_forwarder.py +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_forwarder.py @@ -1,12 +1,24 @@ # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT -"""Executable bus-to-bus gateway forwarder service.""" +"""Executable bus-to-bus gateway forwarder service. + +The wire on both broker legs is the platform canonical +``ModelEventEnvelope``. ``ModelGatewayEnvelope`` is a transform-boundary +model used by the node handlers; it is not a second event-bus envelope and +must never be required from ordinary runtime producers or consumers. +""" from __future__ import annotations +import logging from collections.abc import Awaitable, Callable -from typing import Protocol +from datetime import UTC, datetime +from typing import Literal, Protocol +from uuid import uuid4 +from omnibase_core.models.core.model_envelope_metadata import ModelEnvelopeMetadata +from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope +from omnibase_infra.errors import RuntimeHostError from omnibase_infra.nodes.node_bus_forwarder_effect.handlers import ( HandlerConsumeInbound, HandlerForwardOutbound, @@ -14,14 +26,56 @@ from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( ModelGatewayEnvelope, ModelGatewayForwarderConfig, + ModelGatewayHeartbeat, ) from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( - prefix_topic, + strip_topic_prefix, ) +logger = logging.getLogger(__name__) + +# ``gateway_direction`` tag values that mark an envelope as local-bus-only -- +# the forwarder's own outbound consumer loop (NodeGatewayDelivery polling the +# SAME transport local_bus publishes into, see runtime/gateway_forwarder.py) +# must skip these rather than re-forward them to cloud (OMN-15570/OMN-15742 +# reconciliation finding D1). "cloud-to-local" is an inbound-transformed +# envelope publish_status/publish_heartbeat never emit and needs the same +# skip already established for the inbound leg; "local-mirror" marks a +# direct local-only publish (DEGRADED status, and the G3 heartbeat local +# mirror) that was never meant to leave this cluster at all. +_LOCAL_ONLY_DIRECTIONS = frozenset({"cloud-to-local", "local-mirror"}) + + +def _stamp_local_only( + envelope: ModelEventEnvelope[dict[str, object]], +) -> ModelEventEnvelope[dict[str, object]]: + """Tag a local-bus-only publish so the outbound consumer never re-forwards it. + + Both the DEGRADED status publish (``ServiceGatewayForwarder.publish_status``) + and the G3 heartbeat local mirror (``publish_heartbeat``) go directly onto + the local bus's canonical outbound topic -- exactly the topic the + forwarder's own outbound consumer polls (``NodeGatewayDelivery`` on + ``local_consumer``, the SAME transport object ``local_bus`` publishes into + in the real runtime). Without this tag, ``_forward_outbound_message``'s + loopback skip does not match an untagged envelope, so it falls through to + ``_prepare_outbound`` -- a second cloud publish per heartbeat tick, or a + DEGRADED status leak to cloud (OMN-15570/OMN-15742 reconciliation finding + D1). Module-level (not a method) to stay under the class's method-count + pattern threshold. + """ + metadata = envelope.metadata.model_copy( + update={ + "tags": { + **envelope.metadata.tags, + "gateway_direction": "local-mirror", + } + } + ) + return envelope.model_copy(update={"metadata": metadata}) -class ProtocolGatewayBus(Protocol): - """Structural subset shared by EventBusKafka, EventBusInmemory, and tests.""" + +class ProtocolGatewayPublisher(Protocol): + """Destination publish boundary shared by push and pull transports.""" async def publish( self, @@ -32,101 +86,374 @@ async def publish( ) -> None: """Publish bytes to a topic.""" - async def subscribe( - self, - topic: str, - node_identity: object | None = None, - on_message: Callable[[object], Awaitable[None]] | None = None, - *, - group_id: str | None = None, - **kwargs: object, - ) -> Callable[[], Awaitable[None]]: - """Subscribe to a topic and return an async unsubscribe callback.""" - class ServiceGatewayForwarder: - """Subscribe to mirrored topics on both legs and republish transformed envelopes.""" + """Validate, transform, and republish explicitly polled gateway envelopes.""" def __init__( self, *, config: ModelGatewayForwarderConfig, - local_bus: ProtocolGatewayBus, - cloud_bus: ProtocolGatewayBus, + local_bus: ProtocolGatewayPublisher, + cloud_bus: ProtocolGatewayPublisher, + retry_sleep: Callable[[float], Awaitable[None]] | None = None, ) -> None: self._config = config self._local_bus = local_bus self._cloud_bus = cloud_bus - self._outbound_handler = HandlerForwardOutbound(config) - self._inbound_handler = HandlerConsumeInbound(config) - self._unsubscribe_callbacks: list[Callable[[], Awaitable[None]]] = [] - self._started = False - - async def start(self) -> None: - """Start subscriptions on both bus legs.""" - if self._started: - return - - for topic in self._config.mirror_topics.outbound: - unsubscribe = await self._local_bus.subscribe( - topic=topic, - group_id=self._group_id("outbound"), - on_message=self._forward_outbound_message, - ) - self._unsubscribe_callbacks.append(unsubscribe) - - tenant_slug = self._config.tenant_identity.tenant_slug - for topic in self._config.mirror_topics.inbound: - unsubscribe = await self._cloud_bus.subscribe( - topic=prefix_topic(tenant_slug, topic), - group_id=self._group_id("inbound"), - on_message=self._consume_inbound_message, - ) - self._unsubscribe_callbacks.append(unsubscribe) - - self._started = True + if retry_sleep is None: + import asyncio - async def stop(self) -> None: - """Stop all active subscriptions.""" - callbacks = list(reversed(self._unsubscribe_callbacks)) - self._unsubscribe_callbacks.clear() - self._started = False - for unsubscribe in callbacks: - await unsubscribe() + retry_sleep = asyncio.sleep + self._retry_sleep = retry_sleep + # OMN-15740: the tenant-prefix transform is owned by the contract-declared + # COMPUTE handlers, not re-derived here. The service's job is the trust + # boundary (tag/tenant validation against untrusted bus input) plus the + # I/O; the pure prefix/strip transform and payload stamp are delegated. + self._forward_outbound_handler = HandlerForwardOutbound(config) + self._consume_inbound_handler = HandlerConsumeInbound(config) async def _forward_outbound_message(self, message: object) -> None: + source_topic = self._message_topic(message) envelope = self._decode_message(message) - transformed = self._outbound_handler.forward_outbound(envelope) - await self._cloud_bus.publish( - topic=transformed.wire_topic, + direction = envelope.metadata.tags.get("gateway_direction") + if direction in _LOCAL_ONLY_DIRECTIONS: + logger.debug( + "Skipping gateway loopback on local topic %s (direction=%s)", + source_topic, + direction, + ) + return + transformed, wire_topic = self._prepare_outbound(envelope, source_topic) + await self._publish_with_delivery_retry( + bus=self._cloud_bus, + topic=wire_topic, key=getattr(message, "key", None), value=self._encode_envelope(transformed), headers=getattr(message, "headers", None), ) + async def forward_outbound_message(self, message: object) -> None: + """Validate, transform, and broker-acknowledge one outbound message.""" + await self._forward_outbound_message(message) + + def validate_outbound_message(self, message: object) -> None: + """Validate an outbound trust-boundary message without publishing it.""" + source_topic = self._message_topic(message) + envelope = self._decode_message(message) + direction = envelope.metadata.tags.get("gateway_direction") + if direction not in _LOCAL_ONLY_DIRECTIONS: + self._prepare_outbound(envelope, source_topic) + async def _consume_inbound_message(self, message: object) -> None: + wire_topic = self._message_topic(message) envelope = self._decode_message(message) - transformed = self._inbound_handler.consume_inbound(envelope) - await self._local_bus.publish( - topic=transformed.canonical_topic, + if envelope.metadata.tags.get("gateway_direction") == "local-to-cloud": + logger.debug( + "Skipping gateway loopback on cloud topic %s", + wire_topic, + ) + return + transformed, canonical_topic = self._prepare_inbound(envelope, wire_topic) + await self._publish_with_delivery_retry( + bus=self._local_bus, + topic=canonical_topic, key=getattr(message, "key", None), value=self._encode_envelope(transformed), headers=getattr(message, "headers", None), ) - def _group_id(self, direction: str) -> str: + async def consume_inbound_message(self, message: object) -> None: + """Validate, transform, and broker-acknowledge one inbound message.""" + await self._consume_inbound_message(message) + + def validate_inbound_message(self, message: object) -> None: + """Validate an inbound trust-boundary message without publishing it.""" + wire_topic = self._message_topic(message) + envelope = self._decode_message(message) + if envelope.metadata.tags.get("gateway_direction") != "local-to-cloud": + self._prepare_inbound(envelope, wire_topic) + + @classmethod + def decode_message( + cls, + message: object, + ) -> ModelEventEnvelope[dict[str, object]]: + """Decode the canonical envelope used as the durable dedupe key source.""" + return cls._decode_message(message) + + def _build_status_envelope( + self, + status: Literal["active", "degraded"], + *, + consecutive_failures: int = 0, + detail: str = "", + ) -> tuple[ModelEventEnvelope[dict[str, object]], str]: + """Build the heartbeat/status envelope plus its canonical topic.""" + identity = self._config.tenant_identity + now = datetime.now(UTC) + envelope_id = uuid4() + heartbeat = ModelGatewayHeartbeat( + tenant_id=identity.tenant_slug, + principal_id=identity.principal_id, + status=status, + emitted_at=now, + local_transport_flavor=self._config.local_transport_flavor, + consecutive_failures=consecutive_failures, + detail=detail, + ) + envelope = ModelEventEnvelope[dict[str, object]]( + envelope_id=envelope_id, + envelope_timestamp=now, + correlation_id=envelope_id, + source_tool="gateway-forwarder", + event_type="omnibase-infra.gateway-heartbeat", + payload=heartbeat.model_dump(mode="json"), + metadata=ModelEnvelopeMetadata( + tags={ + "source_tenant_id": str(identity.tenant_id), + "source_tenant_principal_id": identity.principal_id, + } + ), + ) + canonical_topic = next( + topic + for topic in self._config.mirror_topics.outbound + if topic.endswith(".gateway-heartbeat.v1") + ) + return envelope, canonical_topic + + async def publish_heartbeat(self) -> None: + """Publish one tenant-scoped liveness event onto the cloud wire topic.""" + identity = self._config.tenant_identity + envelope, canonical_topic = self._build_status_envelope("active") + # OMN-15740: _prepare_outbound returns (transformed_envelope, wire_topic) -- + # the transform-seam handler delegation G0 wired in, not the pre-G0 single + # return this call site used to unpack. Keep G0's tuple return; G2 only + # adds the reconnect-supervision status publish below. + transformed, wire_topic = self._prepare_outbound(envelope, canonical_topic) + await self._publish_with_delivery_retry( + bus=self._cloud_bus, + topic=wire_topic, + key=str(identity.tenant_id).encode("utf-8"), + value=self._encode_envelope(transformed), + ) + + async def publish_status( + self, + status: Literal["active", "degraded"], + *, + consecutive_failures: int = 0, + detail: str = "", + ) -> None: + """Publish a reconnect-supervision status event onto the LOCAL bus. + + Unlike ``publish_heartbeat`` (which crosses the cloud wire), + reconnect-supervision status -- most importantly a ``DEGRADED`` + transition -- must stay observable while the cloud leg that caused + it is itself unreachable, so this publishes on the local bus using + the same heartbeat topic instead of the cloud leg. + """ identity = self._config.tenant_identity - return f"tenant-{identity.tenant_slug}-gateway-forwarder-{direction}" + envelope, canonical_topic = self._build_status_envelope( + status, + consecutive_failures=consecutive_failures, + detail=detail, + ) + local_only = _stamp_local_only(envelope) + await self._publish_with_delivery_retry( + bus=self._local_bus, + topic=canonical_topic, + key=str(identity.tenant_id).encode("utf-8"), + value=self._encode_envelope(local_only), + ) + + async def _publish_with_delivery_retry( + self, + *, + bus: ProtocolGatewayPublisher, + topic: str, + key: bytes | None, + value: bytes, + headers: object | None = None, + ) -> None: + """Block source acknowledgement until a transient destination recovers. + + ``KafkaTransport.send`` awaits the destination broker acknowledgement. + The gateway adds a process-lifetime retry around that boundary so the + delivery node cannot record its durable marker or commit the source + offset while the message exists on only one broker. Cancellation during + shutdown is deliberately not caught. + """ + delay = self._config.forward_retry_initial_seconds + attempt = 0 + while True: + try: + await bus.publish( + topic=topic, + key=key, + value=value, + headers=headers, + ) + return + except RuntimeHostError as exc: + attempt += 1 + logger.warning( + "Gateway destination unavailable; retaining source message " + "and retrying topic=%s attempt=%d delay_seconds=%.1f " + "error_type=%s", + topic, + attempt, + delay, + type(exc).__name__, + ) + await self._retry_sleep(delay) + delay = min(delay * 2, self._config.forward_retry_max_seconds) @staticmethod - def _decode_message(message: object) -> ModelGatewayEnvelope: + def _decode_message(message: object) -> ModelEventEnvelope[dict[str, object]]: value = getattr(message, "value", message) + if isinstance(value, ModelEventEnvelope): + return ModelEventEnvelope[dict[str, object]].model_validate(value) if isinstance(value, str): value = value.encode("utf-8") if not isinstance(value, bytes): raise TypeError("gateway bus message value must be bytes or string") - return ModelGatewayEnvelope.model_validate_json(value) + return ModelEventEnvelope[dict[str, object]].model_validate_json(value) + + @staticmethod + def _encode_envelope( + envelope: ModelEventEnvelope[dict[str, object]], + ) -> bytes: + return envelope.model_dump_json(exclude_none=True).encode("utf-8") @staticmethod - def _encode_envelope(envelope: ModelGatewayEnvelope) -> bytes: - return envelope.model_dump_json().encode("utf-8") + def _message_topic(message: object) -> str: + topic = getattr(message, "topic", None) + if not isinstance(topic, str) or not topic: + raise TypeError("gateway bus message must carry its source topic") + return topic + + def _prepare_inbound( + self, + envelope: ModelEventEnvelope[dict[str, object]], + wire_topic: str, + ) -> tuple[ModelEventEnvelope[dict[str, object]], str]: + """Validate a cloud command and stamp its config-bound local tenant. + + Trust-boundary validation (topic declared, tags match the config-bound + identity) stays here against the untrusted bus input. The prefix-strip + transform and payload stamp themselves are delegated to the + contract-declared ``HandlerConsumeInbound`` COMPUTE handler (OMN-15740) + so there is exactly one implementation of that transform. + """ + identity = self._config.tenant_identity + canonical_topic = strip_topic_prefix(identity.tenant_slug, wire_topic) + if canonical_topic not in self._config.mirror_topics.inbound: + raise ValueError("canonical_topic is not declared for inbound mirroring") + + tags = envelope.metadata.tags + if tags.get("source_tenant_id") != str(identity.tenant_id): + raise ValueError("envelope tenant_id does not match attached tenant") + if tags.get("source_tenant_principal_id") != str(identity.principal_id): + raise ValueError("envelope principal_id does not match attached tenant") + + gateway_envelope = ModelGatewayEnvelope( + tenant_id=identity.tenant_id, + tenant_slug=identity.tenant_slug, + envelope_id=envelope.envelope_id, + correlation_id=envelope.correlation_id, + event_type=envelope.event_type, + source_topic=wire_topic, + wire_topic=wire_topic, + canonical_topic=canonical_topic, + payload=envelope.payload, + ) + transformed_gateway = self._consume_inbound_handler.consume_inbound( + gateway_envelope + ) + + metadata = envelope.metadata.model_copy( + update={ + "tags": { + **tags, + "gateway_tenant_id": str(identity.tenant_id), + "gateway_tenant_slug": identity.tenant_slug, + "gateway_principal_id": str(identity.principal_id), + "gateway_wire_topic": wire_topic, + "gateway_canonical_topic": canonical_topic, + "gateway_direction": "cloud-to-local", + } + } + ) + return envelope.model_copy( + update={"payload": transformed_gateway.payload, "metadata": metadata} + ), canonical_topic + + def _prepare_outbound( + self, + envelope: ModelEventEnvelope[dict[str, object]], + canonical_topic: str, + ) -> tuple[ModelEventEnvelope[dict[str, object]], str]: + """Validate a local event and bind it to the attached tenant. + + Trust-boundary validation stays here; the wire-topic prefix transform + is delegated to the contract-declared ``HandlerForwardOutbound`` + COMPUTE handler (OMN-15740). + """ + identity = self._config.tenant_identity + if canonical_topic not in self._config.mirror_topics.outbound: + raise ValueError("canonical_topic is not declared for outbound mirroring") + + payload_tenant = envelope.payload.get("tenant_id") + if payload_tenant is not None and payload_tenant != identity.tenant_slug: + raise ValueError( + "outbound payload tenant_id does not match attached tenant" + ) + + tags = envelope.metadata.tags + existing_tenant_id = tags.get("source_tenant_id") + if existing_tenant_id is not None and existing_tenant_id != str( + identity.tenant_id + ): + raise ValueError( + "outbound envelope tenant_id does not match attached tenant" + ) + existing_principal_id = tags.get("source_tenant_principal_id") + if existing_principal_id is not None and existing_principal_id != str( + identity.principal_id + ): + raise ValueError( + "outbound envelope principal_id does not match attached tenant" + ) + + gateway_envelope = ModelGatewayEnvelope( + tenant_id=identity.tenant_id, + tenant_slug=identity.tenant_slug, + envelope_id=envelope.envelope_id, + correlation_id=envelope.correlation_id, + event_type=envelope.event_type, + source_topic=canonical_topic, + wire_topic=canonical_topic, + canonical_topic=canonical_topic, + payload=envelope.payload, + ) + transformed_gateway = self._forward_outbound_handler.forward_outbound( + gateway_envelope + ) + wire_topic = transformed_gateway.wire_topic + + metadata = envelope.metadata.model_copy( + update={ + "tags": { + **tags, + "source_tenant_id": str(identity.tenant_id), + "source_tenant_principal_id": str(identity.principal_id), + "gateway_tenant_slug": identity.tenant_slug, + "gateway_wire_topic": wire_topic, + "gateway_canonical_topic": canonical_topic, + "gateway_direction": "local-to-cloud", + } + } + ) + return envelope.model_copy(update={"metadata": metadata}), wire_topic diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_topic_transform.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_topic_transform.py index fc84ccb71c..8518ccf30f 100644 --- a/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_topic_transform.py +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_topic_transform.py @@ -1,6 +1,18 @@ # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT -"""Tenant-prefix transform helpers for the gateway trust boundary.""" +"""Tenant-prefix transform helpers for the gateway trust boundary. + +OMN-15792 (2026-08-09 operator addressing ruling): ``resolve_physical_topic`` +and its inverse ``resolve_tenant_from_wire_topic`` are THE single runtime +topic resolver -- physical topic addressing resolved from a contract-declared +canonical topic plus optional tenant execution context. Every publish and +subscribe call site that needs tenant-aware physical topic addressing MUST +route through these two functions rather than re-deriving the transform. +This module does not redesign the wire format (still ``tenant-{slug}.``); +it makes ``prefix_topic``/``strip_topic_prefix`` the sole path instead of one +of several independent implementations (OMN-15757/OMN-15778's structural root +cause). +""" from __future__ import annotations @@ -9,7 +21,13 @@ from omnibase_core.validation import validate_topic_suffix RESERVED_TENANT_SLUGS = frozenset({"", "system"}) -_TENANT_SLUG_RE = re.compile(r"^[a-z][a-z0-9-]{1,61}[a-z0-9]$") +_TENANT_SLUG_PATTERN = r"[a-z][a-z0-9-]{1,61}[a-z0-9]" +_TENANT_SLUG_RE = re.compile(rf"^{_TENANT_SLUG_PATTERN}$") +# Inline-cites _TENANT_SLUG_PATTERN rather than hand-duplicating the character +# class (OMN-15759 is the tracked ticket for a cross-repo shared constant; +# until it lands, this is the single in-repo copy the wire-prefix matcher and +# the slug validator both derive from). +_TENANT_WIRE_PREFIX_RE = re.compile(rf"^tenant-({_TENANT_SLUG_PATTERN})\.") def validate_tenant_slug(tenant_slug: str) -> str: @@ -52,3 +70,58 @@ def strip_topic_prefix(tenant_slug: str, wire_topic: str) -> str: raise ValueError("wire_topic does not match attached tenant prefix") canonical_topic = wire_topic[len(prefix) :] return validate_canonical_topic(canonical_topic) + + +def resolve_physical_topic(canonical_topic: str, *, tenant_slug: str | None) -> str: + """Resolve a contract-declared canonical topic to the physical wire topic. + + THE single resolver (OMN-15792) consulted by both the publish path and + the subscribe/dispatch path: contract-declared canonical topic + optional + tenant execution context -> physical topic. + + * ``tenant_slug=None`` -> bare canonical topic (unchanged pass-through + behavior for non-gateway/local-only paths). + * ``tenant_slug`` present -> ``tenant-{slug}.{canonical_topic}``, via + ``prefix_topic`` -- the wire format is not redesigned here. + """ + if tenant_slug is None: + return validate_canonical_topic(canonical_topic) + return prefix_topic(tenant_slug, canonical_topic) + + +def resolve_tenant_from_wire_topic(wire_topic: str) -> tuple[str | None, str]: + """Inverse of ``resolve_physical_topic``: derive ``(tenant_slug, canonical_topic)``. + + THE single resolver's subscribe-side direction (OMN-15792) -- the runtime + dispatch layer calls this instead of re-deriving a tenant prefix with a + private regex. + + Returns ``(None, wire_topic)`` unchanged when ``wire_topic`` does not + start with the ``tenant-`` prefix at all -- never a defaulted or guessed + tenant (Stage-1 warn semantics, matching the OMN-14349 stamp's existing + contract). + + ``tenant-`` is a reserved wire-format prefix: ``validate_canonical_topic`` + already rejects any bare contract-declared topic that starts with it, so + ANY string starting with ``tenant-`` is by construction an attempted + tenant-wire topic, never a coincidentally-named bare canonical topic. + Once that prefix is present, the full ``tenant-.`` shape and the + embedded slug (``validate_tenant_slug`` -- reserved or malformed slugs + included) are both enforced; a shape or slug failure raises rather than + silently falling back to "no tenant". This closes the divergence class + where a malformed-looking ``tenant-`` prefix (wrong case, too short) was + published-side REJECTED but subscribe-side silently passed through + untenanted -- the two directions must agree (both-accept or + both-reject), not just on well-formed-but-reserved slugs. + """ + if not wire_topic.startswith("tenant-"): + return None, wire_topic + match = _TENANT_WIRE_PREFIX_RE.match(wire_topic) + if match is None: + raise ValueError( + "wire_topic starts with the reserved 'tenant-' prefix but does " + f"not match the tenant wire-topic shape: {wire_topic!r}" + ) + slug = validate_tenant_slug(match.group(1)) + canonical_topic = validate_canonical_topic(wire_topic[len(match.group(0)) :]) + return slug, canonical_topic diff --git a/src/omnibase_infra/nodes/node_event_forward_effect/contract.yaml b/src/omnibase_infra/nodes/node_event_forward_effect/contract.yaml index 5b0ffd3155..ee9dd52376 100644 --- a/src/omnibase_infra/nodes/node_event_forward_effect/contract.yaml +++ b/src/omnibase_infra/nodes/node_event_forward_effect/contract.yaml @@ -46,6 +46,8 @@ dependencies: - type: "environment" name: "EVENT_FORWARD_BACKEND_URL" description: "Base URL of the HTTP backend to forward events to" +descriptor: + backend_url: "${env.EVENT_FORWARD_BACKEND_URL}" error_handling: retry_policy: max_retries: 3 diff --git a/src/omnibase_infra/nodes/node_event_forward_effect/contract_descriptor.py b/src/omnibase_infra/nodes/node_event_forward_effect/contract_descriptor.py new file mode 100644 index 0000000000..25b055ec5c --- /dev/null +++ b/src/omnibase_infra/nodes/node_event_forward_effect/contract_descriptor.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Resolve the event-forward backend endpoint from its node contract.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + +from omnibase_infra.runtime.overlay.contract_env_ref import expand_contract_env_refs + +_CONTRACT = Path(__file__).resolve().parent / "contract.yaml" + + +def _load_contract(contract_path: Path) -> dict[str, object]: + # ONEX_EXCLUDE: io_audit - The descriptor is the contract-owned configuration boundary. + with contract_path.open(encoding="utf-8") as contract_file: + raw = yaml.safe_load(contract_file) + if not isinstance(raw, dict): + raise ValueError(f"contract {contract_path} must contain a mapping") + return raw + + +def contract_event_forward_backend_url(contract_path: Path = _CONTRACT) -> str: + """Return the fail-closed event-forward backend URL declared by the contract.""" + descriptor = _load_contract(contract_path).get("descriptor") + if not isinstance(descriptor, dict): + raise ValueError( + f"contract {contract_path} must declare a descriptor mapping with backend_url" + ) + declared = descriptor.get("backend_url") + if not isinstance(declared, str): + raise ValueError( + f"contract {contract_path} must declare a string descriptor.backend_url" + ) + resolved = expand_contract_env_refs(declared).strip() + if not resolved: + raise ValueError( + "descriptor.backend_url resolved empty — configure the event-forward " + "backend through EVENT_FORWARD_BACKEND_URL." + ) + return resolved + + +__all__: list[str] = ["contract_event_forward_backend_url"] diff --git a/src/omnibase_infra/nodes/node_event_forward_effect/handlers/handler_event_forward.py b/src/omnibase_infra/nodes/node_event_forward_effect/handlers/handler_event_forward.py index ade0256e16..d3d08105a0 100644 --- a/src/omnibase_infra/nodes/node_event_forward_effect/handlers/handler_event_forward.py +++ b/src/omnibase_infra/nodes/node_event_forward_effect/handlers/handler_event_forward.py @@ -11,11 +11,13 @@ from __future__ import annotations import logging -import os import httpx from omnibase_infra.enums import EnumHandlerType, EnumHandlerTypeCategory +from omnibase_infra.nodes.node_event_forward_effect.contract_descriptor import ( + contract_event_forward_backend_url, +) from omnibase_infra.nodes.node_event_forward_effect.models.model_event_forward_request import ( ModelEventForwardRequest, ) @@ -68,9 +70,7 @@ async def handle( Returns: ModelEventForwardResult with HTTP status and success flag. """ - backend_url = os.environ.get( # ONEX_EXCLUDE: archive port - "EVENT_FORWARD_BACKEND_URL", "http://localhost:8000" - ) + backend_url = contract_event_forward_backend_url() endpoint_path = _CATEGORY_ENDPOINTS.get( request.category, _CATEGORY_ENDPOINTS["generic"] ) diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/__init__.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/__init__.py new file mode 100644 index 0000000000..4c626bda67 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/__init__.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Gateway attach/session control-plane effect node (OMN-15750, G6). + +Validates a per-tenant Keycloak client-credentials access token, registers a +tenant-bound attach session, and provides heartbeat re-validation (which is +also the revocation path — see ``handlers.handler_gateway_heartbeat``) and +explicit detach. This is the control plane for the productized edge +ingress recommended in ``docs/design/2026-08-08-customer-cloud-connectivity-design.md`` +(candidate A) and ``docs/design/2026-08-08-gateway-node-architecture-lift.md`` (G6). + +Byte-forwarding stays in ``node_bus_forwarder_effect`` — this node owns only +attach/session/revocation, per that assessment's §2 ("what should NOT become +a node"). +""" + +from __future__ import annotations diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/contract.yaml b/src/omnibase_infra/nodes/node_gateway_attach_effect/contract.yaml new file mode 100644 index 0000000000..40e4643306 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/contract.yaml @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 OmniNode Team +# +# ONEX Node Contract — Gateway Attach/Session Control-Plane Effect Node +# +# G6 of the gateway node-architecture lift +# (docs/design/2026-08-08-gateway-node-architecture-lift.md) and the ingress +# node recommended by the connectivity design brief's Open Question 2 +# (docs/design/2026-08-08-customer-cloud-connectivity-design.md, candidate +# A). This node is the control plane only: attach handshake, per-tenant +# Keycloak client-credentials validation, session registration bound to +# tenant_id, heartbeat re-validation (the revocation-detection mechanism), +# and detach. Byte-forwarding stays in node_bus_forwarder_effect per the +# lift assessment's §2 — this node never touches the data-plane sockets. +# +# Related Tickets: +# - OMN-15750: this node +# - OMN-15739: architecture ADOPT ruling (epic OMN-12908) +# - OMN-12911: per-tenant Keycloak client + credential lifecycle (P0B) +# - OMN-15918: hardening follow-ups from CodeRabbit review on #2694 -- +# JWT signature verification (JWKS), heartbeat/detach identity binding, +# atomic session-store transitions, outage-vs-revocation circuit +# breaker. Bumps minor: ModelGatewayDetachRequest gains a required +# access_token field (wire-breaking) and ModelGatewayAttachConfig gains +# keycloak_jwks_ref + circuit_breaker_* fields. +# - OMN-15952: unattended pairing/renewal contract. The renewal cycle a +# persistent/headless runtime must run to survive its own session +# ceiling is now a declared contract term rather than client folklore. +# Bumps minor: ModelGatewayAttachConfig gains renewal_margin_seconds + +# renewal_jitter_seconds, and ModelGatewayAttachResponse gains a +# REQUIRED renewal directive (wire-additive but non-optional -- a +# response without it does not validate). +# +# The mechanism is RE_ATTACH, and that word is load-bearing. expires_at +# is stamped once at attach from min(token exp, max_session_ttl_seconds) +# and no later call moves it; a heartbeat proves liveness and +# non-revocation, it does not buy time. A runtime that wants to keep +# working past expires_at performs a fresh client_credentials grant +# against Keycloak and attaches again, minting a NEW session_id. There +# is no in-place renewal to implement here -- it is refused, because +# extending a live ceiling on the strength of a heartbeat would let a +# credential the IdP has stopped authorizing hold a session open one +# heartbeat at a time. +name: "node_gateway_attach_effect" +contract_name: "node_gateway_attach_effect" +node_name: "node_gateway_attach_effect" +node_type: "EFFECT_GENERIC" +contract_version: + major: 0 + minor: 3 + patch: 0 +node_version: + major: 0 + minor: 3 + patch: 0 +runtime_profiles: + - effects + - canary +description: > + Control-plane effect node for tenant edge attach/session lifecycle. Validates a per-tenant Keycloak client-credentials access token at attach, registers a tenant-bound session, re-validates via RFC 7662 introspection on each heartbeat (the revocation path -- disabling the tenant's Keycloak client is observable within one heartbeat interval), and tears down on explicit detach. Handlers never publish directly; the runtime publishes each handler's typed response onto the contract-declared session-event topic (node_owned_publish, thin-publisher pattern). + +input_model: + name: "ModelGatewayAttachRequest" + module: "omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_request" + description: "Primary (attach) request shape. gateway.heartbeat and gateway.detach use their own typed request models at the handler level, matching node_vector_store_effect's per-operation typing convention." +output_model: + name: "ModelGatewayAttachResponse" + module: "omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_response" + description: "Primary (attach) response shape, embedding the thin-publish session_event payload." +event_bus: + version: + major: 1 + minor: 0 + patch: 0 + subscribe_topics: + - "onex.cmd.omnibase-infra.gateway-attach-request.v1" + - "onex.cmd.omnibase-infra.gateway-heartbeat-request.v1" + - "onex.cmd.omnibase-infra.gateway-detach-request.v1" + publish_topics: + - "onex.evt.omnibase-infra.gateway-session.v1" +config: + name: "ModelGatewayAttachConfig" + module: "omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_config" + description: "Frozen attach-node configuration resolved from contract overlays and secret refs. No broker/broker-adjacent literals here — this node validates identity, it never touches Kafka." + gateway_attach: + keycloak_issuer_ref: "gateway.attach.keycloak.issuer" + keycloak_introspection_ref: "gateway.attach.keycloak.introspection" + keycloak_admin_client_ref: "gateway.attach.keycloak.admin_client_credentials" + keycloak_jwks_ref: "gateway.attach.keycloak.jwks" + required_audience: "gateway-attach" + heartbeat_interval_seconds: 15 + session_degraded_after_seconds: 60 + max_session_ttl_seconds: 3600 + renewal_margin_seconds: 120 + renewal_jitter_seconds: 30 + circuit_breaker_threshold: 5 + circuit_breaker_reset_timeout_seconds: 30.0 +handler_routing: + routing_strategy: "operation_match" + handlers: + - operation: "gateway.attach" + topic: "onex.cmd.omnibase-infra.gateway-attach-request.v1" + handler: + name: "HandlerGatewayAttach" + module: "omnibase_infra.nodes.node_gateway_attach_effect.handlers.handler_gateway_attach" + handler_type: "INFRA" + description: > + Verify a client-credentials access token's signature against Keycloak's JWKS keyset, then validate its claims (issuer, audience, tenant_id/tenant_slug/principal_id), register a new tenant-bound session, return the session plus the ATTACHED session_event and the renewal directive. The directive carries the contract-declared renewal cycle for this session -- mode RE_ATTACH, the jitter window [renew_not_before, renew_at], and the immutable session_expires_at those race -- so an unattended runtime is told how to survive its own ceiling instead of inferring a policy it was never given. + + - operation: "gateway.heartbeat" + topic: "onex.cmd.omnibase-infra.gateway-heartbeat-request.v1" + handler: + name: "HandlerGatewayHeartbeat" + module: "omnibase_infra.nodes.node_gateway_attach_effect.handlers.handler_gateway_heartbeat" + handler_type: "INFRA" + description: > + Verify the presented token's signature and bind it to the stored session's tenant/principal/client identity, then re-validate via Keycloak RFC 7662 introspection. A disabled tenant client makes introspection return active:false on the next call, which tears the session down and emits a REVOKED session_event -- this is the revocation mechanism, not a separate admin call against this node. A Keycloak/JWKS outage raises InfraUnavailableError and leaves the session untouched rather than revoking it. + + - operation: "gateway.detach" + topic: "onex.cmd.omnibase-infra.gateway-detach-request.v1" + handler: + name: "HandlerGatewayDetach" + module: "omnibase_infra.nodes.node_gateway_attach_effect.handlers.handler_gateway_detach" + handler_type: "INFRA" + description: > + Explicit edge-initiated session teardown. Requires a signature-verified access token bound to the stored session's tenant/principal/client identity -- a caller cannot detach a session it does not hold the matching credential for. + +capabilities: + - name: "client_credentials_attach" + description: "Verifies per-tenant Keycloak confidential-client access tokens (client_credentials grant) against the realm's JWKS keyset, then validates claims, at attach time." + - name: "introspection_revocation" + description: "Heartbeat re-validation via RFC 7662 introspection makes Keycloak-side client revocation observable within one heartbeat interval, independent of the token's own exp." + - name: "tenant_bound_session" + description: "Every session carries the immutable tenant_id/principal_id derived from the signature-verified token; handlers never trust caller-declared tenant identity, and heartbeat/detach bind the presented token back to the stored session's identity before acting." + - name: "unattended_renewal_cycle" + description: "Every attach response declares the renewal cycle its client must run (OMN-15952): mode RE_ATTACH, a jitter window opening at renew_not_before and closing at renew_at, and the session ceiling both are strictly before. Renewal is a fresh client_credentials grant plus a fresh attach minting a NEW session_id -- expires_at is stamped once at attach and no path in this node ever moves it, so a persistent/headless runtime never needs a browser session and never gets a lifetime extension from a heartbeat." + - name: "outage_revocation_distinction" + description: "MixinAsyncCircuitBreaker-guarded Keycloak calls (JWKS fetch, introspection) raise InfraUnavailableError on transport failure/non-200/malformed body instead of treating an unreachable IdP as revocation -- a Keycloak outage no longer mass-revokes every active session." + - name: "node_owned_publish" + description: "Handlers never publish; the runtime publishes each handler's typed response (embedding session_event) onto the declared session-event topic." +metadata: + description: "Gateway attach/session control-plane effect node (G6)" + transport_type: "HTTP" + author: "OmniNode Team" + created: "2026-08-08" + updated: "2026-08-14" + tags: + - gateway + - ingress + - attach + - session + - keycloak + - tenant-isolation + known_gaps: + - "Session store is in-process (StoreGatewaySessionMemory) for the first slice — a multi-pod deployment needs a shared backend (Valkey) behind the same ProtocolGatewaySessionStore seam. Tracked as follow-on, not built in OMN-15750." + - "Runtime wiring materializes one shared StoreGatewaySessionMemory, SecretResolver, and ModelGatewayAttachConfig for all three handlers. A future container-owned registration can replace this explicit dependency map when the store moves to a shared Valkey backend." + - "JWKS keys are fetched fresh on every attach/heartbeat/detach call (no in-process cache/TTL). Acceptable for the first slice's call volume; a cache would reduce Keycloak load under sustained heartbeat traffic. Tracked as follow-on, not built here." + - "The renewal contract (OMN-15952) is declared and served here, but the client half is not in this repo: the persistent runtime's own re-grant loop (credential store, issuer pinning, backoff, the failure ladder separating a Keycloak outage from a real revocation) is the consumer's to implement against this directive. This node states the cycle and proves it is served; it cannot prove a given runtime obeys it." + - "Credential delivery for the unattended case -- provisioning the per-tenant ga-* client and revealing its secret at tenant bootstrap -- lives in the onex-api control plane, not here. Until that lands, the renewal cycle this node declares can be exercised with a manually-provisioned client but not obtained end-to-end by a new tenant (OMN-15952 design section 4.1/4.2)." diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/__init__.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/__init__.py new file mode 100644 index 0000000000..e78dfa61de --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/__init__.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Definition-B handlers for node_gateway_attach_effect.""" + +from __future__ import annotations + +from omnibase_infra.nodes.node_gateway_attach_effect.handlers.handler_gateway_attach import ( + HandlerGatewayAttach, +) +from omnibase_infra.nodes.node_gateway_attach_effect.handlers.handler_gateway_detach import ( + HandlerGatewayDetach, +) +from omnibase_infra.nodes.node_gateway_attach_effect.handlers.handler_gateway_heartbeat import ( + HandlerGatewayHeartbeat, +) + +__all__ = [ + "HandlerGatewayAttach", + "HandlerGatewayDetach", + "HandlerGatewayHeartbeat", +] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_attach.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_attach.py new file mode 100644 index 0000000000..87e5a1acad --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_attach.py @@ -0,0 +1,230 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Handler for gateway.attach -- validate token, register session. + +Canonical definition-B EFFECT handler: ``handle(request) -> response``. I/O +(the issuer/JWKS secret-ref resolution, the JWKS fetch, and the session +store write) lives entirely inside this handler and the services it calls, +never in the node's dispatch wiring. + +JWKS fetch (OMN-15918 R1): the token's signature is verified against +Keycloak's real signing keys before any claim is trusted. The fetch itself +is network I/O and stays inline here (imperative-contract-guard's +handlers/-only I/O boundary) wrapped in ``MixinAsyncCircuitBreaker`` so a +Keycloak/JWKS outage fails a single attach attempt (``InfraUnavailableError``, +retry-able) instead of masquerading as a token-validation rejection. +Verification itself (CPU-only, no I/O) lives in +``service_keycloak_token_validator.verify_and_decode_claims``. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from uuid import uuid4 + +import httpx + +from omnibase_infra.enums import ( + EnumHandlerType, + EnumHandlerTypeCategory, + EnumInfraTransportType, +) +from omnibase_infra.errors import InfraUnavailableError, ModelInfraErrorContext +from omnibase_infra.mixins import MixinAsyncCircuitBreaker +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_session_event_type import ( + EnumGatewaySessionEventType, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_session_status import ( + EnumGatewaySessionStatus, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_config import ( + ModelGatewayAttachConfig, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_request import ( + ModelGatewayAttachRequest, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_response import ( + ModelGatewayAttachResponse, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session import ( + ModelGatewaySession, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session_event import ( + ModelGatewaySessionEvent, +) +from omnibase_infra.nodes.node_gateway_attach_effect.services import ( + service_gateway_renewal_policy as renewal_policy, +) +from omnibase_infra.nodes.node_gateway_attach_effect.services import ( + service_keycloak_token_validator as token_validator, +) +from omnibase_infra.nodes.node_gateway_attach_effect.services.protocol_gateway_session_store import ( + ProtocolGatewaySessionStore, +) +from omnibase_infra.runtime.secret_resolver import SecretResolver + +__all__ = ["HandlerGatewayAttach"] + + +class HandlerGatewayAttach(MixinAsyncCircuitBreaker): + """Validate a client-credentials token and register a tenant-bound session.""" + + def __init__( + self, + config: ModelGatewayAttachConfig, + session_store: ProtocolGatewaySessionStore, + secret_resolver: SecretResolver, + ) -> None: + self._config = config + self._session_store = session_store + self._secret_resolver = secret_resolver + self._init_circuit_breaker( + threshold=config.circuit_breaker_threshold, + reset_timeout=config.circuit_breaker_reset_timeout_seconds, + service_name="gateway-attach.keycloak-jwks", + transport_type=EnumInfraTransportType.HTTP, + ) + + @property + def handler_type(self) -> EnumHandlerType: + return EnumHandlerType.INFRA_HANDLER + + @property + def handler_category(self) -> EnumHandlerTypeCategory: + return EnumHandlerTypeCategory.EFFECT + + async def handle( + self, request: ModelGatewayAttachRequest + ) -> ModelGatewayAttachResponse: + issuer_secret = await self._secret_resolver.get_secret_async( + self._config.keycloak_issuer_ref, + required=True, + ) + if issuer_secret is None: + raise token_validator.TokenValidationError( + "Keycloak issuer secret ref resolved to None despite required=True" + ) + expected_issuer = issuer_secret.get_secret_value() + + jwks_keys = await self._fetch_jwks() + claims = token_validator.verify_and_decode_claims( + request.access_token, + jwks_keys, + self._config, + expected_issuer=expected_issuer, + ) + + now = datetime.now(UTC) + token_ttl_seconds = claims.expires_at_epoch - int(now.timestamp()) + if token_ttl_seconds <= 0: + raise token_validator.TokenValidationError("access_token has expired") + session_ttl_seconds = min( + token_ttl_seconds, self._config.max_session_ttl_seconds + ) + expires_at = now + timedelta(seconds=session_ttl_seconds) + + session = ModelGatewaySession( + session_id=uuid4(), + tenant_id=claims.tenant_id, + tenant_slug=claims.tenant_slug, + principal_id=claims.principal_id, + keycloak_client_id=claims.client_id, + edge_instance_id=request.edge_instance_id, + status=EnumGatewaySessionStatus.ACTIVE, + attached_at=now, + last_heartbeat_at=now, + expires_at=expires_at, + ) + await self._session_store.put(session) + + event = ModelGatewaySessionEvent( + event_type=EnumGatewaySessionEventType.ATTACHED, + session_id=session.session_id, + tenant_id=session.tenant_id, + tenant_slug=session.tenant_slug, + principal_id=session.principal_id, + edge_instance_id=session.edge_instance_id, + emitted_at=now, + ) + # OMN-15952: hand the runtime its renewal cycle in the same response + # that gives it the session. The alternative -- publishing the terms + # in documentation and hoping each client implements them -- is what + # left an unattended runtime with no defined behaviour at expiry in + # the first place. Computed from the session just stamped, so the + # directive can never disagree with the ceiling it is racing. + renewal = renewal_policy.build_renewal_directive(session, config=self._config) + + return ModelGatewayAttachResponse( + session=session, + heartbeat_interval_seconds=self._config.heartbeat_interval_seconds, + renewal=renewal, + session_event=event, + ) + + async def _fetch_jwks(self) -> list[dict[str, object]]: + """Fetch the JWKS keyset (RFC 7517). Circuit-breaker guarded. + + Fail-closed but distinguishable: a Keycloak/JWKS outage raises + ``InfraUnavailableError`` (retry-able, never treated as a rejected + token) rather than silently falling back to unsigned trust. + """ + jwks_url_secret = await self._secret_resolver.get_secret_async( + self._config.keycloak_jwks_ref, + required=True, + ) + if jwks_url_secret is None: + raise token_validator.TokenValidationError( + "Keycloak JWKS secret ref resolved to None despite required=True" + ) + jwks_url = jwks_url_secret.get_secret_value() + + async with self._circuit_breaker_lock: + await self._check_circuit_breaker(operation="fetch_jwks") + + try: + async with httpx.AsyncClient(timeout=5.0) as client: + response = await client.get(jwks_url) + except httpx.HTTPError as exc: + async with self._circuit_breaker_lock: + await self._record_circuit_failure(operation="fetch_jwks") + raise InfraUnavailableError( + "Keycloak JWKS endpoint unreachable", + context=ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation="fetch_jwks", + ), + ) from exc + + if response.status_code != 200: + async with self._circuit_breaker_lock: + await self._record_circuit_failure(operation="fetch_jwks") + raise InfraUnavailableError( + f"Keycloak JWKS endpoint returned HTTP {response.status_code}", + context=ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation="fetch_jwks", + ), + ) + + try: + body = response.json() + except ValueError as exc: + async with self._circuit_breaker_lock: + await self._record_circuit_failure(operation="fetch_jwks") + raise InfraUnavailableError( + "Keycloak JWKS response was not valid JSON", + context=ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation="fetch_jwks", + ), + ) from exc + + async with self._circuit_breaker_lock: + await self._reset_circuit_breaker() + + keys = body.get("keys") if isinstance(body, dict) else None + if not isinstance(keys, list) or not keys: + raise token_validator.TokenValidationError( + "Keycloak JWKS response contained no keys" + ) + return keys diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_detach.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_detach.py new file mode 100644 index 0000000000..92463a6889 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_detach.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Handler for gateway.detach -- explicit, edge-initiated teardown. + +OMN-15918 R1 + R2: detach previously took only a session identifier and a +free-text reason -- zero credential, zero identity check, so any caller +holding a session identifier could detach any tenant's session. This +handler now requires the same signature-verified access token as attach and +heartbeat, and binds it to the STORED session's tenant/principal/client +identity before deleting. The JWKS fetch is the only I/O this handler +performs and stays inline here (imperative-contract-guard's handlers/-only +I/O boundary), mirroring ``HandlerGatewayAttach._fetch_jwks`` / +``HandlerGatewayHeartbeat._fetch_jwks``. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import httpx + +from omnibase_infra.enums import ( + EnumHandlerType, + EnumHandlerTypeCategory, + EnumInfraTransportType, +) +from omnibase_infra.errors import InfraUnavailableError, ModelInfraErrorContext +from omnibase_infra.mixins import MixinAsyncCircuitBreaker +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_session_event_type import ( + EnumGatewaySessionEventType, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_session_status import ( + EnumGatewaySessionStatus, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_config import ( + ModelGatewayAttachConfig, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_detach_request import ( + ModelGatewayDetachRequest, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_detach_response import ( + ModelGatewayDetachResponse, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session_event import ( + ModelGatewaySessionEvent, +) +from omnibase_infra.nodes.node_gateway_attach_effect.services import ( + service_keycloak_token_validator as token_validator, +) +from omnibase_infra.nodes.node_gateway_attach_effect.services.protocol_gateway_session_store import ( + ProtocolGatewaySessionStore, +) +from omnibase_infra.runtime.secret_resolver import SecretResolver + +__all__ = ["HandlerGatewayDetach"] + + +class SessionNotFoundError(Exception): + """Raised when detach targets an unknown or already-torn-down session.""" + + +class HandlerGatewayDetach(MixinAsyncCircuitBreaker): + """Tear down a session on explicit edge-initiated detach.""" + + def __init__( + self, + config: ModelGatewayAttachConfig, + session_store: ProtocolGatewaySessionStore, + secret_resolver: SecretResolver, + ) -> None: + self._config = config + self._session_store = session_store + self._secret_resolver = secret_resolver + self._init_circuit_breaker( + threshold=config.circuit_breaker_threshold, + reset_timeout=config.circuit_breaker_reset_timeout_seconds, + service_name="gateway-detach.keycloak-jwks", + transport_type=EnumInfraTransportType.HTTP, + ) + + @property + def handler_type(self) -> EnumHandlerType: + return EnumHandlerType.INFRA_HANDLER + + @property + def handler_category(self) -> EnumHandlerTypeCategory: + return EnumHandlerTypeCategory.EFFECT + + async def handle( + self, request: ModelGatewayDetachRequest + ) -> ModelGatewayDetachResponse: + session = await self._session_store.get(request.session_id) + if session is None: + raise SessionNotFoundError(f"no session {request.session_id}") + + issuer_secret = await self._secret_resolver.get_secret_async( + self._config.keycloak_issuer_ref, required=True + ) + if issuer_secret is None: + raise token_validator.TokenValidationError( + "Keycloak issuer secret ref resolved to None despite required=True" + ) + jwks_keys = await self._fetch_jwks() + claims = token_validator.verify_and_decode_claims( + request.access_token, + jwks_keys, + self._config, + expected_issuer=issuer_secret.get_secret_value(), + ) + if ( + claims.tenant_id != session.tenant_id + or claims.principal_id != session.principal_id + or claims.client_id != session.keycloak_client_id + ): + raise token_validator.TokenValidationError( + "access_token identity does not match the stored session " + "(tenant/principal/client binding mismatch)" + ) + + await self._session_store.delete(request.session_id) + now = datetime.now(UTC) + event = ModelGatewaySessionEvent( + event_type=EnumGatewaySessionEventType.DETACHED, + session_id=session.session_id, + tenant_id=session.tenant_id, + tenant_slug=session.tenant_slug, + principal_id=session.principal_id, + edge_instance_id=session.edge_instance_id, + emitted_at=now, + ) + return ModelGatewayDetachResponse( + session_id=request.session_id, + status=EnumGatewaySessionStatus.DETACHED, + session_event=event, + ) + + async def _fetch_jwks(self) -> list[dict[str, object]]: + """Fetch the JWKS keyset (RFC 7517). Circuit-breaker guarded. + + Mirrors ``HandlerGatewayAttach._fetch_jwks`` -- see that docstring + for the fail-closed-but-distinguishable rationale. + """ + jwks_url_secret = await self._secret_resolver.get_secret_async( + self._config.keycloak_jwks_ref, required=True + ) + if jwks_url_secret is None: + raise token_validator.TokenValidationError( + "Keycloak JWKS secret ref resolved to None despite required=True" + ) + jwks_url = jwks_url_secret.get_secret_value() + + async with self._circuit_breaker_lock: + await self._check_circuit_breaker(operation="fetch_jwks") + + try: + async with httpx.AsyncClient(timeout=5.0) as client: + response = await client.get(jwks_url) + except httpx.HTTPError as exc: + async with self._circuit_breaker_lock: + await self._record_circuit_failure(operation="fetch_jwks") + raise InfraUnavailableError( + "Keycloak JWKS endpoint unreachable", + context=ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation="fetch_jwks", + ), + ) from exc + + if response.status_code != 200: + async with self._circuit_breaker_lock: + await self._record_circuit_failure(operation="fetch_jwks") + raise InfraUnavailableError( + f"Keycloak JWKS endpoint returned HTTP {response.status_code}", + context=ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation="fetch_jwks", + ), + ) + + try: + body = response.json() + except ValueError as exc: + async with self._circuit_breaker_lock: + await self._record_circuit_failure(operation="fetch_jwks") + raise InfraUnavailableError( + "Keycloak JWKS response was not valid JSON", + context=ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation="fetch_jwks", + ), + ) from exc + + async with self._circuit_breaker_lock: + await self._reset_circuit_breaker() + + keys = body.get("keys") if isinstance(body, dict) else None + if not isinstance(keys, list) or not keys: + raise token_validator.TokenValidationError( + "Keycloak JWKS response contained no keys" + ) + return keys diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_heartbeat.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_heartbeat.py new file mode 100644 index 0000000000..b077f02676 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/handlers/handler_gateway_heartbeat.py @@ -0,0 +1,413 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Handler for gateway.heartbeat -- re-validate via Keycloak introspection. + +This is the revocation-detection path: disabling the tenant's Keycloak +client makes ``_introspect`` (RFC 7662) return ``active: false`` on the next +heartbeat, which flips the session to REVOKED and deletes it from the store +-- independent of the presented token's own unexpired ``exp`` claim. + +The introspection HTTP call (and the JWKS fetch, OMN-15918 R1) are inline in +this module (not in a freestanding ``services/`` helper): they are the only +I/O this node performs, and the imperative-contract-guard requires +raw-transport calls to live under ``handlers/``, never in a freestanding +module the guard cannot attribute to a contract-declared handler. + +OMN-15918 hardening in this handler: + - R1: the presented heartbeat token's signature is verified against the + JWKS keyset (same as attach) before its claims are trusted for identity + binding below. + - R2: bind heartbeat identity to the STORED session (tenant_id, + principal_id, keycloak_client_id) from attach time, not the caller's + claims alone -- a token that decodes clean but names a *different* + tenant/principal/client than the session it is heartbeating for is + rejected before introspection ever runs. + - R3 (atomicity): the refreshed session is written with + ``put_if_present`` rather than an unconditional ``put`` -- a concurrent + detach that removed the session during the introspection await window + must not be resurrected by this handler's write. + - R4: introspection failures split into two classes. A genuine Keycloak + ``active: false`` (or client_id mismatch) is real revocation and tears + the session down as before. A transport error, non-200, malformed body, + or open circuit breaker is an *outage* -- ``InfraUnavailableError`` is + raised and the session is left untouched (retry-able), never treated as + revoked. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from uuid import UUID + +import httpx + +from omnibase_infra.enums import ( + EnumHandlerType, + EnumHandlerTypeCategory, + EnumInfraTransportType, +) +from omnibase_infra.errors import InfraUnavailableError, ModelInfraErrorContext +from omnibase_infra.mixins import MixinAsyncCircuitBreaker +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_session_event_type import ( + EnumGatewaySessionEventType, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_session_status import ( + EnumGatewaySessionStatus, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_config import ( + ModelGatewayAttachConfig, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_heartbeat_request import ( + ModelGatewayHeartbeatRequest, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_heartbeat_response import ( + ModelGatewayHeartbeatResponse, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session_event import ( + ModelGatewaySessionEvent, +) +from omnibase_infra.nodes.node_gateway_attach_effect.services import ( + service_keycloak_token_validator as token_validator, +) +from omnibase_infra.nodes.node_gateway_attach_effect.services.protocol_gateway_session_store import ( + ProtocolGatewaySessionStore, +) +from omnibase_infra.runtime.secret_resolver import SecretResolver + +__all__ = ["HandlerGatewayHeartbeat"] + + +class SessionNotFoundError(Exception): + """Raised when a heartbeat targets an unknown or already-torn-down session.""" + + +class GatewayHeartbeatCircuitBreakerGuard(MixinAsyncCircuitBreaker): + """Composition wrapper: lets one handler hold >1 independent circuit breaker. + + ``MixinAsyncCircuitBreaker`` keeps its state on ``self`` (one breaker per + instance). This handler talks to two independently-failing Keycloak + surfaces (the public JWKS endpoint and the admin-credentialed + introspection endpoint), so each gets its own guard instance rather than + sharing one breaker's failure count across unrelated calls. + """ + + def __init__( + self, *, threshold: int, reset_timeout: float, service_name: str + ) -> None: + self._init_circuit_breaker( + threshold=threshold, + reset_timeout=reset_timeout, + service_name=service_name, + transport_type=EnumInfraTransportType.HTTP, + ) + + +class HandlerGatewayHeartbeat: + """Re-validate a session's token via Keycloak introspection.""" + + def __init__( + self, + config: ModelGatewayAttachConfig, + session_store: ProtocolGatewaySessionStore, + secret_resolver: SecretResolver, + ) -> None: + self._config = config + self._session_store = session_store + self._secret_resolver = secret_resolver + self._jwks_circuit = GatewayHeartbeatCircuitBreakerGuard( + threshold=config.circuit_breaker_threshold, + reset_timeout=config.circuit_breaker_reset_timeout_seconds, + service_name="gateway-heartbeat.keycloak-jwks", + ) + self._introspection_circuit = GatewayHeartbeatCircuitBreakerGuard( + threshold=config.circuit_breaker_threshold, + reset_timeout=config.circuit_breaker_reset_timeout_seconds, + service_name="gateway-heartbeat.keycloak-introspection", + ) + + @property + def handler_type(self) -> EnumHandlerType: + return EnumHandlerType.INFRA_HANDLER + + @property + def handler_category(self) -> EnumHandlerTypeCategory: + return EnumHandlerTypeCategory.EFFECT + + async def handle( + self, request: ModelGatewayHeartbeatRequest + ) -> ModelGatewayHeartbeatResponse: + session = await self._session_store.get(request.session_id) + if session is None: + raise SessionNotFoundError(f"no session {request.session_id}") + + # R1 + R2: verify the presented token's signature, then bind it to + # the STORED session identity -- never trust caller-supplied claims + # in isolation. A token that verifies clean but names a different + # tenant/principal/client than the session it targets is rejected + # before introspection (or any store mutation) runs. + issuer_secret = await self._secret_resolver.get_secret_async( + self._config.keycloak_issuer_ref, required=True + ) + if issuer_secret is None: + raise token_validator.TokenValidationError( + "Keycloak issuer secret ref resolved to None despite required=True" + ) + jwks_keys = await self._fetch_jwks() + claims = token_validator.verify_and_decode_claims( + request.access_token, + jwks_keys, + self._config, + expected_issuer=issuer_secret.get_secret_value(), + ) + if ( + claims.tenant_id != session.tenant_id + or claims.principal_id != session.principal_id + or claims.client_id != session.keycloak_client_id + ): + raise token_validator.TokenValidationError( + "access_token identity does not match the stored session " + "(tenant/principal/client binding mismatch)" + ) + + now = datetime.now(UTC) + is_active = await self._introspect( + access_token=request.access_token, + client_id=session.keycloak_client_id, + correlation_id=session.session_id, + ) + + if not is_active: + await self._session_store.delete(session.session_id) + revoked_session = session.model_copy( + update={"status": EnumGatewaySessionStatus.REVOKED} + ) + event = ModelGatewaySessionEvent( + event_type=EnumGatewaySessionEventType.REVOKED, + session_id=session.session_id, + tenant_id=session.tenant_id, + tenant_slug=session.tenant_slug, + principal_id=session.principal_id, + edge_instance_id=session.edge_instance_id, + emitted_at=now, + ) + return ModelGatewayHeartbeatResponse( + session=revoked_session, revoked=True, session_event=event + ) + + elapsed = (now - session.last_heartbeat_at).total_seconds() + status = ( + EnumGatewaySessionStatus.DEGRADED + if elapsed > self._config.session_degraded_after_seconds + else EnumGatewaySessionStatus.ACTIVE + ) + updated_session = session.model_copy( + update={"status": status, "last_heartbeat_at": now} + ) + # R3: put_if_present, not put -- if a concurrent detach removed this + # session during the introspection await above, this write must not + # resurrect it. + still_present = await self._session_store.put_if_present(updated_session) + if not still_present: + raise SessionNotFoundError( + f"session {session.session_id} was detached during heartbeat " + "revalidation" + ) + + event_type = ( + EnumGatewaySessionEventType.HEARTBEAT_DEGRADED + if status is EnumGatewaySessionStatus.DEGRADED + else EnumGatewaySessionEventType.HEARTBEAT_OK + ) + event = ModelGatewaySessionEvent( + event_type=event_type, + session_id=updated_session.session_id, + tenant_id=updated_session.tenant_id, + tenant_slug=updated_session.tenant_slug, + principal_id=updated_session.principal_id, + edge_instance_id=updated_session.edge_instance_id, + emitted_at=now, + ) + return ModelGatewayHeartbeatResponse( + session=updated_session, revoked=False, session_event=event + ) + + async def _fetch_jwks(self) -> list[dict[str, object]]: + """Fetch the JWKS keyset (RFC 7517). Circuit-breaker guarded. + + Mirrors ``HandlerGatewayAttach._fetch_jwks`` -- see that docstring + for the fail-closed-but-distinguishable rationale. + """ + jwks_url_secret = await self._secret_resolver.get_secret_async( + self._config.keycloak_jwks_ref, required=True + ) + if jwks_url_secret is None: + raise token_validator.TokenValidationError( + "Keycloak JWKS secret ref resolved to None despite required=True" + ) + jwks_url = jwks_url_secret.get_secret_value() + + async with self._jwks_circuit._circuit_breaker_lock: + await self._jwks_circuit._check_circuit_breaker(operation="fetch_jwks") + + try: + async with httpx.AsyncClient(timeout=5.0) as client: + response = await client.get(jwks_url) + except httpx.HTTPError as exc: + async with self._jwks_circuit._circuit_breaker_lock: + await self._jwks_circuit._record_circuit_failure(operation="fetch_jwks") + raise InfraUnavailableError( + "Keycloak JWKS endpoint unreachable", + context=ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation="fetch_jwks", + ), + ) from exc + + if response.status_code != 200: + async with self._jwks_circuit._circuit_breaker_lock: + await self._jwks_circuit._record_circuit_failure(operation="fetch_jwks") + raise InfraUnavailableError( + f"Keycloak JWKS endpoint returned HTTP {response.status_code}", + context=ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation="fetch_jwks", + ), + ) + + try: + body = response.json() + except ValueError as exc: + async with self._jwks_circuit._circuit_breaker_lock: + await self._jwks_circuit._record_circuit_failure(operation="fetch_jwks") + raise InfraUnavailableError( + "Keycloak JWKS response was not valid JSON", + context=ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation="fetch_jwks", + ), + ) from exc + + async with self._jwks_circuit._circuit_breaker_lock: + await self._jwks_circuit._reset_circuit_breaker() + + keys = body.get("keys") if isinstance(body, dict) else None + if not isinstance(keys, list) or not keys: + raise token_validator.TokenValidationError( + "Keycloak JWKS response contained no keys" + ) + return keys + + async def _introspect( + self, + *, + access_token: str, + client_id: str, + correlation_id: UUID | None = None, + ) -> bool: + """RFC 7662 token introspection. Returns True iff Keycloak reports ``active``. + + Two failure classes, deliberately not conflated: + - Genuine revocation (a clean HTTP 200 whose body says + ``active: false``, or whose ``client_id`` does not match the + session's) returns ``False`` -- the caller treats this as real + revocation and tears the session down. + - An outage (circuit open, transport error, non-200, or malformed + body) raises ``InfraUnavailableError`` -- the caller must NOT + treat this as revocation. A Keycloak blip must never mass-revoke + every active session on its next heartbeat. + """ + introspection_url_secret = await self._secret_resolver.get_secret_async( + self._config.keycloak_introspection_ref, + required=True, + correlation_id=correlation_id, + ) + admin_client_id_secret = await self._secret_resolver.get_secret_async( + f"{self._config.keycloak_admin_client_ref}.client_id", + required=True, + correlation_id=correlation_id, + ) + admin_client_secret_secret = await self._secret_resolver.get_secret_async( + f"{self._config.keycloak_admin_client_ref}.client_secret", + required=True, + correlation_id=correlation_id, + ) + # required=True guarantees non-None (SecretResolver raises otherwise); + # the return type stays Optional to serve required=False callers + # elsewhere. + if ( + introspection_url_secret is None + or admin_client_id_secret is None + or admin_client_secret_secret is None + ): + raise token_validator.TokenValidationError( + "Keycloak introspection secret refs resolved to None despite required=True" + ) + introspection_url = introspection_url_secret.get_secret_value() + admin_client_id = admin_client_id_secret.get_secret_value() + admin_client_secret = admin_client_secret_secret.get_secret_value() + + error_context = ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation="introspect", + correlation_id=correlation_id, + ) + + async with self._introspection_circuit._circuit_breaker_lock: + await self._introspection_circuit._check_circuit_breaker( + operation="introspect", correlation_id=correlation_id + ) + + try: + async with httpx.AsyncClient(timeout=5.0) as client: + response = await client.post( + introspection_url, + data={ + "token": access_token, + "token_type_hint": "access_token", + "client_id": admin_client_id, + "client_secret": admin_client_secret, + }, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + except httpx.HTTPError as exc: + async with self._introspection_circuit._circuit_breaker_lock: + await self._introspection_circuit._record_circuit_failure( + operation="introspect", correlation_id=correlation_id + ) + raise InfraUnavailableError( + "Keycloak introspection endpoint unreachable", + context=error_context, + ) from exc + + if response.status_code != 200: + async with self._introspection_circuit._circuit_breaker_lock: + await self._introspection_circuit._record_circuit_failure( + operation="introspect", correlation_id=correlation_id + ) + raise InfraUnavailableError( + f"Keycloak introspection endpoint returned HTTP {response.status_code}", + context=error_context, + ) + + try: + body = response.json() + except ValueError as exc: + async with self._introspection_circuit._circuit_breaker_lock: + await self._introspection_circuit._record_circuit_failure( + operation="introspect", correlation_id=correlation_id + ) + raise InfraUnavailableError( + "Keycloak introspection response was not valid JSON", + context=error_context, + ) from exc + + async with self._introspection_circuit._circuit_breaker_lock: + await self._introspection_circuit._reset_circuit_breaker() + + active = body.get("active") + if active is not True: + return False + # Defense in depth: introspection must confirm the same client_id the + # session was attached with. A token re-issued for a *different* + # tenant client must never validate a stale session's heartbeat. + return str(body.get("client_id", "")) == client_id diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/__init__.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/__init__.py new file mode 100644 index 0000000000..f00dac4d3b --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/__init__.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed models for node_gateway_attach_effect.""" + +from __future__ import annotations + +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_renewal_mode import ( + EnumGatewayRenewalMode, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_session_event_type import ( + EnumGatewaySessionEventType, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_session_status import ( + EnumGatewaySessionStatus, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_config import ( + ModelGatewayAttachConfig, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_request import ( + ModelGatewayAttachRequest, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_response import ( + ModelGatewayAttachResponse, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_detach_request import ( + ModelGatewayDetachRequest, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_detach_response import ( + ModelGatewayDetachResponse, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_heartbeat_request import ( + ModelGatewayHeartbeatRequest, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_heartbeat_response import ( + ModelGatewayHeartbeatResponse, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_renewal_directive import ( + ModelGatewayRenewalDirective, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session import ( + ModelGatewaySession, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session_event import ( + ModelGatewaySessionEvent, +) + +__all__ = [ + "EnumGatewayRenewalMode", + "EnumGatewaySessionEventType", + "EnumGatewaySessionStatus", + "ModelGatewayAttachConfig", + "ModelGatewayAttachRequest", + "ModelGatewayAttachResponse", + "ModelGatewayDetachRequest", + "ModelGatewayDetachResponse", + "ModelGatewayHeartbeatRequest", + "ModelGatewayHeartbeatResponse", + "ModelGatewayRenewalDirective", + "ModelGatewaySession", + "ModelGatewaySessionEvent", +] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/enum_gateway_renewal_mode.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/enum_gateway_renewal_mode.py new file mode 100644 index 0000000000..1a1c90ad0b --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/enum_gateway_renewal_mode.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""How an unattended runtime keeps working across attach-token expiry. + +One member today, and the enum exists precisely so that the one member is +*named on the wire* rather than assumed. OMN-15952's design was revised +three times on this exact point, and the final answer is the +counter-intuitive one: + + ``RE_ATTACH`` -- the session's ``expires_at`` is stamped once, at attach, + from ``min(token exp, max_session_ttl_seconds)``. Nothing ever moves it. + A heartbeat proves the session is still alive and still backed by a + non-revoked credential; it does not, and must not, buy the session more + time. So a runtime that wants to keep working past ``expires_at`` performs + a fresh ``client_credentials`` grant against Keycloak and then a fresh + ``gateway.attach`` -- minting a NEW ``session_id``. Continuity across the + boundary is the runtime's property (and the correlation trail's), never + the session record's. + +There is deliberately no ``RENEW_IN_PLACE`` member. It is not unimplemented; +it is refused. Extending a live session's ceiling on the strength of a +heartbeat would let a credential that Keycloak has already stopped +authorizing keep a session alive indefinitely, one heartbeat at a time -- +which is the exact property the fixed ``expires_at`` exists to deny. If a +future lane believes it needs in-place renewal, that is a contract change +with its own security review, not a new enum member. +""" + +from __future__ import annotations + +from enum import Enum + + +class EnumGatewayRenewalMode(str, Enum): + """The renewal mechanism this node's contract declares to its clients.""" + + RE_ATTACH = "RE_ATTACH" + + +__all__ = ["EnumGatewayRenewalMode"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/enum_gateway_session_event_type.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/enum_gateway_session_event_type.py new file mode 100644 index 0000000000..7c3c67c77d --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/enum_gateway_session_event_type.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Session lifecycle event types.""" + +from __future__ import annotations + +from enum import Enum + + +class EnumGatewaySessionEventType(str, Enum): + ATTACHED = "ATTACHED" + HEARTBEAT_OK = "HEARTBEAT_OK" + HEARTBEAT_DEGRADED = "HEARTBEAT_DEGRADED" + REVOKED = "REVOKED" + DETACHED = "DETACHED" + + +__all__ = ["EnumGatewaySessionEventType"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/enum_gateway_session_status.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/enum_gateway_session_status.py new file mode 100644 index 0000000000..3c20b99413 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/enum_gateway_session_status.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Attach session lifecycle states.""" + +from __future__ import annotations + +from enum import Enum + + +class EnumGatewaySessionStatus(str, Enum): + """Lifecycle states of one attach session.""" + + ACTIVE = "ACTIVE" + DEGRADED = "DEGRADED" + DETACHED = "DETACHED" + REVOKED = "REVOKED" + + +__all__ = ["EnumGatewaySessionStatus"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_attach_config.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_attach_config.py new file mode 100644 index 0000000000..0f8b2e3907 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_attach_config.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Frozen configuration for the gateway attach effect node. + +All values resolve from ``contract.yaml`` and contract overlays; secret +material (Keycloak admin credentials) is referenced by name here and +resolved from Infisical at the effect boundary inside +``HandlerGatewayHeartbeat._introspect`` -- never read from a bare env var and +never embedded in this model (operator ruling 2026-08-08: config lives in +contract overlays + Infisical, env for bootstrap only). +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelGatewayAttachConfig(BaseModel): + """Frozen gateway attach node configuration.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + # Keycloak realm issuer used to validate the access token's ``iss`` claim + # and to derive the JWKS endpoint. Resolved indirectly: this is a + # contract ref name, not the literal URL (feedback_all_urls_from_contracts). + keycloak_issuer_ref: str = Field(default="gateway.attach.keycloak.issuer") + # Token introspection endpoint ref (RFC 7662). Introspection, not local + # exp-only validation, is what makes revocation-within-TTL observable: + # a disabled Keycloak client makes the introspection response + # ``active: false`` immediately, independent of the token's own exp. + keycloak_introspection_ref: str = Field( + default="gateway.attach.keycloak.introspection" + ) + keycloak_admin_client_ref: str = Field( + default="gateway.attach.keycloak.admin_client_credentials" + ) + # JWKS endpoint ref (RFC 7517). Fetched at attach time (and re-checked at + # heartbeat time to re-bind identity) so the token's signature is + # verified against Keycloak's real signing keys before any claim is + # trusted -- OMN-15918 R1: attach-time decode previously trusted claims + # from a structurally-valid-but-unsigned/forged token. + keycloak_jwks_ref: str = Field(default="gateway.attach.keycloak.jwks") + # Audience every attach token must carry. + required_audience: str = Field(default="gateway-attach") + # Session lifecycle. Heartbeat interval mirrors the forwarder's + # liveness.heartbeat_interval_seconds (node_bus_forwarder_effect + # contract.yaml) so link-health projections can share one cadence. + heartbeat_interval_seconds: int = Field(default=15, gt=0) + # A session is considered DEGRADED once this many seconds pass with no + # successful heartbeat re-validation. + session_degraded_after_seconds: int = Field(default=60, gt=0) + # Hard ceiling on session lifetime regardless of token exp -- bounds the + # blast radius of a token whose exp claim is misconfigured too far out. + max_session_ttl_seconds: int = Field(default=3600, gt=0) + # OMN-15952 renewal cycle -- the contract-declared terms an unattended + # runtime must obey to survive its own session ceiling. These are + # CLIENT-facing policy (handed back at attach in + # ModelGatewayRenewalDirective), not server-side bounds: nothing on this + # node tears a session down because of them. + # + # How early re-grant + re-attach must be COMPLETE, ahead of the + # session's expires_at. 120s against a 900s attach token leaves ~87% of + # the token's life before renewal starts, while still covering the three + # things that have to fit inside the margin: worst-case clock skew + # between the runtime, Keycloak and this node; the round trip of the + # token grant plus the attach call plus this node's own JWKS + # verification; and at least one backoff-retry of a transient failure. + # A margin sized only to the happy-path round trip is the classic + # expiry-boundary defect -- the token is valid when the request is sent + # and expired when it is validated. + renewal_margin_seconds: int = Field(default=120, gt=0) + # Width of the decorrelation window that opens before renewal_margin. + # A fleet provisioned in one bootstrap batch shares an attach instant, + # so without jitter it also shares a renewal instant and stampedes + # Keycloak's token endpoint every cycle, forever -- the synchronization + # is self-sustaining because a batch that renews together stays + # together. Each runtime picks its own moment uniformly in + # [renew_not_before, renew_at]. Zero is permitted (ge=0) so a + # single-runtime deployment can opt out of spreading it does not need, + # which is why this is not gt=0 like the margin. + renewal_jitter_seconds: int = Field(default=30, ge=0) + # Circuit breaker (MixinAsyncCircuitBreaker) thresholds shared by the + # JWKS fetch (attach + heartbeat) and RFC 7662 introspection (heartbeat) + # HTTP calls to Keycloak -- OMN-15918 R4: distinguishes "Keycloak + # unreachable" (raise InfraUnavailableError, session left untouched) + # from "Keycloak said inactive" (real revocation). + circuit_breaker_threshold: int = Field(default=5, gt=0) + circuit_breaker_reset_timeout_seconds: float = Field(default=30.0, gt=0) + + +__all__ = ["ModelGatewayAttachConfig"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_attach_request.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_attach_request.py new file mode 100644 index 0000000000..64f6aa8d6e --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_attach_request.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Attach request -- the edge dials in with a client-credentials token.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelGatewayAttachRequest(BaseModel): + """Input to ``gateway.attach``: one edge presenting a bearer token.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + # Raw access token obtained by the edge via the Keycloak client_credentials + # grant against its per-tenant confidential client. Never logged verbatim + # by any handler in this node. + access_token: str = Field(min_length=1) + # Caller-declared edge instance identity (host label), used only for + # session bookkeeping/observability -- never trusted for authorization, + # which is derived entirely from the validated token claims. + edge_instance_id: str = Field(min_length=1, max_length=255) + + +__all__ = ["ModelGatewayAttachRequest"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_attach_response.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_attach_response.py new file mode 100644 index 0000000000..cf42129340 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_attach_response.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Attach response.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_renewal_directive import ( + ModelGatewayRenewalDirective, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session import ( + ModelGatewaySession, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session_event import ( + ModelGatewaySessionEvent, +) + + +class ModelGatewayAttachResponse(BaseModel): + """Output of ``gateway.attach``. + + ``session_event`` is the thin-publish payload: the runtime publishes this + node's output onto the contract-declared session-event topic, so the + handler never calls the bus directly (node_owned_publish). + + ``renewal`` is the OMN-15952 addition and is REQUIRED, not optional. An + unattended runtime that attaches and is not told the renewal cycle has + no correct behaviour available to it -- it will either heartbeat into + its own expiry or invent a policy. Making the field optional would let + exactly that case ship silently, so a response without it does not + validate. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + session: ModelGatewaySession + heartbeat_interval_seconds: int + renewal: ModelGatewayRenewalDirective + session_event: ModelGatewaySessionEvent + + +__all__ = ["ModelGatewayAttachResponse"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_detach_request.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_detach_request.py new file mode 100644 index 0000000000..5c85ad1cac --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_detach_request.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Detach request -- explicit, edge-initiated session teardown.""" + +from __future__ import annotations + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelGatewayDetachRequest(BaseModel): + """Input to ``gateway.detach``. + + OMN-15918 R2: ``access_token`` is required so + ``HandlerGatewayDetach.handle`` can bind the caller to the STORED + session's tenant/principal/client identity before deleting -- the + previous shape (``session_id`` + free-text ``reason``, no credential) + let any caller holding a session identifier detach any tenant's session. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + session_id: UUID + access_token: str = Field(min_length=1) + reason: str = Field(min_length=1, max_length=500) + + +__all__ = ["ModelGatewayDetachRequest"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_detach_response.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_detach_response.py new file mode 100644 index 0000000000..23b058ffa2 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_detach_response.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Detach response.""" + +from __future__ import annotations + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_session_status import ( + EnumGatewaySessionStatus, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session_event import ( + ModelGatewaySessionEvent, +) + + +class ModelGatewayDetachResponse(BaseModel): + """Output of ``gateway.detach``.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + session_id: UUID + status: EnumGatewaySessionStatus + session_event: ModelGatewaySessionEvent + + +__all__ = ["ModelGatewayDetachResponse"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_heartbeat_request.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_heartbeat_request.py new file mode 100644 index 0000000000..d0c33570cc --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_heartbeat_request.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Heartbeat request -- also the per-tick revocation re-check.""" + +from __future__ import annotations + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelGatewayHeartbeatRequest(BaseModel): + """Input to ``gateway.heartbeat``. + + Carries a fresh access token (client-credentials tokens are short-lived; + the edge re-mints one per heartbeat cadence) so each heartbeat performs a + real Keycloak introspection call -- this is the mechanism that makes + revocation observable within one heartbeat interval rather than only at + the stale token's original exp. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + session_id: UUID + access_token: str = Field(min_length=1) + + +__all__ = ["ModelGatewayHeartbeatRequest"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_heartbeat_response.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_heartbeat_response.py new file mode 100644 index 0000000000..07a44b5313 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_heartbeat_response.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Heartbeat response.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session import ( + ModelGatewaySession, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session_event import ( + ModelGatewaySessionEvent, +) + + +class ModelGatewayHeartbeatResponse(BaseModel): + """Output of ``gateway.heartbeat``. + + ``revoked`` is True exactly when Keycloak introspection returned + ``active: false`` for the presented token -- the session is torn down + (status flips to REVOKED) in the same handler call, so a caller never + observes a stale ACTIVE session after this response. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + session: ModelGatewaySession + revoked: bool + session_event: ModelGatewaySessionEvent + + +__all__ = ["ModelGatewayHeartbeatResponse"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_renewal_directive.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_renewal_directive.py new file mode 100644 index 0000000000..0aa8a3e584 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_renewal_directive.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""The server-computed renewal cycle handed to the client at attach. + +OMN-15952. Before this model, ``gateway.attach`` told a runtime how often to +heartbeat and nothing else. The runtime could read ``expires_at`` off the +session it got back, but every other term of the renewal cycle -- how early +to start, how much to spread, and above all *what renewal even is* -- was +undeclared. An unattended runtime cannot infer a policy it was never told, +and each independent client that guessed would guess differently. + +This directive makes the cycle a contract term instead of client folklore: + + * ``mode`` names the mechanism (``RE_ATTACH``: re-grant, then attach again + for a NEW ``session_id``). It is on the wire so no client has to assume + that a heartbeat extends anything -- it does not. + * ``renew_at`` is the deadline. Re-grant and re-attach must have COMPLETED + by then, not started. + * ``renew_not_before`` opens the jitter window. A fleet bootstrapped in one + batch must not converge on a single wall-clock second against Keycloak's + token endpoint, so each runtime picks its own moment uniformly in + ``[renew_not_before, renew_at]``. + * ``session_expires_at`` is echoed here rather than only on the session so + the directive is self-contained -- a client that logs, forwards, or + persists only the directive still holds the ceiling it is racing. + +The ordering invariant below (``renew_not_before <= renew_at < +session_expires_at``) is the machine-checkable form of "renewal completes +before expiry, and expiry never moves." It is enforced in the model rather +than in the builder so that any construction path -- handler, test, a future +second caller, or a payload deserialized off the wire -- is subject to it. +""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_renewal_mode import ( + EnumGatewayRenewalMode, +) + + +class ModelGatewayRenewalDirective(BaseModel): + """Server-declared renewal cycle for one attached session.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + mode: EnumGatewayRenewalMode + # Echo of the session ceiling this cycle is racing. Never moved by any + # subsequent call on this session. + session_expires_at: datetime + # Earliest moment the runtime should begin its re-grant + re-attach. + renew_not_before: datetime + # Latest moment by which re-grant + re-attach must have COMPLETED. + renew_at: datetime + # The terms that produced the two timestamps above, echoed so a client + # can recompute the cycle after a clock correction without a second + # round trip, and so drift between the config and the wire is visible. + margin_seconds: int = Field(gt=0) + jitter_seconds: int = Field(ge=0) + + @model_validator(mode="after") + def _check_ordering(self) -> ModelGatewayRenewalDirective: + """Fail closed on any directive that would tell a client to renew late. + + A directive whose ``renew_at`` is at or past ``session_expires_at`` + is worse than no directive: it instructs the runtime to attempt a + re-attach with a session already dead, and reads as a deliberate + policy rather than a bug. Constructing one raises here instead. + """ + if self.renew_not_before > self.renew_at: + raise ValueError( + "renew_not_before must not be later than renew_at " + f"({self.renew_not_before.isoformat()} > {self.renew_at.isoformat()})" + ) + if self.renew_at >= self.session_expires_at: + raise ValueError( + "renew_at must be strictly before session_expires_at -- renewal " + "completes before expiry and never extends it " + f"({self.renew_at.isoformat()} >= " + f"{self.session_expires_at.isoformat()})" + ) + return self + + +__all__ = ["ModelGatewayRenewalDirective"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_session.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_session.py new file mode 100644 index 0000000000..3e7c07a270 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_session.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Attach session record -- the tenant-bound control-plane state unit.""" + +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_session_status import ( + EnumGatewaySessionStatus, +) + + +class ModelGatewaySession(BaseModel): + """Immutable-per-revision record of one tenant edge's attach session.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + session_id: UUID + tenant_id: UUID + tenant_slug: str + principal_id: str + # Keycloak client_id of the per-tenant confidential client, as bound at + # attach time -- carried through heartbeats so the introspection re-check + # always targets the right client without re-deriving it from the token. + keycloak_client_id: str + edge_instance_id: str + status: EnumGatewaySessionStatus + attached_at: datetime + last_heartbeat_at: datetime + # Never later than the underlying access token's exp claim, clamped by + # ModelGatewayAttachConfig.max_session_ttl_seconds. + expires_at: datetime + + +__all__ = ["ModelGatewaySession"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_session_event.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_session_event.py new file mode 100644 index 0000000000..0301cedf19 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/models/model_gateway_session_event.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Session lifecycle event -- the thin-publish payload for the bus. + +Handlers return this alongside their typed response; the node/runtime +owns the actual publish (node_owned_publish, mirroring +node_bus_forwarder_effect's capability of the same name) onto +the contract-declared gateway session-event topic. Handlers never call the bus +directly -- this keeps the effect boundary at the node, not scattered across +handler internals, and lets the eventual link-health projection (G3) +consume one canonical event shape for both the forwarder heartbeat and the +attach control plane. +""" + +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_session_event_type import ( + EnumGatewaySessionEventType, +) + + +class ModelGatewaySessionEvent(BaseModel): + """Session lifecycle fact, one per attach/heartbeat/detach/revoke.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + event_type: EnumGatewaySessionEventType + session_id: UUID + tenant_id: UUID + tenant_slug: str + principal_id: str + edge_instance_id: str + emitted_at: datetime + + +__all__ = ["ModelGatewaySessionEvent"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/node.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/node.py new file mode 100644 index 0000000000..b912e9c1ae --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/node.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Gateway attach/session control-plane effect node.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from omnibase_core.nodes.node_effect import NodeEffect + +if TYPE_CHECKING: + from omnibase_core.models.container import ModelONEXContainer + + +class NodeGatewayAttachEffect(NodeEffect): + """Contract-driven effect node for tenant edge attach/session control.""" + + def __init__(self, container: ModelONEXContainer) -> None: + """Initialize with container dependency injection.""" + super().__init__(container) + + +__all__ = ["NodeGatewayAttachEffect"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/services/__init__.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/services/__init__.py new file mode 100644 index 0000000000..3f0d3db754 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/services/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Effect-boundary services for node_gateway_attach_effect.""" + +from __future__ import annotations diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/services/protocol_gateway_session_store.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/services/protocol_gateway_session_store.py new file mode 100644 index 0000000000..efea427741 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/services/protocol_gateway_session_store.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Session store boundary -- DI seam so the store backend is swappable. + +First slice ships ``StoreGatewaySessionMemory`` (single-process, +adequate for one control-plane pod attaching one tenant). A multi-pod +deployment needs a shared backend (Valkey, matching the rest of the ONEX +runtime session state) behind this same Protocol -- tracked as follow-on, +not built in this slice (see contract.yaml metadata). +""" + +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session import ( + ModelGatewaySession, +) + + +class ProtocolGatewaySessionStore(Protocol): + """Async CRUD boundary for attach sessions.""" + + async def put(self, session: ModelGatewaySession) -> None: ... + + async def get(self, session_id: UUID) -> ModelGatewaySession | None: ... + + async def delete(self, session_id: UUID) -> None: ... + + async def put_if_present(self, session: ModelGatewaySession) -> bool: + """Atomically overwrite a session iff it is still present. + + OMN-15918 R2: closes the heartbeat resurrection race. A handler that + reads a session, awaits network I/O (introspection), and then writes + the refreshed session back must not blindly overwrite -- if a + concurrent detach removed the session during that I/O gap, an + unconditional ``put`` would silently resurrect it (an observable + half-state: the caller detached, but the session reappears ACTIVE on + the next read). ``put_if_present`` performs the presence check and + the write as one atomic step and returns ``False`` (no-op) instead of + resurrecting the row when the session is no longer present. + """ + ... + + +__all__ = ["ProtocolGatewaySessionStore"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/services/service_gateway_renewal_policy.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/services/service_gateway_renewal_policy.py new file mode 100644 index 0000000000..863efc2ed8 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/services/service_gateway_renewal_policy.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Renewal-cycle policy -- when the client must mint its NEXT session. + +OMN-15952. Deliberately a separate module from +``service_gateway_session_policy``, which answers a different question about +a different subject: + + * ``service_gateway_session_policy`` bounds the life of the session that + already exists -- has it passed ``expires_at``, has it gone too long + unverified. Those are the server's decisions, applied on + session-consuming paths, and their outcome is teardown. + * this module computes the cycle the *client* runs so that a successor + session exists before the incumbent one dies. Its outcome is a directive + handed back at attach; it tears nothing down and reads no clock of its + own. + +Merging them would produce one module whose functions answer to two +authorities, and the OMN-15952 review is explicit that a second lifecycle +authority over one session is how a session ends up simultaneously live and +torn down. Keeping the split means the server's bounds cannot quietly become +client advice, or the reverse. + +Everything here is a pure function over a session plus config, with ``now`` +supplied by the caller: the boundary cases (a token so short-lived that the +margin already covers its whole life, a heartbeat exactly at the ceiling) +are then driven directly by tests instead of slept through. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta + +from omnibase_infra.nodes.node_gateway_attach_effect.models.enum_gateway_renewal_mode import ( + EnumGatewayRenewalMode, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_config import ( + ModelGatewayAttachConfig, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_renewal_directive import ( + ModelGatewayRenewalDirective, +) +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session import ( + ModelGatewaySession, +) + + +class ExpiryExtensionError(Exception): + """Raised when a session revision would move ``expires_at``. + + The renewal contract's load-bearing negative: ``expires_at`` is stamped + once at attach and is immutable for the life of that ``session_id``. + Any path that produces a revised session must be able to prove it did + not move the ceiling, which is what ``assert_expiry_not_extended`` + below is for. + """ + + +def build_renewal_directive( + session: ModelGatewaySession, *, config: ModelGatewayAttachConfig +) -> ModelGatewayRenewalDirective: + """Compute the renewal cycle for a freshly attached session. + + ``renew_at`` is ``expires_at - renewal_margin_seconds``; the jitter + window opens ``renewal_jitter_seconds`` before that. + + Both are floored at ``attached_at``, which is what makes the function + total rather than merely usually-correct. A session whose whole life is + shorter than the margin (a token presented with 60 seconds left, or a + deployment that shortens the token lifespan below the configured + margin) would otherwise yield a ``renew_at`` in the past -- and the + model would then reject the directive, turning an unusual-but-valid + attach into a 5xx. Flooring says the honest thing instead: renew + immediately, you have no headroom. The floor cannot collide with the + model's strict ``renew_at < session_expires_at`` invariant, because + ``attached_at < expires_at`` always holds (attach rejects a token with + no remaining lifetime before a session is ever constructed). + """ + renew_at = max( + session.attached_at, + session.expires_at - timedelta(seconds=config.renewal_margin_seconds), + ) + renew_not_before = max( + session.attached_at, + renew_at - timedelta(seconds=config.renewal_jitter_seconds), + ) + return ModelGatewayRenewalDirective( + mode=EnumGatewayRenewalMode.RE_ATTACH, + session_expires_at=session.expires_at, + renew_not_before=renew_not_before, + renew_at=renew_at, + margin_seconds=config.renewal_margin_seconds, + jitter_seconds=config.renewal_jitter_seconds, + ) + + +def is_renewal_due( + session: ModelGatewaySession, + *, + now: datetime, + config: ModelGatewayAttachConfig, +) -> bool: + """True once the runtime is inside the window where it must renew. + + Boundary is inclusive at ``renew_not_before``: at the instant the window + opens, renewal is due. Erring early is the cheap direction -- an early + re-grant costs one token, whereas a late one costs the session. + """ + directive = build_renewal_directive(session, config=config) + return now >= directive.renew_not_before + + +def assert_expiry_not_extended( + previous: ModelGatewaySession, revised: ModelGatewaySession +) -> None: + """Guard that a session revision left the attach-time ceiling alone. + + This is the executable form of the contract's central negative. It is + cheap to state and easy to violate by accident: every session revision + in this node is a ``model_copy(update=...)``, and adding one key to that + dict is all it takes to turn a heartbeat into a lifetime extension. The + resulting defect would be invisible in every existing assertion (status + and timestamps would all look right) and would silently reintroduce the + in-place renewal the design refuses. + + Raises ``ExpiryExtensionError`` on any change to ``expires_at`` -- in + either direction. Shortening is not a safe subset: it is still a second + authority mutating a field whose whole value is that exactly one write, + at attach, ever happens. + """ + if revised.expires_at != previous.expires_at: + raise ExpiryExtensionError( + f"session {previous.session_id} expires_at moved from " + f"{previous.expires_at.isoformat()} to {revised.expires_at.isoformat()}; " + "the attach-time ceiling is immutable -- renewal is re-grant + " + "re-attach (EnumGatewayRenewalMode.RE_ATTACH), never extension" + ) + + +__all__ = [ + "ExpiryExtensionError", + "assert_expiry_not_extended", + "build_renewal_directive", + "is_renewal_due", +] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/services/service_keycloak_token_validator.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/services/service_keycloak_token_validator.py new file mode 100644 index 0000000000..89a682fda2 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/services/service_keycloak_token_validator.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Keycloak client-credentials claim verification for the attach control plane. + +``verify_and_decode_claims`` (attach time, and heartbeat/detach identity +re-check): verifies the JWT's signature against a resolved JWKS keyset, then +validates ``iss``/``aud``/``exp`` and decodes the tenant claim set. This +closes OMN-15918 R1: the previous ``decode_claims`` only base64-decoded the +payload segment and never referenced the token's signature at all, so any +structurally-valid token from any signer (forged or otherwise) that happened +to satisfy the other claims would attach and hold an ACTIVE session for up +to one heartbeat interval before introspection caught it. + +The JWKS *fetch* (RFC 7517, network I/O) is NOT here -- it lives inline in +the calling handler (``HandlerGatewayAttach._fetch_jwks`` / +``HandlerGatewayHeartbeat._fetch_jwks``), matching the pattern +``HandlerGatewayHeartbeat._introspect`` already established: the +imperative-contract-guard requires raw-transport calls to live under +``handlers/``, never in a freestanding module the guard cannot attribute to +a contract-declared handler. This module stays I/O-free by design -- it +receives an already-fetched JWKS keyset and does signature verification +(CPU-only) against it. Do not re-add an httpx call here. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from uuid import UUID + +import jwt +from jwt import PyJWK +from jwt.exceptions import InvalidTokenError + +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_config import ( + ModelGatewayAttachConfig, +) + +_REQUIRED_CLAIMS = ("exp", "iss", "aud", "sub") +_REQUIRED_TENANT_CLAIMS = ("tenant_id", "tenant_slug", "principal_id", "azp") + + +class TokenValidationError(Exception): + """Raised when an access token fails signature, claim, or introspection validation.""" + + +@dataclass(frozen=True) +class ClaimSet: + """Minimal, signature-verified claim set this node relies on.""" + + issuer: str + audience: str + subject: str + tenant_id: UUID + tenant_slug: str + principal_id: str + client_id: str + expires_at_epoch: int + + +def _resolve_signing_key( + jwks_keys: Sequence[Mapping[str, object]], *, kid: str +) -> PyJWK: + matching_jwk = next((key for key in jwks_keys if key.get("kid") == kid), None) + if matching_jwk is None: + raise TokenValidationError( + f"access_token kid {kid!r} not present in the resolved JWKS keyset" + ) + try: + return PyJWK.from_dict(dict(matching_jwk)) + except Exception as exc: + raise TokenValidationError( + "JWKS key material for the token's kid is malformed" + ) from exc + + +def verify_and_decode_claims( + access_token: str, + jwks_keys: Sequence[Mapping[str, object]], + config: ModelGatewayAttachConfig, + *, + expected_issuer: str, +) -> ClaimSet: + """Verify a JWT's signature against the resolved JWKS keyset, then decode its claims. + + Fails closed on every dimension: an unparsable header, a missing/unknown + ``kid``, a bad or absent signature, a wrong ``iss``/``aud``, an expired + ``exp``, or a missing tenant claim all raise ``TokenValidationError`` + before any claim is trusted. ``alg: none`` tokens are rejected implicitly + -- PyJWT never resolves a signing key for ``none`` and the explicit + ``algorithms=`` allowlist below never includes it. + """ + try: + unverified_header = jwt.get_unverified_header(access_token) + except Exception as exc: + raise TokenValidationError("access_token header is not decodable") from exc + + kid = unverified_header.get("kid") + if not kid or not isinstance(kid, str): + raise TokenValidationError("access_token header missing kid") + + alg = unverified_header.get("alg") + if not alg or not isinstance(alg, str) or alg.lower() == "none": + raise TokenValidationError( + f"access_token alg header {alg!r} is not an accepted signing algorithm" + ) + + signing_key = _resolve_signing_key(jwks_keys, kid=kid) + + try: + claims = jwt.decode( + access_token, + key=signing_key, + algorithms=[alg], + audience=config.required_audience, + issuer=expected_issuer, + options={"require": list(_REQUIRED_CLAIMS)}, + ) + except InvalidTokenError as exc: + raise TokenValidationError( + f"access_token signature/claims verification failed: {exc}" + ) from exc + + for required in _REQUIRED_TENANT_CLAIMS: + if required not in claims: + raise TokenValidationError( + f"access_token missing required claim: {required}" + ) + + try: + tenant_id = UUID(str(claims["tenant_id"])) + except ValueError as exc: + raise TokenValidationError("tenant_id claim is not a valid UUID") from exc + + return ClaimSet( + issuer=str(claims["iss"]), + audience=config.required_audience, + subject=str(claims["sub"]), + tenant_id=tenant_id, + tenant_slug=str(claims["tenant_slug"]), + principal_id=str(claims["principal_id"]), + client_id=str(claims["azp"]), + expires_at_epoch=int(claims["exp"]), + ) + + +__all__ = ["ClaimSet", "TokenValidationError", "verify_and_decode_claims"] diff --git a/src/omnibase_infra/nodes/node_gateway_attach_effect/services/store_gateway_session_memory.py b/src/omnibase_infra/nodes/node_gateway_attach_effect/services/store_gateway_session_memory.py new file mode 100644 index 0000000000..cff69d1466 --- /dev/null +++ b/src/omnibase_infra/nodes/node_gateway_attach_effect/services/store_gateway_session_memory.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""In-process session store -- default backend for the first slice.""" + +from __future__ import annotations + +import asyncio +from uuid import UUID + +from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_session import ( + ModelGatewaySession, +) + + +class StoreGatewaySessionMemory: + """Async-safe, single-process session store keyed by session_id.""" + + def __init__(self) -> None: + self._sessions: dict[UUID, ModelGatewaySession] = {} + self._lock = asyncio.Lock() + + async def put(self, session: ModelGatewaySession) -> None: + async with self._lock: + self._sessions[session.session_id] = session + + async def get(self, session_id: UUID) -> ModelGatewaySession | None: + async with self._lock: + return self._sessions.get(session_id) + + async def delete(self, session_id: UUID) -> None: + async with self._lock: + self._sessions.pop(session_id, None) + + async def put_if_present(self, session: ModelGatewaySession) -> bool: + async with self._lock: + if session.session_id not in self._sessions: + return False + self._sessions[session.session_id] = session + return True + + +__all__ = ["StoreGatewaySessionMemory"] diff --git a/src/omnibase_infra/nodes/node_llm_inference_effect/contract.yaml b/src/omnibase_infra/nodes/node_llm_inference_effect/contract.yaml index f1abf4f01e..b56e5a2067 100644 --- a/src/omnibase_infra/nodes/node_llm_inference_effect/contract.yaml +++ b/src/omnibase_infra/nodes/node_llm_inference_effect/contract.yaml @@ -169,7 +169,7 @@ published_events: description: "Content-bearing successful LLM inference response emitted directly from the effect node" topic_config: partitions: 3 - replication_factor: 1 + replication_factor: 2 kafka_config: "retention.ms": "604800000" "cleanup.policy": "delete" @@ -178,13 +178,13 @@ published_events: description: "Per-call metrics emitted after each inference call; may include gpu_seconds, gpu_type, gpu_count, and EnumUsageSource-backed compute_usage_source when the request declares GPU configuration" topic_config: partitions: 3 - replication_factor: 1 + replication_factor: 2 - topic: "onex.evt.omnibase-infra.llm-call-completed.v1" event_type: "LlmCallCompletedInfraEvent" description: "Lightweight per-call metrics for infra-owned consumers; mirrors optional GPU usage evidence fields for infra projections" topic_config: partitions: 3 - replication_factor: 1 + replication_factor: 2 kafka_config: "retention.ms": "604800000" "cleanup.policy": "delete" diff --git a/src/omnibase_infra/nodes/node_registration_orchestrator/contract.yaml b/src/omnibase_infra/nodes/node_registration_orchestrator/contract.yaml index e35430dad1..47f161fc47 100644 --- a/src/omnibase_infra/nodes/node_registration_orchestrator/contract.yaml +++ b/src/omnibase_infra/nodes/node_registration_orchestrator/contract.yaml @@ -525,7 +525,7 @@ published_events: description: "Full catalog snapshot in response to a TopicCatalogQuery" topic_config: partitions: 1 - replication_factor: 1 + replication_factor: 2 kafka_config: "retention.ms": "3600000" "cleanup.policy": "delete" @@ -534,7 +534,7 @@ published_events: description: "Notification published when the topic catalog version changes" topic_config: partitions: 1 - replication_factor: 1 + replication_factor: 2 kafka_config: "retention.ms": "604800000" "cleanup.policy": "delete" diff --git a/src/omnibase_infra/nodes/node_registration_reducer/models/model_payload_ledger_append.py b/src/omnibase_infra/nodes/node_registration_reducer/models/model_payload_ledger_append.py index cf8ffa3376..931bd8e80c 100644 --- a/src/omnibase_infra/nodes/node_registration_reducer/models/model_payload_ledger_append.py +++ b/src/omnibase_infra/nodes/node_registration_reducer/models/model_payload_ledger_append.py @@ -34,8 +34,10 @@ from omnibase_core.types import JsonType -# NOTE: ModelIntentPayloadBase was removed in omnibase_core 0.6.2 -# Using pydantic.BaseModel directly as the base class +# NOTE: This infra-local payload DTO extends pydantic.BaseModel directly (repo +# convention; see docs/standards/ONEX_TERMINOLOGY.md). omnibase_core's +# ModelIntentPayloadBase was never removed — it still exists and bases core's +# closed-set intent payloads. class ModelPayloadLedgerAppend(BaseModel): diff --git a/src/omnibase_infra/nodes/node_registration_reducer/models/model_payload_postgres_upsert_registration.py b/src/omnibase_infra/nodes/node_registration_reducer/models/model_payload_postgres_upsert_registration.py index 0721940aa6..71d485f4db 100644 --- a/src/omnibase_infra/nodes/node_registration_reducer/models/model_payload_postgres_upsert_registration.py +++ b/src/omnibase_infra/nodes/node_registration_reducer/models/model_payload_postgres_upsert_registration.py @@ -18,8 +18,10 @@ from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny -# NOTE: ModelIntentPayloadBase was removed in omnibase_core 0.6.2 -# Using pydantic.BaseModel directly as the base class +# NOTE: This infra-local payload DTO extends pydantic.BaseModel directly (repo +# convention; see docs/standards/ONEX_TERMINOLOGY.md). omnibase_core's +# ModelIntentPayloadBase was never removed — it still exists and bases core's +# closed-set intent payloads. class ModelPayloadPostgresUpsertRegistration(BaseModel): diff --git a/src/omnibase_infra/nodes/node_registration_reducer/registration_reducer.py b/src/omnibase_infra/nodes/node_registration_reducer/registration_reducer.py index 41af67cac1..d5e37c4cd5 100644 --- a/src/omnibase_infra/nodes/node_registration_reducer/registration_reducer.py +++ b/src/omnibase_infra/nodes/node_registration_reducer/registration_reducer.py @@ -529,11 +529,12 @@ class RegistrationReducer: >>> >>> reducer = RegistrationReducer() >>> state = ModelRegistrationState() # Initial idle state + >>> resolved_health_endpoint = "" >>> event = ModelNodeIntrospectionEvent( ... node_id=uuid4(), ... node_type="effect", ... node_version="1.0.0", - ... endpoints={"health": "http://localhost:8080/health"}, + ... endpoints={"health": resolved_health_endpoint}, ... ) >>> output = reducer.reduce(state, event) >>> print(output.result.status) # "pending" diff --git a/src/omnibase_infra/nodes/node_registration_storage_effect/models/model_registration_record.py b/src/omnibase_infra/nodes/node_registration_storage_effect/models/model_registration_record.py index 781642e9bc..345902a148 100644 --- a/src/omnibase_infra/nodes/node_registration_storage_effect/models/model_registration_record.py +++ b/src/omnibase_infra/nodes/node_registration_storage_effect/models/model_registration_record.py @@ -75,12 +75,13 @@ class ModelRegistrationRecord(BaseModel): >>> from uuid import uuid4 >>> from omnibase_core.enums.enum_node_kind import EnumNodeKind >>> from omnibase_core.models.primitives.model_semver import ModelSemVer + >>> resolved_health_endpoint = "" >>> record = ModelRegistrationRecord( ... node_id=uuid4(), ... node_type=EnumNodeKind.EFFECT, ... node_version=ModelSemVer(major=1, minor=0, patch=0), ... capabilities=("registration.storage", "registration.storage.query"), - ... endpoints={"health": "http://localhost:8080/health"}, + ... endpoints={"health": resolved_health_endpoint}, ... metadata={"team": "platform"}, ... created_at=datetime.now(UTC), ... updated_at=datetime.now(UTC), diff --git a/src/omnibase_infra/nodes/node_registry_effect/models/model_registry_request.py b/src/omnibase_infra/nodes/node_registry_effect/models/model_registry_request.py index 55567ead67..934390c64c 100644 --- a/src/omnibase_infra/nodes/node_registry_effect/models/model_registry_request.py +++ b/src/omnibase_infra/nodes/node_registry_effect/models/model_registry_request.py @@ -62,13 +62,14 @@ class ModelRegistryRequest(BaseModel): >>> from uuid import uuid4 >>> from omnibase_core.enums.enum_node_kind import EnumNodeKind >>> from omnibase_core.models.primitives.model_semver import ModelSemVer + >>> resolved_health_endpoint = "" >>> request = ModelRegistryRequest( ... node_id=uuid4(), ... node_type=EnumNodeKind.EFFECT, ... node_version=ModelSemVer(major=1, minor=0, patch=0), ... correlation_id=uuid4(), ... service_name="onex-effect", - ... endpoints={"health": "http://localhost:8080/health"}, + ... endpoints={"health": resolved_health_endpoint}, ... timestamp=datetime(2025, 1, 15, 12, 0, 0, tzinfo=UTC), ... ) >>> request.node_type diff --git a/src/omnibase_infra/nodes/node_remote_agent_invoke_effect/contract.yaml b/src/omnibase_infra/nodes/node_remote_agent_invoke_effect/contract.yaml index d629824e47..882100fe79 100644 --- a/src/omnibase_infra/nodes/node_remote_agent_invoke_effect/contract.yaml +++ b/src/omnibase_infra/nodes/node_remote_agent_invoke_effect/contract.yaml @@ -72,7 +72,7 @@ published_events: description: "Remote-agent lifecycle event emitted by protocol handlers." topic_config: partitions: 3 - replication_factor: 1 + replication_factor: 2 kafka_config: "retention.ms": "604800000" "cleanup.policy": "delete" diff --git a/src/omnibase_infra/nodes/node_row_count_diagnostic_effect/contract.yaml b/src/omnibase_infra/nodes/node_row_count_diagnostic_effect/contract.yaml index 6c43ccda2c..7e754b14c1 100644 --- a/src/omnibase_infra/nodes/node_row_count_diagnostic_effect/contract.yaml +++ b/src/omnibase_infra/nodes/node_row_count_diagnostic_effect/contract.yaml @@ -47,7 +47,7 @@ published_events: description: "Periodic database row count diagnostic snapshot" topic_config: partitions: 1 - replication_factor: 1 + replication_factor: 2 kafka_config: "retention.ms": "604800000" "cleanup.policy": "delete" diff --git a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/contract.yaml b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/contract.yaml index e40585a6e0..135f6017a9 100644 --- a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/contract.yaml +++ b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/contract.yaml @@ -26,15 +26,37 @@ # re-arm signal fields (docker_restart_count, diag_heartbeat_age_seconds) are # surfaced on the assessment. Additive only -- no fleet mutation, no new state # member, no change to the input command or published topics. +# - OMN-15255 (friction F-04): composite readiness. Per-runner `readiness` +# (READY/NOT_READY/UNKNOWN) is a CONJUNCTION over six independently-probed +# signals -- github_registration, docker_health, diag_heartbeat, +# listener_topology, container_stability, disk_capacity -- evaluated in full +# every tick, unlike `state`, which is a first-match-wins precedence pick that +# never evaluates container health or listener topology at all. Adds the +# quarantine set and its strictly narrower bounce-eligible subset, which is now +# the ONLY producer of RESTART_RUNNER: the four state-keyed restart branches +# (CRASH_LOOPING 0.9 / LISTENER_ZOMBIE 0.85 / OFFLINE_IDLE 0.6 / WEDGED 0.5) are +# DELETED. Readiness fails CLOSED (UNKNOWN is not READY); the bounce gate fails +# SAFE (no mutation on indeterminate sources, never a busy runner, never for a +# cause a force-recreate cannot fix). The disk ceiling (90%) is a module +# constant, NOT a new env read -- adding a fourth env-var name to this file +# is exactly what scripts/check-env-reads.sh exists to stop (rule 10: narrow +# the matcher, do not widen the allowlist; OMN-15234 owns that narrowing). +# Still no mutation reachable from this node. +# - OMN-15234: LISTENER_ZOMBIE -> RESTART_RUNNER now requires composite +# corroborating evidence (determinate sources + not busy + registry/container +# corroboration), not the stale-heartbeat flag alone; the uncorroborated case +# records action NONE with the reason. Assessment gains the typed +# corroboration facts (github_status / github_busy / docker_status). Still no +# executor and no fleet mutation -- recommendations only. name: "node_runner_fleet_health_compute" contract_version: major: 1 - minor: 0 - patch: 1 -node_version: "1.0.1" + minor: 1 + patch: 0 +node_version: "1.1.0" node_type: "COMPUTE_GENERIC" description: > - Pure, deterministic classifier of a runner-fleet snapshot into a typed health verdict: per-runner state, fleet aggregates, and recorded (never executed) recommended actions. Uses the canonical def-B typed ModelRunnerFleetHealthEvaluateCommand handler entrypoint. + Pure, deterministic classifier of a runner-fleet snapshot into a typed health verdict: composite per-runner readiness over six signals, the quarantine set and its bounce-eligible subset, precedence health state, fleet aggregates, and recorded (never executed) recommended actions. Uses the canonical def-B typed ModelRunnerFleetHealthEvaluateCommand handler entrypoint. Heartbeat-staleness classification threshold (RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS, env-overridable) defaults to 4500s as of OMN-15233 (1.0.2), raised from 900s. 900s sat below the ~50-minute IDLE _diag write cadence, so this classifier emitted LISTENER_ZOMBIE -> RESTART_RUNNER at confidence 0.85 for idle-but-healthy runners. The default is held identical to docker/runners/healthcheck.sh and runner-monitor.sh so all three surfaces agree on what "stale" means. OMN-15234 (1.1.0) makes the restart recommendation composite rather than flag-driven: a lone stale heartbeat on an otherwise online, idle, running, zero-restart runner is not bounce-eligible. The per-runner assessment carries the typed corroboration facts it was derived from (github_status, github_busy, docker_status) alongside the existing re-arm signals. input_model: name: "ModelRunnerFleetHealthEvaluateCommand" @@ -44,7 +66,7 @@ output_model: name: "ModelRunnerFleetHealthVerdict" module: "omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_runner_fleet_health_verdict" description: >- - Per-runner + fleet-level health verdict with recommended actions, carrying source-determinacy flags (github_source_ok / docker_source_ok / buildx_determinate) so a probe-source outage is distinguishable from a confirmed-healthy fleet (OMN-14228 Slice A). + Per-runner + fleet-level health verdict with recommended actions, carrying source-determinacy flags (github_source_ok / docker_source_ok / buildx_determinate) so a probe-source outage is distinguishable from a confirmed-healthy fleet (OMN-14228 Slice A), plus the OMN-15255 composite readiness surface: per-runner readiness + evaluated signals, ready/not_ready/unknown counts, fleet_ready, quarantined_runners, bounce_eligible_runners, and per-signal fleet rollups. handler_routing: routing_strategy: "payload_type_match" handlers: @@ -74,20 +96,21 @@ published_events: description: "Classified per-runner + fleet-level runner-fleet health verdict" topic_config: partitions: 1 - replication_factor: 1 + replication_factor: 2 kafka_config: "retention.ms": "604800000" "cleanup.policy": "delete" metadata: - description: "Runner-fleet health classifier (OMN-13942 Increment 1 -- read-only detection core) with canonical def-B typed handler entrypoint." + description: "Runner-fleet health classifier + composite readiness surface (OMN-13942 detection core, OMN-15255 readiness/quarantine) with canonical def-B typed handler entrypoint. Read-only: no mutation is reachable from this node." author: "OmniNode Team" license: "MIT" created: "2026-07-04" - updated: "2026-07-20" - ticket: "OMN-14830" + updated: "2026-07-27" + ticket: "OMN-15255" tags: - compute - def-b - health - runner - fleet-maintain + - readiness diff --git a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/handlers/handler_runner_fleet_health_evaluate.py b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/handlers/handler_runner_fleet_health_evaluate.py index eb322dbb60..f0a234ce0a 100644 --- a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/handlers/handler_runner_fleet_health_evaluate.py +++ b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/handlers/handler_runner_fleet_health_evaluate.py @@ -29,6 +29,34 @@ CLOSED on indeterminate health instead of treating a source outage as verified-healthy. No gate/executor logic is added here -- it is precondition data only. + +OMN-15255 (friction F-04) adds the composite readiness surface alongside the +precedence classification. Two things are true at once and must not be +conflated: + + - ``state`` answers "what is the single most severe thing wrong with this + runner" by precedence. First match wins; later signals are not evaluated. + - ``readiness`` answers "may this runner be routed governed work" by + conjunction over six independently-probed signals. Every signal is + evaluated every tick, including the passing ones. + +They legitimately disagree. A GitHub-online runner with a fresh heartbeat, an +unhealthy container and two Runner.Listener processes is ``state=HEALTHY`` +(neither container health nor listener topology is an input to the precedence +chain) and ``readiness=NOT_READY``. That gap is the F-04 finding: at one +closeout GitHub reported 64/64 online while 53 of 64 containers read +docker-unhealthy, and nothing in the system adjudicated. + +The quarantine/bounce split fixes the other half -- the false-positive +restart storm. A stale-looking heartbeat alone can no longer produce a +restart recommendation: a bounce needs a determinate source, an idle runner, +and a failing signal a force-recreate can actually fix. + +OMN-15234: LISTENER_ZOMBIE no longer maps to RESTART_RUNNER on the staleness +flag alone. The classification threshold (4500s, held equal to +``docker/runners/healthcheck.sh``) is a heuristic over the idle ``_diag`` +token-refresh cadence, so the restart *recommendation* additionally requires +composite corroborating evidence -- see ``_zombie_restart_corroboration``. """ from __future__ import annotations @@ -37,12 +65,24 @@ from datetime import UTC, datetime from omnibase_infra.enums import EnumHandlerType, EnumHandlerTypeCategory +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_readiness_signal_outcome import ( + EnumReadinessSignalOutcome, +) from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_recommended_action_type import ( EnumRecommendedActionType, ) from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_runner_fleet_health_state import ( EnumRunnerFleetHealthState, ) +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_runner_readiness_signal import ( + EnumRunnerReadinessSignal, +) +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_runner_readiness_state import ( + EnumRunnerReadinessState, +) +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_readiness_signal_rollup import ( + ModelReadinessSignalRollup, +) from omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_recommended_action import ( ModelRecommendedAction, ) @@ -55,17 +95,67 @@ from omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_runner_health_assessment import ( ModelRunnerHealthAssessment, ) +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_runner_readiness_signal import ( + ModelRunnerReadinessSignal, +) from omnibase_infra.nodes.node_runner_health_snapshot_effect.models.model_runner_fleet_runner_fact import ( ModelRunnerFleetRunnerFact, ) logger = logging.getLogger(__name__) -_DEFAULT_CRASHLOOP_RESTART_THRESHOLD = 5 -_DEFAULT_RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS = 900 -_DEFAULT_WEDGE_QUEUE_AGE_SECONDS = 600 +# Same defaults as the EFFECT + the legacy bash surfaces (runner-monitor.sh, +# healthcheck.sh) so all three surfaces agree on thresholds during the +# trust-building period (OMN-13109/OMN-13912/OMN-13915/OMN-15233). +# +# OMN-15195 removed the env-read form of these three thresholds: they are +# plain constants, not overlay-resolved reads. Nothing constructs this handler +# with overrides, so the values are unchanged by the removal. +_CRASHLOOP_RESTART_THRESHOLD = 5 +# 4500s (75 min) matches docker/runners/healthcheck.sh exactly (OMN-15233). An +# IDLE runner writes _diag only on its ~50-minute OAuth/AAD token refresh, so +# the previous 900s default sat below the healthy idle cadence and classified +# idle-but-healthy runners as LISTENER_ZOMBIE for ~35 of every 50 minutes. +# Keep this value equal to healthcheck.sh's -- a divergence means the node +# verdict and the container healthcheck disagree about what "stale" means. +_RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS = 4500 +_WEDGE_QUEUE_AGE_SECONDS = 600 +# OMN-15255: ceiling on runner-host disk usage. A host past this is out of +# space for checkouts/caches; every container on it is unfit for work even +# though each one still registers online and reports a fresh heartbeat. +# +# Deliberately a plain constant, NOT an env read. The three thresholds above +# are grandfathered pre-gate reads in this file; adding a fourth would be a +# NEW env-var name, which `scripts/check-env-reads.sh` exists to stop and +# which OMN-15234 (PR #2502) narrows the matcher to catch by name rather than +# by file allowlist. Rule 10 says fix the underlying issue instead of widening +# an allowlist, so this value stays a constant until the runner-health +# thresholds get a typed config surface. Tunability is not lost silently -- +# it was never granted. +_RUNNER_READINESS_MAX_DISK_USED_PERCENT = 90.0 _EnumState = EnumRunnerFleetHealthState +_EnumSignal = EnumRunnerReadinessSignal +_EnumOutcome = EnumReadinessSignalOutcome + +# Docker health values that mean "the container itself is fit". ``none`` is a +# PASS, not a gap: it means the image declares no healthcheck, so this signal +# has nothing to assert and the other five carry the verdict. +_HEALTHY_DOCKER_HEALTH_VALUES = frozenset({"healthy", "none"}) + +# Signals a force-recreate can plausibly fix. DISK_CAPACITY is deliberately +# absent -- recreating a container frees no host disk, so bouncing on a full +# disk is pure churn. GITHUB_REGISTRATION is absent as a *standalone* trigger +# per OMN-14057 (raw GitHub offline over-reports under status lag); it only +# contributes when a local signal corroborates it. +_BOUNCE_REMEDIABLE_SIGNALS = frozenset( + { + _EnumSignal.DOCKER_HEALTH, + _EnumSignal.DIAG_HEARTBEAT, + _EnumSignal.LISTENER_TOPOLOGY, + _EnumSignal.CONTAINER_STABILITY, + } +) def _classify_runner( @@ -75,25 +165,22 @@ def _classify_runner( fleet_saturated: bool, buildx_available: bool | None, codeload_throttled: bool, - crashloop_restart_threshold: int, - max_diag_age_seconds: int, - wedge_queue_age_seconds: int, ) -> tuple[_EnumState, str]: """Classify a single runner. Returns (state, detail). Pure, no I/O.""" - if fact.docker_restart_count > crashloop_restart_threshold: + if fact.docker_restart_count > _CRASHLOOP_RESTART_THRESHOLD: return ( _EnumState.CRASH_LOOPING, - f"RestartCount={fact.docker_restart_count} > threshold={crashloop_restart_threshold}", + f"RestartCount={fact.docker_restart_count} > threshold={_CRASHLOOP_RESTART_THRESHOLD}", ) if ( fact.diag_heartbeat_age_seconds is not None - and fact.diag_heartbeat_age_seconds > max_diag_age_seconds + and fact.diag_heartbeat_age_seconds > _RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS ): return ( _EnumState.LISTENER_ZOMBIE, ( f"_diag heartbeat age={fact.diag_heartbeat_age_seconds:.0f}s > " - f"threshold={max_diag_age_seconds}s" + f"threshold={_RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS}s" ), ) if fact.github_status == "offline": @@ -108,7 +195,7 @@ def _classify_runner( if fleet_wedged and not fact.github_busy: return ( _EnumState.WEDGED, - f"fleet-wide: queued job age >= {wedge_queue_age_seconds}s with zero busy runners", + f"fleet-wide: queued job age >= {_WEDGE_QUEUE_AGE_SECONDS}s with zero busy runners", ) if fleet_saturated and fact.github_busy: return ( @@ -118,6 +205,247 @@ def _classify_runner( return _EnumState.HEALTHY, "" +def _signal( + signal: EnumRunnerReadinessSignal, + outcome: EnumReadinessSignalOutcome, + detail: str = "", +) -> ModelRunnerReadinessSignal: + return ModelRunnerReadinessSignal(signal=signal, outcome=outcome, detail=detail) + + +def _github_registration_signal( + fact: ModelRunnerFleetRunnerFact, *, github_source_ok: bool +) -> ModelRunnerReadinessSignal: + if not github_source_ok: + return _signal( + _EnumSignal.GITHUB_REGISTRATION, + _EnumOutcome.UNKNOWN, + "GitHub runners API source failed this tick", + ) + if fact.github_status == "online": + return _signal(_EnumSignal.GITHUB_REGISTRATION, _EnumOutcome.PASS) + return _signal( + _EnumSignal.GITHUB_REGISTRATION, + _EnumOutcome.FAIL, + f"GitHub reports status={fact.github_status!r}, not 'online'", + ) + + +def _docker_health_signal( + fact: ModelRunnerFleetRunnerFact, *, docker_source_ok: bool +) -> ModelRunnerReadinessSignal: + """The signal the legacy precedence classifier never evaluated at all.""" + if not docker_source_ok: + return _signal( + _EnumSignal.DOCKER_HEALTH, + _EnumOutcome.UNKNOWN, + "SSH/Docker source failed this tick", + ) + if fact.docker_status and fact.docker_status != "running": + return _signal( + _EnumSignal.DOCKER_HEALTH, + _EnumOutcome.FAIL, + f"container state={fact.docker_status!r}, not 'running'", + ) + if not fact.docker_health: + return _signal( + _EnumSignal.DOCKER_HEALTH, + _EnumOutcome.UNKNOWN, + "container health not reported by the probe", + ) + if fact.docker_health in _HEALTHY_DOCKER_HEALTH_VALUES: + return _signal(_EnumSignal.DOCKER_HEALTH, _EnumOutcome.PASS) + if fact.docker_health == "starting": + return _signal( + _EnumSignal.DOCKER_HEALTH, + _EnumOutcome.UNKNOWN, + "healthcheck still in start_period (starting)", + ) + return _signal( + _EnumSignal.DOCKER_HEALTH, + _EnumOutcome.FAIL, + f"container health={fact.docker_health!r}", + ) + + +def _diag_heartbeat_signal( + fact: ModelRunnerFleetRunnerFact, *, docker_source_ok: bool +) -> ModelRunnerReadinessSignal: + """OMN-15233: the threshold must bracket a full token-refresh cycle. + + A live idle runner writes ``_diag`` on the listener's token-refresh + cadence, measured at ~50-53 minutes on 2026-07-27. The retired 900s + default therefore read a perfectly healthy idle runner as stale for ~35 of + every 50 minutes -- the false-positive that drove the restart storm this + signal exists to stop. + """ + if fact.diag_heartbeat_age_seconds is None: + return _signal( + _EnumSignal.DIAG_HEARTBEAT, + _EnumOutcome.UNKNOWN, + ( + "SSH/Docker source failed this tick" + if not docker_source_ok + else "no _diag heartbeat age observed" + ), + ) + if fact.diag_heartbeat_age_seconds <= _RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS: + return _signal(_EnumSignal.DIAG_HEARTBEAT, _EnumOutcome.PASS) + return _signal( + _EnumSignal.DIAG_HEARTBEAT, + _EnumOutcome.FAIL, + ( + f"_diag heartbeat age={fact.diag_heartbeat_age_seconds:.0f}s > " + f"threshold={_RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS}s" + ), + ) + + +def _listener_topology_signal( + fact: ModelRunnerFleetRunnerFact, +) -> ModelRunnerReadinessSignal: + if fact.listener_process_count is None or fact.orphaned_listener_count is None: + return _signal( + _EnumSignal.LISTENER_TOPOLOGY, + _EnumOutcome.UNKNOWN, + "listener process topology not observed", + ) + if fact.orphaned_listener_count > 0: + return _signal( + _EnumSignal.LISTENER_TOPOLOGY, + _EnumOutcome.FAIL, + f"{fact.orphaned_listener_count} Runner.Listener process(es) at PPID 1", + ) + if fact.listener_process_count != 1: + return _signal( + _EnumSignal.LISTENER_TOPOLOGY, + _EnumOutcome.FAIL, + f"{fact.listener_process_count} Runner.Listener process(es), expected exactly 1", + ) + return _signal(_EnumSignal.LISTENER_TOPOLOGY, _EnumOutcome.PASS) + + +def _container_stability_signal( + fact: ModelRunnerFleetRunnerFact, *, docker_source_ok: bool +) -> ModelRunnerReadinessSignal: + if not docker_source_ok: + return _signal( + _EnumSignal.CONTAINER_STABILITY, + _EnumOutcome.UNKNOWN, + "SSH/Docker source failed this tick", + ) + if fact.docker_restart_count > _CRASHLOOP_RESTART_THRESHOLD: + return _signal( + _EnumSignal.CONTAINER_STABILITY, + _EnumOutcome.FAIL, + ( + f"RestartCount={fact.docker_restart_count} > " + f"threshold={_CRASHLOOP_RESTART_THRESHOLD}" + ), + ) + return _signal(_EnumSignal.CONTAINER_STABILITY, _EnumOutcome.PASS) + + +def _disk_capacity_signal( + host_disk_used_percent: float | None, +) -> ModelRunnerReadinessSignal: + if host_disk_used_percent is None: + return _signal( + _EnumSignal.DISK_CAPACITY, + _EnumOutcome.UNKNOWN, + "runner-host disk usage not observed", + ) + if host_disk_used_percent < _RUNNER_READINESS_MAX_DISK_USED_PERCENT: + return _signal(_EnumSignal.DISK_CAPACITY, _EnumOutcome.PASS) + return _signal( + _EnumSignal.DISK_CAPACITY, + _EnumOutcome.FAIL, + ( + f"runner-host disk used={host_disk_used_percent:.1f}% >= " + f"ceiling={_RUNNER_READINESS_MAX_DISK_USED_PERCENT:.1f}%" + ), + ) + + +def _evaluate_readiness_signals( + fact: ModelRunnerFleetRunnerFact, + *, + github_source_ok: bool, + docker_source_ok: bool, + host_disk_used_percent: float | None, +) -> tuple[ModelRunnerReadinessSignal, ...]: + """Evaluate every readiness signal for one runner. Pure, no short-circuit. + + Every signal is evaluated even when an earlier one already FAILed. That is + the difference from ``_classify_runner``: a precedence chain stops at the + first match and therefore cannot report that a runner failed three + different ways. + """ + return ( + _github_registration_signal(fact, github_source_ok=github_source_ok), + _docker_health_signal(fact, docker_source_ok=docker_source_ok), + _diag_heartbeat_signal(fact, docker_source_ok=docker_source_ok), + _listener_topology_signal(fact), + _container_stability_signal(fact, docker_source_ok=docker_source_ok), + _disk_capacity_signal(host_disk_used_percent), + ) + + +def _readiness_state( + signals: tuple[ModelRunnerReadinessSignal, ...], +) -> EnumRunnerReadinessState: + """Conjunction: READY only when every signal PASSes.""" + if any(s.outcome == _EnumOutcome.FAIL for s in signals): + return EnumRunnerReadinessState.NOT_READY + if any(s.outcome == _EnumOutcome.UNKNOWN for s in signals): + return EnumRunnerReadinessState.UNKNOWN + return EnumRunnerReadinessState.READY + + +def _quarantine_reason(signals: tuple[ModelRunnerReadinessSignal, ...]) -> str: + return "; ".join( + f"{s.signal.value}: {s.detail}" + for s in signals + if s.outcome == _EnumOutcome.FAIL + ) + + +def _is_bounce_eligible( + fact: ModelRunnerFleetRunnerFact, + signals: tuple[ModelRunnerReadinessSignal, ...], + *, + readiness: EnumRunnerReadinessState, + is_determinate: bool, +) -> bool: + """Decide whether a force-recreate is a defensible remedy for this runner. + + Readiness fails CLOSED (UNKNOWN is not READY, so an unprobeable runner is + not counted as capacity). This gate fails SAFE in the opposite direction: + it mutates nothing on indeterminate evidence, never interrupts a job in + flight, and never bounces for a cause a bounce cannot fix. The asymmetry + is deliberate -- the cost of not routing to a good runner is one idle + runner; the cost of recreating a busy or misread runner is a cancelled CI + job plus the restart storm this ticket exists to stop. + """ + if readiness != EnumRunnerReadinessState.NOT_READY: + return False + if not is_determinate: + return False + if fact.github_busy: + return False + failing = {s.signal for s in signals if s.outcome == _EnumOutcome.FAIL} + remediable = failing & _BOUNCE_REMEDIABLE_SIGNALS + if not remediable: + return False + if remediable == {_EnumSignal.DIAG_HEARTBEAT}: + return ( + fact.github_status != "online" + or fact.docker_status not in ("", "running") + or fact.docker_restart_count > 0 + ) + return True + + def _annotate_indeterminate( detail: str, *, github_source_ok: bool, docker_source_ok: bool ) -> str: @@ -139,37 +467,114 @@ def _annotate_indeterminate( return f"{detail}; {note}" if detail else note +def _zombie_restart_corroboration( + assessment: ModelRunnerHealthAssessment, +) -> str | None: + """Corroborating evidence for a LISTENER_ZOMBIE restart, or None (OMN-15234). + + A stale ``_diag`` heartbeat is a *single* signal, and on an idle runner it + is an expected one: an idle listener writes ``_diag`` only on its ~50-minute + OAuth/AAD token refresh. OMN-15233's live canary measured exactly that -- + on the old 900s threshold, control runners flipped unhealthy at diag ages + of 1120-1237s while the GitHub registry reported 64/64 ONLINE, busy=0 and + every container had ``RestartCount=0``. Raising the threshold to 4500s + shrinks that window; it does not make the flag alone sufficient evidence to + restart a runner, because the threshold is a heuristic over a cadence that + varies with token-refresh timing. + + So the RESTART_RUNNER recommendation now requires the staleness flag PLUS + at least one independent fact that the runner is actually not serving, and + requires that neither probe source failed. The runbook rule this encodes + (docs/runbooks/runner-fleet-listener-liveness.md, OMN-15233): *cross-check + the GitHub runner registry before ANY restart sweep -- if the runner is + online, the flag is the bug.* + + Returns the evidence string when a restart is warranted, or None when the + only thing observed is a stale heartbeat on an otherwise-serving runner. + """ + if not assessment.is_determinate: + # Fail closed (OMN-14228 Slice A): a failed GitHub/Docker probe is not + # evidence of a zombie, and "indeterminate" must never read as "restart". + return None + if assessment.github_busy: + # Restarting a runner mid-job destroys the job. A busy runner writing no + # _diag contradicts itself; the contradiction is not restart evidence. + return None + if assessment.github_status != "online": + return ( + f"github_status={assessment.github_status!r}: the registry does not " + "report this runner online" + ) + if assessment.docker_status not in ("", "running"): + return ( + f"docker_status={assessment.docker_status!r}: the container is not running" + ) + if assessment.docker_restart_count > 0: + return ( + f"docker_restart_count={assessment.docker_restart_count}: the container " + "has restarted at least once" + ) + return None + + def _recommend_for_assessment( assessment: ModelRunnerHealthAssessment, ) -> ModelRecommendedAction | None: - """Map a per-runner assessment to a recommended (never-executed) action.""" - if assessment.state == _EnumState.CRASH_LOOPING: + """Map a per-runner assessment to a recommended (never-executed) action. + + OMN-15255 replaced four state-keyed ``RESTART_RUNNER`` branches + (``CRASH_LOOPING`` 0.9 / ``LISTENER_ZOMBIE`` 0.85 / ``OFFLINE_IDLE`` 0.6 / + ``WEDGED`` 0.5) with one rule: a restart is recommended exactly when the + runner is bounce-eligible. Four independently-tunable confidence + heuristics over the same underlying facts is how a single misread + threshold (the 900s heartbeat window) turned into a fleet-wide restart + storm -- there was no second condition anywhere that could veto it. + + Quarantined-but-not-bounce-eligible still surfaces, as ``NONE`` with the + failing signals as the reason: it is an operator item (host disk, GitHub + status lag), not a restart. + """ + if assessment.bounce_eligible: + if assessment.state == _EnumState.LISTENER_ZOMBIE: + corroboration = _zombie_restart_corroboration(assessment) + if corroboration is not None: + return ModelRecommendedAction( + action_type=EnumRecommendedActionType.RESTART_RUNNER, + target_id=assessment.name, + reason=f"{assessment.detail}; corroborated by {corroboration}", + confidence=0.85, + ) return ModelRecommendedAction( action_type=EnumRecommendedActionType.RESTART_RUNNER, target_id=assessment.name, - reason=assessment.detail, + reason=assessment.quarantine_reason or assessment.detail, confidence=0.9, ) - if assessment.state == _EnumState.LISTENER_ZOMBIE: - return ModelRecommendedAction( - action_type=EnumRecommendedActionType.RESTART_RUNNER, - target_id=assessment.name, - reason=assessment.detail, - confidence=0.85, - ) - if assessment.state == _EnumState.OFFLINE_IDLE: - return ModelRecommendedAction( - action_type=EnumRecommendedActionType.RESTART_RUNNER, - target_id=assessment.name, - reason=assessment.detail, - confidence=0.6, - ) - if assessment.state == _EnumState.WEDGED: + if assessment.quarantined: + if assessment.state == _EnumState.LISTENER_ZOMBIE: + return ModelRecommendedAction( + action_type=EnumRecommendedActionType.NONE, + target_id=assessment.name, + reason=( + "stale _diag heartbeat is not corroborated (registry " + f"status={assessment.github_status!r}, busy=" + f"{assessment.github_busy}, docker_status=" + f"{assessment.docker_status!r}, restarts=" + f"{assessment.docker_restart_count}, determinate=" + f"{assessment.is_determinate}) -- an idle listener writes " + "_diag only on its ~50-min token refresh, so this is not " + "evidence for a bounce (OMN-15234)" + ), + confidence=0.0, + ) return ModelRecommendedAction( - action_type=EnumRecommendedActionType.RESTART_RUNNER, + action_type=EnumRecommendedActionType.NONE, target_id=assessment.name, - reason=assessment.detail, - confidence=0.5, + reason=( + "quarantined; no failing signal a force-recreate can fix -- " + f"{assessment.quarantine_reason}" + ), + confidence=0.0, ) if assessment.state == _EnumState.SATURATED: return ModelRecommendedAction( @@ -202,17 +607,6 @@ class HandlerRunnerFleetHealthEvaluate: output, and no I/O happens anywhere in this class. """ - def __init__( - self, - *, - crashloop_restart_threshold: int = _DEFAULT_CRASHLOOP_RESTART_THRESHOLD, - max_diag_age_seconds: int = _DEFAULT_RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS, - wedge_queue_age_seconds: int = _DEFAULT_WEDGE_QUEUE_AGE_SECONDS, - ) -> None: - self._crashloop_restart_threshold = crashloop_restart_threshold - self._max_diag_age_seconds = max_diag_age_seconds - self._wedge_queue_age_seconds = wedge_queue_age_seconds - @property def handler_type(self) -> EnumHandlerType: return EnumHandlerType.NODE_HANDLER @@ -264,7 +658,7 @@ async def handle( fleet_wedged = ( snapshot.oldest_queued_job_age_seconds is not None - and snapshot.oldest_queued_job_age_seconds >= self._wedge_queue_age_seconds + and snapshot.oldest_queued_job_age_seconds >= _WEDGE_QUEUE_AGE_SECONDS and busy_count == 0 and online_count > 0 ) @@ -283,10 +677,15 @@ async def handle( fleet_saturated=fleet_saturated, buildx_available=snapshot.buildx_available, codeload_throttled=codeload_throttled, - crashloop_restart_threshold=self._crashloop_restart_threshold, - max_diag_age_seconds=self._max_diag_age_seconds, - wedge_queue_age_seconds=self._wedge_queue_age_seconds, ) + signals = _evaluate_readiness_signals( + fact, + github_source_ok=snapshot.github_source_ok, + docker_source_ok=snapshot.docker_source_ok, + host_disk_used_percent=snapshot.host_disk_used_percent, + ) + readiness = _readiness_state(signals) + quarantined = readiness == EnumRunnerReadinessState.NOT_READY assessment = ModelRunnerHealthAssessment( name=fact.name, state=state, @@ -298,6 +697,23 @@ async def handle( is_determinate=is_determinate, docker_restart_count=fact.docker_restart_count, diag_heartbeat_age_seconds=fact.diag_heartbeat_age_seconds, + # OMN-15234: typed corroboration facts the LISTENER_ZOMBIE + # restart recommendation is derived from. Carried on the + # assessment (not read behind the recommender's back) so the + # published verdict shows the evidence the recommendation used. + github_status=fact.github_status, + github_busy=fact.github_busy, + docker_status=fact.docker_status, + readiness=readiness, + signals=signals, + quarantined=quarantined, + quarantine_reason=_quarantine_reason(signals) if quarantined else "", + bounce_eligible=_is_bounce_eligible( + fact, + signals, + readiness=readiness, + is_determinate=is_determinate, + ), ) assessments.append(assessment) if state == _EnumState.CRASH_LOOPING: @@ -318,16 +734,53 @@ async def handle( reason=( f"run {candidate.status} for {candidate.age_seconds:.0f}s in " f"{candidate.repo}, exceeds wedge threshold " - f"({self._wedge_queue_age_seconds}s)" + f"({_WEDGE_QUEUE_AGE_SECONDS}s)" ), confidence=0.4, ) ) + ready_count = sum( + 1 for a in assessments if a.readiness == EnumRunnerReadinessState.READY + ) + not_ready_count = sum( + 1 for a in assessments if a.readiness == EnumRunnerReadinessState.NOT_READY + ) + readiness_unknown_count = sum( + 1 for a in assessments if a.readiness == EnumRunnerReadinessState.UNKNOWN + ) + signal_rollups = tuple( + ModelReadinessSignalRollup( + signal=signal, + fail_count=sum( + 1 + for a in assessments + for s in a.signals + if s.signal == signal and s.outcome == _EnumOutcome.FAIL + ), + unknown_count=sum( + 1 + for a in assessments + for s in a.signals + if s.signal == signal and s.outcome == _EnumOutcome.UNKNOWN + ), + ) + for signal in EnumRunnerReadinessSignal + ) + verdict = ModelRunnerFleetHealthVerdict( correlation_id=correlation_id, evaluated_at=datetime.now(tz=UTC), assessments=tuple(assessments), + ready_count=ready_count, + not_ready_count=not_ready_count, + readiness_unknown_count=readiness_unknown_count, + fleet_ready=ready_count > 0 and not_ready_count == 0, + quarantined_runners=tuple(a.name for a in assessments if a.quarantined), + bounce_eligible_runners=tuple( + a.name for a in assessments if a.bounce_eligible + ), + readiness_signal_rollups=signal_rollups, expected_count=snapshot.expected_count, observed_count=len(snapshot.runners), online_count=online_count, @@ -347,10 +800,16 @@ async def handle( docker_source_ok=snapshot.docker_source_ok, ) logger.info( - "Runner-fleet health verdict: %d/%d online, saturation=%.2f, %d recommended " + "Runner-fleet verdict: %d/%d online, %d READY / %d NOT_READY / %d UNKNOWN, " + "%d quarantined (%d bounce-eligible), saturation=%.2f, %d recommended " "actions (correlation_id=%s)", online_count, snapshot.expected_count, + ready_count, + not_ready_count, + readiness_unknown_count, + len(verdict.quarantined_runners), + len(verdict.bounce_eligible_runners), saturation_ratio, len(recommended_actions), correlation_id, diff --git a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/__init__.py b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/__init__.py index 19fb54e5e9..c027618c25 100644 --- a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/__init__.py +++ b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/__init__.py @@ -2,12 +2,24 @@ # SPDX-License-Identifier: MIT """Models for the runner-fleet health compute node.""" +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_readiness_signal_outcome import ( + EnumReadinessSignalOutcome, +) from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_recommended_action_type import ( EnumRecommendedActionType, ) from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_runner_fleet_health_state import ( EnumRunnerFleetHealthState, ) +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_runner_readiness_signal import ( + EnumRunnerReadinessSignal, +) +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_runner_readiness_state import ( + EnumRunnerReadinessState, +) +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_readiness_signal_rollup import ( + ModelReadinessSignalRollup, +) from omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_recommended_action import ( ModelRecommendedAction, ) @@ -20,12 +32,20 @@ from omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_runner_health_assessment import ( ModelRunnerHealthAssessment, ) +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_runner_readiness_signal import ( + ModelRunnerReadinessSignal, +) __all__ = [ + "EnumReadinessSignalOutcome", "EnumRecommendedActionType", "EnumRunnerFleetHealthState", + "EnumRunnerReadinessSignal", + "EnumRunnerReadinessState", + "ModelReadinessSignalRollup", "ModelRecommendedAction", "ModelRunnerFleetHealthEvaluateCommand", "ModelRunnerFleetHealthVerdict", "ModelRunnerHealthAssessment", + "ModelRunnerReadinessSignal", ] diff --git a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/enum_readiness_signal_outcome.py b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/enum_readiness_signal_outcome.py new file mode 100644 index 0000000000..7c2f6cbd80 --- /dev/null +++ b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/enum_readiness_signal_outcome.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Per-signal outcome for composite runner readiness (OMN-15255). + +Tri-state because two-state loses the distinction that matters: a probe that +returned "bad" and a probe that never returned are different facts, and +collapsing them is exactly the fail-open bug OMN-14228 Slice A fixed at the +source level (a failed docker probe reading as ``docker_restart_count=0``). +""" + +from __future__ import annotations + +from enum import StrEnum + + +class EnumReadinessSignalOutcome(StrEnum): + """Outcome of one readiness signal for one runner.""" + + PASS = "pass" + """The signal was probed and the runner satisfies it.""" + + FAIL = "fail" + """The signal was probed and the runner does NOT satisfy it.""" + + UNKNOWN = "unknown" + """The signal could not be determined (fact absent or its source failed).""" + + +__all__ = ["EnumReadinessSignalOutcome"] diff --git a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/enum_runner_readiness_signal.py b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/enum_runner_readiness_signal.py new file mode 100644 index 0000000000..c11c751b9b --- /dev/null +++ b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/enum_runner_readiness_signal.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""The individual signals composing runner readiness (OMN-15255, friction F-04). + +Each member is an independent question about one runner. Readiness is the +conjunction of all of them -- no single member is sufficient, which is the +whole point: on 2026-07-27T16:40Z the GitHub registry reported 64/64 online +while 53/64 containers read docker-unhealthy. Either surface alone gives the +wrong answer. + +Extending this enum does NOT require a model change: signals are carried as a +tuple of ``ModelRunnerReadinessSignal`` on the assessment. The two remaining +Recommended-4 signals (toolchain/image contract, tiny-governed-job probe) are +not implemented yet and are deliberately absent rather than stubbed PASS. +""" + +from __future__ import annotations + +from enum import StrEnum + + +class EnumRunnerReadinessSignal(StrEnum): + """One independently-probed readiness question.""" + + GITHUB_REGISTRATION = "github_registration" + """GitHub org registry reports this runner ``online``. Necessary: work is + dispatched by GitHub, so an offline runner cannot receive it regardless of + how healthy the container looks locally.""" + + DOCKER_HEALTH = "docker_health" + """The container's Docker health status is ``healthy`` (or ``none`` where + the image declares no healthcheck). Not an input to the legacy precedence + classifier at all -- the blind spot F-04 names.""" + + DIAG_HEARTBEAT = "diag_heartbeat" + """Newest ``_diag/*.log`` write is within the idle-cadence threshold. + OMN-15233: the threshold must bracket a full Runner.Listener token-refresh + cycle (measured ~50-53 min), which the retired 900s default did not.""" + + LISTENER_TOPOLOGY = "listener_topology" + """Exactly one ``Runner.Listener`` process, zero PPID-1 orphans. A + double-listener or an orphan reparented to init produces + ``TaskAgentSessionConflictException`` on the replacement, which the + registry still reports as ``online``.""" + + CONTAINER_STABILITY = "container_stability" + """Docker ``RestartCount`` is at or under the crash-loop threshold. Folded + in as a readiness signal (OMN-15255) so ``RESTART_RUNNER`` has exactly one + producer instead of a state-keyed branch plus a readiness rule.""" + + DISK_CAPACITY = "disk_capacity" + """Runner-host disk usage is under the ceiling. Deliberately NOT + bounce-remediable: recreating a container frees no host disk, so a disk + failure quarantines the runner and recommends no restart.""" + + +__all__ = ["EnumRunnerReadinessSignal"] diff --git a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/enum_runner_readiness_state.py b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/enum_runner_readiness_state.py new file mode 100644 index 0000000000..7c6d300bdc --- /dev/null +++ b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/enum_runner_readiness_state.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Composite per-runner readiness state (OMN-15255). + +Distinct from ``EnumRunnerFleetHealthState``, which is a *precedence* chain -- +the first matching bad state wins and the remaining signals are never +evaluated. Readiness is a *conjunction*: a runner is READY only when every +readiness signal passes, which is what friction F-04 asks for ("'online' +currently means registered, not ready to execute the governed workload"). + +Tri-state on purpose. ``UNKNOWN`` is not a polite ``READY``: a runner whose +signals could not be probed does not count as capacity, and it is also not +``NOT_READY`` -- a missing probe is not evidence of failure, and treating it +as one would quarantine the whole fleet the first time an SSH probe blips. +""" + +from __future__ import annotations + +from enum import StrEnum + + +class EnumRunnerReadinessState(StrEnum): + """Composite readiness verdict for one runner.""" + + READY = "ready" + """Every readiness signal PASSed. The runner may be routed governed work.""" + + NOT_READY = "not_ready" + """At least one readiness signal FAILed. Quarantined; may be bounce-eligible.""" + + UNKNOWN = "unknown" + """No signal FAILed but at least one could not be determined. Not routable, + not quarantined -- absence of evidence is not evidence of failure.""" + + +__all__ = ["EnumRunnerReadinessState"] diff --git a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_readiness_signal_rollup.py b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_readiness_signal_rollup.py new file mode 100644 index 0000000000..fa3fcd745a --- /dev/null +++ b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_readiness_signal_rollup.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Fleet-level rollup of one readiness signal (OMN-15255).""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_runner_readiness_signal import ( + EnumRunnerReadinessSignal, +) + + +class ModelReadinessSignalRollup(BaseModel): + """How many runners FAILed / could not determine one readiness signal. + + This is the fleet view's answer to "which surface disagrees, and about how + many runners" -- the thing an operator previously produced by hand by + diffing ``gh api .../actions/runners`` against ``docker ps`` against + per-container ``_diag`` mtimes. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + signal: EnumRunnerReadinessSignal = Field(..., description="Signal rolled up.") + fail_count: int = Field( + ..., ge=0, description="Runners whose probe returned FAIL for this signal." + ) + unknown_count: int = Field( + ..., ge=0, description="Runners whose probe could not determine this signal." + ) + + +__all__ = ["ModelReadinessSignalRollup"] diff --git a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_runner_fleet_health_verdict.py b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_runner_fleet_health_verdict.py index c6757e812e..28551e523a 100644 --- a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_runner_fleet_health_verdict.py +++ b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_runner_fleet_health_verdict.py @@ -10,6 +10,12 @@ ``buildx_determinate`` so a future remediation gate can fail CLOSED on indeterminate health instead of silently treating a source outage as a verified HEALTHY fleet. + +OMN-15255 (friction F-04) makes this model the single fleet view: composite +per-runner readiness, the quarantine set, the bounce-eligible subset, and +per-signal rollups. It replaces the manual comparison of three surfaces +(GitHub registry vs ``docker ps`` vs per-container ``_diag`` mtimes) that had +no adjudicating third surface. """ from __future__ import annotations @@ -19,6 +25,9 @@ from pydantic import BaseModel, ConfigDict, Field +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_readiness_signal_rollup import ( + ModelReadinessSignalRollup, +) from omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_recommended_action import ( ModelRecommendedAction, ) @@ -71,6 +80,49 @@ class ModelRunnerFleetHealthVerdict(BaseModel): codeload_throttle_signal_count: int = Field( default=0, ge=0, description="Codeload-throttle failure signatures observed." ) + ready_count: int = Field( + default=0, + ge=0, + description=( + "Runners whose composite readiness is READY (OMN-15255). This -- " + "not `online_count` -- is the fleet's usable capacity. On " + "2026-07-27T16:40Z `online_count` was 64 while 53 containers read " + "docker-unhealthy; `ready_count` is the number that reconciles." + ), + ) + not_ready_count: int = Field( + default=0, ge=0, description="Runners with at least one FAILing signal." + ) + readiness_unknown_count: int = Field( + default=0, + ge=0, + description="Runners with no FAILing signal but at least one undetermined.", + ) + fleet_ready: bool = Field( + default=False, + description=( + "True iff at least one runner is READY and none are NOT_READY. " + "A single quarantined runner makes the fleet not-ready as a whole " + "-- it does not mean CI is down, it means the fleet view has an " + "open item." + ), + ) + quarantined_runners: tuple[str, ...] = Field( + default_factory=tuple, + description="Names of runners marked quarantined (readiness NOT_READY).", + ) + bounce_eligible_runners: tuple[str, ...] = Field( + default_factory=tuple, + description=( + "Subset of `quarantined_runners` for which a force-recreate is a " + "defensible remedy. The ONLY source of RESTART_RUNNER " + "recommendations (OMN-15255)." + ), + ) + readiness_signal_rollups: tuple[ModelReadinessSignalRollup, ...] = Field( + default_factory=tuple, + description="Per-signal FAIL/UNKNOWN counts across the fleet.", + ) recommended_actions: tuple[ModelRecommendedAction, ...] = Field( default_factory=tuple, description="Recorded/surfaced remediation recommendations. NEVER executed here.", diff --git a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_runner_health_assessment.py b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_runner_health_assessment.py index ca9049f305..54c4cf72c7 100644 --- a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_runner_health_assessment.py +++ b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_runner_health_assessment.py @@ -7,6 +7,17 @@ re-arm signals that today survive only as free text in ``detail``. This slice does not add any executor or gate logic -- it only stops dropping data a future gate would need. + +OMN-15255 adds the composite readiness verdict (friction F-04): ``state`` is a +precedence pick, ``readiness`` is a conjunction over ``signals``. The two can +legitimately disagree -- a GitHub-online runner with a fresh heartbeat, an +unhealthy container and two listeners is ``state=HEALTHY`` and +``readiness=NOT_READY``, because container health and listener topology are +not inputs to the precedence chain at all. + +OMN-15234 carries GitHub/Docker corroboration facts on the assessment so a +LISTENER_ZOMBIE restart recommendation cannot be derived from stale heartbeat +text alone. """ from __future__ import annotations @@ -16,6 +27,12 @@ from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_runner_fleet_health_state import ( EnumRunnerFleetHealthState, ) +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_runner_readiness_state import ( + EnumRunnerReadinessState, +) +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.model_runner_readiness_signal import ( + ModelRunnerReadinessSignal, +) class ModelRunnerHealthAssessment(BaseModel): @@ -59,6 +76,78 @@ class ModelRunnerHealthAssessment(BaseModel): "probe could not determine an age." ), ) + github_status: str = Field( + ..., + description=( + "GitHub registry status at classification time ('online'/'offline'/" + "'not_registered'). OMN-15234: this is the registry cross-check the " + "LISTENER_ZOMBIE restart recommendation is corroborated against -- " + "'if the registry says online, the stale-heartbeat flag is the bug' " + "(OMN-15233 runbook rule). Typed, not parsed out of `detail`." + ), + ) + github_busy: bool = Field( + ..., + description=( + "Whether GitHub reported a job in flight at classification time. " + "OMN-15234: a busy runner is never recommended for restart on " + "heartbeat staleness -- restarting it would kill a live job." + ), + ) + docker_status: str = Field( + default="", + description=( + "Docker container state at classification time (running/restarting/" + "not_found/...). Empty when the docker probe reported none. " + "OMN-15234: corroborating evidence for the LISTENER_ZOMBIE restart " + "recommendation." + ), + ) + readiness: EnumRunnerReadinessState = Field( + default=EnumRunnerReadinessState.UNKNOWN, + description=( + "Composite readiness (OMN-15255): READY only when every signal in " + "`signals` PASSes. NOT_READY when any FAILs. UNKNOWN when none " + "FAIL but at least one is undetermined. Consumers routing work " + "MUST require READY -- `state == HEALTHY` is NOT equivalent and " + "never was (the precedence chain does not evaluate container " + "health or listener topology at all)." + ), + ) + signals: tuple[ModelRunnerReadinessSignal, ...] = Field( + default_factory=tuple, + description=( + "Every readiness signal evaluated for this runner, PASSing ones " + "included, so a reader can tell 'checked and fine' from 'never " + "checked'." + ), + ) + quarantined: bool = Field( + default=False, + description=( + "True iff readiness is NOT_READY -- i.e. a signal was probed and " + "FAILed. Deliberately NOT set for UNKNOWN: a probe outage is not " + "evidence of runner failure, and quarantining on it would take " + "the fleet down on the first blip. UNKNOWN runners are excluded " + "from routing capacity by `readiness != READY`, which is the " + "fail-closed half of this pair." + ), + ) + quarantine_reason: str = Field( + default="", + description="Failing signal names + observed values. Empty when not quarantined.", + ) + bounce_eligible: bool = Field( + default=False, + description=( + "True iff a force-recreate is a defensible remedy for THIS " + "runner: quarantined, sources determinate, not executing a job, " + "and at least one failing signal is actually fixable by a bounce. " + "Strictly narrower than `quarantined` -- a full host disk or a " + "GitHub status-lag with healthy local evidence (OMN-14057) " + "quarantines without ever recommending a restart." + ), + ) __all__ = ["ModelRunnerHealthAssessment"] diff --git a/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_runner_readiness_signal.py b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_runner_readiness_signal.py new file mode 100644 index 0000000000..216c3ef779 --- /dev/null +++ b/src/omnibase_infra/nodes/node_runner_fleet_health_compute/models/model_runner_readiness_signal.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""One evaluated readiness signal for one runner (OMN-15255).""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_readiness_signal_outcome import ( + EnumReadinessSignalOutcome, +) +from omnibase_infra.nodes.node_runner_fleet_health_compute.models.enum_runner_readiness_signal import ( + EnumRunnerReadinessSignal, +) + + +class ModelRunnerReadinessSignal(BaseModel): + """The evaluated outcome of a single readiness signal. + + Every signal is emitted for every runner on every tick, including the + PASSing ones. A fleet view that only reports failures cannot answer "was + this checked?" -- which is the question the manual three-surface + comparison existed to answer. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + signal: EnumRunnerReadinessSignal = Field( + ..., description="Which readiness question this outcome answers." + ) + outcome: EnumReadinessSignalOutcome = Field( + ..., description="PASS / FAIL / UNKNOWN for this runner." + ) + detail: str = Field( + default="", + description=( + "Observed value and threshold in human-readable form. Populated " + "for FAIL and UNKNOWN; empty for a plain PASS." + ), + ) + + +__all__ = ["ModelRunnerReadinessSignal"] diff --git a/src/omnibase_infra/nodes/node_runner_fleet_maintain_orchestrator/contract.yaml b/src/omnibase_infra/nodes/node_runner_fleet_maintain_orchestrator/contract.yaml index 54ee4c95b0..2189c5987b 100644 --- a/src/omnibase_infra/nodes/node_runner_fleet_maintain_orchestrator/contract.yaml +++ b/src/omnibase_infra/nodes/node_runner_fleet_maintain_orchestrator/contract.yaml @@ -121,7 +121,7 @@ published_events: description: "Terminal event for one runner-fleet-maintain tick (health report only, no mutation)" topic_config: partitions: 1 - replication_factor: 1 + replication_factor: 2 kafka_config: "retention.ms": "604800000" "cleanup.policy": "delete" diff --git a/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/contract.yaml b/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/contract.yaml index 8306a351c3..d05b2dca5b 100644 --- a/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/contract.yaml +++ b/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/contract.yaml @@ -26,18 +26,25 @@ # whose raw facts this node now gathers (classification lives in # node_runner_fleet_health_compute) # - OMN-13932: motivating gap (buildx/codeload blind spots) +# - OMN-15255 (friction F-04): gathers the three facts the composite readiness +# signals need and that no prior probe collected -- container Docker health +# status, Runner.Listener process/orphan (PPID 1) topology, and runner-host +# disk used-percent. All three are read-only inspects/ps/df. Unknowns are +# reported as unknown (empty health, None counts, None disk) and are never +# defaulted to a passing value, so a failed probe becomes an UNKNOWN readiness +# signal downstream rather than a fabricated PASS. name: "node_runner_health_snapshot_effect" contract_version: major: 1 - minor: 0 + minor: 1 patch: 0 node_version: major: 1 - minor: 0 + minor: 1 patch: 0 node_type: "EFFECT_GENERIC" description: > - Effect node for runner-fleet snapshot gathering. Publishes read-only, facts-only health snapshots from GitHub Actions runners + Docker inspection via the event bus. Completed for OMN-13942 (was a contract stub since OMN-6091). + Effect node for runner-fleet snapshot gathering. Publishes read-only, facts-only health snapshots from GitHub Actions runners + Docker inspection via the event bus, including container health status, Runner.Listener process/orphan topology, and runner-host disk usage (OMN-15255). Completed for OMN-13942 (was a contract stub since OMN-6091). input_model: name: "ModelRunnerFleetSnapshotGatherCommand" @@ -46,7 +53,7 @@ input_model: output_model: name: "ModelRunnerFleetSnapshot" module: "omnibase_infra.nodes.node_runner_health_snapshot_effect.models.model_runner_fleet_snapshot" - description: "Facts-only runner-fleet snapshot (no classification)." + description: "Facts-only runner-fleet snapshot (no classification), incl. OMN-15255 readiness facts: per-runner docker_health / listener_process_count / orphaned_listener_count and snapshot-level host_disk_used_percent." handler_routing: routing_strategy: "payload_type_match" handlers: @@ -77,7 +84,7 @@ published_events: description: "Periodic health snapshot from GitHub Actions runners" topic_config: partitions: 1 - replication_factor: 1 + replication_factor: 2 kafka_config: "retention.ms": "604800000" "cleanup.policy": "delete" @@ -87,7 +94,7 @@ published_events: description: "Periodic Docker subnet-pool occupancy snapshot from runner hosts" topic_config: partitions: 1 - replication_factor: 1 + replication_factor: 2 kafka_config: "retention.ms": "604800000" "cleanup.policy": "delete" @@ -96,6 +103,8 @@ metadata: author: "OmniNode Team" license: "MIT" created: "2026-03-31" + updated: "2026-07-27" + ticket: "OMN-15255" tags: - effect - health diff --git a/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/handlers/handler_runner_fleet_snapshot.py b/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/handlers/handler_runner_fleet_snapshot.py index b08809e50d..55d5987273 100644 --- a/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/handlers/handler_runner_fleet_snapshot.py +++ b/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/handlers/handler_runner_fleet_snapshot.py @@ -17,6 +17,9 @@ - Docker buildx availability probe (NEW -- closes an OMN-13932 blind spot) - Codeload-throttle failure-signature probe (NEW -- closes an OMN-13932 blind spot) + - Container health status, Runner.Listener process/orphan topology, and + runner-host disk usage (OMN-15255) -- the three facts the composite + readiness signals need and that no prior probe gathered """ from __future__ import annotations @@ -44,6 +47,10 @@ logger = logging.getLogger(__name__) +# Thresholds are overlay-resolved through ModelRunnerFleetConfig (OMN-15195); +# the values below stay mirrored from runner-monitor.sh / healthcheck.sh +# (OMN-13109, OMN-13912, OMN-13915) so the node and the bash surfaces agree on +# what "stale"/"old" means during the trust-building period. _DEFAULT_WATCH_REPOS = ( "OmniNode-ai/omnibase_infra", "OmniNode-ai/omnibase_core", @@ -58,6 +65,21 @@ ) +def _optional_count(raw: str) -> int | None: + """Parse a probe count, mapping the ``-1``/unparseable sentinels to None. + + ``None`` means "the probe could not look", which the readiness classifier + must render as UNKNOWN. Returning ``0`` here instead would assert "no + listener is running" on a failed exec -- a fabricated FAIL, the mirror of + the fail-open bug OMN-14228 fixed in the other direction. + """ + try: + value = int(raw) + except (TypeError, ValueError): + return None + return None if value < 0 else value + + class HandlerRunnerFleetSnapshot: """Gathers a read-only, facts-only runner-fleet snapshot. @@ -116,13 +138,27 @@ async def _fetch_github_runners(self) -> tuple[list[dict[str, object]], str | No async def _fetch_docker_facts( self, ) -> tuple[dict[str, dict[str, str]], str | None]: - """Fetch Docker container status + RestartCount + _diag heartbeat age via SSH. + """Fetch Docker status, RestartCount, health, listener topology, heartbeat. The heartbeat probe execs into the container to check ``${RUNNER_HOME}/_diag`` (default ``/home/runner/actions-runner``) -- mirroring the anchored-path check OMN-13915 added to ``healthcheck.sh``. Reports ``-1`` when the container/probe cannot determine an age (treated as unknown, not zero, by the caller). + + OMN-15255 adds two facts the composite readiness signals need and the + legacy probe never gathered: + + * ``health`` -- the container's Docker healthcheck status. ``none`` + when the image declares no healthcheck; empty is impossible here + because the inspect template always resolves to one of the two. + * ``listeners`` / ``orphans`` -- live ``Runner.Listener`` process count + and how many of them are reparented to PPID 1. Both report ``-1`` + (unknown) rather than ``0`` when the exec fails, because "no + listener" and "could not look" must not classify the same. + + Every added probe is read-only: ``inspect``, ``ps``, ``grep``. Nothing + in this command can mutate a container. """ prefix = self._config.runner_name_prefix cmd = ( @@ -131,13 +167,27 @@ async def _fetch_docker_facts( "status=$(docker inspect --format '{{.State.Status}}' \"$name\"); " "restart_count=$(docker inspect --format '{{.RestartCount}}' \"$name\"); " "uptime=$(docker ps -a --filter \"name=^/${name}$\" --format '{{.Status}}'); " + "health=$(docker inspect --format " + "'{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' " + '"$name" 2>/dev/null | tail -1); ' + '[ -n "$health" ] || health=unknown; ' + 'listeners=$(docker exec "$name" sh -c ' + "'ps -eo ppid=,args= | grep -c \"[R]unner.Listener\" || true' " + "2>/dev/null | tail -1); " + '[ -n "$listeners" ] || listeners=-1; ' + 'orphans=$(docker exec "$name" sh -c ' + '\'ps -eo ppid=,args= | grep "[R]unner.Listener" ' + '| grep -c "^ *1 " || true\' ' + "2>/dev/null | tail -1); " + '[ -n "$orphans" ] || orphans=-1; ' 'diag_age=$(docker exec "$name" bash -c ' '\'f=$(ls -t "${RUNNER_HOME:-/home/runner/actions-runner}/_diag"/*.log ' "2>/dev/null | head -1); " 'if [ -n "$f" ]; then echo $(( $(date +%s) - $(stat -c %Y "$f") )); ' "else echo -1; fi' 2>/dev/null || echo -1); " - 'printf "%s\\t%s\\t%s\\t%s\\t%s\\n" ' - '"$name" "$status" "$restart_count" "$uptime" "$diag_age"; ' + 'printf "%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n" ' + '"$name" "$status" "$restart_count" "$uptime" "$diag_age" ' + '"$health" "$listeners" "$orphans"; ' "done" ) proc = await asyncio.create_subprocess_exec( @@ -155,17 +205,56 @@ async def _fetch_docker_facts( ) result: dict[str, dict[str, str]] = {} for line in stdout.decode(errors="replace").strip().splitlines(): - parts = line.split("\t", 4) - if len(parts) == 5: - name, status, restart_count, uptime, diag_age = parts - result[name] = { - "status": status, - "restart_count": restart_count, - "uptime": uptime, - "diag_age": diag_age, - } + parts = line.split("\t", 7) + if len(parts) < 5: + continue + # OMN-15255 fields are read positionally and default to the + # unknown sentinels when a shorter line arrives, so a host still + # running the pre-OMN-15255 probe degrades to UNKNOWN readiness + # signals rather than to a fabricated PASS. + padded = parts + ["unknown", "-1", "-1"][len(parts) - 5 :] + name, status, restart_count, uptime, diag_age = padded[:5] + health, listeners, orphans = padded[5:8] + result[name] = { + "status": status, + "restart_count": restart_count, + "uptime": uptime, + "diag_age": diag_age, + "health": health, + "listeners": listeners, + "orphans": orphans, + } return result, None + async def _fetch_host_disk_used_percent(self) -> tuple[float | None, str | None]: + """Probe used-percent of the runner host's Docker filesystem (OMN-15255). + + Host-scoped by nature -- every runner container shares the host's + disk, so this single fact drives the DISK_CAPACITY readiness signal + for the whole fleet. Falls back to ``/`` when ``/var/lib/docker`` is + not a distinct mount. Returns ``None`` (unknown) on any probe failure; + never a fabricated 0.0. + """ + proc = await asyncio.create_subprocess_exec( + "ssh", + self._config.runner_host, + "{ df -P /var/lib/docker 2>/dev/null || df -P /; } " + "| awk 'NR==2 {gsub(/%/, \"\", $5); print $5}'", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + return None, ( + f"host disk probe SSH exit code {proc.returncode}: " + f"{stderr.decode(errors='replace').strip()[:200]}" + ) + raw = stdout.decode(errors="replace").strip() + try: + return float(raw), None + except ValueError: + return None, f"unexpected host disk probe output: {raw!r}" + async def _fetch_queue_facts( self, ) -> tuple[float | None, tuple[ModelZombieRunCandidate, ...], str | None]: @@ -326,18 +415,21 @@ async def handle(self, correlation_id: UUID) -> ModelRunnerFleetSnapshot: queue_result, buildx_result, codeload_result, + disk_result, ) = await asyncio.gather( self._fetch_github_runners(), self._fetch_docker_facts(), self._fetch_queue_facts(), self._fetch_buildx_available(), self._fetch_codeload_throttle_signals(), + self._fetch_host_disk_used_percent(), ) github_runners, gh_error = gh_result docker_facts, docker_error = docker_result oldest_queued_age, zombie_candidates, queue_error = queue_result buildx_available, buildx_error = buildx_result codeload_signal_count, codeload_examples, codeload_error = codeload_result + host_disk_used_percent, disk_error = disk_result source_errors: list[str] = [] for error in ( @@ -346,6 +438,7 @@ async def handle(self, correlation_id: UUID) -> ModelRunnerFleetSnapshot: queue_error, buildx_error, codeload_error, + disk_error, ): if error: source_errors.append(error) @@ -367,6 +460,9 @@ async def handle(self, correlation_id: UUID) -> ModelRunnerFleetSnapshot: "restart_count": "0", "uptime": "", "diag_age": "-1", + "health": "unknown", + "listeners": "-1", + "orphans": "-1", }, ) diag_age_raw = docker.get("diag_age", "-1") @@ -375,6 +471,7 @@ async def handle(self, correlation_id: UUID) -> ModelRunnerFleetSnapshot: restart_count = int(docker.get("restart_count", "0")) except ValueError: restart_count = 0 + docker_health = docker.get("health", "unknown") facts.append( ModelRunnerFleetRunnerFact( name=name, @@ -384,6 +481,13 @@ async def handle(self, correlation_id: UUID) -> ModelRunnerFleetSnapshot: docker_uptime=docker.get("uptime", ""), docker_restart_count=restart_count, diag_heartbeat_age_seconds=diag_age, + docker_health="" if docker_health == "unknown" else docker_health, + listener_process_count=_optional_count( + docker.get("listeners", "-1") + ), + orphaned_listener_count=_optional_count( + docker.get("orphans", "-1") + ), ) ) @@ -400,6 +504,7 @@ async def handle(self, correlation_id: UUID) -> ModelRunnerFleetSnapshot: restart_count = int(docker_info.get("restart_count", "0")) except ValueError: restart_count = 0 + docker_health = docker_info.get("health", "unknown") facts.append( ModelRunnerFleetRunnerFact( name=docker_name, @@ -409,6 +514,13 @@ async def handle(self, correlation_id: UUID) -> ModelRunnerFleetSnapshot: docker_uptime=docker_info.get("uptime", ""), docker_restart_count=restart_count, diag_heartbeat_age_seconds=diag_age, + docker_health="" if docker_health == "unknown" else docker_health, + listener_process_count=_optional_count( + docker_info.get("listeners", "-1") + ), + orphaned_listener_count=_optional_count( + docker_info.get("orphans", "-1") + ), stale_registration=True, error="Docker container exists but not registered in GitHub", ) @@ -422,6 +534,7 @@ async def handle(self, correlation_id: UUID) -> ModelRunnerFleetSnapshot: runners=tuple(facts), oldest_queued_job_age_seconds=oldest_queued_age, zombie_run_candidates=zombie_candidates, + host_disk_used_percent=host_disk_used_percent, buildx_available=buildx_available, codeload_throttle_signal_count=codeload_signal_count, codeload_throttle_examples=codeload_examples, diff --git a/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/models/model_runner_fleet_runner_fact.py b/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/models/model_runner_fleet_runner_fact.py index 704465eb1d..948f6f64c9 100644 --- a/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/models/model_runner_fleet_runner_fact.py +++ b/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/models/model_runner_fleet_runner_fact.py @@ -35,6 +35,34 @@ class ModelRunnerFleetRunnerFact(BaseModel): docker_restart_count: int = Field( default=0, ge=0, description="Docker RestartCount (crash-loop signal)." ) + docker_health: str = Field( + default="", + description=( + "Docker healthcheck status (OMN-15255): 'healthy', 'unhealthy', " + "'starting', or 'none' when the image declares no healthcheck. " + "Empty string means the probe did not report a value -- UNKNOWN, " + "NOT healthy. This is the surface that read unhealthy on 53 of 64 " + "containers while the GitHub registry reported 64/64 online." + ), + ) + listener_process_count: int | None = Field( + default=None, + ge=0, + description=( + "Count of live Runner.Listener processes inside the container " + "(OMN-15255). Exactly 1 is correct; 0 is a dead listener and >1 is " + "the duplicate-listener mode that raises " + "TaskAgentSessionConflictException. None means undetermined." + ), + ) + orphaned_listener_count: int | None = Field( + default=None, + ge=0, + description=( + "Count of Runner.Listener processes reparented to PPID 1 " + "(OMN-15233 orphan mode). None means undetermined." + ), + ) diag_heartbeat_age_seconds: float | None = Field( default=None, description=( diff --git a/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/models/model_runner_fleet_snapshot.py b/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/models/model_runner_fleet_snapshot.py index 8548872245..8896810736 100644 --- a/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/models/model_runner_fleet_snapshot.py +++ b/src/omnibase_infra/nodes/node_runner_health_snapshot_effect/models/model_runner_fleet_snapshot.py @@ -49,6 +49,17 @@ class ModelRunnerFleetSnapshot(BaseModel): default_factory=tuple, description="Queued/in-progress runs observed past the wedge-age threshold.", ) + host_disk_used_percent: float | None = Field( + default=None, + ge=0.0, + description=( + "Used percentage of the runner host's Docker/work filesystem " + "(OMN-15255). Host-scoped, not per-runner: every container on the " + "host shares it, so a full disk fails the DISK_CAPACITY readiness " + "signal for the whole fleet at once. None means the probe could " + "not determine it." + ), + ) buildx_available: bool | None = Field( default=None, description=( diff --git a/src/omnibase_infra/nodes/node_runtime_manifest_reducer/contract.yaml b/src/omnibase_infra/nodes/node_runtime_manifest_reducer/contract.yaml index afffe5c2ed..2cbfd258e6 100644 --- a/src/omnibase_infra/nodes/node_runtime_manifest_reducer/contract.yaml +++ b/src/omnibase_infra/nodes/node_runtime_manifest_reducer/contract.yaml @@ -1,14 +1,26 @@ # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT # ONEX Node Contract - Runtime Manifest Reducer Node (OMN-11197) +# +# OMN-15512 (contract_version 1.1.0): the emitted intent payload +# ModelPayloadInsertRuntimeManifest gained one optional field, +# `attach_readiness: ModelRuntimeAttachReadiness | None` (default None), +# carrying the boot attach-readiness blocker set on the SAME +# runtime-manifest-published envelope this node already subscribes to. +# HandlerPostgresRuntimeManifestInsert flattens it across four new +# runtime_manifests columns (attach_state, attach_required_contracts, +# attach_attached_contracts, attach_not_ready_contracts) added by forward +# migration 095. Additive and backward-compatible: the field is optional and +# a payload omitting it writes attach_state='unknown' (never 'ready'). No +# change to subscribe_topics, handler routing, or the target table. (2026-07-30) contract_version: major: 1 - minor: 0 + minor: 1 patch: 0 -node_version: "1.0.0" +node_version: "1.1.0" name: "node_runtime_manifest_reducer" node_type: "REDUCER_GENERIC" -description: "Append-only reducer that persists runtime manifests to PostgreSQL. Receives runtime-manifest-published events and inserts one row per startup into runtime_manifests." +description: "Append-only reducer that persists runtime manifests to PostgreSQL. Receives runtime-manifest-published events and inserts one row per startup into runtime_manifests, including the boot attach-readiness blocker set (OMN-15512)." input_model: name: "ModelReducerInput" module: "omnibase_core.models.reducer.model_reducer_input" @@ -35,7 +47,7 @@ intent_emission: target_pattern: "postgres://runtime_manifests/{runtime_profile}" payload_model: "ModelPayloadInsertRuntimeManifest" payload_module: "omnibase_infra.nodes.node_runtime_manifest_reducer.models.model_payload_insert_runtime_manifest" - description: "INSERT runtime manifest row into PostgreSQL runtime_manifests table (append-only, never UPDATE)" + description: "INSERT runtime manifest row into PostgreSQL runtime_manifests table (append-only, never UPDATE). Writes 17 columns: the 13 base manifest columns plus the four OMN-15512 attach-readiness columns flattened from the payload's optional attach_readiness field — attach_state (EnumRuntimeReadinessState value — ready/degraded/failed — or 'unknown' when the aggregate is absent), attach_required_contracts and attach_attached_contracts (counts), and attach_not_ready_contracts (JSON array of the blocker-set results). Dedup key (runtime_profile, topology_hash, started_at) is unchanged." node_capabilities: processing: true dependencies: @@ -48,15 +60,18 @@ health_check: endpoint: "/health" interval_seconds: 30 metadata: - description: "Append-only reducer persisting runtime startup manifests to PostgreSQL." + description: "Append-only reducer persisting runtime startup manifests to PostgreSQL. OMN-15512 (contract_version 1.1.0): the intent payload carries an optional attach_readiness aggregate that the handler flattens into four runtime_manifests columns, making the boot NOT-READY blocker set durably queryable instead of log-only." author: "OmniNode Team" created: "2026-05-17" - updated: "2026-05-17" + updated: "2026-07-30" tags: - "reducer" - "runtime-manifest" - "projection" - "append-only" + - "attach-readiness" related_tickets: - "OMN-11197" - "OMN-11188" + - "OMN-15512" + - "OMN-15508" diff --git a/src/omnibase_infra/nodes/node_runtime_manifest_reducer/handlers/handler_postgres_runtime_manifest_insert.py b/src/omnibase_infra/nodes/node_runtime_manifest_reducer/handlers/handler_postgres_runtime_manifest_insert.py index 6c54433710..32e5b9e4f5 100644 --- a/src/omnibase_infra/nodes/node_runtime_manifest_reducer/handlers/handler_postgres_runtime_manifest_insert.py +++ b/src/omnibase_infra/nodes/node_runtime_manifest_reducer/handlers/handler_postgres_runtime_manifest_insert.py @@ -49,8 +49,12 @@ failed_contracts, ownership_violations, image_digest, - started_at -) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + started_at, + attach_state, + attach_required_contracts, + attach_attached_contracts, + attach_not_ready_contracts +) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) ON CONFLICT (runtime_profile, topology_hash, started_at) DO NOTHING RETURNING id; """ @@ -113,6 +117,24 @@ async def _execute_insert( payload: ModelPayloadInsertRuntimeManifest, correlation_id: UUID, ) -> None: + # OMN-15512: flatten the attach-readiness aggregate across four + # columns. `attach_state` stays 'unknown' (the column default) when the + # per-contract interleave never ran — deliberately distinct from + # 'ready', which asserts it ran and every contract attached. + readiness = payload.attach_readiness + if readiness is None: + attach_state = "unknown" + attach_required = 0 + attach_attached = 0 + attach_not_ready_json = "[]" + else: + attach_state = readiness.state.value + attach_required = readiness.required_contracts + attach_attached = readiness.attached_contracts + attach_not_ready_json = json.dumps( + [r.model_dump(mode="json") for r in readiness.not_ready_results] + ) + async with self._pool.acquire() as conn: row = await conn.fetchrow( SQL_INSERT_RUNTIME_MANIFEST, @@ -129,6 +151,10 @@ async def _execute_insert( json.dumps(list(payload.ownership_violations)), payload.image_digest, payload.started_at, + attach_state, + attach_required, + attach_attached, + attach_not_ready_json, ) if row is None: diff --git a/src/omnibase_infra/nodes/node_runtime_manifest_reducer/models/model_payload_insert_runtime_manifest.py b/src/omnibase_infra/nodes/node_runtime_manifest_reducer/models/model_payload_insert_runtime_manifest.py index d01d9b9d24..f2bb4f7ac4 100644 --- a/src/omnibase_infra/nodes/node_runtime_manifest_reducer/models/model_payload_insert_runtime_manifest.py +++ b/src/omnibase_infra/nodes/node_runtime_manifest_reducer/models/model_payload_insert_runtime_manifest.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT -"""Payload model for runtime manifest INSERT intent (OMN-11197).""" +"""Payload model for runtime manifest INSERT intent (OMN-11197 / OMN-15512).""" from __future__ import annotations @@ -9,6 +9,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator +from omnibase_infra.event_bus.model_runtime_attach_readiness import ( + ModelRuntimeAttachReadiness, +) from omnibase_infra.utils.util_pydantic_validators import ( validate_timezone_aware_datetime, ) @@ -41,6 +44,14 @@ class ModelPayloadInsertRuntimeManifest(BaseModel): ownership_violations: list[dict[str, object]] = Field(default_factory=list) image_digest: str | None = Field(default=None) started_at: datetime = Field(...) + # OMN-15512: boot attach-readiness aggregate, carried on the SAME + # runtime-manifest-published envelope. The producer narrows `results` to + # the blocker set (ModelRuntimeAttachReadiness.blockers_only), so this is + # bounded by the not-attached count, not by the 475+ contracts walked at + # boot. The handler flattens it across four runtime_manifests columns — + # this model is `extra="forbid"`, so the producer key name and this field + # name are a matched seam and drift fails loudly at coercion. + attach_readiness: ModelRuntimeAttachReadiness | None = Field(default=None) @field_validator("started_at") @classmethod diff --git a/src/omnibase_infra/nodes/node_setup_validate_effect/contract.yaml b/src/omnibase_infra/nodes/node_setup_validate_effect/contract.yaml index c3281a0daa..9fc9adcf6a 100644 --- a/src/omnibase_infra/nodes/node_setup_validate_effect/contract.yaml +++ b/src/omnibase_infra/nodes/node_setup_validate_effect/contract.yaml @@ -7,9 +7,12 @@ node_type: "EFFECT_GENERIC" contract_version: major: 1 minor: 0 - patch: 1 -node_version: "1.0.1" + patch: 2 +node_version: "1.0.2" changelog: + - version: "1.0.2" + ticket: "OMN-15717" + change: "Annotate the localhost health-check probe URL with a url-authority-ok marker (LOCAL topology probes a host-mapped Compose port on this deployment host) — no behavior change." - version: "1.0.1" ticket: "OMN-10705" change: "Export HandlerServiceValidate via handlers/__init__.py public API" diff --git a/src/omnibase_infra/nodes/node_setup_validate_effect/handlers/handler_service_validate.py b/src/omnibase_infra/nodes/node_setup_validate_effect/handlers/handler_service_validate.py index f5f29669a9..e7705275ff 100644 --- a/src/omnibase_infra/nodes/node_setup_validate_effect/handlers/handler_service_validate.py +++ b/src/omnibase_infra/nodes/node_setup_validate_effect/handlers/handler_service_validate.py @@ -216,7 +216,7 @@ async def execute( health_check_path = local_cfg.health_check_path if health_check_path is not None: - url = f"http://localhost:{host_port}{health_check_path}" + url = f"http://localhost:{host_port}{health_check_path}" # url-authority-ok: LOCAL topology probes a host-mapped Compose port on this deployment host. logger.debug("HTTP health check for %s: %s", svc_name, url) healthy, response_time_ms = await _check_http_health(url, timeout_s) message = f"HTTP {url} {'OK' if healthy else 'UNREACHABLE'}" diff --git a/src/omnibase_infra/nodes/node_topic_migration_executor_effect/contract.yaml b/src/omnibase_infra/nodes/node_topic_migration_executor_effect/contract.yaml index b473830e8f..a7d01a2054 100644 --- a/src/omnibase_infra/nodes/node_topic_migration_executor_effect/contract.yaml +++ b/src/omnibase_infra/nodes/node_topic_migration_executor_effect/contract.yaml @@ -24,7 +24,7 @@ node_version: patch: 0 node_type: "EFFECT_GENERIC" description: > - Effect node that executes a topic migration declared by a ModelTopicMigrationContract. Provisions the new topic via TopicProvisioner, mints the new consumer group via compute_consumer_group_id, and advances the migration through DUAL_WRITE -> DUAL_READ -> CUTOVER -> COMPLETE phases. The drain-proof gate blocks CUTOVER/COMPLETE until the old consumer group has fully drained the old topic. Emits a ModelTopicMigrationLifecycleEvent per advanced phase. + Effect node that executes a topic migration declared by a ModelTopicMigrationContract. Provisions the new topic via TopicProvisioner, passing the command-declared ModelTopicSpec (partitions + replication_factor) through to creation so the new topic is created to the migration's declared durability rather than a provisioner default. ModelTopicProvisioningPolicy resolves an undeclared replication_factor to the target profile's default (the managed durability floor on MSK, never 1) and REFUSES any value below that floor; on a cluster whose measured broker count cannot host the declared value it reduces to the measured capacity, at WARNING (OMN-15395). Mints the new consumer group via compute_consumer_group_id, and advances the migration through DUAL_WRITE -> DUAL_READ -> CUTOVER -> COMPLETE phases. The drain-proof gate blocks CUTOVER/COMPLETE until the old consumer group has fully drained the old topic. Emits a ModelTopicMigrationLifecycleEvent per advanced phase. event_bus: version: @@ -75,7 +75,7 @@ io_operations: dependencies: - name: "topic_provisioner" type: "service" - description: "TopicProvisioner for new-topic creation via ModelTopicSpec" + description: "TopicProvisioner for new-topic creation; the executor passes its built ModelTopicSpec through (spec=), so declared partitions/replication reach CreateTopics (OMN-15395)" required: true - name: "drain_proof_gate" type: "service" diff --git a/src/omnibase_infra/nodes/node_topic_migration_executor_effect/handlers/handler_topic_migration_executor.py b/src/omnibase_infra/nodes/node_topic_migration_executor_effect/handlers/handler_topic_migration_executor.py index 9a8a9187f5..de49e1bd2a 100644 --- a/src/omnibase_infra/nodes/node_topic_migration_executor_effect/handlers/handler_topic_migration_executor.py +++ b/src/omnibase_infra/nodes/node_topic_migration_executor_effect/handlers/handler_topic_migration_executor.py @@ -121,7 +121,12 @@ async def execute( partitions=command.new_topic_partitions, replication_factor=command.new_topic_replication_factor, ) - new_provisioned = await self._provisioner.ensure_topic_exists(spec.suffix) + # OMN-15395 (c): thread the built spec through — dropping it here + # meant the new topic was created at the provisioner's bare default + # replication factor rather than the migration's declared one. + new_provisioned = await self._provisioner.ensure_topic_exists( + spec.suffix, spec=spec + ) detail = ( f"provisioned new topic {new_topic!r} " f"(partitions={command.new_topic_partitions}); " diff --git a/src/omnibase_infra/nodes/node_topic_migration_executor_effect/models/model_topic_migration_command.py b/src/omnibase_infra/nodes/node_topic_migration_executor_effect/models/model_topic_migration_command.py index 9bb9e860cd..3915b9c8a9 100644 --- a/src/omnibase_infra/nodes/node_topic_migration_executor_effect/models/model_topic_migration_command.py +++ b/src/omnibase_infra/nodes/node_topic_migration_executor_effect/models/model_topic_migration_command.py @@ -47,10 +47,15 @@ class ModelTopicMigrationCommand(BaseModel): ge=1, description="Partition count to provision for the new topic", ) - new_topic_replication_factor: int = Field( - default=1, + new_topic_replication_factor: int | None = Field( + default=None, ge=1, - description="Replication factor to provision for the new topic", + description=( + "Replication factor to provision for the new topic. None means the " + "command declares none, and the environment's replication policy " + "resolves it — or refuses, on a managed cluster. It is NOT silently " + "1 (OMN-15395)." + ), ) dual_publish: bool = Field( default=False, diff --git a/src/omnibase_infra/nodes/node_vector_store_effect/contract_descriptor.py b/src/omnibase_infra/nodes/node_vector_store_effect/contract_descriptor.py index f7c8e10a90..8b53502e10 100644 --- a/src/omnibase_infra/nodes/node_vector_store_effect/contract_descriptor.py +++ b/src/omnibase_infra/nodes/node_vector_store_effect/contract_descriptor.py @@ -7,7 +7,7 @@ health probe) connect to (OMN-13558 Wave-1 endpoint→overlay migration). It is declared with the ``${env.VAR}`` overlay convention so an operator overlay / the per-lane service env supplies the real endpoint per lane — never a hardcoded -``http://localhost:6333`` in source. +loopback endpoint in source. Resolution goes through ``expand_contract_env_refs`` — the one sanctioned env-reading boundary in the overlay package — so the handlers never read @@ -40,10 +40,9 @@ def contract_qdrant_url(contract_path: Path = _CONTRACT) -> str: """Return the resolved ``descriptor.qdrant_url`` for the vector-store node. The value is contract-declared (overridable by an operator overlay contract) - via the ``${env.QDRANT_URL}`` convention — never hardcoded in source. Fails - closed: raises ``ValueError`` when the field is absent or resolves to an empty - string, so the vector handlers never silently fall back to - ``http://localhost:6333`` when ``QDRANT_URL`` is unset. + via the ``${env.QDRANT_URL}`` convention — never hardcoded in source. It fails + closed when the field is absent or resolves to an empty string, so vector + handlers never silently fall back to an undeclared local endpoint. """ raw = _load_contract(contract_path) descriptor = raw.get("descriptor") @@ -65,7 +64,7 @@ def contract_qdrant_url(contract_path: Path = _CONTRACT) -> str: "descriptor.qdrant_url resolved empty — set QDRANT_URL (the Qdrant " "HTTP endpoint the vector-store effect connects to). The vector-store " "effect fails closed rather than silently default to " - "http://localhost:6333." + "an undeclared local endpoint." ) return resolved diff --git a/src/omnibase_infra/protocols/protocol_gateway_transport.py b/src/omnibase_infra/protocols/protocol_gateway_transport.py new file mode 100644 index 0000000000..029ca4b2c1 --- /dev/null +++ b/src/omnibase_infra/protocols/protocol_gateway_transport.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""POST seam for the ``onex auth`` gateway client (OMN-15922). + +The gateway client is pure logic over THIS protocol; the one component that +actually opens a socket is the EFFECT adapter that implements it +(``omnibase_infra.gateway.client.gateway_transport_httpx``). Keeping +the seam means the whole credential -> JWT -> Bearer -> attach -> re-attach +cycle is driven in tests by an in-memory fake, with no network and no sleeping, +while the concrete adapter stays small enough to read in one sitting. + +Why a distinct protocol rather than adding ``post`` to ``ProtocolHttpClient``: + ``ProtocolHttpClient`` is ``@runtime_checkable`` and is already satisfied + structurally by adapters implementing exactly ``get``. Adding a method + would silently un-satisfy every one of them -- a breaking change to an + existing seam in service of a new caller. The two POST shapes this client + needs are also genuinely different concerns: an OAuth2 token endpoint takes + ``application/x-www-form-urlencoded`` (RFC 6749 s2.3.1) and the gateway + takes JSON, and collapsing them into one ``post(body)`` would push + content-type assembly into every caller. + +Both methods return ``ProtocolHttpResponse`` -- the response seam ``omnibase_core`` +already defines -- so an implementation reuses whatever adapter it has. Callers +here read ``status`` and ``text()`` only, never ``json()``: the bodies are +parsed through typed Pydantic models, so an ``Any``-returning hop would throw +away the typing the models exist to provide. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Protocol, runtime_checkable + +from omnibase_core.protocols.http.protocol_http_client import ProtocolHttpResponse + +__all__ = ["ProtocolGatewayTransport"] + + +@runtime_checkable +class ProtocolGatewayTransport(Protocol): + """Async POST transport for the token endpoint and the gateway.""" + + async def post_form( + self, + url: str, + *, + form: Mapping[str, str], + headers: Mapping[str, str], + ) -> ProtocolHttpResponse: + """POST ``form`` as ``application/x-www-form-urlencoded``. + + Used for the OAuth2 ``client_credentials`` grant. Implementations MUST + NOT log the form -- it carries ``client_secret``. + + Args: + url: Absolute token-endpoint URL. + form: Form fields to url-encode as the request body. + headers: Headers to send; the implementation owns Content-Type. + + Returns: + The response, with status and body available for the caller to + classify. Implementations must not raise on non-2xx -- fail-closed + classification is the caller's, and it needs the status to say + anything useful about it. + """ + ... + + async def post_json( + self, + url: str, + *, + body: str, + headers: Mapping[str, str], + ) -> ProtocolHttpResponse: + """POST an already-serialized JSON ``body``. + + The body arrives serialized rather than as a mapping so the caller's + Pydantic model stays the single authority on the wire shape (field + order, datetime encoding, alias handling) instead of that being + re-decided by whichever JSON encoder the adapter happens to use. + + Args: + url: Absolute gateway URL. + body: Serialized JSON request body. + headers: Headers to send, including ``Authorization``. + + Returns: + The response, unclassified -- see ``post_form``. + """ + ... diff --git a/src/omnibase_infra/runtime/auto_wiring/__init__.py b/src/omnibase_infra/runtime/auto_wiring/__init__.py index f8e3b15519..8c5f11ef4c 100644 --- a/src/omnibase_infra/runtime/auto_wiring/__init__.py +++ b/src/omnibase_infra/runtime/auto_wiring/__init__.py @@ -19,6 +19,9 @@ EnumQuarantineReason, ) from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + build_unwired_contract_results, + reattach_not_ready_contracts, + run_not_ready_reconciliation_loop, subscribe_wired_contract_topics, wire_from_manifest, ) @@ -75,9 +78,12 @@ "ModelQuarantineRecord", "ModelQuarantinedWiring", "ModelRuntimeProfileOwnershipResult", + "build_unwired_contract_results", "discover_contracts", "discover_contracts_from_paths", "filter_manifest_for_runtime_profile", + "reattach_not_ready_contracts", + "run_not_ready_reconciliation_loop", "subscribe_wired_contract_topics", "wire_from_manifest", ] diff --git a/src/omnibase_infra/runtime/auto_wiring/db_table_validator.py b/src/omnibase_infra/runtime/auto_wiring/db_table_validator.py index d2621f5141..f685c78fb4 100644 --- a/src/omnibase_infra/runtime/auto_wiring/db_table_validator.py +++ b/src/omnibase_infra/runtime/auto_wiring/db_table_validator.py @@ -1,93 +1,77 @@ # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT -"""Warning-based preflight validation for declared db_tables. +"""Typed physical-table preflight for discovered ``db_io`` declarations. -This module is a pre-wiring step that checks whether tables declared in -contract db_io.db_tables actually exist in the target database before -wire_from_manifest is called. - -This is the WARNING phase — does NOT block wiring. Strict blocking is -deferred to Phase 2 once all contracts are complete and validated in -production. Degraded operation is preferable to a hard startup failure -for missing tables. +Global exactly-one ownership is enforced by +``validation.application_relation_ownership``. This optional runtime preflight +has the narrower job of warning when an already-typed table is absent from the +current PostgreSQL connection. It never parses raw contract dictionaries and it +never invents a default database or schema. """ from __future__ import annotations import logging +from collections.abc import Sequence -logger = logging.getLogger(__name__) - - -async def validate_db_tables(contracts: list[dict], db_conn: object) -> list[dict]: - """Preflight check: warn when declared db_tables don't exist in the target database. - - Iterates over each contract's db_io.db_tables declarations and queries - pg_tables to verify existence. Returns structured warning dicts for any - missing table but does NOT raise — wiring continues regardless. +from omnibase_infra.runtime.auto_wiring.models import ModelDiscoveredContract +from omnibase_infra.runtime.auto_wiring.models.model_db_table_validation_warning import ( + ModelDbTableValidationWarning, +) +from omnibase_infra.runtime.auto_wiring.protocol_db_table_catalog_connection import ( + ProtocolDbTableCatalogConnection, +) - This is the warning phase. Strict blocking is Phase 2. +logger = logging.getLogger(__name__) - Args: - contracts: List of raw contract dicts (as loaded from contract.yaml). - Each may contain a ``db_io.db_tables`` list. - db_conn: An asyncpg connection (or compatible mock). Must support - ``await conn.fetchval(query, *args)``. - Returns: - List of warning dicts. Each dict has keys: - - ``reason``: always ``"missing_db_table"`` - - ``severity``: always ``"warning"`` - - ``details``: dict with ``table``, ``database``, and ``node`` keys. - Empty list if all declared tables exist (or no tables are declared). - """ - warnings: list[dict] = [] +async def validate_db_tables( + contracts: Sequence[ModelDiscoveredContract], + db_conn: ProtocolDbTableCatalogConnection, +) -> tuple[ModelDbTableValidationWarning, ...]: + """Warn for typed declared tables absent from the current connection.""" + warnings: list[ModelDbTableValidationWarning] = [] for contract in contracts: - db_io = contract.get("db_io", {}) or {} - db_tables = db_io.get("db_tables", []) or [] - for table_decl in db_tables: - table_name = table_decl.get("name") - if not table_name: - logger.warning( - "Contract %s has a db_tables entry missing 'name'; skipping.", - contract.get("name"), - ) + if contract.db_io is None: + continue + for table in contract.db_io.db_tables: + exists = await _table_exists(db_conn, table.schema, table.name) + if exists: continue - database = table_decl.get("database", "omnidash_analytics") - node_name = contract.get("name") - exists = await _table_exists(db_conn, table_name) - if not exists: - warnings.append( - { - "reason": "missing_db_table", - "severity": "warning", - "details": { - "table": table_name, - "database": database, - "node": node_name, - }, - } - ) - logger.warning( - "Node %s declares table %s but it does not exist in %s. " - "Run migrations before starting this node.", - node_name, - table_name, - database, - ) - return warnings - + warning = ModelDbTableValidationWarning( + table=table.name, + database_ref=table.database_ref, + schema=table.schema, + node=contract.name, + ) + warnings.append(warning) + logger.warning( + "Node %s declares missing table %s.%s in database_ref=%s. " + "Run the declared migration before starting this node.", + contract.name, + table.schema, + table.name, + table.database_ref, + ) + return tuple(warnings) -async def _table_exists(db_conn: object, table_name: str) -> bool: - """Return True if *table_name* exists in pg_tables (public schema). - Uses a parameterised query against the PostgreSQL information schema to - avoid any possibility of SQL injection from contract-sourced table names. - Checks the current connection's database only. - """ - # Why: Optional dependency or runtime adapter exposes this attribute dynamically. - row = await db_conn.fetchval( # type: ignore[attr-defined] - "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename = $1", +async def _table_exists( + db_conn: ProtocolDbTableCatalogConnection, + schema: str, + table_name: str, +) -> bool: + """Check the exact typed schema/name pair on the current connection.""" + row = await db_conn.fetchval( + "SELECT tablename FROM pg_tables WHERE schemaname = $1 AND tablename = $2", + schema, table_name, ) return row is not None + + +__all__ = [ + "ModelDbTableValidationWarning", + "ProtocolDbTableCatalogConnection", + "validate_db_tables", +] diff --git a/src/omnibase_infra/runtime/auto_wiring/discovery.py b/src/omnibase_infra/runtime/auto_wiring/discovery.py index 3df34b26e2..a0fdc43cf8 100644 --- a/src/omnibase_infra/runtime/auto_wiring/discovery.py +++ b/src/omnibase_infra/runtime/auto_wiring/discovery.py @@ -21,6 +21,9 @@ import yaml +from omnibase_core.models.contracts.subcontracts.model_db_ownership_subcontract import ( + ModelDbOwnershipSubcontract, +) from omnibase_infra.runtime.auto_wiring.models import ( ModelAutoWiringManifest, ModelContractVersion, @@ -414,11 +417,14 @@ def _parse_contract( event_bus = ModelEventBusWiring( subscribe_topics=tuple(eb_raw.get("subscribe_topics", [])), publish_topics=tuple(eb_raw.get("publish_topics", [])), + dlq_topics=tuple(eb_raw.get("dlq_topics", [])), + consumer_group=eb_raw.get("consumer_group"), consumer_purpose=eb_raw.get("consumer_purpose"), plugin_managed=_parse_bool_field(eb_raw, "plugin_managed", False), tenant_scoped_ingress=_parse_bool_field( eb_raw, "tenant_scoped_ingress", False ), + terminal_event=eb_raw.get("terminal_event"), ) # Extract handler routing — new format (handler_routing:) or legacy (handler:) @@ -437,6 +443,12 @@ def _parse_contract( handler_routing = _parse_legacy_handler(h_raw) runtime_profiles = _extract_runtime_profiles(raw) + db_io_raw = raw.get("db_io") + db_io = ( + ModelDbOwnershipSubcontract.model_validate(db_io_raw) + if db_io_raw is not None + else None + ) return ModelDiscoveredContract( name=raw.get("name", entry_point_name), @@ -450,10 +462,17 @@ def _parse_contract( package_version=package_version, runtime_profiles=runtime_profiles, compatibility_publish_topics=raw.get("compatibility_publish_topics"), - terminal_event=raw.get("terminal_event"), + terminal_event=( + raw.get("terminal_event") + if raw.get("terminal_event") is not None + else event_bus.terminal_event + if event_bus is not None + else None + ), requires_cloud_gateway=_contract_requires_cloud_gateway(raw), event_bus=event_bus, handler_routing=handler_routing, + db_io=db_io, ) @@ -564,6 +583,15 @@ def _parse_handler_routing( name=em_raw.get("name", ""), module=em_raw.get("module", ""), ) + elif isinstance(em_raw, str): + module_name, separator, model_name = em_raw.rpartition(".") + if not separator or not module_name or not model_name: + raise ValueError( + f"handler_routing.handlers[{index}].event_model must be a " + "fully qualified 'module.Model' string or a {name, module} " + "mapping" + ) + event_model = ModelHandlerRef(name=model_name, module=module_name) entries.append( ModelHandlerRoutingEntry( handler=handler_ref, diff --git a/src/omnibase_infra/runtime/auto_wiring/handler_wiring.py b/src/omnibase_infra/runtime/auto_wiring/handler_wiring.py index 6dccdbd4ea..eb97a6dc1c 100644 --- a/src/omnibase_infra/runtime/auto_wiring/handler_wiring.py +++ b/src/omnibase_infra/runtime/auto_wiring/handler_wiring.py @@ -24,6 +24,7 @@ import asyncio import concurrent.futures +import contextlib import hashlib import importlib import inspect @@ -34,9 +35,10 @@ import re import time from collections import defaultdict -from collections.abc import Awaitable, Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Collection, Iterator, Mapping, Sequence from dataclasses import dataclass, field from datetime import UTC, datetime +from functools import lru_cache from pathlib import Path from typing import ( TYPE_CHECKING, @@ -48,13 +50,25 @@ ) from uuid import UUID, uuid4 -from pydantic import BaseModel, ValidationError +from pydantic import AliasChoices, AliasPath, BaseModel, ValidationError from omnibase_core.enums.enum_core_error_code import EnumCoreErrorCode +from omnibase_core.enums.enum_database_grant_object_type import ( + EnumDatabaseGrantObjectType, +) +from omnibase_core.enums.enum_database_privilege import EnumDatabasePrivilege +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain from omnibase_core.enums.enum_handler_resolution_outcome import ( EnumHandlerResolutionOutcome, ) from omnibase_core.enums.enum_node_kind import EnumNodeKind +from omnibase_core.models.contracts.subcontracts.model_db_table_declaration import ( + ModelDbTableDeclaration, +) +from omnibase_core.models.core.model_deployment_topology import ModelDeploymentTopology +from omnibase_core.models.core.model_deployment_topology_database import ( + ModelDeploymentTopologyDatabase, +) from omnibase_core.models.errors import ModelOnexError from omnibase_core.models.resolver.model_handler_resolver_context import ( ModelHandlerResolverContext, @@ -67,6 +81,7 @@ from omnibase_core.services.service_local_handler_ownership_query import ( ServiceLocalHandlerOwnershipQuery, ) +from omnibase_infra.errors import TopicReplicationPolicyError from omnibase_infra.event_bus.enum_contract_attach_status import ( EnumContractAttachStatus, ) @@ -82,6 +97,9 @@ from omnibase_infra.event_bus.model_topic_set_readiness import ( ModelTopicSetReadiness, ) +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( + resolve_tenant_from_wire_topic, +) from omnibase_infra.protocols.protocol_dispatch_result_applier import ( ProtocolDispatchResultApplier, ) @@ -112,9 +130,25 @@ ModelSkippedEntry, ModelWiringOutcome, ) +from omnibase_infra.runtime.contract_terminal_events import ( + envelope_terminal_payload, + load_terminal_event_topics, +) +from omnibase_infra.runtime.dispatch_envelope_context import ( + current_dispatch_envelope, + current_projection_tenant_authority, +) from omnibase_infra.runtime.models.model_postgres_pool_config import ( ModelPostgresPoolConfig, ) +from omnibase_infra.runtime.projection_tenant_authority import ( + VerifiedProjectionTenantAuthority, + assert_projection_tenant_authority_matches_event, + parse_canonical_tenant_uuid, +) +from omnibase_infra.runtime.protocols.protocol_contract_scoped_dispatch_engine import ( + ProtocolContractScopedDispatchEngine, +) from omnibase_infra.runtime.providers.provider_postgres_pool import ProviderPostgresPool from omnibase_infra.runtime.state_io.state_store_adapter import ( CONTEXTVAR_STATE_IO_ROWS, @@ -122,6 +156,10 @@ StateStoreAdapter, ) from omnibase_infra.shared.tenant_stamp import stamp_verified_tenant_slug +from omnibase_infra.tools.contract_topic_extractor import read_projection_api_topics +from omnibase_infra.topology.physical_schema_mapping import ( + physical_grant_schema_for_table, +) from omnibase_infra.utils.util_retry_optimistic import ( OptimisticConflictError, retry_on_optimistic_conflict, @@ -129,6 +167,37 @@ from omnibase_infra.utils.util_topic_event_type import derive_event_type_from_topic +class BoundaryDlqNotPersistedError(Exception): + """Marks a boundary failure whose DLQ write was NOT confirmed durable. + + OMN-14498 (Lane C): ``_route_swallowed_exception`` used to return normally + even when ``_publish_raw_to_dlq`` reported non-persistence via its + documented ``False`` return. A callback that returns normally IS an ACK -- + ``EventBusKafka._dispatch_to_subscriber`` reads "no exception" as success + and lets the offset advance -- so the message was acknowledged while + existing nowhere durable. That made the OMN-15232 rewind path + (``_rewind_after_unpersisted_dlq``) structurally unreachable for every + auto-wired handler: the boundary swallowed its own failure before the + consumer loop could see it. + + Raising this type in that ONE case (DLQ enabled AND the write confirmed + non-durable) restores the invariant a NACK is supposed to carry: the + offset is withheld and Kafka redelivers. It is deliberately NOT raised + when the DLQ write succeeded, when the flag is off, or when no + DLQ-capable bus is wired -- those paths keep their prior semantics. + """ + + def __init__(self, topic: str, correlation_id: object, cause: Exception) -> None: + super().__init__( + f"boundary DLQ write not persisted; offset must not advance " + f"(topic={topic} correlation_id={correlation_id} " + f"cause={type(cause).__name__})" + ) + self.topic = topic + self.correlation_id = correlation_id + self.cause = cause + + class BoundaryPublishError(Exception): """Marks a result-applier (publish) failure the outbox boundary must PROPAGATE. @@ -731,7 +800,12 @@ async def _callback( else: target_model = _resolve_def_b_input_model_type(handle_method) if target_model is not None: - payload = _extract_dispatch_payload(envelope) + # OMN-16050: pass the registered input model so the unwrap + # STOPS at it. ``ModelEmitRequest`` declares ``payload`` plus + # four transport markers, so a marker-only heuristic unwrapped + # through it and handed the handler the caller's inner + # payload — every node_event_emit_effect command DLQ'd. + payload = _extract_dispatch_payload(envelope, target_model) if isinstance(payload, target_model): dispatch_arg = payload elif isinstance(payload, Mapping): @@ -764,7 +838,13 @@ async def _callback( raw_result, envelope, None, handler_node_kind, published_event_names ) - payload = _extract_dispatch_payload(envelope) + # OMN-16050: resolve the contract-declared event model BEFORE extracting so + # the unwrap can stop at it (same fail-closed rule as the def-B branch + # above). Resolution failure is not fatal here — the existing try/except + # below owns that path — so the hint degrades to None and the extraction + # keeps its pre-OMN-16050 structural behaviour. + payload_target_model = _safe_import_event_model_class(event_model) + payload = _extract_dispatch_payload(envelope, payload_target_model) handler_takes_envelope = _handler_accepts_event_envelope( cast("Callable[..., object]", handle_method) ) @@ -1003,6 +1083,24 @@ def _import_event_model_class(event_model: ModelHandlerRef) -> type[BaseModel]: return cast("type[BaseModel]", model_cls) +def _safe_import_event_model_class( + event_model: ModelHandlerRef | None, +) -> type[BaseModel] | None: + """``_import_event_model_class`` that yields None instead of raising (OMN-16050). + + Used only to hint ``_extract_dispatch_payload`` with the contract-declared + target type. An unimportable/malformed ``event_model`` must not change dispatch + control flow from this call site — the caller's own + ``_import_event_model_class`` inside its try/except still owns that failure. + """ + if event_model is None: + return None + try: + return _import_event_model_class(event_model) + except Exception: # noqa: BLE001 — hint-only resolution, never fatal here + return None + + def _handler_accepts_event_envelope(handle_method: object) -> bool: """Return true when a handler's first parameter is envelope-shaped.""" try: @@ -1138,6 +1236,55 @@ def _coerce_uuid_or_none(value: object) -> object | None: return None +def _ingress_correlation_id(message: object) -> UUID | None: + """Recover the ingress correlation id from a message's TRANSPORT surface. + + OMN-14498: the consume boundary must be able to establish lineage without + decoding the body, because the body is exactly what is unavailable for a + poisoned message. Three transport shapes are supported, in order: + + * ``ModelEventMessage`` -- ``headers.correlation_id`` (a real ``UUID``); + this is what ``EventBusKafka`` hands the callback in production. + * a raw aiokafka ``ConsumerRecord`` -- ``headers`` as an iterable of + ``(str, bytes)`` pairs, matching the ``correlation_id`` header both + ``MixinKafkaDlq`` and ``DLQProducer.replay_message`` write. + * a ``ModelEventEnvelope`` passed directly (legacy in-process call shape) + -- its own ``correlation_id``. + + Returns ``None`` when no lineage is present on the transport, leaving the + caller to fall back to the body and then to minting a fresh id. Never + raises: a malformed header must not take down the consume boundary. + """ + from uuid import UUID as _UUID + + headers = getattr(message, "headers", None) + + header_corr = getattr(headers, "correlation_id", None) + coerced = _coerce_uuid_or_none(header_corr) + if isinstance(coerced, _UUID): + return coerced + + if headers is not None and not isinstance(headers, (str, bytes)): + try: + for entry in headers: + key, value = entry + if key != "correlation_id": + continue + decoded = ( + value.decode("utf-8", errors="replace") + if isinstance(value, bytes) + else value + ) + coerced = _coerce_uuid_or_none(decoded) + if isinstance(coerced, _UUID): + return coerced + except (TypeError, ValueError): + pass + + coerced = _coerce_uuid_or_none(getattr(message, "correlation_id", None)) + return coerced if isinstance(coerced, _UUID) else None + + def _coerce_datetime_or_none(value: object) -> object | None: from datetime import UTC, datetime @@ -1263,11 +1410,21 @@ def _materialize_typed_event_envelope( # Transport-envelope keys the runtime adds around the domain payload. When the # dispatch engine materializes a ModelEventEnvelope to a dict it nests the domain # fields under ``payload`` and carries routing metadata (``partition_key`` etc.) -# alongside. Domain models never declare these keys, so a mapping that carries a -# ``payload`` mapping plus any marker is a transport envelope to unwrap. Mirrors -# omnimarket's ``_ENVELOPE_MARKER_KEYS`` predicate (OMN-12935/12936); the -# auto-wiring kernel unwraps here because it constructs the typed model itself, -# upstream of the handler's own coercion (OMN-12940). +# alongside, so a mapping that carries a ``payload`` mapping plus any marker MAY +# be a transport envelope to unwrap. Mirrors omnimarket's +# ``_ENVELOPE_MARKER_KEYS`` predicate (OMN-12935/12936); the auto-wiring kernel +# unwraps here because it constructs the typed model itself, upstream of the +# handler's own coercion (OMN-12940). +# +# OMN-16050 — this marker set is a NECESSARY, NOT SUFFICIENT signal. The earlier +# text here asserted "domain models never declare these keys"; that invariant is +# FALSE. ``ModelEmitRequest`` (node_event_emit_effect) declares ``payload`` plus +# four of these markers (``event_type``, ``correlation_id``, ``partition_key``, +# ``event_id``), is structurally indistinguishable from a transport envelope, and +# was therefore unwrapped THROUGH — the handler got the caller's inner payload, +# ``model_validate`` raised, and every command DLQ'd. The registered-input-model +# stop condition below (``_is_registered_input_payload``) is what makes the +# heuristic safe: structure alone can never decide this. _ENVELOPE_MARKER_KEYS: frozenset[str] = frozenset( { "partition_key", @@ -1281,11 +1438,13 @@ def _materialize_typed_event_envelope( def _is_transport_envelope(value: object) -> bool: - """True when ``value`` is a transport envelope wrapping a domain payload. + """True when ``value`` is envelope-SHAPED: a ``payload`` mapping plus a marker. - A transport envelope is a mapping that carries a ``payload`` mapping plus at - least one transport marker key. Requiring a marker avoids over-unwrapping a - legitimate domain model that happens to declare its own ``payload`` field. + Structural precondition only. A domain model may legitimately declare both a + ``payload`` mapping and transport-plausible marker fields (OMN-16050), so this + predicate is never sufficient on its own to justify an unwrap — see + ``_is_registered_input_payload``, the fail-closed stop condition applied by + ``_extract_dispatch_payload``. """ return ( isinstance(value, Mapping) @@ -1294,16 +1453,101 @@ def _is_transport_envelope(value: object) -> bool: ) -def _extract_dispatch_payload(envelope: object) -> object: +def _validation_alias_wire_keys(alias: object) -> set[str]: + """Top-level wire keys a pydantic ``validation_alias`` can consume. + + ``validation_alias`` has three shapes and only the plain-string one is a + single key. ``AliasPath("meta", "id")`` consumes the TOP-LEVEL key ``meta`` + (the remaining segments index inside that value), and ``AliasChoices`` holds + a list of alternatives, each itself a string or an ``AliasPath``. + + Missing the non-string shapes is fail-OPEN for OMN-16050: a model aliased + that way would fail ``_is_registered_input_payload``'s key-containment check + even when the candidate IS the registered model, the unwrap would continue + into the caller's payload, and the DLQ defect would return for exactly the + contracts that use richer aliases. + """ + if isinstance(alias, str): + return {alias} + if isinstance(alias, AliasPath): + first = alias.path[0] if alias.path else None + return {first} if isinstance(first, str) else set() + if isinstance(alias, AliasChoices): + keys: set[str] = set() + for choice in alias.choices: + keys |= _validation_alias_wire_keys(choice) + return keys + return set() + + +@lru_cache(maxsize=512) +def _model_declared_wire_keys(model: type[BaseModel]) -> frozenset[str]: + """Every wire key ``model`` can accept: field names plus their input aliases.""" + keys: set[str] = set() + for field_name, model_field in model.model_fields.items(): + keys.add(field_name) + if isinstance(model_field.alias, str): + keys.add(model_field.alias) + keys |= _validation_alias_wire_keys(model_field.validation_alias) + return frozenset(keys) + + +def _is_registered_input_payload( + candidate: object, target_model: type[BaseModel] | None +) -> bool: + """True when ``candidate`` IS the dispatcher's registered input model on the wire. + + The fail-closed stop condition for the recursive unwrap (OMN-16050). A + candidate is claimed by the registered model only when BOTH hold: + + 1. **Key containment** — every key present on the candidate is a declared + field (or input alias) of ``target_model``. A real transport envelope + always carries at least one routing/marker key the domain model does not + declare (``source_tool``, ``envelope_id``, ``__debug_trace``, + ``__bindings``, ``envelope_timestamp``, ...), so this alone keeps the + OMN-12940 double-wrapped case unwrapping. + 2. **Full validation** — the candidate validates as ``target_model``, so a + partial structural coincidence never halts the unwrap short of the domain. + + The cheap set check runs first; ``model_validate`` executes only for the rare + candidate whose keys are entirely owned by the target model. + + Deliberately NOT a marker denylist: dropping ``event_type``/``correlation_id`` + from ``_ENVELOPE_MARKER_KEYS`` would fix ``ModelEmitRequest`` and silently + break every genuine envelope that carries only those markers. This predicate + keys on the CONTRACT-registered target type instead of on key spelling. + """ + if target_model is None or not isinstance(candidate, Mapping): + return False + if not candidate.keys() <= _model_declared_wire_keys(target_model): + return False + try: + target_model.model_validate(dict(candidate)) + except Exception: # noqa: BLE001 — any validation failure means "not the model" + return False + return True + + +def _extract_dispatch_payload( + envelope: object, target_model: type[BaseModel] | None = None +) -> object: # The runtime may deliver a DOUBLE- (or deeper-) wrapped envelope, e.g. # ``{"payload": {"payload": {domain}, ...markers}, "partition_key": None}``. # Unwrap recursively until the domain payload is reached so the kernel's # ``model_validate`` (and the post-handler correlation read) operate on the # domain, not on an intermediate envelope (OMN-12940). + # + # OMN-16050: stop the moment the candidate IS the dispatcher's registered + # input model. ``target_model`` is the contract-declared type the kernel is + # about to construct (the def-B ``handle()`` annotation, or the handler's + # declared ``event_model``); when it is None the caller has no registered + # type in scope and the pre-existing structural behaviour is unchanged. candidate: object = envelope if not isinstance(candidate, Mapping): candidate = getattr(candidate, "payload", candidate) - while _is_transport_envelope(candidate): + while _is_transport_envelope(candidate) and not _is_registered_input_payload( + candidate, target_model + ): candidate = cast("Mapping[str, object]", candidate)["payload"] return candidate @@ -1622,7 +1866,297 @@ class behind OMN-13350 and OMN-14487 (both required a manual "omnidash_analytics": "OMNIDASH_ANALYTICS_DB_URL", } -_OPTIONAL_PROJECTION_DATABASES: frozenset[str] = frozenset({"omnidash_analytics"}) + +@dataclass(frozen=True) +class ProjectionDatabaseBindingTarget: + """One topology-declared workload identity and its secret-free DSN key.""" + + binding_ref: str + database_ref: str + physical_database: str + principal: str + dsn_env: str + + +@dataclass(frozen=True) +class ProjectionCatalogBindingPolicy: + """Composition-root choice of existing topology catalog identities.""" + + read_binding: str | None = None + write_binding: str | None = None + + +@dataclass(frozen=True) +class ProjectionTableTarget: + """Topology-resolved location for one typed table declaration.""" + + table: ModelDbTableDeclaration + database_ref: str + physical_database: str + schema: str + domain: EnumDatabaseSchemaDomain + read_binding: ProjectionDatabaseBindingTarget | None + write_binding: ProjectionDatabaseBindingTarget | None + + +@dataclass(frozen=True) +class ProjectionDatabaseTarget: + """Topology-resolved, per-operation pools for one projection handler.""" + + tables: tuple[ModelDbTableDeclaration, ...] + table_targets: tuple[ProjectionTableTarget, ...] + physical_database: str + + @property + def database_refs(self) -> tuple[str, ...]: + """Return the declared logical database references in stable order.""" + return tuple(sorted({target.database_ref for target in self.table_targets})) + + @property + def schemas(self) -> tuple[str, ...]: + """Return the declared schemas in stable order.""" + return tuple(sorted({target.schema for target in self.table_targets})) + + @property + def domains(self) -> tuple[EnumDatabaseSchemaDomain, ...]: + """Return the topology-derived domains in stable enum-value order.""" + return tuple( + sorted( + {target.domain for target in self.table_targets}, + key=lambda domain: domain.value, + ) + ) + + @property + def bindings(self) -> tuple[ProjectionDatabaseBindingTarget, ...]: + """Return every selected operation binding in stable order.""" + by_ref: dict[str, ProjectionDatabaseBindingTarget] = {} + for table_target in self.table_targets: + for binding in (table_target.read_binding, table_target.write_binding): + if binding is not None: + by_ref[binding.binding_ref] = binding + return tuple(by_ref[key] for key in sorted(by_ref)) + + @property + def dsn_envs(self) -> tuple[str, ...]: + """Return every required DSN environment key in stable order.""" + return tuple(sorted({binding.dsn_env for binding in self.bindings})) + + +_TENANT_PROJECTION_BINDING = "tenant_projection" +_INTERNAL_PROJECTION_BINDING = "omninode_runtime_service" + + +def _resolve_projection_binding( + database: ModelDeploymentTopologyDatabase, + database_ref: str, + binding_ref: str, +) -> ProjectionDatabaseBindingTarget: + """Resolve one explicit workload binding without a physical-DB fallback.""" + binding = database.bindings.get(binding_ref) + if binding is None: + raise ValueError( + f"Projection binding {binding_ref!r} is not declared for " + f"database_ref {database_ref!r}" + ) + if binding.database_ref != database_ref: + raise ValueError( + f"Projection binding {binding_ref!r} resolves to database_ref " + f"{binding.database_ref!r}, expected {database_ref!r}" + ) + principal = database.principals.get(binding.principal) + if principal is None: + raise ValueError( + f"Projection binding {binding_ref!r} references unknown principal " + f"{binding.principal!r}" + ) + if not principal.login or principal.bypass_rls: + raise ValueError( + f"Projection principal {binding.principal!r} must be LOGIN and NOBYPASSRLS" + ) + return ProjectionDatabaseBindingTarget( + binding_ref=binding_ref, + database_ref=database_ref, + physical_database=database.physical_name, + principal=binding.principal, + dsn_env=binding.dsn_env, + ) + + +def _require_projection_binding_privileges( + database: ModelDeploymentTopologyDatabase, + binding: ProjectionDatabaseBindingTarget, + table: ModelDbTableDeclaration, + *, + operation: str, +) -> None: + """Prove the selected topology principal can perform the exact operation.""" + principal = database.principals[binding.principal] + grant_schema = physical_grant_schema_for_table(table.schema, table.name) + has_schema_usage = any( + grant.object_type is EnumDatabaseGrantObjectType.SCHEMA + and grant.schema == grant_schema + and EnumDatabasePrivilege.USAGE in grant.privileges + for grant in principal.grants + ) + required_table_privileges = ( + {EnumDatabasePrivilege.SELECT} + if operation == "read" + else { + # PostgreSQL requires SELECT as well as INSERT/UPDATE for the + # adapter's INSERT ... ON CONFLICT DO UPDATE statement. + EnumDatabasePrivilege.SELECT, + EnumDatabasePrivilege.INSERT, + EnumDatabasePrivilege.UPDATE, + } + ) + granted_table_privileges = { + privilege + for grant in principal.grants + if grant.object_type is EnumDatabaseGrantObjectType.TABLE + and grant.schema == grant_schema + and table.name in grant.objects + for privilege in grant.privileges + } + missing_table_privileges = sorted( + required_table_privileges - granted_table_privileges, + key=lambda privilege: privilege.value, + ) + if not has_schema_usage or missing_table_privileges: + missing = [] + if not has_schema_usage: + missing.append(f"USAGE on schema {grant_schema!r}") + if missing_table_privileges: + names = ", ".join(privilege.value for privilege in missing_table_privileges) + missing.append(f"{names} on table {table.schema}.{table.name}") + raise ValueError( + f"Projection binding {binding.binding_ref!r} principal " + f"{binding.principal!r} lacks declared {operation} privileges: " + + "; ".join(missing) + ) + + +def _projection_operation_bindings( + *, + table: ModelDbTableDeclaration, + database: ModelDeploymentTopologyDatabase, + domain: EnumDatabaseSchemaDomain, + catalog_read_binding: str | None, + catalog_write_binding: str | None, +) -> tuple[ + ProjectionDatabaseBindingTarget | None, + ProjectionDatabaseBindingTarget | None, +]: + """Select explicit read/write identities from domain and table access.""" + needs_read = table.access in {"read", "read_write"} + needs_write = table.access in {"write", "read_write"} + if domain is EnumDatabaseSchemaDomain.TENANT: + binding_ref = _TENANT_PROJECTION_BINDING + read_ref = binding_ref if needs_read else None + write_ref = binding_ref if needs_write else None + elif domain is EnumDatabaseSchemaDomain.OMNINODE_INTERNAL: + binding_ref = _INTERNAL_PROJECTION_BINDING + read_ref = binding_ref if needs_read else None + write_ref = binding_ref if needs_write else None + elif domain is EnumDatabaseSchemaDomain.PLATFORM_CATALOG: + read_ref = catalog_read_binding if needs_read else None + write_ref = catalog_write_binding if needs_write else None + if needs_read and read_ref is None: + raise ValueError( + f"Catalog table {table.name!r} requires an explicit reader binding" + ) + if needs_write and write_ref is None: + raise ValueError( + f"Catalog table {table.name!r} requires an explicit writer binding" + ) + else: # pragma: no cover - enum exhaustiveness guard + raise ValueError(f"Unsupported projection database domain {domain!r}") + + read_binding = ( + _resolve_projection_binding(database, table.database_ref, read_ref) + if read_ref is not None + else None + ) + write_binding = ( + _resolve_projection_binding(database, table.database_ref, write_ref) + if write_ref is not None + else None + ) + if read_binding is not None: + _require_projection_binding_privileges( + database, + read_binding, + table, + operation="read", + ) + if write_binding is not None: + _require_projection_binding_privileges( + database, + write_binding, + table, + operation="write", + ) + return read_binding, write_binding + + +def _resolve_projection_database_target( + db_tables: Sequence[ModelDbTableDeclaration], + topology: ModelDeploymentTopology, + *, + catalog_read_binding: str | None = None, + catalog_write_binding: str | None = None, +) -> ProjectionDatabaseTarget: + """Resolve typed table declarations through the authoritative topology.""" + tables = tuple(db_tables) + if not tables: + raise ValueError("Projection database target requires at least one db_table") + + names = [table.name for table in tables] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ValueError(f"Duplicate db_table declarations: {duplicates!r}") + + table_targets: list[ProjectionTableTarget] = [] + databases_by_physical_name: dict[str, ModelDeploymentTopologyDatabase] = {} + for table in tables: + database = topology.databases.get(table.database_ref) + if database is None: + raise ValueError(f"Unknown database_ref '{table.database_ref}'") + domain = topology.schema_domain(table.database_ref, table.schema) + physical_database = database.physical_name + databases_by_physical_name[physical_database] = database + read_binding, write_binding = _projection_operation_bindings( + table=table, + database=database, + domain=domain, + catalog_read_binding=catalog_read_binding, + catalog_write_binding=catalog_write_binding, + ) + table_targets.append( + ProjectionTableTarget( + table=table, + database_ref=table.database_ref, + physical_database=physical_database, + schema=table.schema, + domain=domain, + read_binding=read_binding, + write_binding=write_binding, + ) + ) + + physical_databases = tuple(sorted(databases_by_physical_name)) + if len(physical_databases) != 1: + raise ValueError( + "Projection handler db_tables require more than one physical database " + f"connection, got {physical_databases!r}; split the handler or provide " + "an explicit multi-adapter boundary" + ) + return ProjectionDatabaseTarget( + tables=tables, + table_targets=tuple(table_targets), + physical_database=physical_databases[0], + ) + _TOPIC_TO_EVENT_TYPE: dict[str, str] = { "node-heartbeat": "heartbeat", @@ -1703,6 +2237,16 @@ def _extract_projection_payload(envelope: object) -> object: return getattr(envelope, "payload", None) +def _extract_projection_envelope_id(envelope: object) -> object | None: + """Return the typed, stable identity of the dispatched event envelope.""" + value = ( + envelope.get("envelope_id") + if isinstance(envelope, dict) + else getattr(envelope, "envelope_id", None) + ) + return _coerce_uuid_or_none(value) + + def _is_raw_event_projection_contract(contract: ModelDiscoveredContract) -> bool: if contract.event_bus is None: return False @@ -1726,36 +2270,16 @@ def _raw_event_projection_enabled( ) -def _read_db_io_tables(contract_path: Path) -> list[dict[str, str]]: - """Read db_io.db_tables from a contract YAML. Returns [] if db_io is absent. - - Raises on YAML parse errors or unexpected file I/O failures so the caller - can mark the contract as broken rather than silently falling back to the - non-projection wiring path. - """ - try: - # Why: Optional integration dependency is validated at runtime but ships incomplete typing. - import yaml # type: ignore[import-untyped] - - with open(contract_path) as f: - raw = yaml.safe_load(f) - except FileNotFoundError: - return [] - if not isinstance(raw, dict): - return [] - db_io = raw.get("db_io") or {} - return list(db_io.get("db_tables") or []) - - def _read_dlq_topics(contract_path: Path) -> list[str]: """Read ``event_bus.dlq_topics`` from a contract YAML. Returns [] if absent. OMN-13548 (D-03): projection handlers declare the DLQ destination for malformed inbound events under ``event_bus.dlq_topics`` (the same field the omnimarket projection runners read). The typed ``ModelEventBusSubcontract`` - does not carry this field, so the wiring reads it from the raw contract YAML - exactly as ``_read_db_io_tables`` reads ``db_io.db_tables``. The DLQ topic is - therefore resolved from the contract — never hardcoded in this module. + does not carry this field, so the wiring reads only this event-bus extension + from the raw contract YAML. Database table locations are already typed on + ``ModelDiscoveredContract`` and are never re-read here. The DLQ topic is + resolved from the contract — never hardcoded in this module. Raises on YAML parse / file I/O failures so a broken contract is surfaced rather than silently degrading to a no-DLQ projection wiring. @@ -1777,16 +2301,16 @@ def _read_dlq_topics(contract_path: Path) -> list[str]: def _contract_declares_db_io(contract: ModelDiscoveredContract) -> bool: - return bool(_read_db_io_tables(contract.contract_path)) + return bool(contract.db_io is not None and contract.db_io.db_tables) def _read_state_io(contract_path: Path) -> dict[str, object]: """Read the top-level ``state_io`` block from a contract YAML. Returns ``{}`` if ``state_io`` is absent. Raises on YAML parse errors or - unexpected file I/O failures — same fail-loud contract as - ``_read_db_io_tables`` — so a malformed contract is surfaced as a broken - contract rather than silently treated as "no state_io". + unexpected file I/O failures, so a malformed contract is surfaced as a + broken contract rather than silently treated as "no state_io". Unlike + ``db_io``, this legacy state subcontract does not yet have a core model. Shape (OMN-14208 opt-in runtime dispatch seam):: @@ -1817,112 +2341,420 @@ def _contract_declares_state_io(contract: ModelDiscoveredContract) -> bool: return bool(_read_state_io(contract.contract_path)) -def _build_sync_db_adapter(db_url: str) -> object: - """Build a synchronous psycopg2-backed DatabaseAdapter from a DSN. +# Tenant-scoped projection tables compare ``tenant_id`` with this +# transaction-local setting in both USING and WITH CHECK policies. +_TENANT_GUC = "app.tenant_id" - Separated from _make_projection_dispatch_callback to allow test patching. - """ - # Why: Optional integration dependency is validated at runtime but ships incomplete typing. - import psycopg2 # type: ignore[import-untyped] - # Why: Optional integration dependency is validated at runtime but ships incomplete typing. - import psycopg2.extras # type: ignore[import-untyped] +def _projection_tenant_context_error(message: str) -> Exception: + from omnibase_infra.errors.error_projection import ProjectionTenantContextError + + return ProjectionTenantContextError(message) + + +def _reject_canonical_tenant_field( + values: Mapping[str, object] | None, *, domain: EnumDatabaseSchemaDomain +) -> None: + if values is not None and "tenant_id" in values: + raise ValueError( + f"{domain.value} operation rejects canonical tenant_id; " + "only tenant-domain operations may carry it" + ) + + +class ProjectionTableOperation: + """Shared SQL mechanics for one topology-resolved table declaration.""" + + def __init__( + self, + adapter: ProjectionDatabaseOperations, + target: ProjectionTableTarget, + ) -> None: + self._adapter = adapter + self._target = target + + def _assert_write_declared(self) -> None: + if self._target.table.access not in {"write", "read_write"}: + raise PermissionError( + f"{self._target.schema}.{self._target.table.name} declares " + f"access={self._target.table.access!r}; write refused" + ) + + def _assert_read_declared(self) -> None: + if self._target.table.access not in {"read", "read_write"}: + raise PermissionError( + f"{self._target.schema}.{self._target.table.name} declares " + f"access={self._target.table.access!r}; read refused" + ) + + def upsert(self, conflict_key: str, row: dict[str, object]) -> bool: + self._assert_write_declared() + return self._adapter._execute_upsert( + self._target, conflict_key, row, tenant_context=None + ) + + def query( + self, filters: dict[str, object] | None = None + ) -> list[dict[str, object]]: + self._assert_read_declared() + return self._adapter._execute_query(self._target, filters, tenant_context=None) + + +class TenantProjectionTableOperation(ProjectionTableOperation): + """Tenant operation whose only authority is a verified capability.""" + + def _context(self) -> VerifiedProjectionTenantAuthority: + return self._adapter._bound_tenant_context() + + def _assert_supplied_tenant( + self, + supplied_tenant: object, + context: VerifiedProjectionTenantAuthority, + *, + operation: str, + ) -> None: + if isinstance(supplied_tenant, UUID): + supplied_uuid = supplied_tenant + elif isinstance(supplied_tenant, str): + supplied_uuid = parse_canonical_tenant_uuid( + supplied_tenant, + authority=f"{self._target.table.name} {operation} compatibility field", + ) + else: + raise _projection_tenant_context_error( + f"{self._target.table.name} {operation} tenant_id does not match " + "verified projection authority" + ) + if supplied_uuid != context.tenant_id: + raise _projection_tenant_context_error( + f"{self._target.table.name} {operation} tenant_id does not match " + "verified projection authority" + ) + + def upsert(self, conflict_key: str, row: dict[str, object]) -> bool: + self._assert_write_declared() + context = self._context() + attributed_row = dict(row) + supplied_tenant = attributed_row.get("tenant_id") + if supplied_tenant is not None: + self._assert_supplied_tenant(supplied_tenant, context, operation="row") + attributed_row["tenant_id"] = context.tenant_id + return self._adapter._execute_upsert( + self._target, + conflict_key, + attributed_row, + tenant_context=context, + ) + + def query( + self, filters: dict[str, object] | None = None + ) -> list[dict[str, object]]: + self._assert_read_declared() + context = self._context() + attributed_filters = dict(filters or {}) + supplied_tenant = attributed_filters.get("tenant_id") + if supplied_tenant is not None: + self._assert_supplied_tenant(supplied_tenant, context, operation="query") + attributed_filters["tenant_id"] = context.tenant_id + return self._adapter._execute_query( + self._target, + attributed_filters, + tenant_context=context, + ) + + +class InternalProjectionTableOperation(ProjectionTableOperation): + """Internal operation that never resolves or sets tenant context.""" + + def upsert(self, conflict_key: str, row: dict[str, object]) -> bool: + _reject_canonical_tenant_field(row, domain=self._target.domain) + conflict_keys = {key.strip() for key in conflict_key.split(",")} + if "source_tenant_id" in conflict_keys: + raise ValueError( + "Internal source_tenant_id is provenance only and cannot be an " + "upsert conflict key" + ) + return super().upsert(conflict_key, row) + + def query( + self, filters: dict[str, object] | None = None + ) -> list[dict[str, object]]: + _reject_canonical_tenant_field(filters, domain=self._target.domain) + return super().query(filters) + + +class CatalogProjectionTableOperation(ProjectionTableOperation): + """Catalog operation enforcing the declaration's explicit access mode.""" - class SyncPsycopg2Adapter: - _conn: object + def upsert(self, conflict_key: str, row: dict[str, object]) -> bool: + _reject_canonical_tenant_field(row, domain=self._target.domain) + return super().upsert(conflict_key, row) - def __init__(self, dsn: str) -> None: - self._dsn = dsn - self._conn = None + def query( + self, filters: dict[str, object] | None = None + ) -> list[dict[str, object]]: + _reject_canonical_tenant_field(filters, domain=self._target.domain) + return super().query(filters) + + +class ProjectionBindingConnections: + """Own per-binding connections, identity attestation, and transactions.""" + + def __init__( + self, + db_urls: Mapping[str, str], + target: ProjectionDatabaseTarget, + psycopg2_module: object, + ) -> None: + required_bindings = {binding.binding_ref for binding in target.bindings} + supplied_bindings = set(db_urls) + if supplied_bindings != required_bindings: + raise ValueError( + "Projection DSN bindings must exactly match the topology target: " + f"required={sorted(required_bindings)!r}, " + f"supplied={sorted(supplied_bindings)!r}" + ) + if any(not isinstance(url, str) or not url for url in db_urls.values()): + raise ValueError("Projection DSN binding values must be non-empty strings") + self._db_urls = dict(db_urls) + self._connections: dict[str, object] = {} + self._closed = False + self._psycopg2 = psycopg2_module - def _get_conn(self) -> object: - if self._conn is None or getattr(self._conn, "closed", False): - conn = psycopg2.connect(self._dsn) + @property + def connections(self) -> dict[str, object]: + """Expose live connections for narrow diagnostics and cleanup proofs.""" + return self._connections + + def get(self, binding: ProjectionDatabaseBindingTarget | None) -> object: + """Return an attested connection for one exact topology binding.""" + self.ensure_open() + if binding is None: + raise PermissionError( + "Projection operation has no declared workload binding" + ) + conn = self._connections.get(binding.binding_ref) + if conn is None or getattr(conn, "closed", False): + connect = self._psycopg2.connect # type: ignore[attr-defined] + conn = connect(self._db_urls[binding.binding_ref]) + try: conn.autocommit = True - self._conn = conn - return self._conn - - def upsert(self, table: str, conflict_key: str, row: dict[str, object]) -> bool: - if not _TABLE_NAME_RE.match(table): - raise ValueError(f"Invalid table name: {table!r}") - conflict_keys = [ - key.strip() for key in conflict_key.split(",") if key.strip() - ] - if not conflict_keys: - raise ValueError("conflict_key must contain at least one column") - bad_conflict_keys = [ - key for key in conflict_keys if not _TABLE_NAME_RE.match(key) - ] - if bad_conflict_keys: - raise ValueError(f"Invalid conflict key: {conflict_key!r}") - conn = self._get_conn() - cols = list(row.keys()) - bad_cols = [c for c in cols if not _TABLE_NAME_RE.match(str(c))] - if bad_cols: - raise ValueError(f"Invalid column names: {bad_cols!r}") - missing_conflict_keys = [key for key in conflict_keys if key not in row] - if missing_conflict_keys: - raise KeyError( - f"row missing conflict key(s): {missing_conflict_keys!r}" + with conn.cursor() as cursor: # type: ignore[attr-defined] + cursor.execute("SELECT current_user, current_database()") + identity = cursor.fetchone() + expected_identity = (binding.principal, binding.physical_database) + if ( + not isinstance(identity, (tuple, list)) + or tuple(identity) != expected_identity + ): + raise PermissionError( + f"Projection binding {binding.binding_ref!r} connected as " + f"{identity!r}, expected {expected_identity!r}" + ) + except BaseException: + conn.close() # type: ignore[attr-defined] + raise + self._connections[binding.binding_ref] = conn + return conn + + def close(self) -> None: + """Deterministically close every per-binding connection.""" + if self._closed: + return + self._closed = True + connections = tuple(self._connections.values()) + self._connections.clear() + for conn in connections: + if not getattr(conn, "closed", False): + conn.close() # type: ignore[attr-defined] + + def ensure_open(self) -> None: + if self._closed: + raise RuntimeError("Projection database adapter is closed") + + @contextlib.contextmanager + def tenant_transaction( + self, conn: object, context: VerifiedProjectionTenantAuthority + ) -> Iterator[None]: + """Set the GUC locally from a validated context, then always end it.""" + conn.autocommit = False # type: ignore[attr-defined] + try: + with conn.cursor() as cursor: # type: ignore[attr-defined] + cursor.execute( + "SELECT set_config(%s, %s, true)", + (_TENANT_GUC, str(context.tenant_id)), ) - quoted_cols = ", ".join(f'"{c}"' for c in cols) - placeholders = ", ".join(f"%({c})s" for c in cols) - conflict_key_set = set(conflict_keys) - updates = ", ".join( - f'"{c}" = EXCLUDED."{c}"' for c in cols if c not in conflict_key_set + yield + conn.commit() # type: ignore[attr-defined] + except BaseException: + conn.rollback() # type: ignore[attr-defined] + raise + finally: + conn.autocommit = True # type: ignore[attr-defined] + + +class ProjectionDatabaseOperations: + """Router over separate table operations selected from typed topology.""" + + def __init__( + self, + db_urls: Mapping[str, str], + target: ProjectionDatabaseTarget, + tenant_authority: VerifiedProjectionTenantAuthority | None, + tenant_event: object | None, + psycopg2_module: object, + extras_module: object, + ) -> None: + self._binding_connections = ProjectionBindingConnections( + db_urls, + target, + psycopg2_module, + ) + # Kept as a read-only diagnostic seam for existing runtime proofs. + self._connections = self._binding_connections.connections + self._extras = extras_module + self._tenant_authority = tenant_authority + self._tenant_event = tenant_event + operation_types: dict[ + EnumDatabaseSchemaDomain, type[ProjectionTableOperation] + ] = { + EnumDatabaseSchemaDomain.TENANT: TenantProjectionTableOperation, + EnumDatabaseSchemaDomain.OMNINODE_INTERNAL: InternalProjectionTableOperation, + EnumDatabaseSchemaDomain.PLATFORM_CATALOG: CatalogProjectionTableOperation, + } + self._operations = { + table_target.table.name: operation_types[table_target.domain]( + self, table_target ) - conflict_columns = ", ".join(f'"{key}"' for key in conflict_keys) - # JSONB adaptation: a dict is always JSON-adapted. A list is - # JSON-adapted per _should_jsonb_wrap_list (suffix convention, - # allowlist, or the OMN-14494 structural any-element-is-dict/list - # heuristic) and otherwise passed raw so genuine Postgres text[] - # ARRAY columns (e.g. swarm_runs.models_used / machines_used) - # keep their array semantics. A JSONB list sent as a Postgres - # ARRAY literal fails the INSERT — which is what silently - # dropped node-generation-completed events before this fix. - adapted_row = { - key: ( - psycopg2.extras.Json(value) - if isinstance(value, dict) - or (isinstance(value, list) and _should_jsonb_wrap_list(key, value)) - else value - ) - for key, value in row.items() - } - # table/conflict_key/cols validated by _TABLE_NAME_RE — not raw user input - parts = [ - f'INSERT INTO "{table}" ({quoted_cols})', - f"VALUES ({placeholders})", - f"ON CONFLICT ({conflict_columns}) DO UPDATE SET {updates}", - ] - insert_sql = " ".join(parts) - # Why: Control flow narrows this union at runtime before the attribute access. - with conn.cursor() as cur: # type: ignore[union-attr, attr-defined] - cur.execute(insert_sql, adapted_row) - return True + for table_target in target.table_targets + } - def query( - self, table: str, filters: dict[str, object] | None = None - ) -> list[dict[str, object]]: - if not _TABLE_NAME_RE.match(table): - raise ValueError(f"Invalid table name: {table!r}") - conn = self._get_conn() - # table validated by _TABLE_NAME_RE — not user input - select_sql = f'SELECT * FROM "{table}"' # noqa: S608 - params: list[object] = [] - if filters: - bad_keys = [k for k in filters if not _TABLE_NAME_RE.match(str(k))] - if bad_keys: - raise ValueError(f"Invalid filter keys: {bad_keys!r}") - clauses = [f'"{k}" = %s' for k in filters] - select_sql += " WHERE " + " AND ".join(clauses) - params = list(filters.values()) - # Why: Control flow narrows this union at runtime before the attribute access. - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: # type: ignore[union-attr, attr-defined] - cur.execute(select_sql, params or None) - return [dict(r) for r in cur.fetchall()] + def _bound_tenant_context(self) -> VerifiedProjectionTenantAuthority: + """Return the already-verified authority or fail before connecting.""" + self._binding_connections.ensure_open() + if self._tenant_authority is None: + raise _projection_tenant_context_error( + "Tenant projection has no cryptographically verified authority" + ) + assert_projection_tenant_authority_matches_event( + self._tenant_authority, + self._tenant_event, + ) + return self._tenant_authority + + def close(self) -> None: + """Release authority and deterministically close every connection.""" + self._tenant_authority = None + self._tenant_event = None + self._binding_connections.close() + + def _operation(self, table: str) -> ProjectionTableOperation: + self._binding_connections.ensure_open() + operation = self._operations.get(table) + if operation is None: + raise ValueError( + f"Projection table {table!r} is not declared by the typed db_io contract" + ) + return operation + + def _adapt_row(self, row: Mapping[str, object]) -> dict[str, object]: + json_adapter = self._extras.Json # type: ignore[attr-defined] + return { + key: ( + json_adapter(value) + if isinstance(value, dict) + or (isinstance(value, list) and _should_jsonb_wrap_list(key, value)) + else value + ) + for key, value in row.items() + } + + def _execute_upsert( + self, + target: ProjectionTableTarget, + conflict_key: str, + row: dict[str, object], + *, + tenant_context: VerifiedProjectionTenantAuthority | None, + ) -> bool: + conflict_keys = [key.strip() for key in conflict_key.split(",") if key.strip()] + if not conflict_keys: + raise ValueError("conflict_key must contain at least one column") + if any(not _TABLE_NAME_RE.fullmatch(key) for key in conflict_keys): + raise ValueError(f"Invalid conflict key: {conflict_key!r}") + cols = list(row) + bad_cols = [column for column in cols if not _TABLE_NAME_RE.fullmatch(column)] + if bad_cols: + raise ValueError(f"Invalid column names: {bad_cols!r}") + missing = [key for key in conflict_keys if key not in row] + if missing: + raise KeyError(f"row missing conflict key(s): {missing!r}") + quoted_cols = ", ".join(f'"{column}"' for column in cols) + placeholders = ", ".join(f"%({column})s" for column in cols) + conflict_columns = ", ".join(f'"{key}"' for key in conflict_keys) + conflict_set = set(conflict_keys) + updates = ", ".join( + f'"{column}" = EXCLUDED."{column}"' + for column in cols + if column not in conflict_set + ) + action = f"DO UPDATE SET {updates}" if updates else "DO NOTHING" + insert_sql = " ".join( + ( + f'INSERT INTO "{target.schema}"."{target.table.name}" ({quoted_cols})', + f"VALUES ({placeholders})", + f"ON CONFLICT ({conflict_columns}) {action}", + ) + ) + conn = self._binding_connections.get(target.write_binding) + adapted_row = self._adapt_row(row) + if tenant_context is None: + with conn.cursor() as cursor: # type: ignore[attr-defined] + cursor.execute(insert_sql, adapted_row) + else: + with self._binding_connections.tenant_transaction(conn, tenant_context): + with conn.cursor() as cursor: # type: ignore[attr-defined] + cursor.execute(insert_sql, adapted_row) + return True - return SyncPsycopg2Adapter(db_url) + def _execute_query( + self, + target: ProjectionTableTarget, + filters: dict[str, object] | None, + *, + tenant_context: VerifiedProjectionTenantAuthority | None, + ) -> list[dict[str, object]]: + # Schema/table originate in validated typed declarations, never request data. + select_sql = f'SELECT * FROM "{target.schema}"."{target.table.name}"' # noqa: S608 + params: list[object] = [] + if filters: + bad_keys = [ + key for key in filters if not _TABLE_NAME_RE.fullmatch(str(key)) + ] + if bad_keys: + raise ValueError(f"Invalid filter keys: {bad_keys!r}") + select_sql += " WHERE " + " AND ".join(f'"{key}" = %s' for key in filters) + params = list(filters.values()) + conn = self._binding_connections.get(target.read_binding) + + def _query() -> list[dict[str, object]]: + cursor_factory = self._extras.RealDictCursor # type: ignore[attr-defined] + with conn.cursor(cursor_factory=cursor_factory) as cursor: # type: ignore[attr-defined] + cursor.execute(select_sql, params or None) + return [dict(record) for record in cursor.fetchall()] + + if tenant_context is None: + return _query() + with self._binding_connections.tenant_transaction(conn, tenant_context): + return _query() + + def upsert(self, table: str, conflict_key: str, row: dict[str, object]) -> bool: + return self._operation(table).upsert(conflict_key, row) + + def query( + self, table: str, filters: dict[str, object] | None = None + ) -> list[dict[str, object]]: + return self._operation(table).query(filters) def _connect_projection_runner_db_if_needed(handler_instance: object) -> None: @@ -2112,7 +2944,7 @@ class ProjectionDispatchSinks: def _make_projection_dispatch_callback( handler_instance: object, - db_tables: list[dict[str, str]], + target: ProjectionDatabaseTarget, subscribe_topics: tuple[str, ...], sinks: ProjectionDispatchSinks | None = None, ) -> DispatcherFunc: @@ -2144,45 +2976,55 @@ def _make_projection_dispatch_callback( terminal_event = sinks.terminal_event dlq_topics = list(sinks.dlq_topics) handler_name = type(handler_instance).__name__ - database = ( - db_tables[0].get("database", "omnidash_analytics") - if db_tables - else "omnidash_analytics" + is_projection_runner = _is_projection_runner_handler(handler_instance) + db_urls = ( + {} + if is_projection_runner + else { + binding.binding_ref: os.environ.get(binding.dsn_env, "") + for binding in target.bindings + } ) - if database not in _DB_URL_ENV_MAP: + missing_bindings = [ + binding + for binding in target.bindings + if not is_projection_runner and not db_urls[binding.binding_ref] + ] + if missing_bindings: raise ValueError( - f"Unknown database {database!r} in contract db_io — " - f"must be one of {sorted(_DB_URL_ENV_MAP)!r}" + "Projection handler requires topology bindings with configured DSNs: " + + ", ".join( + f"{binding.binding_ref}:{binding.dsn_env}" + for binding in missing_bindings + ) ) - db_url_env = _DB_URL_ENV_MAP[database] async def _callback( envelope: ModelEventEnvelope[object], ) -> ModelDispatchResult | None: - if _is_projection_runner_handler(handler_instance): + if is_projection_runner: logger.debug( "Projection runner skipped by DB-injection auto-wiring: handler=%s topic=%s", type(handler_instance).__name__, _extract_projection_topic(envelope) or "unknown", ) return None - db_url = os.environ.get(db_url_env, "") - if not db_url: - log_level = ( - logging.INFO - if database in _OPTIONAL_PROJECTION_DATABASES - else logging.ERROR - ) - logger.log( - log_level, - "Projection handler inactive: %s not set (database=%s)", - db_url_env, - database, - ) - return None projected = False + adapter: object | None = None try: - adapter = _build_sync_db_adapter(db_url) + # MessageDispatchEngine hands callbacks a JSON-safe materialization. + # The original typed envelope is retained only for stable transport + # identity; it is never a tenant-authentication source. Tenant + # operations require the separate cryptographically verified + # capability bound by trusted ingress. + typed_envelope = current_dispatch_envelope() or envelope + tenant_authority = current_projection_tenant_authority() + adapter = _build_projection_db_adapter( + db_urls, + target, + tenant_authority, + typed_envelope, + ) topic = _extract_projection_topic(envelope) event_type = _derive_projection_event_type( topic, @@ -2199,6 +3041,12 @@ async def _callback( input_data["_db"] = adapter input_data["_event_type"] = event_type input_data["_topic"] = topic + envelope_id = _extract_projection_envelope_id(typed_envelope) + if envelope_id is not None: + # Preserve the UUID at the transport boundary. Projection + # handlers may use it as their durable idempotency key instead + # of inventing a fresh identity for every Kafka redelivery. + input_data["_envelope_id"] = envelope_id def _invoke_projection_handler() -> object: _connect_projection_runner_db_if_needed(handler_instance) @@ -2275,6 +3123,11 @@ def _invoke_projection_handler() -> object: handler_name, f"{type(exc).__name__}: {_sanitize_exc(exc)}", ) + finally: + if adapter is not None: + close = getattr(adapter, "close", None) + if callable(close): + close() if projected and event_bus is not None and terminal_event is not None: await _emit_projection_terminal_event(event_bus, terminal_event, envelope) @@ -2284,6 +3137,41 @@ def _invoke_projection_handler() -> object: return _callback +def _build_projection_db_adapter( + db_urls: Mapping[str, str], + target: ProjectionDatabaseTarget, + tenant_authority: VerifiedProjectionTenantAuthority | None, + tenant_event: object | None, +) -> object: + """Build a router whose operations come only from typed topology targets.""" + # Why: Optional integration dependency ships incomplete typing. + import psycopg2 # type: ignore[import-untyped] + + # Why: Optional integration dependency ships incomplete typing. + import psycopg2.extras # type: ignore[import-untyped] + + # Keep UUIDs typed through the adapter and teach psycopg2 the final wire + # conversion, instead of stringifying correlation/tenant IDs in row data. + psycopg2.extras.register_uuid() + + logger.debug( + "Selecting projection adapter: database_refs=%s physical_database=%s " + "schemas=%s domains=%s", + target.database_refs, + target.physical_database, + target.schemas, + [domain.value for domain in target.domains], + ) + return ProjectionDatabaseOperations( + db_urls, + target, + tenant_authority, + tenant_event, + psycopg2, + psycopg2.extras, + ) + + async def _emit_projection_terminal_event( event_bus: object, terminal_event: str, @@ -2472,8 +3360,8 @@ def _make_stateful_dispatch_callback( raise StateIoUnconfiguredError( f"handler_wiring: contract declares state_io (table={table!r}) " f"but {db_url_env} is unset. state_io is a REQUIRED durability " - "seam (OMN-14208) — unlike the optional db_io projection path, " - "it fails closed at wiring time rather than degrading silently." + "seam (OMN-14208) and fails closed at wiring time. Projection " + "db_io topology bindings are likewise wiring-time requirements." ) # Resolved once, at wiring time, not on every dispatch — mirrors the # fail-fast intent of every other _import_handler_class call in this @@ -3176,22 +4064,349 @@ def _raise_if_silent_dispatch_failure( ModelDispatchResult, ) - if not isinstance(result, ModelDispatchResult): - return - if result.status not in ( - EnumDispatchStatus.HANDLER_ERROR, - EnumDispatchStatus.INTERNAL_ERROR, - ): - return - has_applicable_output = bool( - result.output_events or result.output_intents or result.projection_intents - ) - if has_applicable_output: - return - raise HandlerDispatchFailureError( - f"dispatch to topic={topic} returned status={result.status.value} with no " - f"terminal output (dispatcher_id={result.dispatcher_id}): " - f"{result.error_message or 'handler/coercion failure'}" + if not isinstance(result, ModelDispatchResult): + return + if result.status not in ( + EnumDispatchStatus.HANDLER_ERROR, + EnumDispatchStatus.INTERNAL_ERROR, + ): + return + has_applicable_output = bool( + result.output_events or result.output_intents or result.projection_intents + ) + if has_applicable_output: + return + raise HandlerDispatchFailureError( + f"dispatch to topic={topic} returned status={result.status.value} with no " + f"terminal output (dispatcher_id={result.dispatcher_id}): " + f"{result.error_message or 'handler/coercion failure'}" + ) + + +def _normalize_contract_dispatcher_scope( + dispatcher_ids: Collection[str] | None, + *, + contract_name: str, + allow_empty: bool, +) -> frozenset[str]: + """Return a canonical unique dispatcher scope without trusting transport state. + + A contract-owned Kafka callback must never fall back to process-global + dispatch. Two contracts can intentionally consume the same topic under + distinct groups; global fan-out from each callback executes both contracts' + handlers once per group (OMN-15474). The live engine owner registry, not a + serialized wiring report, is authoritative for completeness. + """ + if dispatcher_ids is None: + raise ModelOnexError( + message=( + "handler_wiring: contract-scoped subscription is missing its " + f"dispatcher scope for contract {contract_name!r}; refusing " + "process-global fan-out." + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + raw_dispatcher_ids = tuple(dispatcher_ids) + seen: set[str] = set() + duplicate_ids: set[str] = set() + for dispatcher_id in raw_dispatcher_ids: + if dispatcher_id in seen: + duplicate_ids.add(dispatcher_id) + seen.add(dispatcher_id) + if ( + (not raw_dispatcher_ids and not allow_empty) + or any( + not dispatcher_id or dispatcher_id != dispatcher_id.strip() + for dispatcher_id in raw_dispatcher_ids + ) + or duplicate_ids + ): + raise ModelOnexError( + message=( + "handler_wiring: contract-scoped subscription has an empty or " + f"invalid dispatcher scope for contract {contract_name!r} " + f"(duplicates={sorted(duplicate_ids)}); " + "refusing process-global fan-out." + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + return frozenset(raw_dispatcher_ids) + + +def _require_contract_dispatcher_scope( + dispatcher_ids: Collection[str] | None, + *, + contract_name: str, +) -> frozenset[str]: + """Return a canonical non-empty dispatcher scope before subscribing.""" + return _normalize_contract_dispatcher_scope( + dispatcher_ids, + contract_name=contract_name, + allow_empty=False, + ) + + +def _require_unique_canonical_contract_names( + contract_names: Sequence[str], + *, + identity_source: str, +) -> frozenset[str]: + """Return an exact identity set or reject aliases and duplicate rows.""" + raw_names = tuple(contract_names) + noncanonical_names = tuple( + sorted( + { + contract_name + for contract_name in raw_names + if not contract_name or contract_name != contract_name.strip() + } + ) + ) + if noncanonical_names: + raise ModelOnexError( + message=( + f"handler_wiring: noncanonical {identity_source} contract names " + f"are not valid subscription identities: {noncanonical_names}" + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + + seen: set[str] = set() + duplicate_names: set[str] = set() + for contract_name in raw_names: + if contract_name in seen: + duplicate_names.add(contract_name) + seen.add(contract_name) + if duplicate_names: + raise ModelOnexError( + message=( + f"handler_wiring: duplicate {identity_source} contract names " + "would collapse or schedule repeated consumer attachment: " + f"{sorted(duplicate_names)}" + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + return frozenset(raw_names) + + +def _validate_initial_subscription_contract_identities( + manifest: ModelAutoWiringManifest, + report: ModelAutoWiringReport, +) -> None: + """Require a canonical, exact bijection between report and manifest names. + + OMN-15474 ruling 4 (re-affirmed by OMN-15621 after PR #2609 narrowed this + to a report-subset-of-manifest check, contrary to the ruling). Both + directions are load-bearing for single-owner dispatch: + + 1. **Uniqueness** on each side. A repeated contract name would schedule a + repeated consumer attachment for one identity — the same + execute-the-command-twice class this ticket exists to close. + 2. **report ⊆ manifest.** A report row naming a contract the manifest never + declared is an identity error: it would attach a consumer for a contract + this boot does not own. + 3. **manifest ⊆ report.** A manifest contract with no report row at all is + indistinguishable from one that silently vanished from the wiring + pass — exactly the class of bug that produced process-global dispatch + (OMN-15474). This direction is safe to assert unconditionally because + the report is contractually TOTAL over the manifest it was built from: + :func:`wire_from_manifest` backfills one explicit SKIPPED row per + uncovered contract via :func:`build_unwired_contract_results` before it + returns (see the "OMN-15474 totality post-condition" comment there), so + a contract that failed to wire, was resolver-skipped, or was + quarantined still produces a row — it is simply not ``WIRED``. A + missing row is therefore never legitimate; it means some caller handed + this function a report that was never produced by the real producer + (e.g. a hand-truncated report in a test), or a manifest that differs + from the one the report was built against. Both are boot-time bugs. + + A prior revision of this docstring claimed the reverse direction "refused + the boot outright against the full shipped manifest (missing_from_report + = 118 contracts)". That is not reproducible against the current producer: + a live run of the real main-profile manifest (118 contracts) through + :func:`wire_from_manifest` yields a report that is already exactly total + (0 missing, 0 unexpected) before this check ever runs. The 118 in that + docstring was the full manifest size, not a genuine gap — see OMN-15621. + """ + manifest_names = _require_unique_canonical_contract_names( + tuple(contract.name for contract in manifest.contracts), + identity_source="manifest", + ) + report_names = _require_unique_canonical_contract_names( + tuple(result.contract_name for result in report.results), + identity_source="report", + ) + if report_names != manifest_names: + raise ModelOnexError( + message=( + "handler_wiring: report and manifest contract-name mismatch; " + "initial subscription requires an exact bijection " + f"(missing_from_report={sorted(manifest_names - report_names)}, " + f"unexpected_in_report={sorted(report_names - manifest_names)})" + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + + +def _validate_not_ready_contract_identities( + manifest: ModelAutoWiringManifest, + not_ready_results: Sequence[ModelContractAttachResult], +) -> tuple[ModelContractAttachResult, ...]: + """Return uniquely named NOT_READY rows forming a valid manifest subset.""" + manifest_names = _require_unique_canonical_contract_names( + tuple(contract.name for contract in manifest.contracts), + identity_source="manifest", + ) + pending_results = tuple( + result + for result in not_ready_results + if result.status is EnumContractAttachStatus.NOT_READY + ) + not_ready_names = _require_unique_canonical_contract_names( + tuple(result.contract_name for result in pending_results), + identity_source="NOT_READY", + ) + unexpected_names = not_ready_names.difference(manifest_names) + if unexpected_names: + raise ModelOnexError( + message=( + "handler_wiring: NOT_READY and manifest contract-name mismatch; " + "reattach identities must be a manifest subset " + f"(unexpected={sorted(unexpected_names)})" + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + return pending_results + + +def _require_contract_scoped_dispatch_engine( + dispatch_engine: object, + *, + contract_name: str, +) -> ProtocolContractScopedDispatchEngine: + """Resolve the explicit scoped-dispatch capability before consumer attach.""" + scoped_dispatch = getattr(dispatch_engine, "dispatch_scoped", None) + if not callable(scoped_dispatch): + raise ModelOnexError( + message=( + "handler_wiring: contract-scoped subscription requires an " + f"explicit scoped dispatch capability for contract {contract_name!r}; " + "refusing to attach a consumer that could fail after delivery." + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + return cast("ProtocolContractScopedDispatchEngine", dispatch_engine) + + +def _validate_registered_contract_dispatcher_scope( + dispatch_engine: object, + dispatcher_scope: frozenset[str], + *, + contract_name: str, +) -> frozenset[str]: + """Compare one normalized scope with the complete live engine owner set.""" + scoped_engine = _require_contract_scoped_dispatch_engine( + dispatch_engine, + contract_name=contract_name, + ) + ownership_validator = getattr( + dispatch_engine, + "validate_contract_dispatcher_scope", + None, + ) + if not callable(ownership_validator): + raise ModelOnexError( + message=( + "handler_wiring: contract-scoped subscription requires a " + "dispatcher ownership validation capability for contract " + f"{contract_name!r}; refusing to attach without proving " + "current engine membership." + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + return scoped_engine.validate_contract_dispatcher_scope( + contract_name, + dispatcher_scope, + ) + + +def _require_registered_contract_dispatcher_scope( + dispatch_engine: object, + dispatcher_ids: Collection[str] | None, + *, + contract_name: str, +) -> frozenset[str]: + """Require one non-empty exact scope to exist on the current engine.""" + dispatcher_scope = _require_contract_dispatcher_scope( + dispatcher_ids, + contract_name=contract_name, + ) + return _validate_registered_contract_dispatcher_scope( + dispatch_engine, + dispatcher_scope, + contract_name=contract_name, + ) + + +def _validate_contract_dispatcher_ownership( + dispatch_engine: object, + dispatcher_scopes: Sequence[tuple[str, Collection[str]]], + *, + allow_empty_scopes: bool = False, +) -> None: + """Validate current engine membership and one-contract ownership. + + Reports and persisted NOT_READY results are typed transport artifacts, not + engine authority. Validate every referenced dispatcher before provisioning + or attaching any consumer. One contract may own multiple unique dispatcher + IDs; one dispatcher ID may never be claimed by multiple contracts. + """ + normalized_scopes: list[tuple[str, frozenset[str]]] = [] + owners_by_dispatcher: dict[str, set[str]] = defaultdict(set) + for contract_name, dispatcher_ids in dispatcher_scopes: + dispatcher_scope = _normalize_contract_dispatcher_scope( + dispatcher_ids, + contract_name=contract_name, + allow_empty=allow_empty_scopes, + ) + normalized_scopes.append((contract_name, dispatcher_scope)) + for dispatcher_id in dispatcher_scope: + owners_by_dispatcher[dispatcher_id].add(contract_name) + + multiply_owned = { + dispatcher_id: tuple(sorted(owners)) + for dispatcher_id, owners in owners_by_dispatcher.items() + if len(owners) > 1 + } + if multiply_owned: + raise ModelOnexError( + message=( + "handler_wiring: contract-scoped subscription assigns " + "dispatcher IDs to multiple contracts; refusing consumer " + f"attach: {multiply_owned}" + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + + for contract_name, dispatcher_scope in normalized_scopes: + _validate_registered_contract_dispatcher_scope( + dispatch_engine, + dispatcher_scope, + contract_name=contract_name, + ) + + +async def _dispatch_to_contract_scope( + dispatch_engine: ProtocolContractScopedDispatchEngine, + topic: str, + envelope: ModelEventEnvelope[object], + allowed_dispatcher_ids: frozenset[str], +) -> ModelDispatchResult: + """Dispatch through the engine while preserving callback ownership.""" + return await dispatch_engine.dispatch_scoped( + topic, + envelope, + allowed_dispatcher_ids=allowed_dispatcher_ids, ) @@ -3203,6 +4418,7 @@ def _make_event_bus_callback( tenant_scoped: bool = False, event_bus: object | None = None, propagate_publish_failures: bool = False, + allowed_dispatcher_ids: Collection[str] | None = None, ) -> Callable[..., Awaitable[None]]: """Create a Kafka on_message callback that deserializes and dispatches to engine. @@ -3233,6 +4449,15 @@ def _make_event_bus_callback( from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope + dispatcher_scope = _require_contract_dispatcher_scope( + allowed_dispatcher_ids, + contract_name=topic, + ) + scoped_dispatch_engine = _require_contract_scoped_dispatch_engine( + dispatch_engine, + contract_name=topic, + ) + def _derive_event_type_from_topic(topic: str) -> str | None: parts = topic.split(".") if len(parts) >= 5 and parts[0] == "onex": @@ -3281,7 +4506,12 @@ async def _dispatch_with_bounded_retry( try: if not await _wait_for_dispatch_engine_freeze(topic, dispatch_engine): return - result = await dispatch_engine.dispatch(topic, envelope) + result = await _dispatch_to_contract_scope( + scoped_dispatch_engine, + topic, + envelope, + dispatcher_scope, + ) if result_applier is not None and result is not None: try: await result_applier.apply(result, envelope.correlation_id) @@ -3477,7 +4707,15 @@ def _increment_message_lost_counter() -> None: correlation_id, ) _increment_message_lost_counter() - except Exception as dlq_exc: # noqa: BLE001 — DLQ publish is itself a boundary; never let it crash the consumer + # OMN-14498: a NACK must never ACK the offset. Returning + # normally here IS an ACK -- _dispatch_to_subscriber reads + # "no exception" as success and lets the offset advance -- + # so the record would be acknowledged while existing nowhere + # durable, and the OMN-15232 rewind path would never see it. + raise BoundaryDlqNotPersistedError(topic, correlation_id, exc) + except BoundaryDlqNotPersistedError: + raise + except Exception as dlq_exc: # Best-effort DLQ failed too -- the message IS lost here (gap G1). # Loud, not silent, but not prevented -- see gap G3 above. logger.error( @@ -3491,11 +4729,27 @@ def _increment_message_lost_counter() -> None: correlation_id, ) _increment_message_lost_counter() + # Same invariant as the False-return branch above: the DLQ write + # is not durable, so the offset must be withheld rather than + # advanced over a message that exists nowhere. + raise BoundaryDlqNotPersistedError(topic, correlation_id, exc) from dlq_exc async def callback(message: object) -> None: from uuid import uuid4 - correlation_id: UUID = uuid4() + # OMN-14498: seed lineage from the INGRESS transport headers before + # anything can fail. The body is not a reliable lineage source -- a + # poisoned message (truncated/undecodable JSON) raises inside + # json.loads below, before either body-derived recovery + # (envelope.correlation_id / data["correlation_id"]) can run, and the + # boundary then fell through to the DLQ still holding a freshly + # minted uuid4. That produced a VALID id with the WRONG lineage: the + # DLQ record, and every faithful replay of it, carried a fabricated + # ancestry, so the resulting terminal joined to nothing upstream. + # Precedence is ingress header -> body -> mint, so a decodable + # envelope still wins (it is the authoritative in-band value) and a + # message with no lineage anywhere still gets a usable id. + correlation_id: UUID = _ingress_correlation_id(message) or uuid4() try: raw = getattr(message, "value", None) if raw is not None: @@ -3580,9 +4834,6 @@ async def callback(message: object) -> None: return callback -_TENANT_WIRE_PREFIX_RE = re.compile(r"^tenant-([a-z][a-z0-9-]{1,61}[a-z0-9])\.") - - def _stamp_tenant_id_from_topic_prefix( topic: str, envelope: ModelEventEnvelope[object], @@ -3595,13 +4846,22 @@ def _stamp_tenant_id_from_topic_prefix( payload completely untouched -- never a defaulted or guessed tenant (Stage-1 warn semantics; a missing/self-reported value is handled by the existing OMN-14058 flow downstream, not masked here). + + OMN-15792: this is the subscribe/dispatch-side call site of the single + runtime topic resolver. ``resolve_tenant_from_wire_topic`` is the same + resolver the gateway forwarder's publish-side ``HandlerForwardOutbound`` + resolves through (via ``prefix_topic``) -- previously this function + hand-rolled its own regex extraction with no slug validation, which is + exactly the two-independent-resolvers-disagreeing class OMN-15757/ + OMN-15778 hit. A reserved or malformed slug embedded in a prefix-shaped + topic now raises (routed to the existing swallowed-exception boundary + handling below) instead of being silently stamped. """ - match = _TENANT_WIRE_PREFIX_RE.match(topic) - if match is None: + slug, _canonical_topic = resolve_tenant_from_wire_topic(topic) + if slug is None: return envelope if not isinstance(envelope.payload, dict): return envelope - slug = match.group(1) # OMN-14367: route through the single canonical stamp so this producer and # the gateway forwarder's consume_inbound cannot diverge on the shape again. stamped_payload = stamp_verified_tenant_slug(envelope.payload, slug) @@ -3612,11 +4872,22 @@ def _make_raw_event_projection_callback( topic: str, dispatch_engine: ProtocolDispatchEngine, result_applier: ProtocolDispatchResultApplier, + *, + allowed_dispatcher_ids: Collection[str] | None = None, ) -> Callable[..., Awaitable[None]]: """Create a callback for raw Kafka `ModelEventMessage` projection contracts.""" from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope from omnibase_infra.event_bus.models.model_event_message import ModelEventMessage + dispatcher_scope = _require_contract_dispatcher_scope( + allowed_dispatcher_ids, + contract_name=topic, + ) + scoped_dispatch_engine = _require_contract_scoped_dispatch_engine( + dispatch_engine, + contract_name=topic, + ) + async def callback(message: object) -> None: try: raw_message = ( @@ -3636,7 +4907,12 @@ async def callback(message: object) -> None: ), source_tool=raw_message.headers.source, ) - result = await dispatch_engine.dispatch(topic, envelope) + result = await _dispatch_to_contract_scope( + scoped_dispatch_engine, + topic, + envelope, + dispatcher_scope, + ) if result is not None: await result_applier.apply(result, envelope.correlation_id) except Exception as exc: # noqa: BLE001 — consumer boundary; log and continue @@ -3919,6 +5195,7 @@ def _make_sync_event_publisher( *, event_bus: object, handler_name: str, + terminal_topics: frozenset[str] = frozenset(), ) -> Callable[[str, bytes], None]: """Adapt async runtime event-bus publish to legacy sync handler publishers. @@ -3937,6 +5214,27 @@ def _make_sync_event_publisher( delays (OMN-13658). Scheduling the coroutine back onto the owning kernel loop via ``asyncio.run_coroutine_threadsafe`` keeps every Future on its loop, so the publish completes immediately from any thread. + + ``terminal_topics`` (OMN-15468) carries the publishing contract's declared + terminal topics — every site: ``terminal_event``, ``terminal_events`` and + ``runtime_dispatch.terminal_events``, read through the same function route + discovery uses. A publish to one of those topics is a TERMINAL emission and + is wrapped in a ``ModelEventEnvelope`` here, at the one factory that hands + every def-B handler its publisher. + + Why here and not in the handlers: the other half of this same wiring + (``DispatchResultApplier``) already publishes the def-B return value as a + full envelope, so before this change a single contract emitted its SUCCESS + terminal enveloped and its handler-emitted FAILURE terminal raw — and the + Pattern B broker's terminal path decodes envelopes. Live on the ``.201`` dev + lane at merged ``5dc68190`` (2026-07-30T17:13Z), with #2560 already + subscribing the broker to the failure topic, a forced node-generation + failure still returned ``ok=true`` / ``status=completed`` / ``error=null``, + byte-identical to the success control, because the record waiting on the + failure topic was raw. Fixing that per node would mean editing every handler + that self-publishes a terminal; fixing it here covers the whole declared- + terminal set at once. Any topic that is not a declared terminal is forwarded + byte-for-byte unchanged. """ publish = getattr(event_bus, "publish", None) if not callable(publish): @@ -3956,6 +5254,11 @@ def _make_sync_event_publisher( ) from exc def _publish(topic: str, payload: bytes) -> None: + payload = envelope_terminal_payload( + topic=topic, + payload=payload, + terminal_topics=terminal_topics, + ) result = publish(topic, None, payload) if not inspect.isawaitable(result): return @@ -4125,6 +5428,7 @@ def _materialize_known_handler_dependencies( event_bus: object | None, container: object | None, ownership_query: object | None, + terminal_topics: frozenset[str] = frozenset(), ) -> dict[str, dict[str, object]] | None: """Materialize infra-known constructor deps for core resolver Step 2. @@ -4168,6 +5472,7 @@ def _materialize_known_handler_dependencies( available["event_publisher"] = _make_sync_event_publisher( event_bus=event_bus, handler_name=handler_name, + terminal_topics=terminal_topics, ) if requires_event_consumer and event_bus is not None: available["event_consumer"] = _make_sync_event_consumer( @@ -4457,6 +5762,74 @@ def _live_message_types(pcw: PreparedContractWiring) -> set[str]: return message_types +def _preflight_prepared_registration_ids( + prepared_contracts: Sequence[PreparedContractWiring], + dispatch_engine: object, + *, + dynamic_materialization_authorized: bool = False, +) -> None: + """Validate every derived dispatcher/route ID before the first commit. + + Preparation is deliberately side-effect-free. Preserve that transaction + boundary by detecting cross-contract, same-contract, and normalization + collisions across the complete manifest before any engine registration or + Kafka subscription becomes visible. + """ + dispatcher_origins: dict[str, list[str]] = defaultdict(list) + dispatcher_owners: dict[str, str] = {} + route_origins: dict[str, list[str]] = defaultdict(list) + for prepared_contract in prepared_contracts: + if prepared_contract.skip_result is not None: + continue + contract_name = prepared_contract.contract.name + for prepared in prepared_contract.prepared_wirings: + if prepared.is_skip or prepared.is_quarantined: + continue + origin = f"{contract_name}:{prepared.handler_name}" + dispatcher_origins[prepared.dispatcher_id].append(origin) + dispatcher_owners[prepared.dispatcher_id] = contract_name + for route_id in prepared.route_ids: + route_origins[route_id].append(origin) + + duplicate_dispatcher_ids = { + dispatcher_id: tuple(origins) + for dispatcher_id, origins in dispatcher_origins.items() + if len(origins) > 1 + } + if duplicate_dispatcher_ids: + raise ModelOnexError( + message=( + "handler_wiring: duplicate prepared dispatcher IDs across the " + f"manifest: {duplicate_dispatcher_ids}" + ), + error_code=EnumCoreErrorCode.DUPLICATE_REGISTRATION, + ) + + duplicate_route_ids = { + route_id: tuple(origins) + for route_id, origins in route_origins.items() + if len(origins) > 1 + } + if duplicate_route_ids: + raise ModelOnexError( + message=( + "handler_wiring: duplicate prepared route IDs across the manifest " + f"(including normalized topic IDs): {duplicate_route_ids}" + ), + error_code=EnumCoreErrorCode.DUPLICATE_REGISTRATION, + ) + + from omnibase_infra.runtime.message_dispatch_engine import MessageDispatchEngine + + if isinstance(dispatch_engine, MessageDispatchEngine): + dispatch_engine.validate_registration_batch( + tuple(dispatcher_origins), + tuple(route_origins), + dispatcher_owners=dispatcher_owners, + allow_frozen=dynamic_materialization_authorized, + ) + + def _collect_orchestrator_dispatcher_coverage_gaps( prepared_contracts: list[PreparedContractWiring], failed_gaps: list[str] | None = None, @@ -4497,6 +5870,114 @@ def _assert_orchestrator_dispatcher_coverage( ) +ENV_SINGLE_OWNER_COMMAND_TOPICS = "ONEX_SINGLE_OWNER_COMMAND_TOPICS" + + +def _single_owner_command_topics_strict() -> bool: + """True when the OMN-15474 single-owner command-topic gate must fail closed.""" + return os.environ.get(ENV_SINGLE_OWNER_COMMAND_TOPICS, "").strip().lower() in ( + "1", + "true", + ) + + +def _assert_single_owner_command_topics( + manifest: ModelAutoWiringManifest, +) -> None: + """Fail closed when a COMMAND topic has more than one in-process consumer. + + OMN-15474. A command is an instruction to execute exactly once. Every wired + contract joins its own consumer group (``compute_consumer_group_id`` keys on + node identity), so two contracts subscribed to one ``onex.cmd.*`` topic in + one process means the broker delivers the accepted command to BOTH: the + whole reducer chain runs twice, both executions carry the SAME ingress + correlation id, and every terminal event, projection row, LLM judge call and + cost line is doubled. That is the live defect measured on ``onex-dev`` + (73 duplicated ``(correlation_id, topic)`` pairs in 48h; two quality-gate + evaluations returning DIFFERENT scores for one command). + + ``_detect_duplicate_topics`` already SAW this — it logged + ``Duplicate topic ownership detected`` on every affected boot — but it ran + AFTER Phase 2 had already committed the subscriptions, and only at WARNING. + Detection that arrives after the side effect and cannot fail the boot is not + enforcement ([[feedback_a_rule_is_not_a_mechanism]]). This is the mechanism: + a preflight, before any subscription is attached. + + EVENT topics are deliberately untouched. Fan-out is their contract — many + independent consumers legitimately observe one event on their own groups. + Only the command category carries the execute-exactly-once obligation. + + STRICT MODE IS OFF BY DEFAULT, and that is deliberate. This repo's standing + rule is that a strict invariant "lands AFTER all downstream consumers are + compliant... if a strict gate must ship first, it ships behind an env flag + (default OFF) and is flipped in a separate PR once compliance is merged" + (CLAUDE.md, Testing and CI). Compliance is NOT met today. Measured + 2026-08-01 by running THIS gate's own detection over + ``discover_contracts()`` (109 contracts, omnibase_infra only — + ``omnimarket`` is not installed in that venv, so its contracts are not + discoverable and are NOT counted here), 3 command topics have more than + one in-process owner: + + - ``onex.cmd.omnibase-infra.build-loop-append.v1`` + -> node_build_loop_write_effect, node_ledger_projection_compute + - ``onex.cmd.omnibase-infra.chain-learn.v1`` + -> node_chain_orchestrator, node_chain_retrieval_effect + - ``onex.cmd.platform.request-introspection.v1`` + -> node_ledger_projection_compute, node_registration_orchestrator + + An earlier revision of this docstring claimed "8 (1 in omnibase_infra; + 7 in omnimarket)". That is not reproducible: infra alone is 3, not 1. The + omnimarket figure cannot be measured from this repo's venv at all. A raw, + unfiltered ``contract.yaml`` scan across the infra worktree plus the + canonical omnimarket clone yields 16 topics with >1 declared subscriber, + but that is a strict superset (it applies none of the discovery, package- + activation, or plugin_managed filtering the gate applies). Re-measure with + the gate's own code path in the target deployment before flipping the flag; + do not trust any count in this docstring as the deployed number. + + Raising unconditionally here would refuse the runtime boot on + the very next deploy. So: OFF ⇒ log an ERROR naming every violation + (louder than the pre-existing post-commit WARNING, and now emitted BEFORE + the subscriptions attach); ON ⇒ raise before any side effect. Flip + ``ONEX_SINGLE_OWNER_COMMAND_TOPICS=1`` in a follow-up once those 8 are + resolved. + """ + from omnibase_infra.enums import EnumMessageCategory + + command_topic_owners: dict[str, list[str]] = defaultdict(list) + for contract in manifest.contracts: + if contract.event_bus is None: + continue + for topic in contract.event_bus.subscribe_topics: + if _derive_message_category(topic) == EnumMessageCategory.COMMAND.value: + command_topic_owners[topic].append(contract.name) + + violations = [ + f"{topic} owned by {sorted(owners)}" + for topic, owners in sorted(command_topic_owners.items()) + if len(owners) > 1 + ] + if not violations: + return + + detail = ( + "Command topics are single-owner: a command must execute exactly once, " + "but these command topics have more than one in-process consumer, so " + "every accepted command is dispatched once per owner, under one " + f"correlation id (OMN-15474): {'; '.join(violations)}. Give each " + "command topic exactly one owning contract, or move the additional " + "consumers onto an event topic." + ) + if _single_owner_command_topics_strict(): + raise ModelOnexError(detail, error_code=EnumCoreErrorCode.INVALID_STATE) + logger.error( + "%s (non-strict — set %s=1 to refuse the boot instead of doubling " + "every accepted command on these topics)", + detail, + ENV_SINGLE_OWNER_COMMAND_TOPICS, + ) + + def _detect_duplicate_topics( manifest: ModelAutoWiringManifest, ) -> list[ModelDuplicateTopicOwnership]: @@ -4540,6 +6021,59 @@ def _detect_duplicate_topics( return duplicates +UNWIRED_BACKFILL_REASON = ( + "no wiring result produced for this manifest contract " + "(wiring-report totality backfill, OMN-15474)" +) + + +def build_unwired_contract_results( + manifest: ModelAutoWiringManifest, + *, + reason: str, + already_reported: Collection[str] = (), +) -> tuple[ModelContractWiringResult, ...]: + """Return one explicit "did not wire" row per uncovered manifest contract. + + The wiring report consumed by :func:`subscribe_wired_contract_topics` is + TOTAL over the manifest: every discovered contract either wired, failed, or + carries an explicit :attr:`EnumWiringOutcome.SKIPPED` row naming why it did + not. Totality is what makes the initial-subscription identity check + (:func:`_validate_initial_subscription_contract_identities`) a decision + instead of a guess — a report that merely *omits* a contract is + indistinguishable from one where the contract silently vanished, and + silently-vanished contracts are exactly the class of bug that produced + process-global dispatch (OMN-15474). + + This is the canonical constructor for those rows. Every producer of a + :class:`ModelAutoWiringReport` — including test doubles standing in for the + wiring engine — MUST use it rather than emitting a partial report, so the + "did not wire" set is derived from the manifest the runtime actually holds + rather than hand-mirrored against it. + + Args: + manifest: The manifest the report must be total over. + reason: Human-readable reason recorded on each synthesized row. + already_reported: Contract names that already have a result row. + + Returns: + One SKIPPED result per manifest contract absent from + ``already_reported``, in manifest order. Empty when the report is + already total. + """ + covered = frozenset(already_reported) + return tuple( + ModelContractWiringResult( + contract_name=contract.name, + package_name=contract.package_name, + outcome=EnumWiringOutcome.SKIPPED, + reason=reason, + ) + for contract in manifest.contracts + if contract.name not in covered + ) + + async def wire_from_manifest( manifest: ModelAutoWiringManifest, dispatch_engine: ProtocolDispatchEngine, @@ -4551,6 +6085,8 @@ async def wire_from_manifest( result_appliers_by_contract: Mapping[str, ProtocolDispatchResultApplier] | None = None, materialized_explicit_dependencies: dict[str, dict[str, object]] | None = None, + topology: ModelDeploymentTopology | None = None, + catalog_binding_policy: ProjectionCatalogBindingPolicy | None = None, ) -> ModelAutoWiringReport: """Wire all discovered contracts into the dispatch engine and event bus. @@ -4586,10 +6122,19 @@ async def wire_from_manifest( outputs from auto-wired callbacks. materialized_explicit_dependencies: Optional pre-built constructor dependencies keyed by handler name for resolver Step 2. + topology: Checked-in deployment topology loaded by the composition + boundary. Required for every contract that declares ``db_io``. + catalog_binding_policy: Explicit topology binding names for catalog read + and write operations. Missing choices fail catalog wiring closed. Returns: A :class:`ModelAutoWiringReport` with per-contract outcomes. """ + _require_unique_canonical_contract_names( + tuple(contract.name for contract in manifest.contracts), + identity_source="manifest", + ) + # Construct the resolver + ownership query ONCE per wiring pass from the # manifest itself (OMN-9201). The ownership query is set-membership # against the locally discovered node_name set — no I/O, no SQL. See @@ -4662,6 +6207,8 @@ async def wire_from_manifest( else None, result_appliers_by_contract=result_appliers_by_contract, materialized_explicit_dependencies=materialized_explicit_dependencies, + topology=topology, + catalog_binding_policy=catalog_binding_policy, ) prepared_contracts.append(prepared) except TypeError: @@ -4723,6 +6270,13 @@ async def wire_from_manifest( dispatcher_coverage_failed_gaps, ) + # OMN-15474: single-owner command topics, asserted BEFORE Phase 2 commits any + # subscription. Must stay above the commit loop — the post-commit + # _detect_duplicate_topics warning below is diagnosis, not a gate. + _assert_single_owner_command_topics(manifest) + + _preflight_prepared_registration_ids(prepared_contracts, dispatch_engine) + # Phase 2: All contracts validated — commit registrations and subscriptions. # Failed contracts are included in results so total_failed is accurate. # service_kernel respects the flag before asserting total_failed == 0. @@ -4737,6 +6291,33 @@ async def wire_from_manifest( ) results.append(result) + # OMN-15474 totality post-condition. Phase 1 + Phase 2 above are written so + # that every manifest contract yields exactly one row (a prepared contract + # commits a row, a preparation failure collects one). That is a property of + # two loops, not of this function's signature, so a future refactor can + # break it silently — and the only downstream symptom would be + # subscribe_wired_contract_topics aborting the kernel at boot on a + # report/manifest bijection failure. Backfill instead: any manifest + # contract with no row gets an explicit SKIPPED row naming why, so the + # report this function returns is TOTAL by contract rather than by + # accident. This does NOT relax the downstream identity check — that check + # is unchanged and still rejects report rows with no manifest contract, + # which is the direction no backfill can repair. + unwired_backfill = build_unwired_contract_results( + manifest, + reason=UNWIRED_BACKFILL_REASON, + already_reported=tuple(r.contract_name for r in results), + ) + if unwired_backfill: + logger.error( + "Auto-wiring produced no result row for %d manifest contract(s); " + "backfilling explicit unwired rows to keep the report total " + "(OMN-15474). This is a wiring-engine bug, not a contract bug: %s", + len(unwired_backfill), + sorted(r.contract_name for r in unwired_backfill), + ) + results.extend(unwired_backfill) + duplicates = _detect_duplicate_topics(manifest) for dup in duplicates: @@ -4790,18 +6371,82 @@ async def wire_from_manifest( def _contract_provision_topics(contract: ModelDiscoveredContract) -> tuple[str, ...]: - """Return the topic set this contract owns at boot (OMN-13237, §3.6). + """Return the topic set this contract owns at boot (OMN-13237 §3.6, OMN-15330, + OMN-15832). Subscribe topics (the consumers that attach) UNION the contract's owned - publish topics it must guarantee exist. Names come from the contract's - ``event_bus`` declarations only — never a Python literal. DLQ topics are - handled by the best-effort universe warm (covered across runtimes per §3.6). + publish topics UNION its declared ``event_bus.dlq_topics`` UNION its served + ``projection_api`` topics. Names come from the contract's own declarations + only — never a Python literal. + + OMN-15330 — DLQ topics used to be excluded here and left to the best-effort + universe warm. That delegation broke the moment the warm was switched off: + ``ONEX_BOOT_UNIVERSE_PROVISION=0`` is the standing onex-dev setting (added + after the 2026-07-27 >1000-topic broker near-meltdown), and with the warm + off NOTHING created the declared DLQ topics. The first malformed event then + hit ``[ONEX_CORE_041_INVALID_CONFIGURATION] Topic '' not found on + broker`` inside ``_route_projection_error_to_dlq`` and the record was + dropped — observed live on onex-dev 2026-07-28T16:29Z for + ``onex.dlq.omnimarket.projection-delegation-inference-response-malformed.v1`` + and four siblings. + + The DLQ names are read with ``_read_dlq_topics`` — the SAME reader the + projection auto-wiring uses to build ``ModelProjectionSinks.dlq_topics`` — + so the provisioned string is byte-identical to the routing target by + construction, rather than by a second parser that can drift. DLQ topics + enter the readiness confirm alongside the rest: attaching a consumer whose + dead-letter sink is not ready guarantees silent loss on the first malformed + event, so this fails closed (a NOT_READY contract is retried by the + OMN-15215 reconciliation loop). + + OMN-15832 — the same universe-warm-off gap applies to ``projection_api`` + (``onex.snapshot.*``) topics: nothing else creates them at boot, and the + contract's ``event_bus`` union above never scanned that section at all. + ``read_projection_api_topics`` (``omnibase_infra.tools.contract_topic_extractor``) + is the SAME parser ``ContractTopicExtractor.extract``'s global scan uses — + one source of parsing truth, scoped to ``expose: true`` AND + ``bus_backed: true`` exposures, so the boot provision set can never diverge + from what ``omnimarket.projection.discovery.build_projection_topic_map`` + will actually serve. Explicitly NOT a fix to re-enable + ``ONEX_BOOT_UNIVERSE_PROVISION`` or to have the consumer + (``SnapshotCache``) self-provision its own topics — both remain out of + scope by standing decision; this stays a governed, boot-side, per-contract + addition to the same confirm path DLQ topics already use. """ if contract.event_bus is None: return () ordered = list(contract.event_bus.subscribe_topics) ordered.extend(contract.event_bus.publish_topics) - return tuple(dict.fromkeys(ordered)) + typed_dlq_topics = getattr(contract.event_bus, "dlq_topics", ()) + if typed_dlq_topics: + ordered.extend(typed_dlq_topics) + else: + try: + ordered.extend(_read_dlq_topics(contract.contract_path)) + except Exception: # noqa: BLE001 — per-contract boot boundary + # ``_interleave_contract`` runs under ``asyncio.gather(...)`` with no + # ``return_exceptions=True``, so a raise here would abort the ENTIRE + # boot subscribe pass for every contract. Degrading this one contract + # to its pre-OMN-15330 behaviour (no DLQ provisioning) is strictly less + # bad, and the warning names the contract that needs fixing. + logger.warning( + "Could not read event_bus.dlq_topics for contract '%s' from %s — " + "its DLQ topics will NOT be provisioned at boot (OMN-15330)", + contract.name, + contract.contract_path, + exc_info=True, + ) + try: + ordered.extend(read_projection_api_topics(contract.contract_path)) + except Exception: # noqa: BLE001 — per-contract boot boundary, see DLQ comment above + logger.warning( + "Could not read projection_api topics for contract '%s' from %s — " + "its snapshot topics will NOT be provisioned at boot (OMN-15832)", + contract.name, + contract.contract_path, + exc_info=True, + ) + return tuple(dict.fromkeys(t for t in ordered if t and t.strip())) async def subscribe_wired_contract_topics( @@ -4840,13 +6485,24 @@ async def subscribe_wired_contract_topics( actually subscribed). Backward-compatible: with no *provisioner* the behavior is the original concurrent subscribe (no readiness gate). """ + _validate_initial_subscription_contract_identities(manifest, report) if event_bus is None: return {} + report_dispatcher_scopes = tuple( + (result.contract_name, result.dispatchers_registered) + for result in report.results + ) + _validate_contract_dispatcher_ownership( + dispatch_engine, + report_dispatcher_scopes, + allow_empty_scopes=True, + ) + contract_by_name = {contract.name: contract for contract in manifest.contracts} # Collect eligible contracts in priority order (projection appliers first). - eligible: list[tuple[str, ModelDiscoveredContract]] = [] + eligible: list[tuple[ModelContractWiringResult, ModelDiscoveredContract]] = [] for result in _prioritize_subscription_results( report, result_appliers_by_contract, @@ -4856,6 +6512,18 @@ async def subscribe_wired_contract_topics( contract = contract_by_name.get(result.contract_name) if contract is None: continue + if not result.dispatchers_registered: + # Resolver-owned skips and quarantines intentionally register no + # local dispatcher. They therefore own no consume callback. The + # old path still subscribed them and a process-global dispatch + # could execute some other contract's matching handler; keeping + # them unsubscribed is the only truthful zero-owner state. + logger.info( + "Auto-wiring (deferred): skipping Kafka subscription for " + "contract '%s' because it owns zero dispatchers (OMN-15474)", + contract.name, + ) + continue if _is_raw_event_projection_contract(contract) and ( result_appliers_by_contract is None or contract.name not in result_appliers_by_contract @@ -4869,7 +6537,7 @@ async def subscribe_wired_contract_topics( contract.name, ) continue - eligible.append((result.contract_name, contract)) + eligible.append((result, contract)) knobs = readiness_config or ModelTopicReadinessConfig() # Bounded parallelism across contracts; each contract keeps its own @@ -4877,16 +6545,20 @@ async def subscribe_wired_contract_topics( semaphore = asyncio.Semaphore(knobs.max_concurrent_contract_attach) async def _provision_ready_attach( - name: str, contract: ModelDiscoveredContract + result: ModelContractWiringResult, + contract: ModelDiscoveredContract, ) -> ModelContractAttachResult: async with semaphore: return await _interleave_contract( - name=name, + name=result.contract_name, contract=contract, dispatch_engine=dispatch_engine, event_bus=event_bus, environment=environment, - result_applier=(result_appliers_by_contract or {}).get(name), + result_applier=(result_appliers_by_contract or {}).get( + result.contract_name + ), + allowed_dispatcher_ids=result.dispatchers_registered, provisioner=provisioner, readiness_config=knobs, core_runtime_topics=core_runtime_topics, @@ -4894,7 +6566,7 @@ async def _provision_ready_attach( ) attach_results = await asyncio.gather( - *(_provision_ready_attach(name, contract) for name, contract in eligible) + *(_provision_ready_attach(result, contract) for result, contract in eligible) ) if attach_results_out is not None: @@ -4915,6 +6587,7 @@ async def _interleave_contract( event_bus: object, environment: str, result_applier: ProtocolDispatchResultApplier | None, + allowed_dispatcher_ids: Collection[str] | None, provisioner: ProtocolTopicProvisioner | None, readiness_config: ModelTopicReadinessConfig, core_runtime_topics: frozenset[str] = frozenset(), @@ -4925,6 +6598,11 @@ async def _interleave_contract( The order invariant is enforced here: every ``ensure_topic_exists`` for the contract precedes its readiness confirm, which precedes consumer attach. """ + dispatcher_scope = _require_registered_contract_dispatcher_scope( + dispatch_engine, + allowed_dispatcher_ids, + contract_name=name, + ) provision_topics = _contract_provision_topics(contract) readiness: ModelTopicSetReadiness | None = None @@ -4933,6 +6611,13 @@ async def _interleave_contract( for topic in provision_topics: try: await provisioner.ensure_topic_exists(topic_name=topic) + except TopicReplicationPolicyError: + # OMN-15395: a durability-policy violation is fail-closed and + # must escape this best-effort boundary. Attaching a consumer to + # a contract whose topics were silently skipped because one of + # them declares RF1 on MSK is the exact outcome the policy + # exists to prevent. + raise except Exception: # noqa: BLE001 — boundary: per-contract, never fatal logger.warning( "Topic provisioning failed for contract '%s' topic '%s' " @@ -4970,6 +6655,7 @@ async def _interleave_contract( return ModelContractAttachResult( contract_name=name, status=EnumContractAttachStatus.NOT_READY, + dispatcher_ids=tuple(sorted(dispatcher_scope)), readiness=readiness, detail=f"readiness {readiness.status.value}", ) @@ -4982,6 +6668,7 @@ async def _interleave_contract( event_bus=event_bus, environment=environment, result_applier=result_applier, + allowed_dispatcher_ids=dispatcher_scope, core_runtime_topics=core_runtime_topics, core_runtime_owners=core_runtime_owners, ) @@ -4995,6 +6682,7 @@ async def _interleave_contract( return ModelContractAttachResult( contract_name=name, status=EnumContractAttachStatus.FAILED, + dispatcher_ids=tuple(sorted(dispatcher_scope)), readiness=readiness, detail=type(exc).__name__, ) @@ -5002,11 +6690,229 @@ async def _interleave_contract( return ModelContractAttachResult( contract_name=name, status=EnumContractAttachStatus.ATTACHED, + dispatcher_ids=tuple(sorted(dispatcher_scope)), topics_subscribed=tuple(topics_subscribed), readiness=readiness, ) +# Bounded background NOT_READY reconciliation (OMN-15215, OMN-13237 follow-up). +DEFAULT_NOT_READY_RETRY_INITIAL_DELAY_SECONDS: float = 30.0 +DEFAULT_NOT_READY_RETRY_BACKOFF_SECONDS: float = 30.0 +DEFAULT_NOT_READY_RETRY_MAX_ATTEMPTS: int = 5 + + +async def reattach_not_ready_contracts( + manifest: ModelAutoWiringManifest, + not_ready_results: Sequence[ModelContractAttachResult], + dispatch_engine: ProtocolDispatchEngine, + event_bus: object | None, + environment: str = "dev", + result_appliers_by_contract: Mapping[str, ProtocolDispatchResultApplier] + | None = None, + *, + provisioner: ProtocolTopicProvisioner | None = None, + readiness_config: ModelTopicReadinessConfig | None = None, + core_runtime_topics: frozenset[str] = frozenset(), + core_runtime_owners: Mapping[str, str] | None = None, +) -> tuple[dict[str, tuple[str, ...]], tuple[ModelContractAttachResult, ...]]: + """Re-attempt provision -> confirm-ready -> attach for contracts still NOT_READY. + + OMN-15215 (CONFIRMED root cause): ``subscribe_wired_contract_topics`` makes + exactly ONE provision->confirm->attach attempt per contract via + ``_interleave_contract``. A contract whose topic metadata has not converged + within the bounded readiness poll (``ModelTopicReadinessConfig``, 30s/60 + attempts by default) is recorded NOT_READY and its consumer attach is + skipped — PERMANENTLY, for the rest of the process lifetime, because + nothing ever calls ``_interleave_contract`` for it again. For a + wide-topic-count contract (e.g. ``node_ledger_projection_compute``'s 26 + topics, OMN-15006/OMN-15168) a transient cold-broker topic-creation race on + a handful of just-provisioned topics starves the ENTIRE contract's + consumer: since ``_subscribe_contract_topics`` subscribes a contract's + topics as one all-or-nothing unit, zero of its 26 topics ever get a Kafka + consumer group — not even the ones unrelated to the race. Live evidence + (fresh stability-test boot, 2026-07-27): ``NOT-READY: topic metadata did + not converge (status=not_ready failures=[4 OCC governance topics])`` + logged exactly once at boot, followed by a ZERO count of "Auto-wired + subscription ... node=node_ledger_projection_compute" log lines across the + container's entire observed lifetime (3 separate contract-discovery + passes, ~11 minutes) — the OMN-13237 "runtime stays live" framing implies + eventual recoverability that was never actually implemented. + + This is NOT the ``handler_routing_loader`` "Unknown routing_strategy + 'topic_match'" fallback warning (that code path is a separate, informational + ``RuntimeContractConfigLoader`` boot-summary pass — its output is never + consumed by ``auto_wiring``'s wire/attach decision, confirmed by a real, + unmocked repro: the current ``discovery.py`` + ``handler_wiring.py`` path + already wires and attaches 26/26 topic_match entries for + ``node_ledger_projection_compute`` via the topic-folded dispatcher-ID + derivation from OMN-14580/OMN-13825). Fixing the loader's + ``VALID_ROUTING_STRATEGIES`` alone would NOT have unblocked OMN-15169 — + only closing this NOT_READY-has-no-retry gap does. + + Re-runs the SAME provision->confirm->attach interleave + (``_interleave_contract``) for each contract still in NOT_READY status, + returning newly-attached topics and updated per-contract results. Callers + invoke this repeatedly (bounded, with backoff — see + ``run_not_ready_reconciliation_loop``) until every contract attaches or a + bounded retry budget is exhausted. + """ + pending_results = _validate_not_ready_contract_identities( + manifest, + not_ready_results, + ) + if event_bus is None: + return {}, () + + contract_by_name = {contract.name: contract for contract in manifest.contracts} + still_not_ready_names = tuple(result.contract_name for result in pending_results) + if not still_not_ready_names: + return {}, () + + _validate_contract_dispatcher_ownership( + dispatch_engine, + tuple( + (result.contract_name, result.dispatcher_ids) for result in pending_results + ), + ) + + knobs = readiness_config or ModelTopicReadinessConfig() + semaphore = asyncio.Semaphore(knobs.max_concurrent_contract_attach) + + not_ready_by_name = {result.contract_name: result for result in pending_results} + + async def _retry_one(name: str) -> ModelContractAttachResult | None: + contract = contract_by_name.get(name) + previous_result = not_ready_by_name.get(name) + if contract is None or previous_result is None: + return None + async with semaphore: + return await _interleave_contract( + name=name, + contract=contract, + dispatch_engine=dispatch_engine, + event_bus=event_bus, + environment=environment, + result_applier=(result_appliers_by_contract or {}).get(name), + allowed_dispatcher_ids=previous_result.dispatcher_ids, + provisioner=provisioner, + readiness_config=knobs, + core_runtime_topics=core_runtime_topics, + core_runtime_owners=core_runtime_owners, + ) + + retried = await asyncio.gather( + *(_retry_one(name) for name in still_not_ready_names) + ) + results = tuple(r for r in retried if r is not None) + + newly_subscribed: dict[str, tuple[str, ...]] = { + r.contract_name: r.topics_subscribed + for r in results + if r.status is EnumContractAttachStatus.ATTACHED + } + return newly_subscribed, results + + +async def run_not_ready_reconciliation_loop( + manifest: ModelAutoWiringManifest, + initial_not_ready: Sequence[ModelContractAttachResult], + dispatch_engine: ProtocolDispatchEngine, + event_bus: object | None, + environment: str = "dev", + result_appliers_by_contract: Mapping[str, ProtocolDispatchResultApplier] + | None = None, + *, + provisioner: ProtocolTopicProvisioner | None = None, + readiness_config: ModelTopicReadinessConfig | None = None, + core_runtime_topics: frozenset[str] = frozenset(), + core_runtime_owners: Mapping[str, str] | None = None, + initial_delay_seconds: float = DEFAULT_NOT_READY_RETRY_INITIAL_DELAY_SECONDS, + backoff_seconds: float = DEFAULT_NOT_READY_RETRY_BACKOFF_SECONDS, + max_attempts: int = DEFAULT_NOT_READY_RETRY_MAX_ATTEMPTS, + on_attempt: Callable[ + [dict[str, tuple[str, ...]], tuple[ModelContractAttachResult, ...]], None + ] + | None = None, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, +) -> tuple[ModelContractAttachResult, ...]: + """Bounded background retry of NOT_READY contracts (OMN-15215). + + Sleeps ``initial_delay_seconds``, then re-attempts every still-NOT_READY + contract via ``reattach_not_ready_contracts``, up to ``max_attempts`` times + with ``backoff_seconds`` between attempts. Stops early once every contract + has attached. Never raises on a still-NOT_READY outcome — this preserves + the OMN-13237 fail-open boot contract (a contract that never converges + stays degraded, not crash-looping); this loop makes "runtime stays live" + actually recoverable instead of a permanent skip. ``on_attempt`` is an + optional caller hook (e.g. to fold newly-subscribed topics into shared + boot-time bookkeeping such as topic-collision detection) invoked after + each attempt with ``(newly_subscribed, results)``. ``sleep`` is injectable + so tests can drive the loop without real wall-clock delay. + """ + validated_not_ready = _validate_not_ready_contract_identities( + manifest, + initial_not_ready, + ) + _validate_contract_dispatcher_ownership( + dispatch_engine, + tuple( + (result.contract_name, result.dispatcher_ids) + for result in validated_not_ready + ), + ) + pending: dict[str, ModelContractAttachResult] = { + r.contract_name: r for r in validated_not_ready + } + if not pending: + return () + + await sleep(initial_delay_seconds) + latest: dict[str, ModelContractAttachResult] = {} + for attempt in range(1, max_attempts + 1): + if not pending: + break + newly_subscribed, results = await reattach_not_ready_contracts( + manifest, + tuple(pending.values()), + dispatch_engine, + event_bus, + environment, + result_appliers_by_contract, + provisioner=provisioner, + readiness_config=readiness_config, + core_runtime_topics=core_runtime_topics, + core_runtime_owners=core_runtime_owners, + ) + for result in results: + latest[result.contract_name] = result + if result.status is EnumContractAttachStatus.ATTACHED: + pending.pop(result.contract_name, None) + else: + pending[result.contract_name] = result + if on_attempt is not None: + on_attempt(newly_subscribed, results) + logger.info( + "NOT_READY reconciliation attempt %d/%d: resolved=%d remaining=%d " + "(OMN-15215)", + attempt, + max_attempts, + len(newly_subscribed), + len(pending), + ) + if pending and attempt < max_attempts: + await sleep(backoff_seconds) + + if pending: + logger.warning( + "NOT_READY reconciliation exhausted after %d attempts, still " + "not-ready: %s (OMN-15215/OMN-13237, runtime stays live degraded)", + max_attempts, + sorted(pending), + ) + return tuple(latest.values()) + + def _prioritize_subscription_results( report: ModelAutoWiringReport, result_appliers_by_contract: Mapping[str, ProtocolDispatchResultApplier] @@ -5048,6 +6954,8 @@ def _prepare_contract_wiring( pre_resolved_handlers: dict[str, object] | None = None, result_appliers_by_contract: Mapping[str, ProtocolDispatchResultApplier] | None = None, + topology: ModelDeploymentTopology | None = None, + catalog_binding_policy: ProjectionCatalogBindingPolicy | None = None, ) -> PreparedContractWiring: """Prepare one contract for wiring — NO side effects. @@ -5149,6 +7057,8 @@ def _prepare_contract_wiring( container=container, materialized_explicit_dependencies=materialized_explicit_dependencies, pre_resolved_handlers=pre_resolved_handlers, + topology=topology, + catalog_binding_policy=catalog_binding_policy, ) prepared_wirings.append(prepared) except TypeError: @@ -5210,6 +7120,7 @@ async def _commit_contract_wiring( *, subscribe_immediately: bool = True, result_applier: ProtocolDispatchResultApplier | None = None, + dynamic_materialization_authorized: bool = False, ) -> ModelContractWiringResult: """Commit a validated PreparedContractWiring to the engine and event bus. @@ -5233,7 +7144,12 @@ async def _commit_contract_wiring( quarantined: list[ModelQuarantinedWiring] = [] for prepared in pcw.prepared_wirings: - dispatcher_id, route_ids = _commit_handler_wiring(prepared, dispatch_engine) + dispatcher_id, route_ids = _commit_handler_wiring( + prepared, + dispatch_engine, + owner_contract_name=contract.name, + dynamic_materialization_authorized=dynamic_materialization_authorized, + ) if prepared.is_quarantined: assert prepared.quarantine_reason is not None # narrow for mypy quarantined.append( @@ -5298,7 +7214,12 @@ async def _commit_contract_wiring( quarantined_handlers=tuple(quarantined), ) - if subscribe_immediately and event_bus is not None and pcw.subscription_topics: + if ( + subscribe_immediately + and event_bus is not None + and pcw.subscription_topics + and dispatchers_registered + ): topics_subscribed.extend( await _subscribe_contract_topics( contract=contract, @@ -5306,8 +7227,20 @@ async def _commit_contract_wiring( event_bus=event_bus, environment=pcw.environment, result_applier=result_applier, + allowed_dispatcher_ids=dispatchers_registered, ) ) + elif ( + subscribe_immediately + and event_bus is not None + and pcw.subscription_topics + and not dispatchers_registered + ): + logger.info( + "Auto-wiring: skipping Kafka subscription for contract '%s' " + "because it owns zero dispatchers (OMN-15474)", + contract.name, + ) # OMN-9457: when every prepared handler was quarantined, report SKIPPED # with reason "all handlers quarantined" — there is nothing wired on @@ -5353,6 +7286,7 @@ async def _subscribe_contract_topics( event_bus: object, environment: str, result_applier: ProtocolDispatchResultApplier | None = None, + allowed_dispatcher_ids: Collection[str] | None = None, core_runtime_topics: frozenset[str] = frozenset(), core_runtime_owners: Mapping[str, str] | None = None, ) -> list[str]: @@ -5375,6 +7309,12 @@ async def _subscribe_contract_topics( if contract.event_bus is None or not contract.event_bus.subscribe_topics: return [] + dispatcher_scope = _require_registered_contract_dispatcher_scope( + dispatch_engine, + allowed_dispatcher_ids, + contract_name=contract.name, + ) + from omnibase_infra.enums import EnumConsumerGroupPurpose from omnibase_infra.models import ModelNodeIdentity from omnibase_infra.runtime.event_bus_subcontract_wiring import ( @@ -5416,6 +7356,15 @@ async def _subscribe_contract_topics( output_topic=output_topic, output_topic_map=output_topic_map, allowed_output_topics=contract.event_bus.publish_topics, + # OMN-15468 AC2: hand the applier the contract's DECLARED failure + # terminal so a returned model that states a failure verdict cannot + # be republished onto the success terminal by map-miss fallback. + # Read through the same single reader the Pattern B broker's + # subscription set is built from, so the two cannot disagree about + # which topics are terminal for this contract. + failure_terminal_topics=_declared_failure_terminal_topics( + contract, success_topic=output_topic + ), ) node_identity = ModelNodeIdentity( env=environment, @@ -5465,6 +7414,7 @@ async def _subscribe_contract_topics( # Why: Runtime wiring validates and narrows this payload shape before use. dispatch_engine, # type: ignore[arg-type] effective_result_applier, + allowed_dispatcher_ids=dispatcher_scope, ) else: callback = _make_event_bus_callback( @@ -5481,6 +7431,7 @@ async def _subscribe_contract_topics( # (redeliver) instead of being log-and-discarded. Non-state_io # contracts keep the historical swallow behavior unchanged. propagate_publish_failures=_contract_declares_state_io(contract), + allowed_dispatcher_ids=dispatcher_scope, ) topic_callbacks.append((topic, callback)) @@ -5511,6 +7462,38 @@ async def _subscribe_one( return topics_subscribed +def _declared_failure_terminal_topics( + contract: ModelDiscoveredContract, + *, + success_topic: str, +) -> tuple[str, ...]: + """Return the contract's declared FAILURE terminal topics (OMN-15468 AC2). + + A failure terminal is any contract-declared terminal topic that is (a) not + the success terminal the applier falls back to and (b) actually publishable + by this contract. Both conditions matter: publishing to an undeclared topic + would violate the contract's own publish allowlist, and re-routing to the + success terminal would be a no-op. + + Read through :func:`load_terminal_event_topics` — the SAME reader the + Pattern B broker's subscription set is built from — so the applier's idea of + which topics are terminal cannot drift from the broker's. A second + hand-rolled reader here is exactly the seam mismatch that produced this + ticket. + """ + if contract.event_bus is None or not contract.event_bus.publish_topics: + return () + publishable = set(contract.event_bus.publish_topics) + declared = load_terminal_event_topics(contract.contract_path) + return tuple( + sorted( + topic + for topic in declared + if topic != success_topic and topic in publishable + ) + ) + + def _select_dispatch_result_output_topic( contract: ModelDiscoveredContract, ) -> str | None: @@ -5536,6 +7519,9 @@ async def _wire_single_contract( event_bus: object | None, environment: str, container: object | None = None, + topology: ModelDeploymentTopology | None = None, + catalog_binding_policy: ProjectionCatalogBindingPolicy | None = None, + dynamic_materialization_authorized: bool = False, ) -> ModelContractWiringResult: """Wire a single discovered contract into the dispatch engine. @@ -5560,8 +7546,20 @@ async def _wire_single_contract( event_bus=event_bus, environment=environment, container=container, + topology=topology, + catalog_binding_policy=catalog_binding_policy, + ) + _preflight_prepared_registration_ids( + (prepared,), + dispatch_engine, + dynamic_materialization_authorized=dynamic_materialization_authorized, + ) + return await _commit_contract_wiring( + prepared, + dispatch_engine, + event_bus, + dynamic_materialization_authorized=dynamic_materialization_authorized, ) - return await _commit_contract_wiring(prepared, dispatch_engine, event_bus) def _prepare_handler_wiring( @@ -5575,6 +7573,8 @@ def _prepare_handler_wiring( container: object | None = None, materialized_explicit_dependencies: (dict[str, dict[str, object]] | None) = None, pre_resolved_handlers: dict[str, object] | None = None, + topology: ModelDeploymentTopology | None = None, + catalog_binding_policy: ProjectionCatalogBindingPolicy | None = None, ) -> PreparedWiring: """Prepare one handler entry — delegates construction to the resolver. @@ -5681,6 +7681,11 @@ def _prepare_handler_wiring( event_bus=event_bus, container=_effective_container, ownership_query=ownership_query, + # OMN-15468: the publishing contract's own declared terminal topics, + # read through the same function route discovery uses, so the wiring's + # notion of "this publish is a terminal" cannot drift from the set the + # Pattern B broker subscribes to. + terminal_topics=load_terminal_event_topics(contract.contract_path), ) def _quarantine_prepared( @@ -5836,7 +7841,7 @@ def _quarantine_prepared( # after boundary hook and returns a normal ModelDispatchResult through # the standard result-applier path. A contract declaring both is a # wiring-time contract defect, not a case to silently prioritize one arm. - db_tables = _read_db_io_tables(contract.contract_path) + db_tables = tuple(contract.db_io.db_tables) if contract.db_io is not None else () state_io = _read_state_io(contract.contract_path) if db_tables and state_io: raise ModelOnexError( @@ -5845,6 +7850,18 @@ def _quarantine_prepared( "(OMN-14208); a contract must declare exactly one." ) if db_tables: + if topology is None: + raise ModelOnexError( + f"handler_wiring: contract {contract.name!r} declares db_io but " + "wire_from_manifest received no checked-in ModelDeploymentTopology" + ) + catalog_policy = catalog_binding_policy or ProjectionCatalogBindingPolicy() + target = _resolve_projection_database_target( + db_tables, + topology, + catalog_read_binding=catalog_policy.read_binding, + catalog_write_binding=catalog_policy.write_binding, + ) subscribe_topics = ( contract.event_bus.subscribe_topics if contract.event_bus else () ) @@ -5856,13 +7873,21 @@ def _quarantine_prepared( else None ) # OMN-13548 (D-03): resolve the malformed-event DLQ destination from the - # contract's event_bus.dlq_topics (not the typed subcontract, which omits - # the field) so a projection handler error routes to the bus instead of - # being logged + dropped. Never hardcoded here. - projection_dlq_topics = _read_dlq_topics(contract.contract_path) + # contract's typed event-bus declaration. Filesystem-discovered legacy + # contracts retain the raw-YAML fallback during the migration window. + typed_dlq_topics = ( + getattr(contract.event_bus, "dlq_topics", ()) + if contract.event_bus is not None + else () + ) + projection_dlq_topics = ( + list(typed_dlq_topics) + if typed_dlq_topics + else _read_dlq_topics(contract.contract_path) + ) callback = _make_projection_dispatch_callback( handler_instance, - db_tables, + target, subscribe_topics, sinks=ProjectionDispatchSinks( event_bus=event_bus, @@ -5874,7 +7899,7 @@ def _quarantine_prepared( "Auto-wired projection handler with DB injection: handler=%s db_tables=%s " "terminal_event=%s dlq_topics=%s", handler_ref.name, - [t.get("name") for t in db_tables], + [table.name for table in target.tables], projection_terminal_event, projection_dlq_topics, ) @@ -5993,6 +8018,8 @@ def _quarantine_prepared( def _commit_handler_wiring( prepared: PreparedWiring, dispatch_engine: object, + *, + owner_contract_name: str | None = None, dynamic_materialization_authorized: bool = False, ) -> tuple[str, list[str]]: """Register a prepared handler wiring with the dispatch engine (side effects only). @@ -6012,6 +8039,9 @@ def _commit_handler_wiring( the private dynamic registration methods are used instead of the standard ones. This flag MUST only be set by ``materialize_cached_contract()`` after full contract validation — never by general application code (OMN-11246). + ``owner_contract_name`` records the contract provenance used by the + pre-subscribe ownership validator. Auto-wiring callers always supply it; + direct/manual registrations remain deliberately unowned. Returns: Tuple of (dispatcher_id, list of route_ids registered). Returns @@ -6041,6 +8071,7 @@ def _commit_handler_wiring( category=prepared.category, message_types=prepared.message_types, payload_type_matcher=prepared.payload_type_matcher, + owner_contract_name=owner_contract_name, ) for route in prepared.routes: engine._register_route_dynamic(route) @@ -6051,6 +8082,7 @@ def _commit_handler_wiring( category=prepared.category, message_types=prepared.message_types, payload_type_matcher=prepared.payload_type_matcher, + owner_contract_name=owner_contract_name, ) for route in prepared.routes: engine.register_route(route) @@ -6065,6 +8097,8 @@ def _wire_handler_entry( dispatch_engine: object, event_bus: object | None = None, container: object | None = None, + topology: ModelDeploymentTopology | None = None, + catalog_binding_policy: ProjectionCatalogBindingPolicy | None = None, ) -> tuple[str, list[str]]: """Prepare and immediately commit one handler entry (single-contract shortcut). @@ -6086,5 +8120,11 @@ def _wire_handler_entry( ownership_query=ownership_query, event_bus=event_bus, container=container, + topology=topology, + catalog_binding_policy=catalog_binding_policy, + ) + return _commit_handler_wiring( + prepared, + dispatch_engine, + owner_contract_name=contract.name, ) - return _commit_handler_wiring(prepared, dispatch_engine) diff --git a/src/omnibase_infra/runtime/auto_wiring/introspection_manifest_identity.py b/src/omnibase_infra/runtime/auto_wiring/introspection_manifest_identity.py new file mode 100644 index 0000000000..3e45c5e446 --- /dev/null +++ b/src/omnibase_infra/runtime/auto_wiring/introspection_manifest_identity.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Bind runtime build identity onto the introspection manifest (OMN-10856). + +Extracted as a standalone pure function — analogous to +``manifest_builder.build_runtime_manifest`` — so the identity-binding seam +served over ``/v1/introspection/manifest`` is unit-testable without driving +the full ``service_kernel.bootstrap()`` sequence. The kernel calls exactly +this function at its ``health_server.attach_manifest(...)`` call site +(``service_kernel.py`` ~L3199-3208), so a test that asserts on this +function's output is asserting on the artifact that runs. + +This module does NOT read process environment variables itself. +``scripts/check-env-reads.sh`` restricts new env-var reads to an approved +boundary set that ``service_kernel.py`` is on and this module is not; the +kernel resolves ``ONEX_IMAGE_DIGEST`` / ``ONEX_DEPLOYMENT_SHA`` there and +passes the already-classified ``ModelRuntimeBuildSha`` values in. +``ONEX_IMAGE_DIGEST`` is chosen to reuse the existing single source of +truth: it is already read by ``manifest_builder.build_runtime_manifest`` +(the separate ``runtime_manifests`` projection pathway, +OMN-11196/OMN-11197) via ``service_kernel.py``'s +``publish_runtime_manifest(image_digest=...)`` call — reusing the same name +means one env var injection serves both the projection pathway and the +introspection HTTP surface, not two divergently named ones. +``ONEX_DEPLOYMENT_SHA`` has no prior reader in this repo; as of +OMN-10856 neither var is injected anywhere in ``omninode_infra``'s +``k8s/onex-dev/runtime/`` Deployments (verified by direct grep) — both +surface as absent-with-reason until the companion env-injection change +lands there. +""" + +from __future__ import annotations + +from omnibase_infra.runtime.auto_wiring.models.model_auto_wiring_manifest import ( + ModelAutoWiringManifest, +) +from omnibase_infra.runtime.auto_wiring.models.model_runtime_build_sha import ( + ModelRuntimeBuildSha, +) + +ENV_VAR_IMAGE_SHA = "ONEX_IMAGE_DIGEST" +ENV_VAR_DEPLOYMENT_SHA = "ONEX_DEPLOYMENT_SHA" + + +def bind_introspection_manifest_identity( + manifest: ModelAutoWiringManifest, + *, + runtime_profile: str, + image_sha: ModelRuntimeBuildSha, + deployment_sha: ModelRuntimeBuildSha, +) -> ModelAutoWiringManifest: + """Return a copy of ``manifest`` with build identity bound (OMN-10856). + + Reuses ``manifest.contracts`` / ``manifest.errors`` verbatim — topology + is never re-derived here, only the identity fields are added — so this + stays the single source for what auto-discovery found. + + Args: + manifest: The (filtered or discovery-only) auto-wiring manifest to + enrich. Its ``contracts``/``errors`` are carried through + unchanged. + runtime_profile: The resolved ``RUNTIME_PROFILE`` identity (e.g. + from ``load_runtime_profile().name``). + image_sha: Already-classified build SHA (present or + absent-with-reason) — the caller resolves this from + ``ENV_VAR_IMAGE_SHA``. + deployment_sha: Already-classified deployment SHA — the caller + resolves this from ``ENV_VAR_DEPLOYMENT_SHA``. + + Returns: + A new ``ModelAutoWiringManifest`` with all three identity fields set. + """ + return ModelAutoWiringManifest( + contracts=manifest.contracts, + errors=manifest.errors, + runtime_profile=runtime_profile, + image_sha=image_sha, + deployment_sha=deployment_sha, + ) + + +__all__: list[str] = [ + "ENV_VAR_DEPLOYMENT_SHA", + "ENV_VAR_IMAGE_SHA", + "bind_introspection_manifest_identity", +] diff --git a/src/omnibase_infra/runtime/auto_wiring/models/__init__.py b/src/omnibase_infra/runtime/auto_wiring/models/__init__.py index 7bcfeb62ce..f54fe4b5f4 100644 --- a/src/omnibase_infra/runtime/auto_wiring/models/__init__.py +++ b/src/omnibase_infra/runtime/auto_wiring/models/__init__.py @@ -49,6 +49,9 @@ from omnibase_infra.runtime.auto_wiring.models.model_quarantine_record import ( ModelQuarantineRecord, ) +from omnibase_infra.runtime.auto_wiring.models.model_runtime_build_sha import ( + ModelRuntimeBuildSha, +) __all__ = [ "HandshakeFailureReason", @@ -65,4 +68,5 @@ "ModelLifecycleHookResult", "ModelLifecycleHooks", "ModelQuarantineRecord", + "ModelRuntimeBuildSha", ] diff --git a/src/omnibase_infra/runtime/auto_wiring/models/model_auto_wiring_manifest.py b/src/omnibase_infra/runtime/auto_wiring/models/model_auto_wiring_manifest.py index 0c9a0b431e..f122871aa1 100644 --- a/src/omnibase_infra/runtime/auto_wiring/models/model_auto_wiring_manifest.py +++ b/src/omnibase_infra/runtime/auto_wiring/models/model_auto_wiring_manifest.py @@ -12,13 +12,35 @@ from omnibase_infra.runtime.auto_wiring.models.model_discovery_error import ( ModelDiscoveryError, ) +from omnibase_infra.runtime.auto_wiring.models.model_runtime_build_sha import ( + ModelRuntimeBuildSha, +) + + +def _unbound_build_sha() -> ModelRuntimeBuildSha: + """Default for legacy/discovery-only construction (OMN-10856). + + Distinct from an env-lookup miss: this manifest was constructed before + any identity binding was attempted at all (e.g. ``discover_contracts()`` + or one of the ~50 pre-existing test call sites that only care about + ``contracts``/``errors``). ``bind_introspection_manifest_identity()`` is + the single place that resolves the real SHA-from-env values. + """ + return ModelRuntimeBuildSha( + value=None, + absent_reason="manifest constructed without a build-identity binding", + ) class ModelAutoWiringManifest(BaseModel): """Complete manifest produced by contract auto-discovery. Contains all successfully discovered contracts and any errors - encountered during scanning. Pure data — no side effects. + encountered during scanning, plus the runtime build identity (OMN-10856) + that binds this reported topology to a specific deployed process: + which runtime profile produced it, and which image/deployment build it + came from. Pure data — no side effects; identity resolution (env var + reads) happens in ``bind_introspection_manifest_identity``, not here. """ model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) @@ -31,6 +53,28 @@ class ModelAutoWiringManifest(BaseModel): default_factory=tuple, description="Errors encountered during discovery", ) + runtime_profile: str = Field( + default="", + description=( + "RUNTIME_PROFILE identity this manifest was built for (e.g. " + "'workers', 'effects', 'main'). Empty string means the manifest " + "was constructed before profile binding (discovery-only)." + ), + ) + image_sha: ModelRuntimeBuildSha = Field( + default_factory=_unbound_build_sha, + description=( + "Container image SHA/digest this runtime was built from, or an " + "explicit absent-with-reason marker (OMN-10856)." + ), + ) + deployment_sha: ModelRuntimeBuildSha = Field( + default_factory=_unbound_build_sha, + description=( + "Deployment/source revision SHA this runtime was deployed from, " + "or an explicit absent-with-reason marker (OMN-10856)." + ), + ) @property def total_discovered(self) -> int: diff --git a/src/omnibase_infra/runtime/auto_wiring/models/model_db_table_validation_warning.py b/src/omnibase_infra/runtime/auto_wiring/models/model_db_table_validation_warning.py new file mode 100644 index 0000000000..9d4e82fd15 --- /dev/null +++ b/src/omnibase_infra/runtime/auto_wiring/models/model_db_table_validation_warning.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed warning for a declared table absent from a catalog connection.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict + + +class ModelDbTableValidationWarning(BaseModel): + """One declared table absent from the current physical connection.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + reason: Literal["missing_db_table"] = "missing_db_table" + severity: Literal["warning"] = "warning" + table: str + database_ref: str + schema: str # type: ignore[assignment] + node: str + + +__all__ = ["ModelDbTableValidationWarning"] diff --git a/src/omnibase_infra/runtime/auto_wiring/models/model_discovered_contract.py b/src/omnibase_infra/runtime/auto_wiring/models/model_discovered_contract.py index 557a091372..ae3052de43 100644 --- a/src/omnibase_infra/runtime/auto_wiring/models/model_discovered_contract.py +++ b/src/omnibase_infra/runtime/auto_wiring/models/model_discovered_contract.py @@ -8,6 +8,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator +from omnibase_core.models.contracts.subcontracts.model_db_ownership_subcontract import ( + ModelDbOwnershipSubcontract, +) from omnibase_infra.runtime.auto_wiring.models.model_contract_version import ( ModelContractVersion, ) @@ -77,6 +80,13 @@ class ModelDiscoveredContract(BaseModel): handler_routing: ModelHandlerRouting | None = Field( default=None, description="Handler routing if declared" ) + db_io: ModelDbOwnershipSubcontract | None = Field( + default=None, + description=( + "Typed database ownership declarations. Table locations are validated " + "at discovery and never re-read as raw YAML during handler wiring." + ), + ) @field_validator("runtime_profiles", mode="before") @classmethod diff --git a/src/omnibase_infra/runtime/auto_wiring/models/model_event_bus_wiring.py b/src/omnibase_infra/runtime/auto_wiring/models/model_event_bus_wiring.py index 61b3325313..421f0e8bf6 100644 --- a/src/omnibase_infra/runtime/auto_wiring/models/model_event_bus_wiring.py +++ b/src/omnibase_infra/runtime/auto_wiring/models/model_event_bus_wiring.py @@ -20,6 +20,14 @@ class ModelEventBusWiring(BaseModel): default_factory=tuple, description="Topics this node publishes to", ) + dlq_topics: tuple[str, ...] = Field( + default_factory=tuple, + description="Contract-declared dead-letter topics", + ) + consumer_group: str | None = Field( + default=None, + description="Optional contract-declared consumer group identifier", + ) consumer_purpose: str | None = Field( default=None, description="Optional contract-declared consumer purpose", @@ -45,3 +53,7 @@ class ModelEventBusWiring(BaseModel): "behavior change for every non-opted-in contract." ), ) + terminal_event: str | None = Field( + default=None, + description="Optional terminal event topic declared by the event bus", + ) diff --git a/src/omnibase_infra/runtime/auto_wiring/models/model_runtime_build_sha.py b/src/omnibase_infra/runtime/auto_wiring/models/model_runtime_build_sha.py new file mode 100644 index 0000000000..a2e778ff96 --- /dev/null +++ b/src/omnibase_infra/runtime/auto_wiring/models/model_runtime_build_sha.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Runtime build-identity SHA value with explicit absent-with-reason semantics. + +OMN-10856: the auto-wiring manifest served over ``/v1/introspection/manifest`` +had no way to bind reported topology to a specific deployed build (no image +SHA, no deployment SHA). A build-identity value that is simply +``str | None = None`` (the pattern used by +``omnibase_core.models.runtime_manifest.model_runtime_manifest.ModelRuntimeManifest.image_digest``) +cannot distinguish "the environment variable was genuinely unset" from any +other reason a caller might pass ``None`` — this model makes that +distinction load-bearing: exactly one of ``value`` / ``absent_reason`` is +non-``None`` at all times, so absence always carries an explanation instead +of silently reading as "unknown". + +This module deliberately does NOT read process environment variables +itself — ``scripts/check-env-reads.sh`` restricts new env-var reads to an +approved boundary set (``service_kernel.py``, ``runtime/overlay/``, etc.) +and this is a plain data/models module. Callers resolve the raw string +(typically via the standard library's environment getter in +``service_kernel.py``, which is on that allowlist) and pass it to +:meth:`from_raw`. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class ModelRuntimeBuildSha(BaseModel): + """A single build-identity SHA (image digest or deployment/source revision). + + Fail-fast, never a silent default: :meth:`from_raw` turns an unset (or + blank) source value into ``value=None`` with a typed ``absent_reason`` + naming exactly why — never a fabricated placeholder like ``"unknown"``. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + value: str | None = Field( + default=None, + description="The resolved SHA/digest value, or None when absent.", + ) + absent_reason: str | None = Field( + default=None, + description=( + "Reason the value is absent. Must be None when value is present, " + "and must be set (non-blank) when value is None." + ), + ) + + @model_validator(mode="after") + def _validate_presence_xor_reason(self) -> ModelRuntimeBuildSha: + if self.value is None and not self.absent_reason: + raise ValueError( + "ModelRuntimeBuildSha requires a non-blank absent_reason " + "when value is None — absence must always be explained, " + "never silently defaulted." + ) + if self.value is not None and self.absent_reason is not None: + raise ValueError( + "ModelRuntimeBuildSha.absent_reason must be None when value " + "is present — a real value cannot also carry an absence " + "reason." + ) + return self + + @property + def is_present(self) -> bool: + """True when a real SHA/digest value was resolved.""" + return self.value is not None + + @classmethod + def present(cls, value: str) -> ModelRuntimeBuildSha: + """Construct a present value. Raises if ``value`` is blank.""" + stripped = value.strip() + if not stripped: + raise ValueError( + "ModelRuntimeBuildSha.present() requires a non-blank value" + ) + return cls(value=stripped, absent_reason=None) + + @classmethod + def absent(cls, reason: str) -> ModelRuntimeBuildSha: + """Construct an explicit absent-with-reason marker.""" + return cls(value=None, absent_reason=reason) + + @classmethod + def from_raw(cls, raw: str | None, *, source_name: str) -> ModelRuntimeBuildSha: + """Classify an already-fetched raw source value (fail-fast, no env I/O). + + ``raw`` is typically the environment lookup for ``source_name``, + resolved by the caller (e.g. ``service_kernel.py``, the approved + env-read boundary). + A blank (whitespace-only) value is treated the same as unset. + + Args: + raw: The raw value, or None if the source had nothing set. + source_name: Human-readable name of where ``raw`` came from + (e.g. the env var name), used only in the absent reason. + """ + if raw is None or raw.strip() == "": + return cls.absent(f"{source_name} is not set") + return cls.present(raw) + + +__all__: list[str] = ["ModelRuntimeBuildSha"] diff --git a/src/omnibase_infra/runtime/auto_wiring/protocol_db_table_catalog_connection.py b/src/omnibase_infra/runtime/auto_wiring/protocol_db_table_catalog_connection.py new file mode 100644 index 0000000000..96b6175458 --- /dev/null +++ b/src/omnibase_infra/runtime/auto_wiring/protocol_db_table_catalog_connection.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Minimal catalog connection protocol for typed table validation.""" + +from typing import Protocol + + +class ProtocolDbTableCatalogConnection(Protocol): + """Minimal asyncpg-compatible catalog query surface.""" + + async def fetchval(self, query: str, *args: object) -> object | None: ... + + +__all__ = ["ProtocolDbTableCatalogConnection"] diff --git a/src/omnibase_infra/runtime/contract_loaders/handler_routing_loader.py b/src/omnibase_infra/runtime/contract_loaders/handler_routing_loader.py index 09cc4d5ece..a7cd4bbc00 100644 --- a/src/omnibase_infra/runtime/contract_loaders/handler_routing_loader.py +++ b/src/omnibase_infra/runtime/contract_loaders/handler_routing_loader.py @@ -337,12 +337,24 @@ def load_and_validate_contract_yaml(contract_path: Path) -> ModelContractNodeTyp # Valid routing strategies for handler routing contracts. -# Currently only "payload_type_match" is implemented. Additional strategies -# such as "first_match" or "all_match" may be added in future versions. -# Unknown strategies will trigger a warning and fall back to "payload_type_match". +# "payload_type_match" and "topic_match" (OMN-15215) are both contract-legal +# values live in the auto_wiring consumer-attach path +# (omnibase_infra.runtime.auto_wiring.models.ModelHandlerRouting, +# OMN-14580/OMN-13825/OMN-14594) — node_ledger_projection_compute has declared +# routing_strategy: "topic_match" since OMN-14594. This module's +# ModelRoutingSubcontract does NOT carry topic_match's per-entry topic +# disambiguation (see model_routing_subcontract.py docstring); it only backs +# the informational RuntimeContractConfigLoader boot-summary pass. Prior to +# OMN-15215 this set omitted "topic_match", so every boot logged a false +# "Unknown routing_strategy 'topic_match' ... Using 'payload_type_match' as +# default" warning for that contract — confirmed live on a fresh +# omnibase-infra-stability-test boot, 2026-07-27. Additional strategies such +# as "first_match" or "all_match" may be added in future versions. Unknown +# strategies still trigger a warning and fall back to "payload_type_match". VALID_ROUTING_STRATEGIES: frozenset[str] = frozenset( { - "payload_type_match", # Match by payload type (default and only implemented) + "payload_type_match", # Match by payload type (default) + "topic_match", # Match by contract-declared per-entry topic (OMN-14594) } ) diff --git a/src/omnibase_infra/runtime/contract_terminal_events.py b/src/omnibase_infra/runtime/contract_terminal_events.py new file mode 100644 index 0000000000..224820b4d9 --- /dev/null +++ b/src/omnibase_infra/runtime/contract_terminal_events.py @@ -0,0 +1,389 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Contract-declared terminal events: one reader, one on-the-wire shape (OMN-15468). + +Two things must agree about a contract's terminal events, and before this module +they were written twice and agreed only by accident: + +1. **Which topics are terminal.** ``runtime_local_ingress`` reads them to build + the Pattern B broker's subscription set. The def-B auto-wiring needs the same + answer to know which handler publishes are terminal emissions. A second + hand-rolled reader in the wiring layer would be free to drift from the + broker's view of the same contract — the exact class of seam mismatch that + produced this ticket. :func:`extract_terminal_event_topics` is the single + reader; ``runtime_local_ingress._extract_terminal_events`` delegates to it. + +2. **What a terminal record looks like on the bus.** ``DispatchResultApplier`` + publishes the def-B return value as a full :class:`ModelEventEnvelope`. A + handler that emits its own terminal through the wiring-injected + ``event_publisher`` published the bytes it was handed — RAW, un-enveloped. + Same contract, same terminal pair, two different wire shapes depending on + which half of the wiring did the publishing. + + Live proof (``.201`` dev lane, 2026-07-30T17:13Z, merged ``5dc68190`` — the + readback that reopened OMN-15468 after #2560): for correlation + ``4a5e0730-…-000000000002`` the SUCCESS topic carried a full envelope + (``event_type='omnimarket.node-generation-completed'``, 19 envelope keys, + ``payload.contract_passed=False``) while the FAILURE topic carried + ``{"correlation_id": …, "task_description": …, "attempts": [...]}`` with no + ``event_type``, no ``envelope_id`` and no ``payload`` wrapper. #2560 had + already made the broker subscribe to the failure topic (proven by the + runtime's own ``Updating subscribed topics to: frozenset({'…-failed.v1'})`` + log line at both probe start times), so the subscription was live and the + record was sitting on it — but the shape is not one the broker's + envelope-decoding terminal path accepts, and the outer ``/skill`` response + stayed ``ok=true`` / ``status=completed`` / ``error=null``, byte-identical to + the success control. + +:func:`envelope_terminal_payload` closes (2) at the ONE factory that hands every +def-B handler its publisher, so the fix lands for every contract that declares a +terminal event rather than per node. Field parity with the applier's success +envelope is by construction: the same ``derive_event_type_from_topic`` helper +stamps ``event_type``, the same ``uuid5``-from-correlation scheme mints +``envelope_id``, and the same ``correlation_id``/``envelope_timestamp`` fields +are populated. + +3. **Whether a terminal record is telling the truth.** ``DispatchResultApplier`` + routed the def-B return value by class name alone: a model missing the + contract's ``published_events`` map fell back to the SUCCESS terminal + whatever verdict its payload carried, and the Pattern B broker then derived + ``completed`` from the arrival topic. :func:`resolve_terminal_verdict` reads + the verdict the payload already states and + :func:`apply_failure_terminal_guard` re-routes it to the contract's declared + failure terminal — fail-closed, and only when the model states a failure. + This is OMN-15468 acceptance criterion 2, which the earlier revision of this + docstring correctly recorded as still open; it is closed here. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +from collections.abc import Container, Iterable, Mapping, Sequence +from datetime import UTC, datetime +from pathlib import Path +from uuid import UUID, uuid5 + +import yaml + +from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope +from omnibase_infra.utils import derive_event_type_from_topic + +logger = logging.getLogger(__name__) + +# A record carrying any of these keys is already an envelope; wrapping it again +# would nest a terminal inside a terminal and break the broker's single unwrap. +_ENVELOPE_MARKER_KEYS: frozenset[str] = frozenset( + {"envelope_id", "envelope_version", "payload_type", "envelope_timestamp"} +) + +__all__ = [ + "apply_failure_terminal_guard", + "envelope_terminal_payload", + "extract_terminal_event_topics", + "load_terminal_event_topics", + "resolve_terminal_verdict", + "terminal_event_topics_from_declaration", +] + +# Status strings that name a terminal outcome, in the vocabulary the runtime +# already uses on the wire (``ModelDispatchBusTerminalResult.status`` is +# completed/failed/timeout; producers additionally use error/cancelled). +_FAILED_STATUS_VALUES: frozenset[str] = frozenset( + {"failed", "failure", "timeout", "timed_out", "error", "cancelled", "canceled"} +) +_SUCCEEDED_STATUS_VALUES: frozenset[str] = frozenset( + {"completed", "complete", "succeeded", "success", "ok"} +) + + +def _safe_optional_string(value: object) -> str | None: + if isinstance(value, str): + normalized = value.strip() + return normalized or None + return None + + +def terminal_event_topics_from_declaration(declaration: object) -> tuple[str, ...]: + """Normalize one ``terminal_events`` declaration into success-first topics. + + A mapping declaration is emitted with its ``success`` entry FIRST, regardless + of YAML key order, because the Pattern B broker treats + ``terminal_events[0]`` as the success topic whenever the contract has no + top-level ``terminal_event`` (``_status_for_terminal_topic``). Leaving that + to mapping order would make a terminal's completed-vs-failed meaning depend + on how the contract author happened to sort two YAML keys. + """ + + if isinstance(declaration, dict): + ordered: list[object] = [] + if "success" in declaration: + ordered.append(declaration["success"]) + ordered.extend(value for key, value in declaration.items() if key != "success") + values: Iterable[object] = ordered + elif isinstance(declaration, list | tuple): + values = declaration + else: + values = () + + topics: list[str] = [] + for value in values: + topic = _safe_optional_string(value) + if topic is not None: + topics.append(topic) + return tuple(topics) + + +def extract_terminal_event_topics(raw: Mapping[object, object]) -> tuple[str, ...]: + """Return all contract-declared terminal topics, success-first, de-duplicated. + + Reads three declaration sites, in success-first order: + + 1. top-level ``terminal_event`` (single success topic), + 2. top-level ``terminal_events`` (mapping or sequence), + 3. ``runtime_dispatch.terminal_events`` (OMN-15468 / #2560). + + Site 3 is the address external clients — the dashboard included — dispatch + through, and it is where **51** contracts declare their FAILURE terminal and + nowhere else. + + PROVENANCE — every number here is a measurement, not a constant. Framing: the + RAW corpus of 384 ``src/omnimarket/nodes/*/contract.yaml`` files at + ``omnimarket@aea0c33dd89fb82fdca33aac7149992a21c46d43`` (``origin/dev``), + measured 2026-07-30, no discovery filter. Re-derive rather than copy forward: + 51 = contracts whose ``runtime_dispatch.terminal_events`` normalizes + non-empty; 30 of those 51 declare no top-level ``terminal_event`` *or* + ``terminal_events``; 17 of those 30 also clear the route-discovery filter. + These drift as contracts land. + """ + + terminal_events: list[str] = [] + terminal_event = _safe_optional_string(raw.get("terminal_event")) + if terminal_event is not None: + terminal_events.append(terminal_event) + + terminal_events.extend( + terminal_event_topics_from_declaration(raw.get("terminal_events")) + ) + + runtime_dispatch = raw.get("runtime_dispatch") + if isinstance(runtime_dispatch, dict): + terminal_events.extend( + terminal_event_topics_from_declaration( + runtime_dispatch.get("terminal_events") + ) + ) + + return tuple(dict.fromkeys(terminal_events)) + + +def load_terminal_event_topics(contract_path: Path | None) -> frozenset[str]: + """Read a contract file and return its declared terminal topics. + + Fail-open by design: an unreadable or non-mapping contract yields the empty + set, which restores the pre-OMN-15468 publish behavior (bytes forwarded + verbatim) rather than failing a wiring that has nothing to do with terminals. + The contract discovery path already validated this file; a failure here means + the file moved or the runtime lost read access, and a handler must still be + constructible in that state. + """ + + if contract_path is None: + return frozenset() + path = contract_path + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + logger.warning( + "contract_terminal_events: could not read terminal declarations from " + "%s (%s) — handler publishes will be forwarded un-enveloped", + path, + type(exc).__name__, + ) + return frozenset() + if not isinstance(raw, Mapping): + return frozenset() + return frozenset(extract_terminal_event_topics(raw)) + + +def resolve_terminal_verdict(event: object) -> bool | None: + """Read a returned model's OWN terminal verdict. ``None`` means unknown. + + OMN-15468 AC2. ``DispatchResultApplier`` routes a definition-B return value + to the contract's SUCCESS terminal whenever the model's class name misses + the ``published_events`` map — the payload's verdict is never consulted. + Live on 2026-07-30 that republished a ``contract_passed=false`` benchmark + onto ``node-generation-completed.v1``, and the Pattern B broker, which + derives status purely from the arrival topic, reported ``ok=true`` for a run + that had failed. This function is the missing read. + + Fields are consulted in decreasing order of explicitness. Each is a field + ONEX producers already carry; nothing new is required of a handler to be + covered: + + 1. ``terminal_failure_cause`` — a typed non-``None`` cause is an + unambiguous failure declaration (the delegate-skill seam, OMN-15469). + 2. ``status`` — the wire vocabulary the broker itself terminalizes on. + 3. ``ok`` / ``success`` — explicit booleans. + 4. ``contract_passed`` — the generation-benchmark verdict field from this + ticket's original live reproduction. + + Returns ``None`` when the model declares no verdict at all, which is the + common case and MUST leave routing exactly as it was: this is a fail-closed + correction for models that state a failure, never a guess about models that + state nothing. + """ + cause = getattr(event, "terminal_failure_cause", None) + if cause is not None: + return False + + status = getattr(event, "status", None) + if isinstance(status, str): + normalized = status.strip().lower() + if normalized in _FAILED_STATUS_VALUES: + return False + if normalized in _SUCCEEDED_STATUS_VALUES: + return True + + for attribute in ("ok", "success", "contract_passed"): + value = getattr(event, attribute, None) + if isinstance(value, bool): + return value + + return None + + +def apply_failure_terminal_guard( + event: object, + topic: str, + *, + success_topic: str, + failure_terminal_topics: Sequence[str], +) -> str: + """Re-route a failure-verdict return value off the SUCCESS terminal. + + OMN-15468 AC2 — the residual PR #2578 left open. Class-based routing (a + handler returning a ``…Failed`` variant declared in ``published_events``) is + the primary mechanism and this guard never overrides it: it fires ONLY when + the resolution already landed on the contract's success terminal. It is the + fail-closed backstop for the case that produced the live defect — a returned + model whose class misses the map, which falls back to the success terminal + no matter what verdict the payload carries. + + Three conditions must ALL hold, so the guard cannot fire speculatively: + + * the resolved topic is the contract's success terminal; + * the contract declares exactly ONE failure terminal (two or more is + ambiguous — there is no basis to choose, so routing is left alone and the + ambiguity is logged rather than guessed at); + * the model declares an explicit failure verdict + (:func:`resolve_terminal_verdict` returns ``False``; ``None`` — no verdict + field at all, the common case — changes nothing). + + Lives here rather than on ``DispatchResultApplier`` so that *which topics + are terminal* and *what a failure verdict is* stay in the one module that + already owns both questions for the broker's subscription set. + """ + if topic != success_topic: + return topic + if len(failure_terminal_topics) != 1: + if failure_terminal_topics and resolve_terminal_verdict(event) is False: + logger.warning( + "Failure-verdict output event left on the success terminal: " + "contract declares %d failure terminals, cannot disambiguate " + "(OMN-15468)", + len(failure_terminal_topics), + extra={ + "output_event_type": type(event).__name__, + "success_topic": topic, + "failure_topics": list(failure_terminal_topics), + }, + ) + return topic + if resolve_terminal_verdict(event) is not False: + return topic + failure_topic = failure_terminal_topics[0] + logger.warning( + "Re-routing failure-verdict output event from success terminal %s to " + "declared failure terminal %s (OMN-15468)", + topic, + failure_topic, + extra={ + "output_event_type": type(event).__name__, + "success_topic": topic, + "failure_topic": failure_topic, + }, + ) + return failure_topic + + +def _is_already_envelope(body: Mapping[str, object]) -> bool: + return bool(_ENVELOPE_MARKER_KEYS & set(body)) and "payload" in body + + +def envelope_terminal_payload( + *, + topic: str, + payload: bytes, + terminal_topics: Container[str], +) -> bytes: + """Wrap a terminal-topic publish in a ``ModelEventEnvelope``; else pass through. + + Field parity with the success path (``DispatchResultApplier.apply``) is the + point, so each field is derived the same way that applier derives it: + + * ``payload`` — the publisher's own JSON body, unmodified. + * ``correlation_id`` — the body's top-level ``correlation_id``. + * ``event_type`` — ``derive_event_type_from_topic(topic)``, the shared + helper the applier calls, so a failure terminal is stamped ``…-failed`` + exactly as the success terminal is stamped ``…-completed``. + * ``envelope_id`` — ``uuid5`` in the correlation's namespace, so a + redelivered terminal mints the SAME id and stays de-duplicable. + * ``envelope_timestamp`` — emission time (UTC). + + Pass-through (byte-identical, zero behavior change) when any of these hold — + each is a case where wrapping would either be wrong or unverifiable: + + * ``topic`` is not a contract-declared terminal topic (ordinary command / + side-effect publishes keep their existing shape), + * the body is not a JSON object, + * the body carries no parseable ``correlation_id`` — an envelope whose + correlation cannot be set is undeliverable to any waiting caller anyway, + * the body is already an envelope (never double-wrap). + """ + + if topic not in terminal_topics: + return payload + + try: + decoded: object = json.loads(payload.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + return payload + if not isinstance(decoded, dict): + return payload + body: dict[str, object] = decoded + if _is_already_envelope(body): + return payload + + raw_correlation = body.get("correlation_id") + if raw_correlation is None: + return payload + try: + correlation_id = UUID(str(raw_correlation)) + except (TypeError, ValueError): + return payload + + envelope: ModelEventEnvelope[object] = ModelEventEnvelope[object]( + envelope_id=uuid5( + correlation_id, + f"{topic}:{hashlib.sha256(payload).hexdigest()}", + ), + payload=body, + correlation_id=correlation_id, + envelope_timestamp=datetime.now(UTC), + ) + event_type = derive_event_type_from_topic(topic) + if event_type is not None: + envelope = envelope.model_copy(update={"event_type": event_type}) + return envelope.model_dump_json().encode("utf-8") diff --git a/src/omnibase_infra/runtime/dispatch_envelope_context.py b/src/omnibase_infra/runtime/dispatch_envelope_context.py new file mode 100644 index 0000000000..cc547b8707 --- /dev/null +++ b/src/omnibase_infra/runtime/dispatch_envelope_context.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""In-process typed context channels for one materialized dispatch.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar + +from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope +from omnibase_infra.runtime.projection_tenant_authority import ( + VerifiedProjectionTenantAuthority, +) + +# The dispatch payload remains strictly JSON-safe. The original typed envelope +# travels beside it only to preserve transport identity such as envelope_id. It +# is explicitly NOT an authentication source. +_CURRENT_DISPATCH_ENVELOPE: ContextVar[ModelEventEnvelope[object] | None] = ContextVar( + "onex_current_dispatch_envelope", + default=None, +) + +# Authentication is a separate channel. Only the opaque capability minted by +# canonical signature verification may be bound here. +_CURRENT_PROJECTION_TENANT_AUTHORITY: ContextVar[ + VerifiedProjectionTenantAuthority | None +] = ContextVar("onex_current_projection_tenant_authority", default=None) + + +@contextmanager +def bind_dispatch_envelope(envelope: object) -> Iterator[None]: + """Bind only a typed envelope and restore the prior context on every exit.""" + authoritative = envelope if isinstance(envelope, ModelEventEnvelope) else None + token = _CURRENT_DISPATCH_ENVELOPE.set(authoritative) + try: + yield + finally: + _CURRENT_DISPATCH_ENVELOPE.reset(token) + + +def current_dispatch_envelope() -> ModelEventEnvelope[object] | None: + """Return the typed envelope bound to the current dispatcher invocation.""" + return _CURRENT_DISPATCH_ENVELOPE.get() + + +@contextmanager +def bind_projection_tenant_authority( + authority: VerifiedProjectionTenantAuthority, +) -> Iterator[None]: + """Bind one verified capability and restore the prior value on exit.""" + if type(authority) is not VerifiedProjectionTenantAuthority: + raise TypeError("projection tenant authority must be a verified capability") + token = _CURRENT_PROJECTION_TENANT_AUTHORITY.set(authority) + try: + yield + finally: + _CURRENT_PROJECTION_TENANT_AUTHORITY.reset(token) + + +def current_projection_tenant_authority() -> VerifiedProjectionTenantAuthority | None: + """Return the capability bound by a trusted ingress verification boundary.""" + return _CURRENT_PROJECTION_TENANT_AUTHORITY.get() + + +__all__ = [ + "bind_dispatch_envelope", + "bind_projection_tenant_authority", + "current_dispatch_envelope", + "current_projection_tenant_authority", +] diff --git a/src/omnibase_infra/runtime/event_bus_subcontract_wiring.py b/src/omnibase_infra/runtime/event_bus_subcontract_wiring.py index 7ad7e2e5b5..ec4bd96ded 100644 --- a/src/omnibase_infra/runtime/event_bus_subcontract_wiring.py +++ b/src/omnibase_infra/runtime/event_bus_subcontract_wiring.py @@ -132,6 +132,20 @@ ) +# Contract ownership metadata is consumed by auto-wiring before this loader runs. +# It is intentionally not part of ``ModelEventBusSubcontract``, which models the +# transport behavior used by ``EventBusSubcontractWiring``. Preserve strict +# validation for every other key while removing only the explicitly supported +# ownership fields at this adapter boundary. +_EVENT_BUS_WIRING_METADATA_FIELDS = frozenset( + { + "consumer_purpose", + "plugin_managed", + "tenant_scoped_ingress", + } +) + + def validate_topic(topic: str, deny_patterns: tuple[str, ...] = ()) -> None: """Validate a topic name against contract-declared deny patterns [OMN-6342]. @@ -1299,7 +1313,32 @@ def load_event_bus_subcontract( ) return None - return ModelEventBusSubcontract.model_validate(event_bus_data) + if not isinstance(event_bus_data, dict): + _logger.warning( + "event_bus section is not a dict in %s: got %s", + contract_path, + type(event_bus_data).__name__, + ) + return None + + supported_fields = set(ModelEventBusSubcontract.model_fields) + unknown_fields = ( + set(event_bus_data) - supported_fields - _EVENT_BUS_WIRING_METADATA_FIELDS + ) + if unknown_fields: + _logger.warning( + "Invalid event_bus subcontract in %s: unsupported fields: %s", + contract_path, + sorted(unknown_fields), + ) + return None + + subcontract_data = { + key: value + for key, value in event_bus_data.items() + if key in supported_fields + } + return ModelEventBusSubcontract.model_validate(subcontract_data) except yaml.YAMLError as e: _logger.warning( diff --git a/src/omnibase_infra/runtime/gateway_canary_probe.py b/src/omnibase_infra/runtime/gateway_canary_probe.py new file mode 100644 index 0000000000..8f500040fa --- /dev/null +++ b/src/omnibase_infra/runtime/gateway_canary_probe.py @@ -0,0 +1,280 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Path-verifying canary probe for the gateway container healthcheck (OMN-15741). + +Replaces ``test -f /tmp/gateway-forwarder-ready`` (a sentinel written once at +startup and never re-verified) with a real produce+readback round trip against +each broker leg, using the exact transport and credentials real gateway +traffic uses (``KafkaTransport`` over the resolved ``local_bus``/``cloud_bus`` +config). This is the failure mode of the 2026-08-04 outage: the process was up +and the ready-file was present for four days while the cloud leg forwarded 0% +-- a liveness signal that cannot see past "the process started once." + +The probe checks the local leg and the cloud leg independently and reports +each leg's outcome on its own line, so ``docker inspect ...State.Health.Log`` +distinguishes "local leg healthy, cloud leg dead" from "both legs healthy" -- +the acceptance criterion this ticket names explicitly. Overall exit is +non-zero if either leg fails. + +To avoid spamming either broker, a real check only runs once every +contract-declared ``canary.cadence_seconds``; between real checks this process +reports the last real result from ``--state-file`` (default +``/tmp/gateway-canary-probe-state.json``), refreshed on every real run. This +process never depends on the long-running forwarder process being alive or +reachable -- it dials the brokers directly, so a healthy forwarder container +whose cloud leg is actually dead is caught even though ``docker ps`` and the +old ready-file check would both call it fine. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import sys +import time +from collections.abc import Sequence +from pathlib import Path +from typing import NamedTuple +from uuid import uuid4 + +from omnibase_infra.event_bus.kafka_transport import KafkaTransport +from omnibase_infra.event_bus.models.config import ModelKafkaEventBusConfig +from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( + ModelGatewayCanaryConfig, + ModelGatewayForwarderRuntimeConfig, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( + prefix_topic, +) +from omnibase_infra.runtime.gateway_forwarder import ( + load_gateway_forwarder_runtime_config, +) + +logger = logging.getLogger(__name__) + +_CANARY_HEADER_NAME = "canary-correlation-id" +_POLL_SLICE_MS = 2000 + + +class ModelCanaryLegResult(NamedTuple): + """Outcome of one leg's real produce+readback attempt.""" + + leg: str + passed: bool + detail: str + + +async def check_canary_leg( + *, + leg: str, + bus_config: ModelKafkaEventBusConfig, + topic: str, + canary: ModelGatewayCanaryConfig, + transport_factory: type[KafkaTransport] = KafkaTransport, +) -> ModelCanaryLegResult: + """Produce one tiny canary record to ``topic`` and confirm readback. + + Uses a fresh, unique consumer group so every invocation joins at + ``auto_offset_reset="latest"`` -- the probe only ever waits for the record + it just produced, never replays history, so repeated runs cannot build up + unbounded lag on the dedicated canary topic. + """ + correlation_id = uuid4().hex.encode("ascii") + group = f"gateway-canary-probe-{leg}-{uuid4().hex}" + transport = transport_factory( + config=bus_config, + group=group, + topics=(topic,), + auto_offset_reset="latest", + ) + started = False + try: + try: + await asyncio.wait_for( + transport.start(), timeout=canary.produce_deadline_seconds + ) + except Exception as exc: # noqa: BLE001 -- any failure here means the leg is dead + return ModelCanaryLegResult( + leg=leg, passed=False, detail=f"{leg} leg connect failed: {exc}" + ) + started = True + + try: + await asyncio.wait_for( + transport.send( + topic, + None, + correlation_id, + {_CANARY_HEADER_NAME: correlation_id}, + ), + timeout=canary.produce_deadline_seconds, + ) + except Exception as exc: # noqa: BLE001 -- any failure here means the leg is dead + return ModelCanaryLegResult( + leg=leg, passed=False, detail=f"{leg} leg produce failed: {exc}" + ) + + deadline_at = time.monotonic() + canary.readback_deadline_seconds + while time.monotonic() < deadline_at: + remaining_ms = max(int((deadline_at - time.monotonic()) * 1000), 1) + try: + messages = await transport.poll( + max_messages=32, + timeout_ms=min(remaining_ms, _POLL_SLICE_MS), + ) + except Exception as exc: # noqa: BLE001 -- any failure here means the leg is dead + return ModelCanaryLegResult( + leg=leg, passed=False, detail=f"{leg} leg readback failed: {exc}" + ) + for message in messages: + if message.value == correlation_id: + return ModelCanaryLegResult( + leg=leg, + passed=True, + detail=f"{leg} leg produce+readback confirmed", + ) + return ModelCanaryLegResult( + leg=leg, + passed=False, + detail=( + f"{leg} leg readback timed out after " + f"{canary.readback_deadline_seconds}s -- canary record was " + "produced but never read back" + ), + ) + finally: + if started: + try: + await transport.close() + except Exception: # noqa: BLE001 -- cleanup must not mask the real result + logger.warning("canary probe: %s leg transport close failed", leg) + + +async def run_canary_check( + config: ModelGatewayForwarderRuntimeConfig, +) -> tuple[ModelCanaryLegResult, ModelCanaryLegResult]: + """Check the local leg and cloud leg concurrently; return both outcomes. + + The cloud-leg wire topic carries the tenant prefix, matching exactly what + real outbound traffic does (``prefix_topic``) -- the probe is not a + parallel code path, it drives the same transform real traffic drives. + """ + canary = config.forwarder.canary + tenant_slug = config.forwarder.tenant_identity.tenant_slug + local_result, cloud_result = await asyncio.gather( + check_canary_leg( + leg="local", + bus_config=config.local_bus, + topic=canary.topic, + canary=canary, + ), + check_canary_leg( + leg="cloud", + bus_config=config.cloud_bus, + topic=prefix_topic(tenant_slug, canary.topic), + canary=canary, + ), + ) + return local_result, cloud_result + + +def _load_cached_state(state_path: Path, cadence_seconds: int) -> str | None: + """Return the cached report if it is still within cadence, else ``None``.""" + try: + raw = json.loads(state_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + checked_at = raw.get("checked_at") + passed = raw.get("passed") + report = raw.get("report") + if not isinstance(checked_at, (int, float)) or not isinstance(passed, bool): + return None + if not isinstance(report, str): + return None + if time.time() - checked_at >= cadence_seconds: + return None + return report if passed else f"CANARY_STALE_FAILURE_CACHED\n{report}" + + +def _write_state( + state_path: Path, *, passed: bool, report: str, checked_at: float +) -> None: + payload = {"checked_at": checked_at, "passed": passed, "report": report} + try: + state_path.write_text(json.dumps(payload), encoding="utf-8") + except OSError: + # State-file persistence is a spam-avoidance optimization, not a + # correctness requirement -- a write failure must not turn a real + # PASS into a probe crash. + logger.warning("canary probe: failed to persist state to %s", state_path) + + +async def probe( + config: ModelGatewayForwarderRuntimeConfig, + *, + state_path: Path, + force: bool = False, +) -> tuple[bool, str]: + """Return ``(passed, report)``, consulting/refreshing the cadence cache.""" + canary = config.forwarder.canary + if not force: + cached = _load_cached_state(state_path, canary.cadence_seconds) + if cached is not None: + return not cached.startswith("CANARY_STALE_FAILURE_CACHED"), cached + + local_result, cloud_result = await run_canary_check(config) + passed = local_result.passed and cloud_result.passed + report = "\n".join( + f"{'PASS' if result.passed else 'FAIL'}: {result.detail}" + for result in (local_result, cloud_result) + ) + _write_state(state_path, passed=passed, report=report, checked_at=time.time()) + return passed, report + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Path-verifying canary healthcheck: produce+readback a canary " + "record across the local and cloud gateway legs." + ) + ) + parser.add_argument( + "--config", + type=Path, + required=True, + help="Path to the resolved, typed gateway forwarder YAML", + ) + parser.add_argument( + "--state-file", + type=Path, + default=Path("/tmp/gateway-canary-probe-state.json"), # noqa: S108 -- container-local scratch state + help="Cadence cache so repeated healthcheck ticks do not spam the brokers", + ) + parser.add_argument( + "--force", + action="store_true", + help="Bypass the cadence cache and always run a real produce+readback check", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> None: + """CLI entrypoint. Exits non-zero when either leg's path is dead.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + args = _build_parser().parse_args(argv) + config = load_gateway_forwarder_runtime_config(args.config) + passed, report = asyncio.run( + probe(config, state_path=args.state_file, force=args.force) + ) + print(report) # noqa: T201 -- captured by `docker inspect ...State.Health.Log` + sys.exit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/src/omnibase_infra/runtime/gateway_forwarder.py b/src/omnibase_infra/runtime/gateway_forwarder.py new file mode 100644 index 0000000000..3d375e6cfc --- /dev/null +++ b/src/omnibase_infra/runtime/gateway_forwarder.py @@ -0,0 +1,525 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Standalone process entrypoint for the hybrid gateway bus forwarder.""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import logging +import random +import signal +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Literal + +import yaml +from aiokafka.errors import KafkaError + +from omnibase_core.protocols.runtime.protocol_transport_producer import ( + ProtocolTransportProducer, +) +from omnibase_infra.errors import InfraUnavailableError +from omnibase_infra.event_bus.kafka_transport import KafkaTransport +from omnibase_infra.event_bus.models import ModelEventHeaders +from omnibase_infra.idempotency import StoreIdempotencySqlite +from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( + ModelGatewayForwarderConfig, + ModelGatewayForwarderRuntimeConfig, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_delivery import ( + NodeGatewayDelivery, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_forwarder import ( + ServiceGatewayForwarder, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( + prefix_topic, +) + +logger = logging.getLogger(__name__) + +_GATEWAY_CONTRACT_NAME = "node_bus_forwarder_effect" +_DEFAULT_GATEWAY_CONTRACT_PATH = ( + Path(__file__).parents[1] / "nodes" / _GATEWAY_CONTRACT_NAME / "contract.yaml" +) + + +def load_gateway_forwarder_runtime_config( + config_path: Path, + *, + contract_path: Path = _DEFAULT_GATEWAY_CONTRACT_PATH, +) -> ModelGatewayForwarderRuntimeConfig: + """Load and validate one explicit two-leg forwarder configuration.""" + raw_object: object = yaml.safe_load(config_path.read_text(encoding="utf-8")) + if not isinstance(raw_object, dict): + raise ValueError("gateway forwarder config must be a YAML mapping") + raw: dict[str, object] = {str(key): value for key, value in raw_object.items()} + _materialize_contract_mirror_topics(raw, contract_path) + _materialize_contract_canary_config(raw, contract_path) + return ModelGatewayForwarderRuntimeConfig.model_validate(raw) + + +def _load_gateway_forwarder_config_block( + contract_path: Path, selector: object +) -> dict[str, object]: + """Read and validate ``config.gateway_forwarder`` from the node contract.""" + contract_object: object = yaml.safe_load(contract_path.read_text(encoding="utf-8")) + if not isinstance(contract_object, dict): + raise ValueError("gateway node contract must be a YAML mapping") + contract: dict[str, object] = { + str(key): value for key, value in contract_object.items() + } + if contract.get("contract_name") != selector: + raise ValueError("gateway node contract does not match mirror_topic_set") + contract_config = contract.get("config") + if not isinstance(contract_config, dict): + raise ValueError("gateway node contract is missing config") + gateway_config = contract_config.get("gateway_forwarder") + if not isinstance(gateway_config, dict): + raise ValueError("gateway node contract is missing gateway_forwarder") + return {str(key): value for key, value in gateway_config.items()} + + +def _materialize_contract_mirror_topics( + raw: dict[str, object], + contract_path: Path, +) -> None: + """Resolve the named fixed topic set from the node contract. + + Resolved deployment YAML intentionally cannot repeat raw topic literals. + The node contract is their sole authority; the tenant config names that + contract and this boundary copies its validated inbound/outbound set into + the frozen runtime model before either broker starts. + """ + forwarder_object = raw.get("forwarder") + if not isinstance(forwarder_object, dict): + raise ValueError("gateway forwarder config requires a forwarder mapping") + forwarder: dict[str, object] = { + str(key): value for key, value in forwarder_object.items() + } + raw["forwarder"] = forwarder + if "mirror_topics" in forwarder: + raise ValueError( + "resolved gateway config must name mirror_topic_set instead of " + "redeclaring topic literals" + ) + selector = forwarder.pop("mirror_topic_set", None) + if selector != _GATEWAY_CONTRACT_NAME: + raise ValueError( + f"mirror_topic_set must be {_GATEWAY_CONTRACT_NAME!r}, got {selector!r}" + ) + + gateway_config = _load_gateway_forwarder_config_block(contract_path, selector) + mirror_topics_object = gateway_config.get("mirror_topics") + if not isinstance(mirror_topics_object, dict): + raise ValueError("gateway node contract mirror_topics must be a mapping") + forwarder["mirror_topics"] = { + str(key): value for key, value in mirror_topics_object.items() + } + + +def _materialize_contract_canary_config( + raw: dict[str, object], + contract_path: Path, +) -> None: + """Resolve the canary probe topic/cadence/deadlines from the node contract. + + Same authority pattern as ``_materialize_contract_mirror_topics``: resolved + deployment YAML names the contract via ``canary_topic_set`` and may not + redeclare the canary block inline, so the contract stays the sole source + of the canary topic and its cadence/deadlines (OMN-15741). + """ + forwarder_object = raw["forwarder"] + if not isinstance(forwarder_object, dict): + raise ValueError("gateway forwarder config requires a forwarder mapping") + forwarder: dict[str, object] = forwarder_object + if "canary" in forwarder: + raise ValueError( + "resolved gateway config must name canary_topic_set instead of " + "redeclaring the canary block" + ) + selector = forwarder.pop("canary_topic_set", None) + if selector != _GATEWAY_CONTRACT_NAME: + raise ValueError( + f"canary_topic_set must be {_GATEWAY_CONTRACT_NAME!r}, got {selector!r}" + ) + + gateway_config = _load_gateway_forwarder_config_block(contract_path, selector) + canary_object = gateway_config.get("canary") + if not isinstance(canary_object, dict): + raise ValueError( + "gateway node contract is missing config.gateway_forwarder.canary" + ) + forwarder["canary"] = {str(key): value for key, value in canary_object.items()} + + +async def run_gateway_forwarder( + config: ModelGatewayForwarderRuntimeConfig, + *, + shutdown_event: asyncio.Event, + ready_path: Path | None = None, +) -> None: + """Run the bridge until ``shutdown_event`` is set, then close both legs.""" + tenant_slug = config.forwarder.tenant_identity.tenant_slug + local_transport = KafkaTransport( + config=config.local_bus, + group=f"tenant-{tenant_slug}-gateway-forwarder-outbound", + topics=config.forwarder.mirror_topics.outbound, + auto_offset_reset=config.local_bus.auto_offset_reset, + ) + cloud_transport = KafkaTransport( + config=config.cloud_bus, + group=f"tenant-{tenant_slug}-gateway-forwarder-inbound", + topics=tuple( + prefix_topic(tenant_slug, topic) + for topic in config.forwarder.mirror_topics.inbound + ), + auto_offset_reset=config.cloud_bus.auto_offset_reset, + ) + local_bus = TransportGatewayBus(local_transport) + cloud_bus = TransportGatewayBus(cloud_transport) + idempotency_store = StoreIdempotencySqlite(config.forwarder.dedupe_store_path) + forwarder = ServiceGatewayForwarder( + config=config.forwarder, + local_bus=local_bus, + cloud_bus=cloud_bus, + ) + delivery = NodeGatewayDelivery( + config=config.forwarder, + forwarder=forwarder, + local_consumer=local_transport, + cloud_consumer=cloud_transport, + idempotency_store=idempotency_store, + ) + + if ready_path is not None: + ready_path.unlink(missing_ok=True) + + store_started = False + started_transports: list[KafkaTransport] = [] + delivery_started = False + heartbeat_task: asyncio.Task[None] | None = None + try: + await idempotency_store.start() + store_started = True + await local_transport.start() + started_transports.append(local_transport) + await cloud_transport.start() + started_transports.append(cloud_transport) + await delivery.start() + delivery_started = True + heartbeat_task = asyncio.create_task( + _run_heartbeat_loop(forwarder, config, shutdown_event), + name="gateway-forwarder-heartbeat", + ) + + if ready_path is not None: + ready_path.write_text("ready\n", encoding="utf-8") + identity = config.forwarder.tenant_identity + logger.info( + "Gateway forwarder ready for tenant_id=%s tenant_slug=%s", + identity.tenant_id, + identity.tenant_slug, + ) + await _supervise_gateway_delivery( + forwarder=forwarder, + delivery=delivery, + heartbeat_task=heartbeat_task, + shutdown_event=shutdown_event, + config=config.forwarder, + ) + finally: + if ready_path is not None: + ready_path.unlink(missing_ok=True) + if heartbeat_task is not None and not heartbeat_task.done(): + heartbeat_task.cancel() + if heartbeat_task is not None: + await asyncio.gather(heartbeat_task, return_exceptions=True) + if delivery_started: + await delivery.stop() + for transport in reversed(started_transports): + await transport.close() + if store_started: + await idempotency_store.close() + + +async def _supervise_gateway_delivery( + *, + forwarder: ServiceGatewayForwarder, + delivery: NodeGatewayDelivery, + heartbeat_task: asyncio.Task[None], + shutdown_event: asyncio.Event, + config: ModelGatewayForwarderConfig, +) -> None: + """Keep the delivery loop alive across cloud-leg faults, no terminal exit. + + A delivery-loop failure (e.g. the cloud broker leg dropping) previously + propagated straight out of ``run_gateway_forwarder`` and ended the + process. It is now retried in place with bounded exponential backoff + and jitter. Once the failure has persisted past the contract-declared + ``degraded_after_seconds`` window, one ``DEGRADED`` status event is + published (locally -- see ``ServiceGatewayForwarder.publish_status``) + so the failure is observable on the bus rather than only in restart + counts. A restart only clears the failure window once the delivery + loop has stayed up for a full ``heartbeat_interval_seconds`` recovery + window without failing again -- a bare ``delivery.start()`` call + succeeding proves the coroutines were scheduled, not that the cloud + leg is actually reachable again, so it is deliberately not treated as + recovery on its own. The process still exits on shutdown, on the + heartbeat task failing unexpectedly, or on the delivery loop returning + without either an exception or a shutdown signal (both are + unrecoverable/programmer errors, not connectivity faults). + """ + consecutive_failures = 0 + first_failure_at: datetime | None = None + degraded_emitted = False + shutdown_wait_task = asyncio.create_task( + shutdown_event.wait(), name="gateway-shutdown-wait" + ) + try: + while True: + delivery_wait_task = asyncio.create_task( + delivery.wait(), name="gateway-delivery-health" + ) + recovery_task: asyncio.Task[None] | None = None + if consecutive_failures > 0: + recovery_task = asyncio.create_task( + asyncio.sleep(config.heartbeat_interval_seconds), + name="gateway-delivery-recovery-confirm", + ) + waitables: set[asyncio.Task[object]] = { + delivery_wait_task, + shutdown_wait_task, + heartbeat_task, + } + if recovery_task is not None: + waitables.add(recovery_task) + try: + done, _ = await asyncio.wait( + waitables, return_when=asyncio.FIRST_COMPLETED + ) + if shutdown_wait_task in done: + return + if heartbeat_task in done: + await heartbeat_task + return + if ( + recovery_task is not None + and recovery_task in done + and delivery_wait_task not in done + ): + # Survived a full heartbeat interval without a new + # failure -- treat the connection as recovered. + consecutive_failures = 0 + first_failure_at = None + if degraded_emitted: + await _publish_gateway_status(forwarder, status="active") + degraded_emitted = False + continue + exc = delivery_wait_task.exception() + if exc is None: + raise RuntimeError( + "gateway delivery loop exited without a shutdown signal" + ) + finally: + if recovery_task is not None and not recovery_task.done(): + recovery_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await recovery_task + if not delivery_wait_task.done(): + delivery_wait_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await delivery_wait_task + + consecutive_failures += 1 + now = datetime.now(UTC) + if first_failure_at is None: + first_failure_at = now + elapsed_seconds = (now - first_failure_at).total_seconds() + logger.warning( + "Gateway delivery loop failed; reconnect attempt=%d " + "elapsed_seconds=%.1f error_type=%s error=%s", + consecutive_failures, + elapsed_seconds, + type(exc).__name__, + exc, + ) + + degraded_threshold = config.degraded_after_seconds + if not degraded_emitted and elapsed_seconds >= degraded_threshold: + await _publish_gateway_status( + forwarder, + status="degraded", + consecutive_failures=consecutive_failures, + detail=f"{type(exc).__name__}: {exc}", + ) + degraded_emitted = True + + delay = _compute_reconnect_delay_seconds(config, consecutive_failures) + shutdown_fired = await _sleep_or_shutdown(delay, shutdown_event) + if shutdown_fired: + return + + await delivery.stop() + try: + await delivery.start() + except Exception: + logger.exception("Gateway delivery restart failed; will retry") + continue + finally: + if not shutdown_wait_task.done(): + shutdown_wait_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await shutdown_wait_task + + +def _compute_reconnect_delay_seconds( + config: ModelGatewayForwarderConfig, + attempt: int, +) -> float: + """Bounded exponential backoff with additive jitter, contract-declared.""" + exponential = config.reconnect_backoff_initial_seconds * (2 ** (attempt - 1)) + capped = min(exponential, config.reconnect_backoff_max_seconds) + jitter = random.uniform(0, config.reconnect_backoff_jitter_seconds) + return capped + jitter + + +async def _sleep_or_shutdown(delay: float, shutdown_event: asyncio.Event) -> bool: + """Sleep for ``delay`` seconds; return True if shutdown fired first.""" + try: + await asyncio.wait_for(shutdown_event.wait(), timeout=delay) + except TimeoutError: + return False + return True + + +async def _publish_gateway_status( + forwarder: ServiceGatewayForwarder, + *, + status: Literal["active", "degraded"], + consecutive_failures: int = 0, + detail: str = "", +) -> None: + """Best-effort status publish -- must never itself take down supervision.""" + try: + await forwarder.publish_status( + status, + consecutive_failures=consecutive_failures, + detail=detail, + ) + except Exception: + logger.exception("Gateway %s status publish failed", status) + + +class TransportGatewayBus: + """Adapt the pull transport producer to the forwarder's publish boundary.""" + + def __init__(self, producer: ProtocolTransportProducer) -> None: + self._producer = producer + + async def publish( + self, + topic: str, + key: bytes | None, + value: bytes, + headers: object | None = None, + ) -> None: + encoded_headers: Mapping[str, bytes] + if headers is None: + encoded_headers = {} + elif isinstance(headers, Mapping): + if not all( + isinstance(header_key, str) and isinstance(header_value, bytes) + for header_key, header_value in headers.items() + ): + raise TypeError( + "gateway transport headers must map string keys to bytes" + ) + encoded_headers = { + header_key: header_value + for header_key, header_value in headers.items() + if isinstance(header_key, str) and isinstance(header_value, bytes) + } + elif isinstance(headers, ModelEventHeaders): + encoded_headers = { + header_key: str(header_value).encode("utf-8") + for header_key, header_value in headers.model_dump( + mode="json", + exclude_none=True, + ).items() + } + else: + raise TypeError("gateway transport headers must map string keys to bytes") + try: + await self._producer.send(topic, key, value, encoded_headers) + except KafkaError as exc: + raise InfraUnavailableError( + f"gateway destination broker unavailable for topic {topic}" + ) from exc + + +async def _run_heartbeat_loop( + forwarder: ServiceGatewayForwarder, + config: ModelGatewayForwarderRuntimeConfig, + shutdown_event: asyncio.Event, +) -> None: + """Emit immediately, then at the contract-declared liveness cadence.""" + interval = config.forwarder.heartbeat_interval_seconds + while not shutdown_event.is_set(): + await forwarder.publish_heartbeat() + try: + await asyncio.wait_for(shutdown_event.wait(), timeout=interval) + except TimeoutError: + continue + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run one tenant-scoped local/cloud event-bus forwarder", + ) + parser.add_argument( + "--config", + type=Path, + required=True, + help="Path to the resolved, typed gateway forwarder YAML", + ) + parser.add_argument( + "--ready-file", + type=Path, + default=None, + help=( + "Optional readiness sentinel written only after both broker transports " + "and delivery loops start" + ), + ) + return parser + + +async def _async_main(args: argparse.Namespace) -> None: + config = load_gateway_forwarder_runtime_config(args.config) + shutdown_event = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, shutdown_event.set) + await run_gateway_forwarder( + config, + shutdown_event=shutdown_event, + ready_path=args.ready_file, + ) + + +def main(argv: Sequence[str] | None = None) -> None: + """CLI entrypoint.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + args = _build_parser().parse_args(argv) + asyncio.run(_async_main(args)) + + +if __name__ == "__main__": + main() diff --git a/src/omnibase_infra/runtime/health/container_healthcheck.py b/src/omnibase_infra/runtime/health/container_healthcheck.py new file mode 100644 index 0000000000..bcebf65f2b --- /dev/null +++ b/src/omnibase_infra/runtime/health/container_healthcheck.py @@ -0,0 +1,409 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Semantic container healthcheck for runtime lanes (OMN-15217). + +The pre-OMN-15217 container healthcheck was:: + + test: ["CMD", "curl", "-sf", ":8085/health"] + +``curl -sf`` asserts one thing: the response code is < 400. That is a liveness +probe wearing a health probe's name. Observed on the stability lane +2026-07-27T12:58Z: ``docker ps`` reported ``Up 3 hours (healthy)`` for +``omninode-stability-test-runtime`` while that same runtime logged +``Runtime health check: status=DEGRADED contracts=296 errors=4`` every five +minutes. A promotion gate, an operator, or a proof packet citing that green is +citing a false signal — which is what blocked the OMN-15181 prod bootstrap. + +This module reads the runtime's *semantic* health instead: the +``details.runtime_health`` block published by +:mod:`omnibase_infra.runtime.health.runtime_health_block`, which carries the +``ServiceRuntimeHealthMonitor`` verdict. + +Usage as a container healthcheck (compose):: + + healthcheck: + test: ["CMD", "python", "/usr/local/bin/onex-container-healthcheck", + "--degraded-policy", "fail"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 1800s + +``Dockerfile.runtime`` installs this file at +``/usr/local/bin/onex-container-healthcheck``. It is invoked as a *file*, not as +``python -m omnibase_infra...``: importing the package chain costs ~6.8s inside +the runtime image against a 10s probe timeout, while this stdlib-only module +starts in ~0.12s (both measured in-container 2026-07-27). That is why nothing +here imports from ``omnibase_infra`` — a unit test pins the property. + +``start_period`` and ``retries`` are deliberately preserved from the shallow +check: contract discovery and Kafka group joins take many minutes on a cold +runtime, and the monitor's first verdict lands one check interval after boot. +Within ``start_period`` Docker reports ``starting``, so a not-yet-computed +verdict never flaps a booting container. + +Usage as a proof reader (gates, promotion checks):: + + python -m omnibase_infra.runtime.health.container_healthcheck \\ + --url --require-verdict --json + +(off the container's critical path, so the import cost is irrelevant there) + +``--require-verdict`` fails closed when no monitor verdict is present: for a +liveness probe an absent verdict is "not known yet" (pass, do not restart), but +for a *proof* consumer an absent verdict is "cannot prove healthy" (fail). +Same evaluator, different policy, one code path. + +Exit codes: ``0`` healthy, ``1`` not healthy (Docker's unhealthy signal). +""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.error +import urllib.request +from collections.abc import Mapping, Sequence +from typing import Literal + +DEGRADED_POLICY_FAIL = "fail" +DEGRADED_POLICY_WARN = "warn" + +_DEFAULT_TIMEOUT_SECONDS = 5.0 +_DEFAULT_PORT = 8085 +_MAX_RESPONSE_BYTES = 4 * 1024 * 1024 + +# Mirrors runtime_health_block.RUNTIME_HEALTH_DETAIL_KEY. Duplicated as a plain +# literal (with a seam test pinning the two together) so the healthcheck process +# imports nothing but the standard library — it runs on every probe interval +# inside a container with a 10s timeout, and package import cost is not a budget +# a health probe should spend. +RUNTIME_HEALTH_DETAIL_KEY = "runtime_health" + +EXIT_HEALTHY = 0 +EXIT_UNHEALTHY = 1 + +_VerdictLiteral = Literal["PASS", "FAIL"] + + +class ContainerHealthVerdict: + """Result of evaluating a ``/health`` response. + + Attributes: + verdict: ``PASS`` or ``FAIL``. + reason: Stable machine-greppable reason code (e.g. ``runtime_degraded``). + detail: Human-readable detail for container logs / ``docker inspect``. + """ + + __slots__ = ("detail", "reason", "verdict") + + def __init__(self, verdict: _VerdictLiteral, reason: str, detail: str = "") -> None: + self.verdict = verdict + self.reason = reason + self.detail = detail + + @property + def exit_code(self) -> int: + """Process exit code — ``0`` for PASS, ``1`` for FAIL.""" + return EXIT_HEALTHY if self.verdict == "PASS" else EXIT_UNHEALTHY + + def as_dict(self) -> dict[str, object]: + """Return a JSON-serializable view (for ``--json``).""" + return { + "verdict": self.verdict, + "reason": self.reason, + "detail": self.detail, + "exit_code": self.exit_code, + } + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return ( + f"ContainerHealthVerdict(verdict={self.verdict!r}, " + f"reason={self.reason!r}, detail={self.detail!r})" + ) + + +def evaluate_health_response( + *, + http_status: int | None, + payload: Mapping[str, object] | None, + degraded_policy: str = DEGRADED_POLICY_FAIL, + require_verdict: bool = False, + max_verdict_age_seconds: float | None = None, +) -> ContainerHealthVerdict: + """Evaluate a ``/health`` response into a container health verdict. + + Pure function — no I/O, no clock, no environment reads — so the full verdict + table is testable hermetically against recorded payloads. + + Args: + http_status: HTTP status code, or ``None`` if the endpoint was + unreachable / the response was not parseable as JSON. + payload: Decoded ``/health`` JSON body, or ``None``. + degraded_policy: ``fail`` (default) treats a DEGRADED runtime as + unhealthy; ``warn`` reports it but still exits 0. + require_verdict: When ``True``, a missing monitor verdict fails closed. + max_verdict_age_seconds: When set, a verdict older than this is treated + as stale. Stale fails closed only under ``require_verdict``, because + a liveness probe must not restart a container merely for a monitor + cycle that has not landed yet. + + Returns: + A :class:`ContainerHealthVerdict`. + """ + if http_status is None: + return ContainerHealthVerdict( + "FAIL", "probe_unreachable", "health endpoint unreachable or unparseable" + ) + if http_status >= 400: + return ContainerHealthVerdict( + "FAIL", "http_error", f"health endpoint returned HTTP {http_status}" + ) + if payload is None: + return ContainerHealthVerdict( + "FAIL", "payload_missing", "health endpoint returned no JSON body" + ) + + fail_on_degraded = degraded_policy != DEGRADED_POLICY_WARN + + top_status = str(payload.get("status", "")).lower() + if top_status == "unhealthy": + return ContainerHealthVerdict( + "FAIL", "runtime_unhealthy", "health payload reports status=unhealthy" + ) + + details_raw = payload.get("details") + details: Mapping[str, object] = ( + details_raw if isinstance(details_raw, Mapping) else {} + ) + + # --- Semantic verdict from ServiceRuntimeHealthMonitor ------------------- + verdict_raw = details.get(RUNTIME_HEALTH_DETAIL_KEY) + verdict_block: Mapping[str, object] | None = ( + verdict_raw if isinstance(verdict_raw, Mapping) else None + ) + + if verdict_block is None: + if require_verdict: + return ContainerHealthVerdict( + "FAIL", + "verdict_absent", + "health payload carries no runtime_health verdict — health is " + "unknown, not proven", + ) + else: + verdict_status = str(verdict_block.get("status", "")).upper() + + if max_verdict_age_seconds is not None: + age = _coerce_float(verdict_block.get("age_seconds")) + if require_verdict and (age is None or age > max_verdict_age_seconds): + age_text = "unknown" if age is None else f"{age:.0f}s" + return ContainerHealthVerdict( + "FAIL", + "verdict_stale", + f"runtime_health verdict age {age_text} exceeds " + f"{max_verdict_age_seconds:.0f}s", + ) + + if verdict_status == "CRITICAL": + return ContainerHealthVerdict( + "FAIL", + "runtime_critical", + _describe_dimensions(verdict_block, "CRITICAL") + or "runtime health monitor reports CRITICAL", + ) + if verdict_status == "DEGRADED": + detail = ( + _describe_dimensions(verdict_block, "DEGRADED") + or "runtime health monitor reports DEGRADED" + ) + if fail_on_degraded: + return ContainerHealthVerdict("FAIL", "runtime_degraded", detail) + return ContainerHealthVerdict("PASS", "runtime_degraded_warn", detail) + + # --- Process-level degradation (handlers failed to instantiate) ---------- + if top_status == "degraded" or bool(details.get("degraded", False)): + detail = "health payload reports a degraded runtime process" + if fail_on_degraded: + return ContainerHealthVerdict("FAIL", "process_degraded", detail) + return ContainerHealthVerdict("PASS", "process_degraded_warn", detail) + + return ContainerHealthVerdict("PASS", "healthy", "runtime healthy") + + +def _describe_dimensions( + verdict_block: Mapping[str, object], status: str +) -> str | None: + """Summarize the dimensions matching ``status`` for the failure detail.""" + dimensions = verdict_block.get("dimensions") + if not isinstance(dimensions, Sequence) or isinstance(dimensions, str | bytes): + return None + parts: list[str] = [] + for dimension in dimensions: + if not isinstance(dimension, Mapping): + continue + if str(dimension.get("status", "")).upper() != status: + continue + name = str(dimension.get("name", "unknown")) + detail = str(dimension.get("detail", "")).strip() + parts.append(f"{name}: {detail}" if detail else name) + if not parts: + return None + return f"runtime health {status} — " + "; ".join(parts) + + +def _coerce_float(value: object) -> float | None: + """Best-effort float coercion for values arriving from JSON.""" + if isinstance(value, bool): + return None + if isinstance(value, int | float): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return None + return None + + +def fetch_health( + url: str, timeout_seconds: float +) -> tuple[int | None, Mapping[str, object] | None]: + """Fetch and decode the health endpoint. + + Returns: + ``(http_status, payload)``. ``http_status`` is ``None`` when the + endpoint could not be reached at all; ``payload`` is ``None`` when the + body was absent or not a JSON object. + """ + try: + # Localhost-only container probe; the URL comes from the container's own + # configuration, never from a request. + with urllib.request.urlopen(url, timeout=timeout_seconds) as response: # noqa: S310 + status = int(response.status) + body = response.read(_MAX_RESPONSE_BYTES) + except urllib.error.HTTPError as exc: + try: + body = exc.read(_MAX_RESPONSE_BYTES) + except Exception: # noqa: BLE001 — boundary: error body is best-effort + body = b"" + return int(exc.code), _decode(body) + except Exception: # noqa: BLE001 — boundary: any transport failure is unreachable + return None, None + + return status, _decode(body) + + +def _decode(body: bytes) -> Mapping[str, object] | None: + """Decode a JSON object body, or ``None`` when it is not one.""" + if not body: + return None + try: + decoded = json.loads(body.decode("utf-8")) + except (ValueError, UnicodeDecodeError): + return None + return decoded if isinstance(decoded, dict) else None + + +# Default probe target: this container's own health endpoint on the loopback +# interface. A container healthcheck checks the process it runs beside — there +# is no routing authority for "myself", and resolving a service address here +# would probe a *different* replica and report the wrong container's health. +# Lanes that move the listener off 8085 (ONEX_HTTP_PORT) must pass --url in the +# compose healthcheck; this process reads no environment (the check-env-reads +# gate keeps configuration in the overlay, and a healthcheck's configuration +# belongs on its command line where `docker inspect` can show it). +DEFAULT_HEALTH_URL = f"http://localhost:{_DEFAULT_PORT}/health" # url-authority-ok: container self-probe on loopback + + +def build_parser() -> argparse.ArgumentParser: + """Build the CLI parser (exposed for tests).""" + parser = argparse.ArgumentParser( + prog="container_healthcheck", + description=( + "Semantic container healthcheck — consumes the runtime's own health " + "verdict instead of trusting an HTTP 200 (OMN-15217)." + ), + ) + parser.add_argument( + "--url", + default=DEFAULT_HEALTH_URL, + help=f"Health endpoint URL (default: {DEFAULT_HEALTH_URL})", + ) + parser.add_argument( + "--degraded-policy", + choices=[DEGRADED_POLICY_FAIL, DEGRADED_POLICY_WARN], + default=DEGRADED_POLICY_FAIL, + help="How to treat a DEGRADED runtime verdict (default: fail)", + ) + parser.add_argument( + "--require-verdict", + action="store_true", + default=False, + help=( + "Fail closed when no runtime_health verdict is present or it is " + "stale — use for proof/promotion readers, not for liveness probes" + ), + ) + parser.add_argument( + "--max-verdict-age-seconds", + type=float, + default=None, + help="Treat a verdict older than this as stale (default: no age limit)", + ) + parser.add_argument( + "--timeout-seconds", + type=float, + default=_DEFAULT_TIMEOUT_SECONDS, + help="HTTP timeout in seconds (default: 5)", + ) + parser.add_argument( + "--json", + action="store_true", + help="Emit the verdict as JSON instead of a single human-readable line", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """CLI entrypoint. Returns the process exit code.""" + args = build_parser().parse_args(argv) + + http_status, payload = fetch_health(args.url, args.timeout_seconds) + verdict = evaluate_health_response( + http_status=http_status, + payload=payload, + degraded_policy=args.degraded_policy, + require_verdict=args.require_verdict, + max_verdict_age_seconds=args.max_verdict_age_seconds, + ) + + # Written to stdout (not logged): Docker captures healthcheck output into + # the container's health log, which is where `docker inspect` surfaces the + # reason a container went unhealthy. + if args.json: + line = json.dumps(verdict.as_dict(), sort_keys=True) + else: + line = f"{verdict.verdict} [{verdict.reason}] {verdict.detail}".rstrip() + sys.stdout.write(line + "\n") + return verdict.exit_code + + +if __name__ == "__main__": # pragma: no cover - process entrypoint + sys.exit(main()) + + +__all__: list[str] = [ + "DEGRADED_POLICY_FAIL", + "DEGRADED_POLICY_WARN", + "DEFAULT_HEALTH_URL", + "EXIT_HEALTHY", + "EXIT_UNHEALTHY", + "RUNTIME_HEALTH_DETAIL_KEY", + "ContainerHealthVerdict", + "build_parser", + "evaluate_health_response", + "fetch_health", + "main", +] diff --git a/src/omnibase_infra/runtime/health/runtime_health_block.py b/src/omnibase_infra/runtime/health/runtime_health_block.py new file mode 100644 index 0000000000..b5f145ce95 --- /dev/null +++ b/src/omnibase_infra/runtime/health/runtime_health_block.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Projection of the runtime health monitor verdict onto the HTTP health payload (OMN-15217). + +``ServiceRuntimeHealthMonitor`` computes the runtime's *semantic* health — +contract-discovery errors, consumer-group coverage, topic coverage — every +``RUNTIME_HEALTH_CHECK_INTERVAL`` seconds. Before OMN-15217 that verdict only +reached the container logs and the ``runtime-health-check.v1`` Kafka topic; it +was never joined to the ``/health`` HTTP payload that Docker, operators, and +promotion gates actually read. + +The observed consequence (stability lane, 2026-07-27T12:58Z): ``/health`` +returned ``{"status": "healthy", "details": {"healthy": true, "degraded": +false}}`` with HTTP 200 while the monitor logged +``status=DEGRADED ... errors=4`` every five minutes. Both the HTTP status code +*and* the payload body were green, so no consumer — however carefully it parsed +the response — could see the degradation. + +This module is the single seam that closes that gap: + +* :func:`build_runtime_health_block` renders the monitor's latest verdict into + the ``details.runtime_health`` block of the ``/health`` payload. +* :func:`fold_runtime_verdict_into_status` degrades the payload's top-level + ``status`` field when the monitor reports DEGRADED/CRITICAL. + +Deliberate non-change: the HTTP *status code* is untouched. ``/health`` is also +the liveness probe watched by ``autoheal``; a restart-immune degradation (four +contracts that fail to import will fail to import again after a restart) must +not turn into a restart loop. Honest body, unchanged liveness code. Container +health strictness is a separate, per-lane opt-in — see +:mod:`omnibase_infra.runtime.health.container_healthcheck`. + +.. versionadded:: 0.39.0 +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Literal, cast + +from omnibase_core.types import JsonType + +if TYPE_CHECKING: + from omnibase_infra.event_bus.enum_runtime_readiness_state import ( + EnumRuntimeReadinessState, + ) + from omnibase_infra.models.health.model_runtime_health_check_event import ( + ModelRuntimeHealthCheckEvent, + ) + +# Key under ``/health`` -> ``details`` carrying the monitor verdict. Consumers +# (container healthcheck, promotion gate, dashboards) key off this constant +# rather than a literal so the seam has exactly one name. +RUNTIME_HEALTH_DETAIL_KEY = "runtime_health" + +_SEMANTIC_STATUS = Literal["HEALTHY", "DEGRADED", "CRITICAL"] + +# Max number of dimension entries rendered into the payload. The monitor emits +# a small fixed set today; the cap keeps a future fan-out from bloating a +# response served on every Docker probe interval. +_MAX_DIMENSIONS = 32 + + +def build_runtime_health_block( + event: ModelRuntimeHealthCheckEvent | None, + *, + now: datetime | None = None, +) -> dict[str, JsonType] | None: + """Render the monitor's latest verdict as a ``/health`` details block. + + Args: + event: The most recent health-check event, or ``None`` when the monitor + has not completed a cycle yet (or is not running in this profile). + now: Injected clock for deterministic tests. Defaults to ``datetime.now(UTC)``. + + Returns: + A JSON-serializable block, or ``None`` when no verdict exists yet. + ``None`` is a distinct state from HEALTHY and consumers that require a + verdict must treat it as unknown, never as healthy. + """ + if event is None: + return None + + current = now or datetime.now(UTC) + observed_at = event.timestamp + if observed_at.tzinfo is None: + observed_at = observed_at.replace(tzinfo=UTC) + age_seconds = max(0.0, (current - observed_at).total_seconds()) + + block: dict[str, JsonType] = { + "status": event.status, + "observed_at": observed_at.isoformat(), + "age_seconds": round(age_seconds, 3), + "contract_count": event.contract_count, + "discovery_error_count": event.discovery_error_count, + "consumer_group_count": event.consumer_group_count, + "empty_consumer_group_count": event.empty_consumer_group_count, + "subscribe_topic_count": event.subscribe_topic_count, + "uncovered_topic_count": event.uncovered_topic_count, + "dimensions": cast( + "JsonType", + [ + { + "name": dimension.name, + "status": dimension.status, + "detail": dimension.detail, + } + for dimension in event.dimensions[:_MAX_DIMENSIONS] + ], + ), + } + return block + + +def fold_runtime_verdict_into_status( + payload_status: Literal["healthy", "degraded", "unhealthy"], + verdict_status: str | None, +) -> Literal["healthy", "degraded", "unhealthy"]: + """Degrade the payload status when the monitor reports a worse verdict. + + The runtime process can be perfectly alive (handlers registered, event bus + connected) while the runtime is semantically degraded (contracts failing to + load). The reported status is the worse of the two. + + Args: + payload_status: Status derived from ``RuntimeHostProcess.health_check()``. + verdict_status: ``HEALTHY`` / ``DEGRADED`` / ``CRITICAL`` from the + monitor, or ``None`` when no verdict exists yet (status unchanged — + absence is not evidence of degradation, and consumers that need a + verdict assert its presence explicitly). + + Returns: + The worse of the two statuses. + """ + if payload_status == "unhealthy" or verdict_status is None: + return payload_status + if verdict_status == "CRITICAL": + return "unhealthy" + if verdict_status == "DEGRADED": + return "degraded" + return payload_status + + +def fold_attach_readiness_into_status( + payload_status: Literal["healthy", "degraded", "unhealthy"], + readiness_state: EnumRuntimeReadinessState | None, +) -> Literal["healthy", "degraded", "unhealthy"]: + """Degrade the payload status when a boot-wired consumer failed to attach. + + OMN-15642. Before this, ``ModelRuntimeAttachReadiness`` (OMN-15512) reached + only ``details.components.runtime_wiring`` — a nested detail nothing + upstream reads — while the top-level ``status`` field stayed ``"healthy"`` + with HTTP 200. That is the exact class of gap OMN-15217 already closed for + the ``ServiceRuntimeHealthMonitor`` verdict (see the module docstring): a + runtime can boot fully, with EVERY rollout/digest/dashboard/staleness gate + green, while one Kafka consumer contract silently never attaches its + subscription (raises inside ``subscribe_wired_contract_topics``, caught + per-contract and downgraded to a non-fatal ``ModelContractAttachResult`` — + see ``omnibase_infra.event_bus.model_runtime_attach_readiness``). A + projection or reducer that stops consuming produces zero new rows for + whatever it writes, invisible to every liveness/rollout check that never + queries the wiring detail. + + Caller scoping (remediation, read before wiring this into a new caller): + this fold is used by ``ServiceHealth._handle_health_detailed`` ONLY, not + ``_handle_health``. ``ModelRuntimeAttachReadiness``'s own docstring states + "the readiness endpoint reports attach status ONLY — it is not a source of + truth for contract lifecycle", and OMN-13237 deliberately designed a + NOT_READY/DEGRADED contract to be recorded and skipped, never fatal, never + a restart/redeploy trigger. ``/health`` (unlike ``/health/detailed``) is a + hard, no-tolerance boot/deploy gate for four real automated consumers — + ``.github/workflows/reusable-runtime-boot.yml``, + ``scripts/deploy-agent/deploy_agent/executor.py``, + ``scripts/runtime_build/verify_stability_refresh.py`` / + ``verify_dev_refresh.py`` — that assert ``status == "healthy"`` with no + DEGRADED tolerance. Folding a documented-non-fatal DEGRADED into + ``/health``'s gated status would convert every ordinary NOT_READY skip + (e.g. one unprovisioned topic, a documented live condition on onex-dev — + OMN-15330) into a hard boot/deploy failure. Route new visibility needs + through ``/health/detailed`` or the existing + ``details.components.runtime_wiring`` block, not through this fold on + ``/health``. + + Args: + payload_status: Status derived from ``RuntimeHostProcess.health_check()``, + already folded with :func:`fold_runtime_verdict_into_status`. + readiness_state: The boot attach-readiness aggregate's tri-state, or + ``None`` before the kernel has attached it (status unchanged -- + absence is not evidence of degradation, matching + :func:`fold_runtime_verdict_into_status`'s ``None`` handling). + + Returns: + The worse of the two statuses. Deliberately does NOT change the HTTP + status code -- see the module docstring's "Deliberate non-change". + """ + if payload_status == "unhealthy" or readiness_state is None: + return payload_status + # Local import: EnumRuntimeReadinessState is TYPE_CHECKING-only above so + # this module carries no runtime import-time dependency on the event_bus + # package for callers that never pass a readiness_state. + from omnibase_infra.event_bus.enum_runtime_readiness_state import ( + EnumRuntimeReadinessState as _EnumRuntimeReadinessState, + ) + + if readiness_state is _EnumRuntimeReadinessState.FAILED: + return "unhealthy" + if readiness_state is _EnumRuntimeReadinessState.DEGRADED: + return "degraded" + return payload_status + + +__all__: list[str] = [ + "RUNTIME_HEALTH_DETAIL_KEY", + "build_runtime_health_block", + "fold_attach_readiness_into_status", + "fold_runtime_verdict_into_status", +] diff --git a/src/omnibase_infra/runtime/kafka_contract_source.py b/src/omnibase_infra/runtime/kafka_contract_source.py index 73d337b1a8..ddfdadde4b 100644 --- a/src/omnibase_infra/runtime/kafka_contract_source.py +++ b/src/omnibase_infra/runtime/kafka_contract_source.py @@ -7,15 +7,16 @@ Part of OMN-1654: KafkaContractSource (cache + discovery). -**Beta Implementation**: Cache-only model. Does NOT wire business subscriptions -dynamically. Discovered contracts take effect on next restart. +Validated registration events are cached first. The runtime may then materialize a +cached contract into its live dispatch engine through the explicitly authorized +post-freeze path; rejected contracts never mutate the engine. Contract Event Flow: 1. External system publishes ModelContractRegisteredEvent to platform topic 2. KafkaContractSource receives event via baseline-wired subscription 3. Contract YAML is parsed and cached as ModelHandlerDescriptor - 4. Next call to discover_handlers() returns cached descriptors - 5. Runtime restart applies new handler configuration + 4. The runtime delegates live materialization back to this source + 5. Typed auto-wiring registers routes and subscriptions transactionally Event Topics (Platform Reserved): - Registration: {env}.{TOPIC_SUFFIX_CONTRACT_REGISTERED} @@ -116,6 +117,9 @@ ) if TYPE_CHECKING: + from omnibase_core.models.core.model_deployment_topology import ( + ModelDeploymentTopology, + ) from omnibase_infra.runtime.auto_wiring.models import ModelDiscoveredContract from omnibase_infra.runtime.auto_wiring.models.model_event_bus_wiring import ( ModelEventBusWiring, @@ -146,21 +150,20 @@ def _resolve_handler_class_from_routing( Market node contracts do not declare ``metadata.handler_class`` (the form ModelHandlerContract reads). Instead they declare the handler module under - ``handler_routing.handlers[].handler.module`` and/or a top-level - ``handler.module``, with the class name under ``.name`` (routing form) or - ``.class`` (top-level form). This helper joins module + class into the - ``module.ClassName`` path the live materializer imports. + ``handler_routing.handlers[].handler.module``, with the class name under + ``.name``. This helper joins module + class into the ``module.ClassName`` + path the live materializer imports. - The routing form is preferred over the top-level form because routing is the - canonical dispatch surface; the top-level ``handler`` block is a convenience - declaration. + A legacy top-level ``handler`` block is intentionally not accepted. The + canonical ``ModelHandlerContract`` rejects undeclared fields, so accepting + that block here would bypass the typed contract boundary. Args: contract_data: Parsed contract YAML as a mapping. Returns: - Fully qualified ``module.ClassName`` path, or None when neither the - routing nor the top-level handler block carries a resolvable module. + Fully qualified ``module.ClassName`` path, or None when the routing + declaration does not carry a resolvable module. """ def _join(module: object, class_name: object) -> str | None: @@ -186,13 +189,6 @@ def _join(module: object, class_name: object) -> str | None: if resolved is not None: return resolved - # Fall back to the top-level form: handler.{module,class} - top_level = contract_data.get("handler") - if isinstance(top_level, dict): - resolved = _join(top_level.get("module"), top_level.get("class")) - if resolved is not None: - return resolved - return None @@ -272,22 +268,58 @@ def parse( if not contract_data: raise ValueError("Contract YAML is empty or invalid") + # Validate the canonical handler contract without discarding typed node + # extensions retained in ``contract_config``. Core deliberately forbids + # unknown root fields, including the retired top-level ``handler`` shape. + # Node contracts carry three infra-owned typed extensions which core's + # handler shell deliberately does not model. Validate them independently, + # retain their exact source payload in ``contract_config``, then strip them + # only from the strict core-shell validation view. The retired top-level + # ``handler`` shape remains unstripped and therefore fail-closed. + validation_data = contract_data + if isinstance(contract_data, dict): + from omnibase_core.models.contracts.subcontracts.model_db_ownership_subcontract import ( + ModelDbOwnershipSubcontract, + ) + from omnibase_infra.runtime.auto_wiring.discovery import ( + _parse_handler_routing, + ) + from omnibase_infra.runtime.auto_wiring.models.model_event_bus_wiring import ( + ModelEventBusWiring, + ) + + db_io_raw = contract_data.get("db_io") + if db_io_raw is not None: + ModelDbOwnershipSubcontract.model_validate(db_io_raw) + event_bus_raw = contract_data.get("event_bus") + if event_bus_raw is not None: + ModelEventBusWiring.model_validate(event_bus_raw) + handler_routing_raw = contract_data.get("handler_routing") + if handler_routing_raw is not None: + if not isinstance(handler_routing_raw, dict): + raise ValueError("handler_routing must be a mapping") + _parse_handler_routing(handler_routing_raw) + validation_data = { + key: value + for key, value in contract_data.items() + if key not in {"db_io", "event_bus", "handler_routing"} + } + # Validate against ModelHandlerContract - contract = ModelHandlerContract.model_validate(contract_data) + contract = ModelHandlerContract.model_validate(validation_data) - # Extract handler_class from metadata section - # NOTE: handler_class is read from metadata for handler-shaped contracts - # (root-level extra fields are ignored by ModelHandlerContract). - handler_class = None + # Extract handler_class from the typed field first, then fall back to + # legacy metadata for older dynamic-registration payloads. + handler_class = contract.handler_class if isinstance(contract_data, dict): metadata = contract_data.get("metadata", {}) - if isinstance(metadata, dict): + if handler_class is None and isinstance(metadata, dict): handler_class = metadata.get("handler_class") # Fallback for node-shaped (market) contracts: these declare the handler - # module under handler_routing.handlers[].handler.module (and/or a - # top-level handler.module) rather than metadata.handler_class. Join - # module + class into the fully qualified path the materializer imports. + # module under handler_routing.handlers[].handler.module rather than + # metadata.handler_class. Join module + class into the fully qualified + # path the materializer imports. if handler_class is None and isinstance(contract_data, dict): handler_class = _resolve_handler_class_from_routing(contract_data) @@ -1191,105 +1223,51 @@ def _build_event_bus_wiring( ) eb_raw = config.get("event_bus") - if not isinstance(eb_raw, dict): + if isinstance(eb_raw, dict): + return ModelEventBusWiring.model_validate(eb_raw) + + consumed_raw = config.get("yaml_consumed_events") + published_raw = config.get("yaml_published_events") + subscribe_topics = ( + tuple( + str(item.get("event_type")) + for item in consumed_raw + if isinstance(item, dict) and item.get("event_type") + ) + if isinstance(consumed_raw, list) + else () + ) + publish_topics = ( + tuple( + str(item.get("event_type") or item.get("topic")) + for item in published_raw + if isinstance(item, dict) + and (item.get("event_type") or item.get("topic")) + ) + if isinstance(published_raw, list) + else () + ) + if not subscribe_topics and not publish_topics: return None - - sub_raw = eb_raw.get("subscribe_topics") - pub_raw = eb_raw.get("publish_topics") - cp_raw = eb_raw.get("consumer_purpose") return ModelEventBusWiring( - subscribe_topics=tuple(sub_raw) if isinstance(sub_raw, list) else (), - publish_topics=tuple(pub_raw) if isinstance(pub_raw, list) else (), - consumer_purpose=cp_raw if isinstance(cp_raw, str) else None, - plugin_managed=bool(eb_raw.get("plugin_managed", False)), + subscribe_topics=subscribe_topics, + publish_topics=publish_topics, + consumer_purpose=None, + plugin_managed=False, ) @staticmethod def _build_handler_routing( config: Mapping[str, object], ) -> ModelHandlerRouting | None: - from omnibase_infra.runtime.auto_wiring.models.model_handler_ref import ( - ModelHandlerRef, - ) - from omnibase_infra.runtime.auto_wiring.models.model_handler_routing import ( - ModelHandlerRouting, - ) - from omnibase_infra.runtime.auto_wiring.models.model_handler_routing_entry import ( - ModelHandlerRoutingEntry, + from omnibase_infra.runtime.auto_wiring.discovery import ( + _parse_handler_routing, ) hr_raw = config.get("handler_routing") if not isinstance(hr_raw, dict): return None - - entries: list[ModelHandlerRoutingEntry] = [] - handlers_raw = hr_raw.get("handlers") - - # When a handlers list is present it takes priority. When absent, - # fall back to the ``default_handler`` shorthand. - if not isinstance(handlers_raw, list): - default_handler = hr_raw.get("default_handler") - if ( - isinstance(default_handler, str) - and default_handler - and ":" in default_handler - ): - module_ref, class_name = default_handler.rsplit(":", 1) - # Bare name (no dots) like ``handler`` is kept as-is; fully - # qualified paths (``pkg.sub.module``) are used verbatim. - entries.append( - ModelHandlerRoutingEntry( - handler=ModelHandlerRef( - name=class_name.strip(), - module=module_ref.strip(), - ), - ) - ) - else: - for handler_item in handlers_raw: - if not isinstance(handler_item, dict): - continue - handler_data = handler_item.get("handler") or {} - if not isinstance(handler_data, dict): - continue - event_model_ref = None - event_model_data = handler_item.get("event_model") - if isinstance(event_model_data, dict): - event_model_ref = ModelHandlerRef( - name=event_model_data.get("name", ""), - module=event_model_data.get("module", ""), - ) - entries.append( - ModelHandlerRoutingEntry( - handler=ModelHandlerRef( - name=handler_data.get("name", ""), - module=handler_data.get("module", ""), - ), - event_model=event_model_ref, - operation=handler_item.get("operation"), - event_type=handler_item.get("event_type"), - message_category=handler_item.get("message_category"), - # OMN-15188: without this, a topic_match contract with - # several handler_routing entries sharing the same - # operation + handler across distinct topics (a - # legitimate shape, e.g. node_codegen_outcome_reducer) - # loses per-entry topic disambiguation on this - # materialization path. _derive_handler_entry_key folds - # topic into the dispatcher-ID digest only when it is - # present (OMN-14580); every entry here would otherwise - # collapse to the SAME operation-only key and crash - # bootstrap with ONEX_CORE_064_DUPLICATE_REGISTRATION on - # the 2nd handler_routing entry. Keep in parity with - # discovery.py's _parse_handler_routing (OMN-13825), - # which already threads this field through. - topic=handler_item.get("topic"), - ) - ) - - return ModelHandlerRouting( - routing_strategy=hr_raw.get("routing_strategy", "payload_type_match"), - handlers=tuple(entries), - ) + return _parse_handler_routing(hr_raw) def _build_materialization_contract( self, @@ -1297,6 +1275,9 @@ def _build_materialization_contract( descriptor: ModelHandlerDescriptor, environment: str | None, ) -> ModelDiscoveredContract: + from omnibase_core.models.contracts.subcontracts.model_db_ownership_subcontract import ( + ModelDbOwnershipSubcontract, + ) from omnibase_infra.runtime.auto_wiring.models import ModelDiscoveredContract from omnibase_infra.runtime.auto_wiring.models.model_contract_version import ( ModelContractVersion, @@ -1305,6 +1286,21 @@ def _build_materialization_contract( ver = descriptor.version effective_env = environment or self._environment config = descriptor.contract_config or {} + db_io_raw = config.get("db_io") + db_io = ( + ModelDbOwnershipSubcontract.model_validate(db_io_raw) + if db_io_raw is not None + else None + ) + event_bus = self._build_event_bus_wiring(config) + root_terminal_event = config.get("terminal_event") + terminal_event = ( + root_terminal_event + if isinstance(root_terminal_event, str) + else event_bus.terminal_event + if event_bus is not None + else None + ) return ModelDiscoveredContract( name=node_name, node_type=descriptor.handler_kind, @@ -1318,8 +1314,10 @@ def _build_materialization_contract( contract_path=Path(f"/kafka/{effective_env}/{node_name}/contract.yaml"), entry_point_name=f"kafka.{node_name}", package_name="dynamic", - event_bus=self._build_event_bus_wiring(config), + terminal_event=terminal_event, + event_bus=event_bus, handler_routing=self._build_handler_routing(config), + db_io=db_io, ) async def materialize_cached_contract( @@ -1329,6 +1327,7 @@ async def materialize_cached_contract( event_bus: object | None = None, environment: str | None = None, container: object | None = None, + topology: ModelDeploymentTopology | None = None, ) -> ModelDynamicMaterializationResult: """Materialize a cached contract descriptor into the live dispatch engine. @@ -1350,6 +1349,8 @@ async def materialize_cached_contract( event_bus: Optional event bus for Kafka subscriptions. environment: Override environment (defaults to self._environment). container: Optional DI container for handler resolution. + topology: Checked-in deployment topology. Required when the + dynamically registered contract declares ``db_io``. Returns: ModelDynamicMaterializationResult with enum status and topology data. @@ -1417,6 +1418,8 @@ async def materialize_cached_contract( event_bus=event_bus, environment=effective_env, container=container, + topology=topology, + dynamic_materialization_authorized=True, ) if result.outcome == EnumWiringOutcome.WIRED: diff --git a/src/omnibase_infra/runtime/manifest_builder.py b/src/omnibase_infra/runtime/manifest_builder.py index 93e98c44f2..e7b5726393 100644 --- a/src/omnibase_infra/runtime/manifest_builder.py +++ b/src/omnibase_infra/runtime/manifest_builder.py @@ -1,16 +1,25 @@ # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT -"""Build a ModelRuntimeManifest from auto-wiring results (OMN-11196). +"""Build and publish the runtime manifest snapshot (OMN-11196 / OMN-15512). Called once at the end of the bootstrap sequence, after all startup phases: contract discovery, ownership validation, handler registration, and topic ownership. Produces a deterministic, hash-stable snapshot of the runtime topology for observability and drift detection. + +OMN-15512: the snapshot also carries the boot attach-readiness aggregate, so +the NOT-READY blocker set (contract name + the topics whose readiness confirm +failed) reaches the ``runtime_manifests`` projection instead of dying in the +log stream. :func:`publish_runtime_manifest` is the seam a test can drive with +a recording bus — the kernel calls exactly this function, so a test that +asserts on the captured envelope is asserting on the artifact that runs. """ from __future__ import annotations from datetime import UTC, datetime +from typing import TYPE_CHECKING +from uuid import UUID from omnibase_infra.runtime.auto_wiring.models.model_auto_wiring_manifest import ( ModelAutoWiringManifest, @@ -21,32 +30,47 @@ ModelContractWiringResult, ) +if TYPE_CHECKING: + from omnibase_infra.event_bus.model_runtime_attach_readiness import ( + ModelRuntimeAttachReadiness, + ) + from omnibase_infra.protocols import ProtocolEventBusLike + from omnibase_infra.runtime.models.model_runtime_manifest_published import ( + ModelRuntimeManifestPublished, + ) + def build_runtime_manifest( report: ModelAutoWiringReport, manifest: ModelAutoWiringManifest, runtime_profile: str, image_digest: str | None = None, + attach_readiness: ModelRuntimeAttachReadiness | None = None, ) -> object: - """Build a ModelRuntimeManifest from auto-wiring results. + """Build the published runtime manifest from auto-wiring results. Extracts wired/skipped/failed contracts, topics, and handlers from the wiring report and discovered manifest, then returns a frozen - ModelRuntimeManifest ready for publication on the event bus. + ModelRuntimeManifestPublished ready for publication on the event bus. - The import of ModelRuntimeManifest is deferred so that this module can be - imported before omnibase_core PR #1098 lands (the model is gated behind a - try/import in service_kernel.py as well). + The import of the core manifest models is deferred so that this module can + be imported before omnibase_core PR #1098 lands (the model is gated behind + a try/import in service_kernel.py as well). Args: report: The wiring report produced by wire_from_manifest(). manifest: The filtered auto-wiring manifest (post-quarantine). runtime_profile: The RUNTIME_PROFILE value (e.g. "main"). image_digest: Optional OCI image digest for the running container. + attach_readiness: Boot attach-readiness aggregate (OMN-15512). Narrowed + to the blocker set before publication — see + ``ModelRuntimeAttachReadiness.blockers_only``. ``None`` when the + per-contract interleave did not run at all. Returns: - A ModelRuntimeManifest instance (typed as object to allow graceful - fallback when the model is not yet available in omnibase_core). + A ModelRuntimeManifestPublished instance (typed as object to allow + graceful fallback when the base model is not available in + omnibase_core). Raises: ImportError: If omnibase_core.models.runtime_manifest is not installed. @@ -57,8 +81,8 @@ def build_runtime_manifest( from omnibase_core.models.runtime_manifest.model_manifest_handler import ( ModelManifestHandler, ) - from omnibase_core.models.runtime_manifest.model_runtime_manifest import ( - ModelRuntimeManifest, + from omnibase_infra.runtime.models.model_runtime_manifest_published import ( + ModelRuntimeManifestPublished, ) results_by_outcome: dict[str, list[ModelContractWiringResult]] = { @@ -133,7 +157,7 @@ def _to_manifest_contract( ) ) - return ModelRuntimeManifest( + return ModelRuntimeManifestPublished( runtime_profile=runtime_profile, contracts=wired_contracts, owned_command_topics=frozenset(owned_command_topics), @@ -144,4 +168,70 @@ def _to_manifest_contract( ownership_violations=(), image_digest=image_digest, started_at=datetime.now(tz=UTC), + attach_readiness=( + attach_readiness.blockers_only() if attach_readiness is not None else None + ), + ) + + +async def publish_runtime_manifest( + *, + event_bus: ProtocolEventBusLike, + report: ModelAutoWiringReport, + manifest: ModelAutoWiringManifest, + runtime_profile: str, + topic: str, + correlation_id: UUID, + image_digest: str | None = None, + attach_readiness: ModelRuntimeAttachReadiness | None = None, +) -> ModelRuntimeManifestPublished: + """Build the boot snapshot and publish it on the runtime-manifest topic. + + Extracted from ``service_kernel`` step 9.8 (OMN-15512) so the publish seam + is drivable by a test with a recording bus. The kernel calls this exact + function, so a test that asserts on the captured envelope payload asserts + on the artifact that runs — not on a surrogate. + + Args: + event_bus: The runtime event bus (``publish_envelope``). + report: The wiring report produced by wire_from_manifest(). + manifest: The filtered auto-wiring manifest (post-quarantine). + runtime_profile: The RUNTIME_PROFILE value (e.g. "main"). + topic: Resolved topic for SUFFIX_RUNTIME_MANIFEST_PUBLISHED. + correlation_id: The boot correlation id, propagated onto the envelope. + image_digest: Optional OCI image digest for the running container. + attach_readiness: Boot attach-readiness aggregate (OMN-15512). + + Returns: + The published payload, so callers and tests can assert on exactly what + went onto the bus. + + Raises: + ImportError: If omnibase_core.models.runtime_manifest is not installed. + """ + from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope + from omnibase_infra.runtime.models.model_runtime_manifest_published import ( + ModelRuntimeManifestPublished, + ) + + payload = build_runtime_manifest( + report=report, + manifest=manifest, + runtime_profile=runtime_profile, + image_digest=image_digest, + attach_readiness=attach_readiness, + ) + if not isinstance(payload, ModelRuntimeManifestPublished): # pragma: no cover + raise TypeError( + "build_runtime_manifest must return ModelRuntimeManifestPublished, " + f"got {type(payload).__name__}" + ) + + envelope: ModelEventEnvelope[object] = ModelEventEnvelope( + payload=payload, + correlation_id=correlation_id, + event_type="runtime-manifest-published", + source_tool="service_kernel", ) + await event_bus.publish_envelope(envelope=envelope, topic=topic) + return payload diff --git a/src/omnibase_infra/runtime/message_dispatch_engine.py b/src/omnibase_infra/runtime/message_dispatch_engine.py index b56aea403c..5de916bae8 100644 --- a/src/omnibase_infra/runtime/message_dispatch_engine.py +++ b/src/omnibase_infra/runtime/message_dispatch_engine.py @@ -136,7 +136,9 @@ import logging import threading import time -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Collection, Mapping +from contextvars import copy_context +from dataclasses import dataclass from datetime import UTC, datetime from typing import TYPE_CHECKING, TypedDict, Unpack, cast, overload from uuid import UUID, uuid4 @@ -182,6 +184,7 @@ from omnibase_infra.runtime._enum_coercion import coerce_message_category from omnibase_infra.runtime.binding_resolver import OperationBindingResolver from omnibase_infra.runtime.dispatch_context_enforcer import DispatchContextEnforcer +from omnibase_infra.runtime.dispatch_envelope_context import bind_dispatch_envelope from omnibase_infra.utils import sanitize_error_message _VALIDATION_DETAIL_MAX_LENGTH = 500 @@ -300,6 +303,17 @@ def _get_route_dispatcher_id(route: object) -> str: raise AttributeError(msg) +def _find_duplicate_identifiers(values: Collection[str]) -> frozenset[str]: + """Return identifiers repeated within one claimed registration boundary.""" + seen: set[str] = set() + duplicates: set[str] = set() + for value in values: + if value in seen: + duplicates.add(value) + seen.add(value) + return frozenset(duplicates) + + # Minimum number of parameters for a dispatcher to be considered context-aware. # Context-aware dispatchers have signature: (envelope, context, ...) # Non-context-aware dispatchers have signature: (envelope) @@ -392,6 +406,9 @@ class DispatchEntryInternal: instead of fanning the message out to every sibling handler (OMN-12416). ``None`` preserves the legacy string-only matching for operation-only or untyped dispatchers. + owner_contract_name: Contract that registered this dispatcher through + auto-wiring. ``None`` marks a manual/global dispatcher that cannot + be used as a contract-owned subscription scope. """ __slots__ = ( @@ -402,6 +419,7 @@ class DispatchEntryInternal: "message_types", "node_kind", "operation_bindings", + "owner_contract_name", "payload_type_matcher", ) @@ -415,6 +433,7 @@ def __init__( accepts_context: bool = False, operation_bindings: ModelOperationBindingsSubcontract | None = None, payload_type_matcher: Callable[[object], bool] | None = None, + owner_contract_name: str | None = None, ) -> None: self.dispatcher_id = dispatcher_id self.dispatcher = dispatcher @@ -425,10 +444,25 @@ def __init__( self.operation_bindings = ( operation_bindings # Declarative bindings for this dispatcher ) + self.owner_contract_name = owner_contract_name # None means "not type-scoped" — legacy string-only matching applies. self.payload_type_matcher = payload_type_matcher +@dataclass( + frozen=True, + slots=True, +) # internal-dataclass-ok: dispatch-engine-internal routing criteria +class DispatchMatchCriteriaInternal: + """Cohesive criteria for the private dispatcher-matching pass.""" + + topic: str + category: EnumMessageCategory + message_type: str + payload: object | None = None + allowed_dispatcher_ids: frozenset[str] | None = None + + class PayloadScopingOutcomeInternal: """Outcome of the OMN-12416 type-scoping pass over candidate dispatchers. @@ -689,6 +723,7 @@ def register_dispatcher( node_kind: None = None, operation_bindings: ModelOperationBindingsSubcontract | None = None, payload_type_matcher: Callable[[object], bool] | None = None, + owner_contract_name: str | None = None, ) -> None: ... # Stub: no node_kind -> DispatcherFunc (no context) @overload @@ -702,6 +737,7 @@ def register_dispatcher( node_kind: EnumNodeKind, operation_bindings: ModelOperationBindingsSubcontract | None = None, payload_type_matcher: Callable[[object], bool] | None = None, + owner_contract_name: str | None = None, ) -> None: ... # Stub: with node_kind -> ContextAwareDispatcherFunc (gets context) def register_dispatcher( @@ -713,6 +749,7 @@ def register_dispatcher( node_kind: EnumNodeKind | None = None, operation_bindings: ModelOperationBindingsSubcontract | None = None, payload_type_matcher: Callable[[object], bool] | None = None, + owner_contract_name: str | None = None, ) -> None: """ Register a message dispatcher. @@ -749,6 +786,9 @@ def register_dispatcher( message to the single handler whose ``event_model`` matches the payload type rather than fanning out to every sibling handler (OMN-12416). When None, legacy string-only matching applies. + owner_contract_name: Exact auto-wiring contract owner. Manual/global + registrations leave this unset and cannot later be cited by a + contract-owned subscription scope. Raises: ModelOnexError: If engine is frozen (INVALID_STATE) @@ -827,6 +867,7 @@ def register_dispatcher( node_kind=node_kind, operation_bindings=operation_bindings, payload_type_matcher=payload_type_matcher, + owner_contract_name=owner_contract_name, ) def _validate_dispatcher_registration( @@ -864,6 +905,7 @@ def _register_dispatcher_unlocked( node_kind: EnumNodeKind | None, operation_bindings: ModelOperationBindingsSubcontract | None, payload_type_matcher: Callable[[object], bool] | None = None, + owner_contract_name: str | None = None, ) -> None: if dispatcher_id in self._dispatchers: raise ModelOnexError( @@ -871,6 +913,14 @@ def _register_dispatcher_unlocked( "Cannot register duplicate dispatcher ID.", error_code=EnumCoreErrorCode.DUPLICATE_REGISTRATION, ) + if owner_contract_name is not None and ( + not owner_contract_name + or owner_contract_name != owner_contract_name.strip() + ): + raise ModelOnexError( + message="Dispatcher owner contract name must be non-empty and canonical.", + error_code=EnumCoreErrorCode.INVALID_PARAMETER, + ) accepts_context = self._dispatcher_accepts_context(dispatcher) entry = DispatchEntryInternal( @@ -882,6 +932,7 @@ def _register_dispatcher_unlocked( accepts_context=accepts_context, operation_bindings=operation_bindings, payload_type_matcher=payload_type_matcher, + owner_contract_name=owner_contract_name, ) self._dispatchers[dispatcher_id] = entry @@ -971,6 +1022,7 @@ def _register_dispatcher_dynamic( category: EnumMessageCategory, message_types: set[str] | None = None, payload_type_matcher: Callable[[object], bool] | None = None, + owner_contract_name: str | None = None, ) -> None: """Register a dispatcher post-freeze for dynamic contract materialization. @@ -1002,6 +1054,7 @@ def _register_dispatcher_dynamic( node_kind=None, operation_bindings=None, payload_type_matcher=payload_type_matcher, + owner_contract_name=owner_contract_name, ) self._logger.info( @@ -1096,6 +1149,8 @@ async def dispatch( self, topic: str, envelope: ModelEventEnvelope[object], + *, + allowed_dispatcher_ids: Collection[str] | None = None, ) -> ModelDispatchResult: """ Dispatch a message to matching dispatchers. @@ -1114,6 +1169,11 @@ async def dispatch( Args: topic: The topic the message was received on (e.g., "dev.user.events.v1") envelope: The message envelope to dispatch + allowed_dispatcher_ids: Optional exact dispatcher scope for a wired + contract subscription. When supplied, only routes owned by these + dispatcher IDs may execute. An empty or unknown scope fails + closed. ``None`` preserves process-global fan-out for direct + runtime callers that are not contract subscription callbacks. Returns: ModelDispatchResult with dispatch status, metrics, and dispatcher outputs @@ -1159,6 +1219,28 @@ async def dispatch( error_code=EnumCoreErrorCode.INVALID_PARAMETER, ) + dispatcher_scope: frozenset[str] | None = None + if allowed_dispatcher_ids is not None: + dispatcher_scope = frozenset(allowed_dispatcher_ids) + if not dispatcher_scope: + raise ModelOnexError( + message=( + "Contract-scoped dispatch requires at least one allowed " + "dispatcher ID; refusing process-global fan-out." + ), + error_code=EnumCoreErrorCode.INVALID_PARAMETER, + ) + unknown_dispatcher_ids = dispatcher_scope.difference(self._dispatchers) + if unknown_dispatcher_ids: + raise ModelOnexError( + message=( + "Contract-scoped dispatch references dispatcher IDs that " + "are not registered on this engine: " + f"{sorted(unknown_dispatcher_ids)}" + ), + error_code=EnumCoreErrorCode.ITEM_NOT_REGISTERED, + ) + # Start timing start_time = time.perf_counter() dispatch_id = uuid4() @@ -1284,10 +1366,13 @@ async def dispatch( # multi-handler contract routes each message to the single matching # handler instead of fanning out to every sibling (OMN-12416). matching_dispatchers, scoping_outcome = self._find_matching_dispatchers( - topic=topic, - category=topic_category, - message_type=message_type, - payload=envelope_payload, + DispatchMatchCriteriaInternal( + topic=topic, + category=topic_category, + message_type=message_type, + payload=envelope_payload, + allowed_dispatcher_ids=dispatcher_scope, + ) ) # Log routing decision at DEBUG level @@ -1625,6 +1710,11 @@ async def dispatch( # Use empty string sentinel internally to avoid str | None union matched_route_id: str = "" for route in self._routes.values(): + if ( + dispatcher_scope is not None + and _get_route_dispatcher_id(route) not in dispatcher_scope + ): + continue if route.matches(topic, topic_category, message_type): matched_route_id = route.route_id break @@ -1759,6 +1849,26 @@ async def dispatch( output_events=[], ) + async def dispatch_scoped( + self, + topic: str, + envelope: ModelEventEnvelope[object], + *, + allowed_dispatcher_ids: Collection[str], + ) -> ModelDispatchResult: + """Dispatch through an explicit, non-empty contract-owned scope. + + This distinct capability keeps the published ``ProtocolDispatchEngine`` + call shape backward compatible. Contract subscription wiring resolves + this method before attaching a consumer, so an engine that implements + only the process-global protocol cannot fail after receiving a record. + """ + return await self.dispatch( + topic, + envelope, + allowed_dispatcher_ids=allowed_dispatcher_ids, + ) + async def dispatch_with_transaction( self, *, @@ -1873,10 +1983,7 @@ async def dispatch_with_transaction( def _find_matching_dispatchers( self, - topic: str, - category: EnumMessageCategory, - message_type: str, - payload: object | None = None, + criteria: DispatchMatchCriteriaInternal, ) -> tuple[list[DispatchEntryInternal], PayloadScopingOutcomeInternal]: """ Find all dispatchers that match the given criteria. @@ -1898,12 +2005,8 @@ def _find_matching_dispatchers( legacy string-only matching and are unaffected. Args: - topic: The topic to match - category: The message category - message_type: The specific message type - payload: The message payload, used for event_model type-scoping. - When None (e.g. legacy callers without a payload), type-scoping - is skipped and string-only matching applies. + criteria: Topic, category, message type, payload, and optional exact + dispatcher scope used for this matching pass. Returns: Tuple of (matching dispatcher entries (may be empty), scoping @@ -1913,6 +2016,12 @@ def _find_matching_dispatchers( existed but its event_model rejected the payload), plus the real validation detail for the latter. """ + topic = criteria.topic + category = criteria.category + message_type = criteria.message_type + payload = criteria.payload + allowed_dispatcher_ids = criteria.allowed_dispatcher_ids + matching_dispatchers: list[DispatchEntryInternal] = [] seen_dispatcher_ids: set[str] = set() scoping_outcome = PayloadScopingOutcomeInternal() @@ -1931,6 +2040,11 @@ def _find_matching_dispatchers( # Get the dispatcher for this route dispatcher_id = _get_route_dispatcher_id(route) + if ( + allowed_dispatcher_ids is not None + and dispatcher_id not in allowed_dispatcher_ids + ): + continue if dispatcher_id in seen_dispatcher_ids: # Avoid duplicate dispatcher execution continue @@ -2164,55 +2278,68 @@ async def _execute_dispatcher( # Note: context is only non-None when entry.accepts_context is True, # so checking `context is not None` is sufficient to determine whether # to pass context to the dispatcher. - if inspect.iscoroutinefunction(dispatcher): - if context is not None: - # NOTE: Dispatcher signature varies - context param may be optional. - # Return type depends on dispatcher implementation (dict or model). - # Why: Runtime factory dispatch accepts this dynamic constructor shape. - return await dispatcher(envelope_for_handler, context) # type: ignore[call-arg,no-any-return] # NOTE: dispatcher signature varies - # NOTE: Return type depends on dispatcher implementation (dict or model). - # Why: Dispatcher boundary returns adapter output whose concrete type is runtime-defined. - return await dispatcher(envelope_for_handler) # type: ignore[no-any-return] # NOTE: dispatcher return type varies - else: - # Sync dispatcher execution via ThreadPoolExecutor - # ----------------------------------------------- - # WARNING: Sync dispatchers MUST be non-blocking (< 100ms execution). - # Blocking dispatchers can exhaust the thread pool, causing: - # - Starvation of other sync dispatchers - # - Delayed async dispatcher scheduling - # - Potential deadlocks under high load - # - # For blocking I/O operations, use async dispatchers instead. - loop = asyncio.get_running_loop() - - if context is not None: - # Context-aware sync dispatcher - sync_ctx_dispatcher = cast( - "_SyncContextAwareDispatcherFunc", dispatcher - ) - # NOTE: run_in_executor arg-type check fails because envelope_for_handler - # is dict[str, JsonType] but dispatcher expects dict[str, object]. - # JsonType is a subset of object, so this is safe at runtime. - return await loop.run_in_executor( - None, - # Why: Runtime wiring validates and narrows this payload shape before use. - sync_ctx_dispatcher, # type: ignore[arg-type] - envelope_for_handler, - context, - ) + # The handler still receives the canonical JSON-safe materialization. + # Bind the original typed envelope beside it only for transport identity + # (for example envelope_id). Tenant authentication uses a separate, + # cryptographically verified capability and never derives from this model. + with bind_dispatch_envelope(envelope): + if inspect.iscoroutinefunction(dispatcher): + if context is not None: + # NOTE: Dispatcher signature varies - context param may be optional. + # Return type depends on dispatcher implementation (dict or model). + # Why: Runtime factory dispatch accepts this dynamic constructor shape. + return await dispatcher(envelope_for_handler, context) # type: ignore[call-arg,no-any-return] # NOTE: dispatcher signature varies + # NOTE: Return type depends on dispatcher implementation (dict or model). + # Why: Dispatcher boundary returns adapter output whose concrete type is runtime-defined. + return await dispatcher(envelope_for_handler) # type: ignore[no-any-return] # NOTE: dispatcher return type varies else: - # Cast to sync-only type - safe because iscoroutinefunction check above - # guarantees this branch only executes for non-async callables - sync_dispatcher = cast("_SyncDispatcherFunc", dispatcher) - # NOTE: run_in_executor arg-type check fails because envelope_for_handler - # is dict[str, JsonType] but dispatcher expects dict[str, object]. - # JsonType is a subset of object, so this is safe at runtime. - return await loop.run_in_executor( - None, - # Why: Runtime wiring validates and narrows this payload shape before use. - sync_dispatcher, # type: ignore[arg-type] - envelope_for_handler, - ) + # Sync dispatcher execution via ThreadPoolExecutor + # ----------------------------------------------- + # WARNING: Sync dispatchers MUST be non-blocking (< 100ms execution). + # Blocking dispatchers can exhaust the thread pool, causing: + # - Starvation of other sync dispatchers + # - Delayed async dispatcher scheduling + # - Potential deadlocks under high concurrent load + # + # For blocking I/O operations, use async dispatchers instead. + loop = asyncio.get_running_loop() + + if context is not None: + # Context-aware sync dispatcher + sync_ctx_dispatcher = cast( + "_SyncContextAwareDispatcherFunc", dispatcher + ) + payload_for_sync = cast("dict[str, object]", envelope_for_handler) + execution_context = copy_context() + + def _invoke_sync_dispatcher_with_context() -> DispatcherOutput: + return execution_context.run( + sync_ctx_dispatcher, + payload_for_sync, + context, + ) + + return await loop.run_in_executor( + None, + _invoke_sync_dispatcher_with_context, + ) + else: + # Cast to sync-only type - safe because iscoroutinefunction check above + # guarantees this branch only executes for non-async callables + sync_dispatcher = cast("_SyncDispatcherFunc", dispatcher) + payload_for_sync = cast("dict[str, object]", envelope_for_handler) + execution_context = copy_context() + + def _invoke_sync_dispatcher() -> DispatcherOutput: + return execution_context.run( + sync_dispatcher, + payload_for_sync, + ) + + return await loop.run_in_executor( + None, + _invoke_sync_dispatcher, + ) def _create_context_for_entry( self, @@ -2862,6 +2989,187 @@ def dispatcher_count(self) -> int: """Get the number of registered dispatchers.""" return len(self._dispatchers) + def validate_registration_batch( + self, + dispatcher_ids: Collection[str], + route_ids: Collection[str], + *, + dispatcher_owners: Mapping[str, str], + allow_frozen: bool = False, + ) -> None: + """Prove a complete prepared batch is collision-free before mutation. + + Auto-wiring prepares every dispatcher and route before committing any + side effects. This boundary validates the complete derived identifier + batch, including normalization collisions, conflicts with the live + registry, and immutable contract-owner conflicts, while holding the + same lock used by registration. + """ + raw_dispatcher_ids = tuple(dispatcher_ids) + raw_route_ids = tuple(route_ids) + for label, identifiers in ( + ("dispatcher", raw_dispatcher_ids), + ("route", raw_route_ids), + ): + invalid_ids = tuple( + identifier + for identifier in identifiers + if not identifier or identifier != identifier.strip() + ) + if invalid_ids: + raise ModelOnexError( + message=( + f"Prepared {label} IDs must be non-empty and canonical: " + f"{invalid_ids}" + ), + error_code=EnumCoreErrorCode.INVALID_PARAMETER, + ) + duplicate_ids = _find_duplicate_identifiers(identifiers) + if duplicate_ids: + raise ModelOnexError( + message=( + f"Auto-wiring batch contains duplicate prepared {label} IDs: " + f"{sorted(duplicate_ids)}" + ), + error_code=EnumCoreErrorCode.DUPLICATE_REGISTRATION, + ) + + prepared_dispatcher_ids = frozenset(raw_dispatcher_ids) + if frozenset(dispatcher_owners) != prepared_dispatcher_ids or any( + not owner or owner != owner.strip() for owner in dispatcher_owners.values() + ): + raise ModelOnexError( + message=( + "Prepared dispatcher ownership must provide one canonical " + "contract owner for every prepared dispatcher ID." + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + + with self._registration_lock: + if ( + self._frozen + and not allow_frozen + and (raw_dispatcher_ids or raw_route_ids) + ): + raise ModelOnexError( + message=( + "Cannot commit prepared registration batch: " + "MessageDispatchEngine is frozen." + ), + error_code=EnumCoreErrorCode.INVALID_STATE, + ) + existing_dispatcher_ids = frozenset(raw_dispatcher_ids).intersection( + self._dispatchers + ) + if existing_dispatcher_ids: + raise ModelOnexError( + message=( + "Prepared dispatcher IDs are already registered on this engine: " + f"{sorted(existing_dispatcher_ids)}" + ), + error_code=EnumCoreErrorCode.DUPLICATE_REGISTRATION, + ) + existing_route_ids = frozenset(raw_route_ids).intersection(self._routes) + if existing_route_ids: + raise ModelOnexError( + message=( + "Prepared route IDs are already registered on this engine: " + f"{sorted(existing_route_ids)}" + ), + error_code=EnumCoreErrorCode.DUPLICATE_REGISTRATION, + ) + prepared_owner_names = frozenset(dispatcher_owners.values()) + existing_owner_scopes = { + owner_name: tuple( + sorted( + dispatcher_id + for dispatcher_id, entry in self._dispatchers.items() + if entry.owner_contract_name == owner_name + ) + ) + for owner_name in prepared_owner_names + } + conflicting_owner_scopes = { + owner_name: dispatcher_scope + for owner_name, dispatcher_scope in existing_owner_scopes.items() + if dispatcher_scope + } + if conflicting_owner_scopes: + raise ModelOnexError( + message=( + "Prepared registration would extend immutable live " + "contract-owner dispatcher scopes: " + f"{conflicting_owner_scopes}" + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + + def validate_contract_dispatcher_scope( + self, + contract_name: str, + dispatcher_ids: Collection[str], + ) -> frozenset[str]: + """Prove an exact scope against the complete immutable owner snapshot.""" + raw_dispatcher_ids = tuple(dispatcher_ids) + if ( + not contract_name + or contract_name != contract_name.strip() + or any( + not dispatcher_id or dispatcher_id != dispatcher_id.strip() + for dispatcher_id in raw_dispatcher_ids + ) + or _find_duplicate_identifiers(raw_dispatcher_ids) + ): + raise ModelOnexError( + message=( + "Contract dispatcher scope requires a canonical contract name " + "and unique canonical dispatcher IDs." + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + dispatcher_scope = frozenset(raw_dispatcher_ids) + with self._registration_lock: + unknown_dispatcher_ids = dispatcher_scope.difference(self._dispatchers) + if unknown_dispatcher_ids: + raise ModelOnexError( + message=( + "Contract-scoped subscription references dispatcher IDs " + "that are not registered on this engine for contract " + f"{contract_name!r}: {sorted(unknown_dispatcher_ids)}" + ), + error_code=EnumCoreErrorCode.ITEM_NOT_REGISTERED, + ) + invalid_owners = { + dispatcher_id: self._dispatchers[dispatcher_id].owner_contract_name + for dispatcher_id in dispatcher_scope + if self._dispatchers[dispatcher_id].owner_contract_name != contract_name + } + if invalid_owners: + raise ModelOnexError( + message=( + "Contract-scoped subscription references dispatcher IDs " + f"not owned by contract {contract_name!r}: {invalid_owners}" + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + complete_owner_scope = frozenset( + dispatcher_id + for dispatcher_id, entry in self._dispatchers.items() + if entry.owner_contract_name == contract_name + ) + if dispatcher_scope != complete_owner_scope: + raise ModelOnexError( + message=( + "Contract-scoped subscription does not match the complete " + f"live owner set for contract {contract_name!r}: " + f"missing={sorted(complete_owner_scope - dispatcher_scope)}, " + f"unexpected={sorted(dispatcher_scope - complete_owner_scope)}" + ), + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + ) + return complete_owner_scope + def __str__(self) -> str: """Human-readable string representation.""" return ( diff --git a/src/omnibase_infra/runtime/models/model_runtime_manifest_published.py b/src/omnibase_infra/runtime/models/model_runtime_manifest_published.py new file mode 100644 index 0000000000..4e6c3d5954 --- /dev/null +++ b/src/omnibase_infra/runtime/models/model_runtime_manifest_published.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Wire payload for ``onex.evt.omnibase-infra.runtime-manifest-published.v1``. + +This is the canonical boot snapshot published once per startup at +``service_kernel`` step 9.8. It is ``ModelRuntimeManifest`` (what the runtime +WIRED) plus one additive field, ``attach_readiness`` (what actually ATTACHED, +and for every contract that did not, which topics failed readiness confirm). + +Why a subclass and not a second model +------------------------------------- +``ModelRuntimeManifest`` is ``frozen``/``extra="forbid"`` and lives in +``omnibase_core``, which ``omnibase_infra`` consumes at an immutable pinned rev +(``pyproject.toml [tool.uv.sources]``). Every attach model +(``ModelRuntimeAttachReadiness``, ``ModelContractAttachResult``, +``ModelTopicSetReadiness``) is ``omnibase_infra``-resident and core cannot +import infra (layering: compat -> core -> spi -> infra), so the field cannot be +added to the base without first relocating the attach models into core. + +Subclassing keeps ONE model per shape — every base field is inherited, none is +redeclared — and is wire-compatible: the serialized payload is byte-identical +to the previous one plus a single ``attach_readiness`` key. Existing consumers +(``node_runtime_manifest_reducer`` here, ``node_redeploy_orchestrator`` in +omnimarket, which declares the subscription but decodes no typed model) read +the keys they already read. ``contract_hash`` / ``topology_hash`` are inherited +computed fields and are unchanged by this addition, so manifest dedup and drift +detection keep their existing values. + +Residual (tracked on OMN-15512): when ``omnibase_core`` next cuts a release +that carries the attach models, fold ``attach_readiness`` onto +``ModelRuntimeManifest`` itself and delete this subclass. + +Related Tickets: + - OMN-11196: Emit the runtime manifest snapshot at boot. + - OMN-11197: Persist it to the ``runtime_manifests`` projection. + - OMN-13237: Per-contract provision -> confirm-ready -> attach interleave. + - OMN-15512: Fold the attach-readiness aggregate onto this payload. +""" + +from __future__ import annotations + +from pydantic import Field + +from omnibase_core.models.runtime_manifest.model_runtime_manifest import ( + ModelRuntimeManifest, +) +from omnibase_infra.event_bus.model_runtime_attach_readiness import ( + ModelRuntimeAttachReadiness, +) + + +class ModelRuntimeManifestPublished(ModelRuntimeManifest): + """Runtime manifest snapshot carrying the boot attach-readiness aggregate. + + Attributes: + attach_readiness: Boot attach-readiness aggregate. ``None`` only when + the per-contract interleave did not run (auto-wiring disabled or + failed before subscribe), which is distinct from "ran and every + contract attached" — that case carries a ``READY`` aggregate with + an empty ``results``. The published copy is narrowed to the blocker + set via :meth:`ModelRuntimeAttachReadiness.blockers_only`, so + ``required_contracts - attached_contracts == len(results)``. + """ + + attach_readiness: ModelRuntimeAttachReadiness | None = Field(default=None) + + +__all__: list[str] = ["ModelRuntimeManifestPublished"] diff --git a/src/omnibase_infra/runtime/models/model_runtime_process_policy.py b/src/omnibase_infra/runtime/models/model_runtime_process_policy.py index 9b263d83b4..5f473b16f5 100644 --- a/src/omnibase_infra/runtime/models/model_runtime_process_policy.py +++ b/src/omnibase_infra/runtime/models/model_runtime_process_policy.py @@ -24,12 +24,13 @@ class ModelRuntimeProcessPolicy(BaseModel): omnimemory_memgraph_host: str = "" publish_introspection: bool = False # OMN-12990: contract-declared replica count for this runtime process. - # The base compose worker deploy default is ${WORKER_REPLICAS:-0}, which - # silently scales the worker to zero on a plain compose recreate. Each lane - # pins this value via {PROFILE}_WORKER_REPLICAS (rendered into - # runtime-policy.env) and the lane overrides reference it fail-fast, so a - # recreate that omits the policy env fails loudly instead of dropping the - # worker with no signal. + # The base compose worker deploy default used to be a bare + # ${WORKER_REPLICAS:-0}, which silently scaled the worker to zero on a plain + # compose recreate. Each lane pins this value via {PROFILE}_WORKER_REPLICAS + # (rendered into runtime-policy.env); every lane compose surface now + # references it fail-fast -- the base infra file for dev (OMN-14968) and the + # stability-test/prod overlays -- so a recreate that omits the policy env + # fails loudly instead of dropping the worker with no signal. replicas: int = Field(default=1, ge=1) @field_validator("capabilities") diff --git a/src/omnibase_infra/runtime/projection_tenant_authority.py b/src/omnibase_infra/runtime/projection_tenant_authority.py new file mode 100644 index 0000000000..61ff81b4ea --- /dev/null +++ b/src/omnibase_infra/runtime/projection_tenant_authority.py @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Cryptographically verified tenant authority for projection writes. + +The public event models in this repository are data containers, not proof that +authentication occurred. In particular, ``ModelEventEnvelope`` security +context and metadata fields can be supplied by any deserializer. This module +therefore mints a sealed-construction capability only after the canonical core +``ModelMessageEnvelope`` signature verifies. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING, Protocol, runtime_checkable +from uuid import UUID + +from pydantic import BaseModel + +if TYPE_CHECKING: + from omnibase_core.protocols.crypto.protocol_key_provider import ( + ProtocolKeyProvider, + ) + + +_AUTHORITY_MINT = object() + + +def _tenant_context_error(message: str) -> Exception: + from omnibase_infra.errors.error_projection import ProjectionTenantContextError + + return ProjectionTenantContextError(message) + + +def parse_canonical_tenant_uuid(value: object, *, authority: str) -> UUID: + """Parse a non-zero canonical UUID without accepting slugs or sentinels.""" + if not isinstance(value, str) or value != value.strip(): + raise _tenant_context_error( + f"Projection tenant context from {authority} is not a canonical UUID" + ) + try: + tenant_id = UUID(value) + except ValueError as exc: + raise _tenant_context_error( + f"Projection tenant context from {authority} is malformed" + ) from exc + if str(tenant_id) != value or tenant_id.int == 0: + raise _tenant_context_error( + f"Projection tenant context from {authority} is not a canonical UUID" + ) + return tenant_id + + +@dataclass(frozen=True, slots=True, init=False) +class VerifiedProjectionTenantAuthority: + """Opaque, in-process capability produced by signature verification. + + The constructor is intentionally sealed. Callers obtain an instance only + through :func:`verify_signed_projection_tenant_authority`; request-shaped + dictionaries and materialized envelopes cannot be coerced into this type. + """ + + tenant_id: UUID + trace_id: UUID + runtime_id: str + realm: str + bus_id: str + emitted_at: datetime + event_envelope_id: UUID + event_payload_hash: str + + def __init__( + self, + *, + tenant_id: UUID, + trace_id: UUID, + runtime_id: str, + realm: str, + bus_id: str, + emitted_at: datetime, + event_envelope_id: UUID, + event_payload_hash: str, + _mint: object, + ) -> None: + if _mint is not _AUTHORITY_MINT: + raise TypeError( + "VerifiedProjectionTenantAuthority can only be minted by " + "signature verification" + ) + object.__setattr__(self, "tenant_id", tenant_id) + object.__setattr__(self, "trace_id", trace_id) + object.__setattr__(self, "runtime_id", runtime_id) + object.__setattr__(self, "realm", realm) + object.__setattr__(self, "bus_id", bus_id) + object.__setattr__(self, "emitted_at", emitted_at) + object.__setattr__(self, "event_envelope_id", event_envelope_id) + object.__setattr__(self, "event_payload_hash", event_payload_hash) + + +@runtime_checkable +class ProtocolProjectionTenantBindingResolver(Protocol): + """Authoritative mapping from a verified signer scope to one tenant UUID.""" + + def resolve_tenant_id( + self, + *, + runtime_id: str, + realm: str, + bus_id: str, + ) -> UUID | None: + """Return the tenant authorized for this signer scope, or ``None``.""" + ... + + +def verify_signed_projection_tenant_authority( + envelope: object, + key_provider: ProtocolKeyProvider, + tenant_binding_resolver: ProtocolProjectionTenantBindingResolver, +) -> VerifiedProjectionTenantAuthority: + """Verify a core signed envelope and mint its canonical UUID authority. + + ``ModelMessageEnvelope`` signs the tenant identifier together with the + runtime, realm, bus, trace, timestamp, and payload hash. Unknown signers, + bad signatures, altered payloads/metadata, missing tenants, and non-UUID + tenants all fail closed before a database adapter can be built. + """ + from omnibase_core.models.envelope.model_message_envelope import ( + ModelMessageEnvelope, + ) + + if not isinstance(envelope, ModelMessageEnvelope): + raise _tenant_context_error( + "Tenant projection authority requires a signed ModelMessageEnvelope" + ) + try: + verified = envelope.verify_signature(key_provider) + except Exception as exc: + raise _tenant_context_error( + "Projection tenant envelope signature verification failed" + ) from exc + if not verified: + raise _tenant_context_error( + "Projection tenant envelope signature verification failed" + ) + tenant_id = parse_canonical_tenant_uuid( + envelope.tenant_id, + authority="verified signed message envelope", + ) + authorized_tenant_id = tenant_binding_resolver.resolve_tenant_id( + runtime_id=envelope.runtime_id, + realm=envelope.realm, + bus_id=envelope.bus_id, + ) + if not isinstance(authorized_tenant_id, UUID): + raise _tenant_context_error( + "Projection envelope signer has no authoritative tenant binding" + ) + if authorized_tenant_id != tenant_id: + raise _tenant_context_error( + "Projection envelope tenant does not match its signer binding" + ) + from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope + + event_envelope = envelope.payload + if not isinstance(event_envelope, ModelEventEnvelope): + try: + event_envelope = ModelEventEnvelope[object].model_validate(event_envelope) + except Exception as exc: + raise _tenant_context_error( + "Signed projection authority must wrap a valid ModelEventEnvelope" + ) from exc + if ( + not isinstance(event_envelope.correlation_id, UUID) + or event_envelope.correlation_id != envelope.trace_id + ): + raise _tenant_context_error( + "Signed projection trace does not match event correlation UUID" + ) + return VerifiedProjectionTenantAuthority( + tenant_id=tenant_id, + trace_id=envelope.trace_id, + runtime_id=envelope.runtime_id, + realm=envelope.realm, + bus_id=envelope.bus_id, + emitted_at=envelope.emitted_at, + event_envelope_id=event_envelope.envelope_id, + event_payload_hash=envelope.signature.payload_hash, + _mint=_AUTHORITY_MINT, + ) + + +def assert_projection_tenant_authority_matches_event( + authority: VerifiedProjectionTenantAuthority, + event_envelope: object, +) -> None: + """Bind a verified capability to the exact event being dispatched.""" + from omnibase_core.crypto.crypto_blake3_hasher import hash_canonical_json + from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope + + if not isinstance(event_envelope, ModelEventEnvelope): + raise _tenant_context_error( + "Verified tenant authority requires a typed dispatch envelope" + ) + if ( + event_envelope.envelope_id != authority.event_envelope_id + or event_envelope.correlation_id != authority.trace_id + ): + raise _tenant_context_error( + "Verified tenant authority does not match the dispatched envelope" + ) + payload: BaseModel = event_envelope + actual_hash = hash_canonical_json(payload.model_dump(mode="json")) + if actual_hash != authority.event_payload_hash: + raise _tenant_context_error( + "Verified tenant authority payload does not match the dispatched envelope" + ) + + +__all__ = [ + "ProtocolProjectionTenantBindingResolver", + "VerifiedProjectionTenantAuthority", + "assert_projection_tenant_authority_matches_event", + "parse_canonical_tenant_uuid", + "verify_signed_projection_tenant_authority", +] diff --git a/src/omnibase_infra/runtime/protocols/protocol_contract_scoped_dispatch_engine.py b/src/omnibase_infra/runtime/protocols/protocol_contract_scoped_dispatch_engine.py new file mode 100644 index 0000000000..75efacb68c --- /dev/null +++ b/src/omnibase_infra/runtime/protocols/protocol_contract_scoped_dispatch_engine.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Protocol for dispatch engines that support exact contract-owned scopes.""" + +from __future__ import annotations + +from collections.abc import Collection +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +if TYPE_CHECKING: + from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope + from omnibase_infra.models.dispatch.model_dispatch_result import ( + ModelDispatchResult, + ) + +__all__ = ["ProtocolContractScopedDispatchEngine"] + + +@runtime_checkable +class ProtocolContractScopedDispatchEngine(Protocol): + """Dispatch through an explicit set of contract-owned dispatcher IDs.""" + + def validate_contract_dispatcher_scope( + self, + contract_name: str, + dispatcher_ids: Collection[str], + ) -> frozenset[str]: + """Return a validated scope or raise before consumer side effects.""" + ... + + async def dispatch_scoped( + self, + topic: str, + envelope: ModelEventEnvelope[object], + *, + allowed_dispatcher_ids: Collection[str], + ) -> ModelDispatchResult: ... diff --git a/src/omnibase_infra/runtime/protocols/protocol_delegation_dispatch_port.py b/src/omnibase_infra/runtime/protocols/protocol_delegation_dispatch_port.py index d5a0f32502..4bfc3089d5 100644 --- a/src/omnibase_infra/runtime/protocols/protocol_delegation_dispatch_port.py +++ b/src/omnibase_infra/runtime/protocols/protocol_delegation_dispatch_port.py @@ -19,13 +19,18 @@ async def dispatch( prompt: str, task_type: str, correlation_id: UUID, - max_tokens: int, + max_tokens: int | None, source_file_path: str | None, source_session_id: str | None, wait: bool, quality_contract_mode: str, acceptance_criteria: tuple[str, ...], tenant_id: str | None = None, + backend_id: str | None = None, + response_contract: dict[str, object] | None = None, + system_prompt: str | None = None, + temperature: float | None = None, + response_format: dict[str, object] | None = None, ) -> dict[str, object]: ... diff --git a/src/omnibase_infra/runtime/render_bifrost_delegation_contract.py b/src/omnibase_infra/runtime/render_bifrost_delegation_contract.py index afbff01485..ad3eab5db4 100644 --- a/src/omnibase_infra/runtime/render_bifrost_delegation_contract.py +++ b/src/omnibase_infra/runtime/render_bifrost_delegation_contract.py @@ -511,7 +511,24 @@ def _resolve_target_path( return target_path configured_path = env.get("BIFROST_CONTRACT_PATH") if configured_path is None: - return _DEFAULT_TARGET_PATH + # OMN-15628: no silent target-path default. The previous body + # returned _DEFAULT_TARGET_PATH here, so a caller that forgot to bind + # BIFROST_CONTRACT_PATH (or a direct-call bypass of the + # entrypoint-runtime.sh `[ -n "${BIFROST_CONTRACT_PATH:-}" ]` guard) + # would still render/write a contract with no attributable cause — + # the exact write-side half of the silent-fallback defect class the + # read-side reducer fix (this ticket) closes. A service that + # deliberately skips rendering binds BIFROST_CONTRACT_PATH="" (the + # existing, still-supported "explicit empty = disabled" signal + # below) rather than leaving the key unset. + raise ProtocolConfigurationError( + "BIFROST_CONTRACT_PATH is not bound; render_bifrost_delegation_contract " + "refuses to fall back to the packaged default target path " + f"({_DEFAULT_TARGET_PATH}) (CLAUDE.md rule 8 — no silent config " + "fallback, OMN-15628). Set BIFROST_CONTRACT_PATH explicitly " + "(empty string to deliberately skip rendering for this service), " + "or pass target_path explicitly." + ) stripped_path = configured_path.strip() if not stripped_path: return None @@ -533,10 +550,12 @@ def render_bifrost_delegation_contract( source_path: Path to the packaged bifrost_delegation.yaml source. Defaults to the omnimarket-bundled copy. target_path: Path to write the rendered contract on the runtime volume. - Defaults to /app/data/delegation/bifrost_delegation.yaml. - Pass ``None`` to accept the env-configured target; returns None - when BIFROST_CONTRACT_PATH resolves to an empty path (services - that deliberately skip rendering, e.g. projection-api). + Pass ``None`` to accept the env-configured target: resolves to + ``Path(BIFROST_CONTRACT_PATH)`` when that env var is a non-empty + string, returns ``None`` when it is bound to an explicit empty + string (services that deliberately skip rendering, e.g. + projection-api), and raises ``ProtocolConfigurationError`` when it + is unbound entirely (OMN-15628 — no silent default target path). environ: Mapping used for env-var reads. Defaults to os.environ. verify_endpoints: When True, probe each populated endpoint via GET /v1/models and reject backends whose model is not listed. @@ -554,8 +573,11 @@ def render_bifrost_delegation_contract( Raises: ProtocolConfigurationError: On any rendering failure — missing source, - malformed YAML, schema validation failure, or zero populated - endpoints (OMN-12814: fail-loud, never returns a silent empty result). + malformed YAML, schema validation failure, zero populated + endpoints (OMN-12814: fail-loud, never returns a silent empty + result), or an unbound BIFROST_CONTRACT_PATH when ``target_path`` + is not passed explicitly (OMN-15628: no silent default target + path). FileNotFoundError: If the source contract cannot be found. ValueError: If the source YAML fails schema validation. """ diff --git a/src/omnibase_infra/runtime/required_image_config_paths.py b/src/omnibase_infra/runtime/required_image_config_paths.py new file mode 100644 index 0000000000..6498d4aa40 --- /dev/null +++ b/src/omnibase_infra/runtime/required_image_config_paths.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Single typed source of the config files that MUST exist in the runtime image. + +OMN-15676. Three separate incidents have now had the same shape: a config file +is tracked in the repo, every source-level check passes, and the defect exists +only in the *built image* because no ``COPY`` ships the file: + +* the grants fixture (OMN-6698 / OMN-12726 class -- deploy scripts omitting + required build-context paths), +* ``routing_tiers.yaml`` (OMN-15645 -- baked only after the image that pinned + it had already been built, forcing OMN-15623 to pin a site-packages literal), +* ``runner_fleet.yaml`` (OMN-15676 -- this module's reason for existing; the + deployed runtime raised ``FileNotFoundError: /app/config/runner_fleet.yaml`` + while the repo looked perfectly correct). + +Nothing tested the built artifact, so no source review, static sweep, or unit +suite could see any of the three. This registry is the single typed source that +``scripts/ci/assert_image_config_paths.py`` reads to assert ``test -f`` for +every entry **inside the built image**, wired into the runtime image builds +before the push step so a missing ``COPY`` cannot reach a registry. + +Scope -- deliberately narrow, and the exclusions are the load-bearing part: + +* IN SCOPE: files under ``/app/config/`` that the runtime resolves at startup. + ``/app/config/`` carries no volume or bind-mount entry anywhere in the compose + lane files or the onex-dev manifests, so image-baked content there is never + shadowed at container start. That property is exactly why the OMN-15645 bake + chose ``/app/config/`` over ``/app/contracts/``; see the block comment above + the ``routing_tiers.yaml`` COPY in ``docker/Dockerfile.runtime``. +* OUT OF SCOPE ``/app/data/``: rendered at boot by ``entrypoint-runtime.sh`` + (``render_bifrost_delegation_contract`` / ``render_secret_resolver_config``) + *after* volumes are mounted, and mount-covered by design in both compose and + k8s (the OMN-12945 emptyDir shadow lives here). Asserting image-baked content + under ``/app/data/`` would assert the wrong thing: absence there is correct. +* OUT OF SCOPE ``/app/contracts/``: bind-mounted read-only from the host by the + compose runtime services, so host content legitimately shadows the baked tree. + +Adding an entry here is the whole cost of protecting a new startup-resolved +config path. If a path is resolved from an env-var pin, record the pin in +``resolved_by`` so the binding is greppable from one place. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelRequiredImageConfigPath(BaseModel): + """One config file that must be present in the built runtime image.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + image_path: str = Field( + ..., + min_length=1, + description=( + "Absolute in-image path asserted with `test -f` inside the built " + "image. Must be absolute and must not contain shell metacharacters." + ), + ) + resolved_by: str = Field( + ..., + min_length=1, + description=( + "The startup surface that resolves this path -- a dotted callable, " + "or an env-var pin name plus where it is bound." + ), + ) + why_required: str = Field( + ..., + min_length=1, + description="What breaks at runtime when the file is absent from the image.", + ) + ticket: str = Field( + ..., + min_length=1, + description="Ticket that added or repaired this requirement.", + ) + + +REQUIRED_IMAGE_CONFIG_PATHS: tuple[ModelRequiredImageConfigPath, ...] = ( + ModelRequiredImageConfigPath( + image_path="/app/config/runner_fleet.yaml", + resolved_by=( + "omnibase_infra.observability.runner_health.model_runner_fleet_config." + "default_runner_fleet_config_path (RUNNER_FLEET_CONFIG_PATH override, " + "else repo-root-relative -- which under the image's PYTHONPATH=/app/src " + "resolves to /app/config/runner_fleet.yaml)" + ), + why_required=( + "HandlerRunnerFleetSnapshot.__init__ calls load_runner_fleet_config(), " + "which raises FileNotFoundError rather than falling back to embedded " + "lab values. The handler is instantiated during auto-wiring, so under " + "ONEX_WIRING_STRICT_MODE=1 (bound on onex-dev) the absence is a boot " + "failure, not a degraded mode." + ), + ticket="OMN-15676", + ), + ModelRequiredImageConfigPath( + image_path="/app/config/delegation/routing_tiers.yaml", + resolved_by=( + "DELEGATION_ROUTING_TIERS_PATH -- pinned in docker-compose.infra.yml " + "x-runtime-env and in the onex-dev ConfigMap plus the three runtime " + "Deployments (omninode_infra k8s/onex-dev/runtime/)" + ), + why_required=( + "The delegation routing reducer fails closed when the tiers file the " + "pin names does not exist. Baked from the installed omnimarket package " + "to a stable path so the pin never embeds a python3.X site-packages " + "directory that a base-image Python bump silently invalidates." + ), + ticket="OMN-15645", + ), +) + + +def required_image_config_paths() -> tuple[str, ...]: + """Return just the in-image paths, in declaration order.""" + return tuple(entry.image_path for entry in REQUIRED_IMAGE_CONFIG_PATHS) + + +__all__ = [ + "REQUIRED_IMAGE_CONFIG_PATHS", + "ModelRequiredImageConfigPath", + "required_image_config_paths", +] diff --git a/src/omnibase_infra/runtime/runtime_host_process.py b/src/omnibase_infra/runtime/runtime_host_process.py index 91415374f5..3c34858d55 100644 --- a/src/omnibase_infra/runtime/runtime_host_process.py +++ b/src/omnibase_infra/runtime/runtime_host_process.py @@ -84,6 +84,7 @@ async def main() -> None: ProtocolConfigurationError, RuntimeHostError, SecretResolutionError, + TopicReplicationPolicyError, UnknownHandlerTypeError, ) @@ -178,6 +179,9 @@ async def main() -> None: if TYPE_CHECKING: from omnibase_core.container import ModelONEXContainer + from omnibase_core.models.core.model_deployment_topology import ( + ModelDeploymentTopology, + ) from omnibase_core.models.envelope.model_message_envelope import ( ModelMessageEnvelope, ) @@ -803,6 +807,7 @@ def __init__( dispatch_engine: MessageDispatchEngine | None = None, runtime_node_graph_config: ModelRuntimeNodeGraphConfig | None = None, prefetch_policy: PrefetchPolicy = "disabled", + deployment_topology: ModelDeploymentTopology | None = None, ) -> None: """Initialize the runtime host process. @@ -995,6 +1000,11 @@ async def create_runtime() -> RuntimeHostProcess: The dispatch engine must be frozen (freeze() called) before RuntimeHostProcess.start() is invoked. The kernel handles this by freezing after all plugins have registered their dispatchers. + + deployment_topology: Authoritative checked-in application database + topology. Required for dynamic or filesystem-discovered contracts + that declare ``db_io``; event namespace configuration is never used + as a topology selector. """ # Store container reference for dependency resolution self._container: ModelONEXContainer | None = container @@ -1002,6 +1012,7 @@ async def create_runtime() -> RuntimeHostProcess: # from runtime contract YAMLs instead of module-level DEFAULT_* constants. # Env-var override layer is handled by ModelRuntimeNodeGraphConfig.from_contracts_dir(). self._runtime_node_graph_config = runtime_node_graph_config + self._deployment_topology = deployment_topology # Handler registry (container-based DI or singleton fallback) self._handler_registry: RegistryProtocolBinding | None = handler_registry @@ -2529,6 +2540,7 @@ async def _dispatch_local_ingress_request( validated_payload = validate_runtime_local_ingress_payload( route, request.payload, + correlation_id=correlation_id, ) except ValidationError as exc: return ModelLocalRuntimeIngressResponse( @@ -3407,6 +3419,59 @@ async def _materialize_handler_live( Added as part of OMN-1989 live contract materialization. """ try: + # Typed database projections use the same contract-driven wiring path + # as cold boot. The historical registry-only path below cannot inject + # projection database adapters and would therefore register a handler + # that is present but unusable. Both baseline and post-freeze callbacks + # reach this method only after KafkaContractSource has cached the exact + # descriptor, so delegate the whole transaction back to that source. + contract_config = descriptor.contract_config or {} + if contract_config.get("db_io") is not None: + source = getattr(self, "_kafka_contract_source", None) + dispatch_engine = getattr(self, "_dispatch_engine", None) + if source is None or dispatch_engine is None: + logger.warning( + "Cannot materialize db_io contract without Kafka source and " + "dispatch engine", + extra={ + "node_name": node_name, + "correlation_id": str(correlation_id), + }, + ) + return False + + from omnibase_infra.runtime.enums.enum_materialization_status import ( + EnumMaterializationStatus, + ) + + materialization = await source.materialize_cached_contract( + node_name=node_name, + dispatch_engine=dispatch_engine, + event_bus=getattr(self, "_event_bus", None), + environment=source.environment, + container=getattr(self, "_container", None), + topology=getattr(self, "_deployment_topology", None), + ) + accepted = materialization.status in { + EnumMaterializationStatus.MATERIALIZED, + EnumMaterializationStatus.ALREADY_MATERIALIZED, + } + if not accepted: + logger.warning( + "Typed db_io contract materialization rejected", + extra={ + "node_name": node_name, + "status": materialization.status.value, + "reason": ( + materialization.reason.value + if materialization.reason is not None + else None + ), + "correlation_id": str(correlation_id), + }, + ) + return accepted + # Step 1: Early-return if no handler_class if descriptor.handler_class is None: logger.debug( @@ -3694,6 +3759,10 @@ async def _wire_live_handler_subscriptions( for _topic in subcontract.subscribe_topics: try: await _provisioner.ensure_topic_exists(topic_name=_topic) + except TopicReplicationPolicyError: + # OMN-15395: durability violations are fail-closed + # and must escape this best-effort boundary. + raise except Exception: # noqa: BLE001 — boundary: best-effort logger.warning( "Topic pre-provisioning failed for live contract " diff --git a/src/omnibase_infra/runtime/runtime_local_ingress.py b/src/omnibase_infra/runtime/runtime_local_ingress.py index a0c2c113c0..c7a2bf62d3 100644 --- a/src/omnibase_infra/runtime/runtime_local_ingress.py +++ b/src/omnibase_infra/runtime/runtime_local_ingress.py @@ -10,14 +10,22 @@ import logging import os import stat -from collections.abc import Awaitable, Callable, Iterable, Sequence +import typing +from collections.abc import Awaitable, Callable, Sequence +from copy import deepcopy from pathlib import Path -from typing import cast +from types import UnionType +from typing import cast, get_args, get_origin +from uuid import UUID import yaml -from pydantic import BaseModel, ConfigDict, ValidationError +from pydantic import AliasChoices, AliasPath, BaseModel, ConfigDict, ValidationError from omnibase_core.types import JsonType +from omnibase_infra.runtime.contract_terminal_events import ( + extract_terminal_event_topics, + terminal_event_topics_from_declaration, +) from omnibase_infra.runtime.event_bus_subcontract_wiring import ( EventBusSubcontractWiring, ) @@ -33,6 +41,9 @@ logger = logging.getLogger(__name__) +_RuntimeIngressAliasPath = tuple[object, ...] +_MISSING_ALIAS_VALUE = object() + def _preferred_request_name(raw: object) -> str: if not isinstance(raw, dict): @@ -318,28 +329,77 @@ def _local_ingress_routes_equivalent( ) -def _extract_terminal_events(raw: dict[object, object]) -> tuple[str, ...]: - """Return all contract-declared terminal topics for local ingress waits.""" +def _terminal_event_topics_from_declaration(declaration: object) -> tuple[str, ...]: + """Normalize one ``terminal_events`` declaration into success-first topics. - terminal_events: list[str] = [] - terminal_event = _safe_optional_string(raw.get("terminal_event")) - if terminal_event is not None: - terminal_events.append(terminal_event) + Delegates to the shared reader in + :mod:`omnibase_infra.runtime.contract_terminal_events` (OMN-15468). The + normalization itself is unchanged; it moved so the def-B auto-wiring can ask + the SAME question about the SAME contract and cannot answer it differently + from the broker's subscription set. + """ - raw_terminal_events = raw.get("terminal_events") - if isinstance(raw_terminal_events, dict): - values: Iterable[object] = raw_terminal_events.values() - elif isinstance(raw_terminal_events, list | tuple): - values = raw_terminal_events - else: - values = () + return terminal_event_topics_from_declaration(declaration) - for value in values: - topic = _safe_optional_string(value) - if topic is not None: - terminal_events.append(topic) - return tuple(dict.fromkeys(terminal_events)) +def _extract_terminal_events(raw: dict[object, object]) -> tuple[str, ...]: + """Return all contract-declared terminal topics for local ingress waits. + + Reads three declaration sites, in success-first order: + + 1. top-level ``terminal_event`` (single success topic), + 2. top-level ``terminal_events`` (mapping or sequence), + 3. ``runtime_dispatch.terminal_events`` (OMN-15468). + + Site 3 is the address external clients — the dashboard included — dispatch + through, and it is where **51** contracts declare their FAILURE terminal and + nowhere else. It was previously unread, so those routes reached the Pattern B + broker carrying only their success topic even though the broker is built to + race every declared terminal concurrently (OMN-13118/13128). The + consequences were both wrong and indistinguishable from each other: a node + that correctly published its failure terminal either timed out (the broker + was not subscribed to the topic the terminal landed on) or was reported as + ``completed`` (the def-B wiring republishes the returned model onto the + contract's success ``terminal_event`` irrespective of the payload verdict). + + Of those 51, **30** declare no top-level ``terminal_event`` *or* + ``terminal_events``, so this function returned an EMPTY tuple for them and + the broker rejected the command outright with "does not declare terminal + events"; the other 21 kept a working success topic and lost only the failure + one. **17** of the 30 also clear the route-discovery filter in + ``discover_runtime_local_ingress_routes`` above (a mapping ``event_bus`` + whose ``subscribe_topics`` yield a ``_select_command_topic``), so 17 were + live, undispatchable ``/skill`` routes; the remaining 13 are latent + declarations discovery never reaches. + + PROVENANCE — every number above is a measurement, not a constant. Framing: + the RAW corpus of 384 ``src/omnimarket/nodes/*/contract.yaml`` files at + ``omnimarket@aea0c33dd89fb82fdca33aac7149992a21c46d43`` (``origin/dev``), + measured 2026-07-30, no discovery filter applied except where "17" says so. + Re-derive rather than copy forward: 51 = contracts whose + ``runtime_dispatch.terminal_events`` normalizes non-empty (all 51 carry an + explicit ``failure`` key and at least one topic absent from the top level); + 30 = those 51 whose top-level ``terminal_event`` and ``terminal_events`` are + both empty; 17 = those 30 for which ``_select_command_topic`` returns a + topic. These drift as contracts land. An earlier revision of this docstring + asserted an unsourced "24 of the 51", which reproduces under no framing — + the defect was the missing provenance, not only the wrong digit. + + Live reproduction that motivated this (.201 dev lane, 2026-07-30): correlation + ``4a5e0730-0000-4000-8000-000000000002`` returned outer ``ok=true`` / + ``status=completed`` while the payload it carried held + ``contract_passed=false`` with empty ``contract_yaml``/``handler_source``, and + two correct failure terminals sat unread on + ``onex.evt.omnimarket.node-generation-failed.v1``. + + Reader body lifted to + :func:`omnibase_infra.runtime.contract_terminal_events.extract_terminal_event_topics` + (OMN-15468 slice 2) so route discovery and the def-B publish seam read the + contract through ONE function. This wrapper is the discovery-side name and + stays as the route builder's call site. + """ + + return extract_terminal_event_topics(raw) def _package_scoped_route_aliases( @@ -458,15 +518,283 @@ def _handler_event_type( def validate_runtime_local_ingress_payload( route: ModelRuntimeLocalIngressRoute, payload: dict[str, JsonType], + *, + correlation_id: UUID, ) -> dict[str, JsonType]: - """Validate and JSON-normalize an ingress payload against its route contract.""" + """Validate and JSON-normalize a payload under ingress correlation authority. + + The outer local-ingress request owns the correlation identifier. When the + route's typed input model declares ``correlation_id``, stamp that authority + before validation so a model default factory cannot mint a second workflow + identity. A caller-supplied value is accepted only when it is the same UUID. + """ model_cls = _load_route_input_model(route) if model_cls is None: return payload - model = model_cls.model_validate(payload) - return cast("dict[str, JsonType]", model.model_dump(mode="json", exclude_none=True)) + if "correlation_id" not in model_cls.model_fields: + model = model_cls.model_validate(payload) + return cast( + "dict[str, JsonType]", model.model_dump(mode="json", exclude_none=True) + ) + + correlation_field = model_cls.model_fields["correlation_id"] + validation_alias_paths = _validation_alias_paths(correlation_field.validation_alias) + correlation_alias_path_candidates: list[_RuntimeIngressAliasPath] = [ + ("correlation_id",) + ] + if isinstance(correlation_field.alias, str): + correlation_alias_path_candidates.append((correlation_field.alias,)) + correlation_alias_path_candidates.extend(validation_alias_paths) + correlation_alias_paths = tuple(dict.fromkeys(correlation_alias_path_candidates)) + + for alias_path in correlation_alias_paths: + raw_correlation_id = _read_alias_path(payload, alias_path) + if raw_correlation_id is _MISSING_ALIAS_VALUE: + continue + try: + payload_correlation_id = UUID(str(raw_correlation_id)) + except ValueError as exc: + raise ValueError( + "Local ingress payload correlation_id must be a valid UUID" + ) from exc + if payload_correlation_id != correlation_id: + raise ValueError( + "Local ingress payload correlation_id conflicts with the " + "authoritative request correlation_id" + ) + + declares_uuid = _annotation_contains_uuid(correlation_field.annotation) + authoritative_correlation_id: object = ( + correlation_id if declares_uuid else str(correlation_id) + ) + validate_by_alias = model_cls.model_config.get("validate_by_alias") is not False + injection_path = ( + validation_alias_paths[0] + if validate_by_alias and validation_alias_paths + else ("correlation_id",) + ) + authoritative_payload = cast("dict[str, object]", deepcopy(payload)) + + removable_alias_paths: list[_RuntimeIngressAliasPath] = [] + for alias_path in correlation_alias_paths: + if alias_path == injection_path: + continue + if ( + len(alias_path) > 1 + and len(injection_path) > 1 + and alias_path[0] == injection_path[0] + ): + if _read_alias_path(authoritative_payload, alias_path) is not ( + _MISSING_ALIAS_VALUE + ): + _write_alias_path( + authoritative_payload, + alias_path, + authoritative_correlation_id, + ) + continue + removable_alias_paths.append(alias_path) + _remove_alias_paths(authoritative_payload, tuple(removable_alias_paths)) + _write_alias_path( + authoritative_payload, + injection_path, + authoritative_correlation_id, + ) + + model = model_cls.model_validate(authoritative_payload) + normalized_payload = cast( + "dict[str, JsonType]", + model.model_dump(mode="json", exclude_none=True, by_alias=False), + ) + try: + validated_correlation_id = UUID(str(normalized_payload.get("correlation_id"))) + except ValueError as exc: + raise ValueError( + "Validated local ingress correlation_id must be a valid UUID" + ) from exc + if validated_correlation_id != correlation_id: + raise ValueError( + "Validated local ingress correlation_id conflicts with the " + "authoritative request correlation_id" + ) + return normalized_payload + + +def _annotation_contains_uuid(annotation: object) -> bool: + """Return whether an annotation accepts ``UUID`` as a top-level value.""" + + if annotation is UUID: + return True + origin = get_origin(annotation) + if origin is typing.Annotated: + annotation_args = get_args(annotation) + return bool(annotation_args) and _annotation_contains_uuid(annotation_args[0]) + if origin in (typing.Union, UnionType): + return any(_annotation_contains_uuid(arg) for arg in get_args(annotation)) + return False + + +def _validation_alias_paths(alias: object) -> tuple[_RuntimeIngressAliasPath, ...]: + """Return the concrete input paths represented by one Pydantic alias.""" + + if alias is None: + return () + if isinstance(alias, str): + return ((alias,),) + if isinstance(alias, AliasPath): + return (tuple(alias.path),) + if isinstance(alias, AliasChoices): + return tuple(tuple(path) for path in alias.convert_to_aliases()) + raise TypeError( + "Local ingress correlation_id declares an unsupported validation alias" + ) + + +def _read_alias_path( + payload: object, + alias_path: _RuntimeIngressAliasPath, +) -> object: + """Read an alias path without treating a present ``None`` as missing.""" + + current = payload + for segment in alias_path: + if isinstance(segment, str): + if not isinstance(current, dict) or segment not in current: + return _MISSING_ALIAS_VALUE + current = current[segment] + continue + if not isinstance(segment, int): + raise TypeError( + "Local ingress correlation alias segments must be str or int" + ) + if not isinstance(current, list): + return _MISSING_ALIAS_VALUE + try: + current = current[segment] + except IndexError: + return _MISSING_ALIAS_VALUE + return current + + +def _remove_alias_paths( + payload: object, + alias_paths: tuple[_RuntimeIngressAliasPath, ...], +) -> None: + """Remove alternate correlation aliases while preserving sibling payload data.""" + + grouped: dict[object, list[_RuntimeIngressAliasPath]] = {} + for alias_path in alias_paths: + if alias_path: + grouped.setdefault(alias_path[0], []).append(alias_path[1:]) + + if isinstance(payload, dict): + for segment, tails in grouped.items(): + if not isinstance(segment, str) or segment not in payload: + continue + if any(not tail for tail in tails): + payload.pop(segment) + continue + child = payload[segment] + _remove_alias_paths(child, tuple(tail for tail in tails if tail)) + if isinstance(child, (dict, list)) and not child: + payload.pop(segment) + return + + if not isinstance(payload, list): + return + indexed_tails: dict[int, list[_RuntimeIngressAliasPath]] = {} + for segment, tails in grouped.items(): + if not isinstance(segment, int): + continue + index = segment if segment >= 0 else len(payload) + segment + if 0 <= index < len(payload): + indexed_tails.setdefault(index, []).extend(tails) + for index in sorted(indexed_tails, reverse=True): + tails = indexed_tails[index] + if any(not tail for tail in tails): + payload.pop(index) + continue + child = payload[index] + _remove_alias_paths(child, tuple(tail for tail in tails if tail)) + if isinstance(child, (dict, list)) and not child: + payload.pop(index) + + +def _write_alias_path( + payload: dict[str, object], + alias_path: _RuntimeIngressAliasPath, + value: object, +) -> None: + """Write the authoritative value through a Pydantic validation alias path.""" + + if not alias_path or not isinstance(alias_path[0], str): + raise TypeError( + "Local ingress correlation_id validation alias must start with a string" + ) + + current: object = payload + for position, segment in enumerate(alias_path): + is_leaf = position == len(alias_path) - 1 + if isinstance(segment, str): + if not isinstance(current, dict): + raise ValueError( + "Local ingress payload correlation_id alias path conflicts " + "with the payload structure" + ) + if is_leaf: + current[segment] = value + return + next_segment = alias_path[position + 1] + if segment not in current: + current[segment] = [] if isinstance(next_segment, int) else {} + child = current[segment] + expected_type = list if isinstance(next_segment, int) else dict + if not isinstance(child, expected_type): + raise ValueError( + "Local ingress payload correlation_id alias path conflicts " + "with the payload structure" + ) + current = child + continue + + if not isinstance(segment, int): + raise TypeError( + "Local ingress correlation alias segments must be str or int" + ) + if not isinstance(current, list): + raise ValueError( + "Local ingress payload correlation_id alias path conflicts " + "with the payload structure" + ) + if segment < 0: + missing_slots = max(0, -segment - len(current)) + if missing_slots: + current[:0] = [None] * missing_slots + index = len(current) + segment + else: + missing_slots = max(0, segment + 1 - len(current)) + if missing_slots: + current.extend([None] * missing_slots) + index = segment + if is_leaf: + current[index] = value + return + next_segment = alias_path[position + 1] + child = current[index] + expected_type = list if isinstance(next_segment, int) else dict + if child is None: + child = [] if expected_type is list else {} + current[index] = child + if not isinstance(child, expected_type): + raise ValueError( + "Local ingress payload correlation_id alias path conflicts " + "with the payload structure" + ) + current = child + + raise AssertionError("unreachable correlation_id alias path write") def _load_route_input_model( diff --git a/src/omnibase_infra/runtime/service_delegation_dispatch_port.py b/src/omnibase_infra/runtime/service_delegation_dispatch_port.py index 7f12221570..24b586bd2f 100644 --- a/src/omnibase_infra/runtime/service_delegation_dispatch_port.py +++ b/src/omnibase_infra/runtime/service_delegation_dispatch_port.py @@ -102,6 +102,67 @@ def _select_delegation_route( ) +def _resolve_delegation_provenance(normalized: Mapping[str, object]) -> str: + """Resolve where the delegation actually ran, from the terminal's own fields. + + OMN-15471: this used to be ``normalized.get("provider", "local")``. No real + ``delegation-completed.v1`` payload carries a ``provider`` key — that event + models the resolved serving endpoint as ``endpoint_url`` plus + ``cost_tier_name`` — so the literal default fired on EVERY bus-path + delegation and stamped ``provider="local"`` on the durable terminal. A + Gemini-routed result (``endpoint_url`` = the Google Generative Language API, + ``cost_tier_name`` = ``cheap_cloud``) was recorded as a local-provider run: + 39/39 ``delegate-skill-completed.v1`` rows read ``local`` on the onex-dev + lane and not one of them ran on a local model. + + Provenance is therefore derived ONLY from facts the terminal payload + actually carries, in descending order of how directly they identify the + serving endpoint: + + 1. ``provider`` — an explicit upstream stamp, if a producer ever sets one. + 2. ``endpoint_url`` — the host that was really called. This is the strongest + available provenance fact: it cannot read as local for a cloud call, and + for a genuinely local backend it is the private/loopback address, so the + local case stays identifiable. + 3. ``cost_tier_name`` / ``cost_tier_type`` — the resolved routing tier, used + only when no endpoint identity survived into the terminal. + + When none of those resolve, the return is the empty string. That is + deliberate: an absent provenance must stay absent so the consumer + (``handler_delegate_skill._response_from_result``, which reads + ``delegated_to or endpoint_url or ""``) falls through its own chain instead + of inheriting a fabricated deployment class. Never invent one here. + """ + + for key in ("provider", "endpoint_url", "cost_tier_name", "cost_tier_type"): + value = normalized.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def _resolve_measured_actual_cost(normalized: Mapping[str, object]) -> float | None: + """Resolve non-negative measured spend from the canonical terminal fields. + + The cumulative total is authoritative for a current terminal, while the + final-attempt value keeps older/defaulted terminals compatible. Taking the + maximum also enforces the domain invariant that total spend cannot be less + than the final attempt, without turning a genuine free-local ``0/0`` into an + absent measurement. + """ + + costs: list[float] = [] + for key in ("cumulative_attempt_cost", "final_attempt_cost"): + value = normalized.get(key) + if ( + isinstance(value, int | float) + and not isinstance(value, bool) + and value >= 0.0 + ): + costs.append(float(value)) + return max(costs) if costs else None + + def _normalize_result_payload( *, status: str, @@ -125,13 +186,22 @@ def _normalize_result_payload( if error_message: normalized["error_message"] = error_message normalized.setdefault("model_name", normalized.get("model_used", "")) - normalized.setdefault("delegated_to", normalized.get("provider", "local")) + # OMN-15471: derive real provenance; never default to the literal "local". + normalized.setdefault("delegated_to", _resolve_delegation_provenance(normalized)) normalized.setdefault( "quality_gate_passed", normalized.get("quality_passed", False) ) normalized.setdefault("input_tokens", normalized.get("prompt_tokens", 0)) normalized.setdefault("output_tokens", normalized.get("completion_tokens", 0)) normalized.setdefault("delegation_latency_ms", normalized.get("latency_ms", 0)) + # OMN-15520: the workflow terminal owns measured actual cost. Total cost + # across an escalation ladder is cumulative; single-attempt/legacy + # terminals expose only the final attempt. Preserve an explicit zero by + # checking for None rather than truthiness, and overwrite any stale + # consumer-shaped ``cost_usd`` with the upstream measurement when present. + actual_cost = _resolve_measured_actual_cost(normalized) + if actual_cost is not None: + normalized["cost_usd"] = actual_cost return normalized @@ -169,7 +239,7 @@ async def dispatch( prompt: str, task_type: str, correlation_id: UUID, - max_tokens: int, + max_tokens: int | None, source_file_path: str | None, source_session_id: str | None, wait: bool, @@ -177,9 +247,41 @@ async def dispatch( quality_contract_mode: str = "extend_task_class", acceptance_criteria: tuple[str, ...] = (), tenant_id: str | None = None, + backend_id: str | None = None, + response_contract: dict[str, object] | None = None, + system_prompt: str | None = None, + temperature: float | None = None, + response_format: dict[str, object] | None = None, ) -> dict[str, object]: """Dispatch a delegation request and return the terminal result payload.""" + # OmniMarket's consumer-facing handler always supplies these optional + # arguments. The deployed bus model does not expose the completion-shaping + # fields yet, so None preserves the existing route while explicit requests + # fail closed instead of being silently dropped at this boundary. + for feature_name, feature_value in ( + ("system_prompt", system_prompt), + ("temperature", temperature), + ("response_format", response_format), + ): + if feature_value is not None: + raise NotImplementedError( + f"{feature_name} is not yet supported on the deployed bus " + "dispatch path (RuntimeDelegationDispatchPort); threading it " + "requires the canonical delegation request wire to carry it " + "end to end (OMN-15482)" + ) + if backend_id is not None: + raise NotImplementedError( + "backend_id pin is not yet supported on the deployed bus " + "dispatch path (RuntimeDelegationDispatchPort)" + ) + if response_contract is not None: + raise NotImplementedError( + "response_contract is not yet supported on the deployed bus " + "dispatch path (RuntimeDelegationDispatchPort)" + ) + routes = self._resolved_routes() selected = _select_delegation_route(routes) request_payload: dict[str, object] = { diff --git a/src/omnibase_infra/runtime/service_dispatch_result_applier.py b/src/omnibase_infra/runtime/service_dispatch_result_applier.py index ab70a58c70..3d03af3166 100644 --- a/src/omnibase_infra/runtime/service_dispatch_result_applier.py +++ b/src/omnibase_infra/runtime/service_dispatch_result_applier.py @@ -60,6 +60,9 @@ from omnibase_infra.models.errors.model_infra_error_context import ( ModelInfraErrorContext, ) +from omnibase_infra.runtime.contract_terminal_events import ( + apply_failure_terminal_guard, +) from omnibase_infra.topics import topic_keys from omnibase_infra.topics.service_topic_registry import ServiceTopicRegistry from omnibase_infra.utils import derive_event_type_from_topic, sanitize_error_message @@ -180,6 +183,7 @@ def __init__( output_event_handler: Callable[[BaseModel], Awaitable[BaseModel | None]] | None = None, allowed_output_topics: Iterable[str] | None = None, + failure_terminal_topics: Iterable[str] | None = None, ) -> None: """Initialize the dispatch result applier. @@ -214,6 +218,13 @@ class name is not in the map fall back to ``output_topic``. topics. This should include ``event_bus.publish_topics`` so per-instance embedded topics can select any declared terminal topic, even when multiple terminal outcomes share one event model. + failure_terminal_topics: Optional contract-declared FAILURE terminal + topics (``runtime_dispatch.terminal_events.failure`` and the + non-success entries of a top-level ``terminal_events`` map). + When exactly one is declared, a returned model that resolves to + the contract's SUCCESS terminal but declares a failure verdict + is re-routed there instead (OMN-15468 AC2). Empty/absent ⇒ the + guard cannot fire and routing is byte-for-byte unchanged. """ self._event_bus = event_bus self._output_topic = output_topic @@ -228,6 +239,15 @@ class name is not in the map fall back to ``output_topic``. for topic in (allowed_output_topics or ()) if isinstance(topic, str) and topic.strip() } + self._failure_terminal_topics: tuple[str, ...] = tuple( + dict.fromkeys( + topic.strip() + for topic in (failure_terminal_topics or ()) + if isinstance(topic, str) + and topic.strip() + and topic.strip() != output_topic + ) + ) @property def published_events_map(self) -> dict[str, str]: @@ -314,9 +334,17 @@ class name, and falls back to ``_output_topic``. The resolved topic string. """ embedded_topic = self._resolve_embedded_output_topic(event) - if embedded_topic is not None: - return embedded_topic - return self._resolve_mapped_output_topic(event) + resolved = ( + embedded_topic + if embedded_topic is not None + else self._resolve_mapped_output_topic(event) + ) + return apply_failure_terminal_guard( + event, + resolved, + success_topic=self._output_topic, + failure_terminal_topics=self._failure_terminal_topics, + ) def _resolve_embedded_output_topic(self, event: BaseModel) -> str | None: """Return a declared topic the event itself names, if present. @@ -382,6 +410,7 @@ def _allowed_output_topics(self) -> set[str]: *self._output_topic_map.values(), *self._topic_router.values(), *self._allowed_output_topic_set, + *self._failure_terminal_topics, } @staticmethod @@ -704,6 +733,19 @@ async def apply( self._resolve_mapped_output_topic(output_event), ) ) + # OMN-15468 AC2: last stop before publish — a return value + # that declares a failure verdict never leaves on the + # contract's SUCCESS terminal. Applied here rather than only + # inside _resolve_output_topic because this loop resolves + # the topic inline (topic_router is consulted here and + # nowhere else), so guarding only the helper would leave the + # path that actually publishes unguarded. + resolved_topic = apply_failure_terminal_guard( + output_event, + resolved_topic, + success_topic=self._output_topic, + failure_terminal_topics=self._failure_terminal_topics, + ) # OMN-12116: Derive event_type from the resolved topic so # multi-step FSM orchestrators can match the dispatcher diff --git a/src/omnibase_infra/runtime/service_kernel.py b/src/omnibase_infra/runtime/service_kernel.py index c654e5790d..da2f37e968 100644 --- a/src/omnibase_infra/runtime/service_kernel.py +++ b/src/omnibase_infra/runtime/service_kernel.py @@ -70,6 +70,12 @@ import yaml if TYPE_CHECKING: + from omnibase_core.models.core.model_deployment_topology import ( + ModelDeploymentTopology, + ) + from omnibase_infra.event_bus.model_runtime_attach_readiness import ( + ModelRuntimeAttachReadiness as ModelRuntimeAttachReadinessType, + ) from omnibase_infra.event_bus.model_topic_readiness_config import ( ModelTopicReadinessConfig, ) @@ -96,6 +102,7 @@ SchemaFingerprintMismatchError, SchemaFingerprintMissingError, ServiceResolutionError, + TopicReplicationPolicyError, ) # OMN-7077: EventBusInmemory is migrating to omnibase_core. @@ -157,7 +164,10 @@ RegistryDomainPlugin, ) from omnibase_infra.runtime.runtime_host_process import RuntimeHostProcess -from omnibase_infra.runtime.runtime_profile import load_runtime_profile +from omnibase_infra.runtime.runtime_profile import ( + load_runtime_profile, + resolve_secret_resolver_config_path, +) from omnibase_infra.runtime.util_container_wiring import ( wire_infrastructure_services, ) @@ -355,6 +365,32 @@ def _get_contracts_dir() -> Path: return Path(DEFAULT_CONTRACTS_DIR) +def _load_runtime_database_topology() -> ModelDeploymentTopology | None: + """Resolve only the explicit application-database topology profile. + + ``ONEX_ENVIRONMENT`` and ``KAFKA_ENVIRONMENT`` are event namespaces and are + intentionally not consulted here. An absent profile leaves non-database + runtimes unchanged; a runtime that owns a ``db_io`` contract is rejected + after discovery unless this function returned a checked-in topology. + """ + profile = os.environ.get("ONEX_DATABASE_TOPOLOGY_PROFILE") + if profile is None: + return None + + from omnibase_infra.topology import load_topology_profile + + return load_topology_profile(profile) + + +def _should_auto_create_missing_topics( + *, + validation_is_valid: bool, + universe_warm_enabled: bool, +) -> bool: + """Keep the universe-wide auto-create retry behind the universe-warm gate.""" + return universe_warm_enabled and not validation_is_valid + + def resolve_topic_readiness_config() -> ModelTopicReadinessConfig: """Resolve the per-contract boot-interleave readiness knobs (OMN-13237). @@ -417,8 +453,14 @@ def _int_env(name: str, default: int) -> int: def _build_runtime_handler_dependencies( postgres_pool: object | None, kafka_bootstrap_servers: str | None = None, + gateway_secret_resolver_config_path: Path | None = None, ) -> dict[str, dict[str, object]] | None: - """Build constructor dependencies for runtime-owned handlers.""" + """Build constructor dependencies for runtime-owned handlers. + + Gateway session handlers must share one session store and one resolver. + The resolver is built from the deploy-rendered, typed configuration artifact; + no handler may infer secret names or read secret values directly. + """ dependencies: dict[str, dict[str, object]] = {} if postgres_pool is not None: dependencies.update( @@ -447,6 +489,63 @@ def _build_runtime_handler_dependencies( "quarantine_producer": DLQQuarantineProducer(dlq_replay_config), } + if gateway_secret_resolver_config_path: + from omnibase_infra.nodes.node_gateway_attach_effect.models.model_gateway_attach_config import ( + ModelGatewayAttachConfig, + ) + from omnibase_infra.nodes.node_gateway_attach_effect.services.store_gateway_session_memory import ( + StoreGatewaySessionMemory, + ) + from omnibase_infra.runtime.models.model_secret_resolver_config import ( + ModelSecretResolverConfig, + ) + from omnibase_infra.runtime.secret_resolver import SecretResolver + + config_path = gateway_secret_resolver_config_path + try: + raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + secret_resolver_config = ModelSecretResolverConfig.model_validate( + raw_config + ) + except (OSError, ValueError, yaml.YAMLError) as exc: + raise ProtocolConfigurationError( + "Gateway attach dependency wiring requires a valid rendered " + f"secret-resolver config at {config_path}" + ) from exc + + gateway_config = ModelGatewayAttachConfig() + required_gateway_refs = { + gateway_config.keycloak_issuer_ref, + gateway_config.keycloak_introspection_ref, + gateway_config.keycloak_jwks_ref, + f"{gateway_config.keycloak_admin_client_ref}.client_id", + f"{gateway_config.keycloak_admin_client_ref}.client_secret", + } + mapped_refs = { + mapping.logical_name for mapping in secret_resolver_config.mappings + } + missing_refs = sorted(required_gateway_refs - mapped_refs) + if missing_refs: + raise ProtocolConfigurationError( + "Gateway attach dependency wiring is missing explicit " + f"secret-resolver mappings: {', '.join(missing_refs)}" + ) + + session_store = StoreGatewaySessionMemory() + secret_resolver = SecretResolver(config=secret_resolver_config) + shared_dependencies = { + "config": gateway_config, + "session_store": session_store, + "secret_resolver": secret_resolver, + } + dependencies.update( + { + "HandlerGatewayAttach": dict(shared_dependencies), + "HandlerGatewayHeartbeat": dict(shared_dependencies), + "HandlerGatewayDetach": dict(shared_dependencies), + } + ) + if not dependencies: return None return dependencies @@ -857,6 +956,7 @@ async def bootstrap() -> int: llm_health_service: ServiceLlmEndpointHealth | None = None wiring_health_checker: WiringHealthChecker | None = None wiring_health_task: asyncio.Task[None] | None = None + not_ready_reconciliation_task: asyncio.Task[None] | None = None triage_unsub: Callable[[], Awaitable[None]] | None = None build_loop_db_handler = None # HandlerDb | None, assigned inside try block baselines_task: asyncio.Task[None] | None = None @@ -970,6 +1070,7 @@ async def bootstrap() -> int: # Pass correlation_id for consistent tracing across initialization sequence config_start_time = time.time() config = load_runtime_config(contracts_dir, correlation_id=correlation_id) + deployment_topology = _load_runtime_database_topology() config_duration = time.time() - config_start_time # Log only safe config fields (no credentials or sensitive data) # Full config.model_dump() could leak passwords, API keys, connection strings @@ -1231,6 +1332,9 @@ async def _cb_transition_handler( # per-contract confirm carries all owned consumers (W2 evidence). The # provisioner instance is reused by the Phase B interleave. topic_provisioner: object | None = None + _universe_warm_enabled = ( + os.environ.get("ONEX_BOOT_UNIVERSE_PROVISION", "1") != "0" + ) if use_kafka: _contracts_root = _get_contracts_dir() _skill_manifests_root: Path | None = None @@ -1275,9 +1379,6 @@ async def _cb_transition_handler( ) # OMN-13237: universe warm is best-effort and demoted; the # per-contract confirm (Phase B) gates consumer attach. - _universe_warm_enabled = ( - os.environ.get("ONEX_BOOT_UNIVERSE_PROVISION", "1") != "0" - ) if not _universe_warm_enabled: logger.info( "Topic provisioning: universe warm DISABLED " @@ -1308,6 +1409,11 @@ async def _cb_transition_handler( provisioning_result["failed"] or "none", correlation_id, ) + except TopicReplicationPolicyError: + # OMN-15395: a durability-policy violation is fail-closed and + # must escape this best-effort boundary — the whole point of the + # distinct error class is that it is not degradable to a warning. + raise except Exception: # noqa: BLE001 — boundary: logs warning and degrades logger.warning( "Topic provisioning failed (best-effort, non-blocking) " @@ -1336,7 +1442,10 @@ async def _cb_transition_handler( correlation_id=correlation_id, log_missing=False, ) - if not validation_result.is_valid: + if _should_auto_create_missing_topics( + validation_is_valid=validation_result.is_valid, + universe_warm_enabled=_universe_warm_enabled, + ): # OMN-7810: Auto-create missing topics before failing strict # validation. This handles topics that were added to the # provisioning registry but not yet created on the broker @@ -1375,6 +1484,9 @@ async def _cb_transition_handler( "(correlation_id=%s)", correlation_id, ) + except TopicReplicationPolicyError: + # OMN-15395: fail-closed past the best-effort boundary. + raise except Exception: # noqa: BLE001 logger.warning( "Auto-create missing topics failed (best-effort) " @@ -1383,17 +1495,17 @@ async def _cb_transition_handler( exc_info=True, ) - if strict_topic_validation: - raise RuntimeError( - f"Missing topics: {validation_result.missing_topics}" - ) - if not validation_result.is_valid: - logger.warning( - "Topic validation: %d missing (non-blocking) " - "(correlation_id=%s)", - len(validation_result.missing_topics), - correlation_id, - ) + if strict_topic_validation and not validation_result.is_valid: + raise RuntimeError( + f"Missing topics: {validation_result.missing_topics}" + ) + if not validation_result.is_valid: + logger.warning( + "Topic validation: %d missing (non-blocking) " + "(correlation_id=%s)", + len(validation_result.missing_topics), + correlation_id, + ) except RuntimeError: raise except Exception: # noqa: BLE001 — boundary: logs warning and degrades @@ -1634,6 +1746,12 @@ async def _baselines_loop() -> None: _savings_topic = _savings_config.produce_topic _savings_input_topics = list(_savings_config.consumed_topics) + _savings_node_identity = ModelNodeIdentity( + env=environment, + service=config.name or "onex-kernel", + node_name="savings-estimator", + version="v1", + ) async def _savings_consumer_loop() -> None: """Consume input events and produce savings estimates.""" @@ -1655,8 +1773,8 @@ async def _savings_on_message( await event_bus.subscribe( _input_topic, + node_identity=_savings_node_identity, on_message=_savings_on_message, - group_id=f"savings-estimator.{_input_topic}", ) except Exception: # noqa: BLE001 logger.warning( @@ -2463,6 +2581,11 @@ async def _savings_on_message( auto_wiring_manifest_for_subscriptions = None auto_wiring_manifest_discovered = None # OMN-11198: full discovery result lifecycle_executor = None + # OMN-15512: boot attach-readiness aggregate, hoisted to bootstrap scope + # so step 9.8 can fold it onto the runtime-manifest snapshot. Stays None + # when the per-contract interleave never ran, which is distinct from + # "ran and everything attached" (that carries a READY aggregate). + attach_readiness: ModelRuntimeAttachReadinessType | None = None try: from omnibase_infra.runtime.auto_wiring import ( LifecycleHookExecutor, @@ -2583,6 +2706,18 @@ async def _savings_on_message( errors=manifest.errors, ) auto_wiring_manifest_for_subscriptions = filtered_manifest + db_io_contracts = tuple( + contract.name + for contract in filtered_manifest.contracts + if contract.db_io is not None and contract.db_io.db_tables + ) + if db_io_contracts and deployment_topology is None: + raise RuntimeHostError( + "Auto-wiring discovered db_io contracts but " + "ONEX_DATABASE_TOPOLOGY_PROFILE is not set; an explicit " + "checked-in database topology profile is required for: " + f"{sorted(db_io_contracts)}" + ) # OMN-12409: Wire result appliers for all manifest contracts that # declare published_events but are not yet in auto_wiring_result_appliers. @@ -2783,9 +2918,17 @@ async def _savings_on_message( correlation_id, ) + gateway_secret_resolver_config_path_raw = ( + resolve_secret_resolver_config_path() + ) runtime_handler_dependencies = _build_runtime_handler_dependencies( registration_service.postgres_pool, kafka_bootstrap_servers if use_kafka else None, + gateway_secret_resolver_config_path=( + Path(gateway_secret_resolver_config_path_raw) + if gateway_secret_resolver_config_path_raw + else None + ), ) # 5. Wire handlers into dispatch engine @@ -2798,6 +2941,7 @@ async def _savings_on_message( subscribe_immediately=False, result_appliers_by_contract=auto_wiring_result_appliers, materialized_explicit_dependencies=(runtime_handler_dependencies), + topology=deployment_topology, ) auto_wiring_duration = time.time() - auto_wiring_start @@ -2891,6 +3035,21 @@ async def _savings_on_message( port=http_port, version=KERNEL_VERSION, ) + # OMN-15217: publish the runtime health monitor's verdict on /health. + # The monitor is started above (step 3) and owns the semantic view of + # runtime health — contract discovery errors, consumer-group coverage, + # topic coverage. Without this line that verdict never leaves the + # container logs and /health reports a DEGRADED runtime as healthy. + # When the monitor did not start (non-Kafka profiles, startup failure) + # the provider is left unset and the payload carries a null verdict. + if runtime_health_monitor is not None: + health_server.set_runtime_health_provider( + lambda: ( + runtime_health_monitor.latest_event + if runtime_health_monitor is not None + else None + ) + ) health_start_time = time.time() await health_server.start() health_start_duration = time.time() - health_start_time @@ -3087,6 +3246,9 @@ async def _savings_on_message( runtime_node_graph_config=node_graph_config, # OMN-10587: Wire prefetch policy from runtime profile. prefetch_policy=kernel_profile.prefetch_policy, + # OMN-15418: Thread the same checked-in topology used by cold boot + # into post-freeze Kafka contract materialization. + deployment_topology=deployment_topology, ) runtime_create_duration = time.time() - runtime_create_start_time logger.debug( @@ -3123,6 +3285,36 @@ async def _savings_on_message( else auto_wiring_manifest_discovered ) if _introspection_manifest is not None: + # OMN-10856: bind runtime profile + image/deployment SHA identity + # onto the served manifest so a reported topology can be tied to + # a specific deployed build. contracts/errors are carried through + # unchanged (single source — see bind_introspection_manifest_identity). + # Env reads happen HERE, not in the auto_wiring package, because + # this file is the approved env-read boundary + # (scripts/check-env-reads.sh). ONEX_IMAGE_DIGEST reuses the same + # var already read below at publish_runtime_manifest(image_digest=...) + # (OMN-11196/OMN-11197) rather than inventing a second name for + # the same concept. + from omnibase_infra.runtime.auto_wiring.introspection_manifest_identity import ( + ENV_VAR_DEPLOYMENT_SHA, + ENV_VAR_IMAGE_SHA, + bind_introspection_manifest_identity, + ) + from omnibase_infra.runtime.auto_wiring.models.model_runtime_build_sha import ( + ModelRuntimeBuildSha, + ) + + _introspection_manifest = bind_introspection_manifest_identity( + _introspection_manifest, + runtime_profile=kernel_profile.name, + image_sha=ModelRuntimeBuildSha.from_raw( + os.environ.get(ENV_VAR_IMAGE_SHA), source_name=ENV_VAR_IMAGE_SHA + ), + deployment_sha=ModelRuntimeBuildSha.from_raw( + os.environ.get(ENV_VAR_DEPLOYMENT_SHA), + source_name=ENV_VAR_DEPLOYMENT_SHA, + ), + ) health_server.attach_manifest(_introspection_manifest) # OMN-13768: the long-running Kafka subscription work (per-contract @@ -3142,6 +3334,9 @@ async def _savings_on_message( # process. The aggregate tri-state is logged for operator visibility. from typing import cast as _cast + from omnibase_infra.event_bus.enum_contract_attach_status import ( + EnumContractAttachStatus, + ) from omnibase_infra.event_bus.model_contract_attach_result import ( ModelContractAttachResult, ) @@ -3196,6 +3391,16 @@ async def _savings_on_message( _attach_readiness = ModelRuntimeAttachReadiness.from_results( tuple(_attach_results) ) + # OMN-15512: hand the aggregate to the enclosing bootstrap scope so + # step 9.8 folds it onto the runtime-manifest snapshot. Before this + # it died at the logger.info below, so the only way to read the + # NOT-READY blocker set was `docker logs | grep NOT-READY` — which + # is literally how OMN-15508 had to be diagnosed. + attach_readiness = _attach_readiness + # Counts also go onto the EXISTING /health/detailed components map. + # No new endpoint and no new producer: the authoritative, queryable + # copy is the runtime_manifests projection, not this endpoint. + health_server.attach_readiness(_attach_readiness) logger.info( "Per-contract boot interleave: state=%s attached=%d/%d " "(OMN-13237) (correlation_id=%s)", @@ -3262,6 +3467,63 @@ async def _savings_on_message( correlation_id, ) + # OMN-15215: the boot interleave above makes exactly ONE + # provision->confirm->attach attempt per contract; a contract left + # NOT_READY (transient broker topic-metadata-convergence race, + # OMN-13237) is otherwise skipped for the rest of the process + # lifetime — no consumer group is ever created for it. Schedule a + # bounded background retry so "runtime stays live" is actually + # recoverable instead of a permanent skip. + _not_ready_at_boot = tuple( + r + for r in _attach_results + if r.status is EnumContractAttachStatus.NOT_READY + ) + if _not_ready_at_boot: + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + run_not_ready_reconciliation_loop, + ) + + async def _reconcile_not_ready_contracts() -> None: + assert auto_wiring_manifest_for_subscriptions is not None + try: + await run_not_ready_reconciliation_loop( + auto_wiring_manifest_for_subscriptions, + _not_ready_at_boot, + dispatch_engine, + event_bus, + environment, + auto_wiring_result_appliers, + provisioner=_cast( + "ProtocolTopicProvisioner | None", + topic_provisioner, + ), + readiness_config=resolve_topic_readiness_config(), + core_runtime_topics=core_runtime_topics, + core_runtime_owners=core_runtime_owners, + ) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 — boundary: background reconciliation must never crash boot + logger.warning( + "NOT_READY reconciliation loop raised, giving up " + "for this boot (correlation_id=%s)", + correlation_id, + exc_info=True, + ) + + not_ready_reconciliation_task = asyncio.create_task( + _reconcile_not_ready_contracts(), + name="not-ready-contract-reconciliation", + ) + logger.info( + "NOT_READY reconciliation scheduled for %d contract(s): " + "%s (OMN-15215, correlation_id=%s)", + len(_not_ready_at_boot), + sorted(r.contract_name for r in _not_ready_at_boot), + correlation_id, + ) + # --- Pass 2: Start consumers for ready plugins only --- # ready_plugins is a subset of activated_plugins: only plugins that # completed wire_handlers() successfully. This prevents starting @@ -3270,7 +3532,17 @@ async def _savings_on_message( plugin_id = plugin.plugin_id try: consumer_result = await plugin.start_consumers(plugin_config) - if consumer_result and consumer_result.unsubscribe_callbacks: + if not consumer_result.success: + logger.warning( + "Plugin '%s' failed to start consumers: %s (correlation_id=%s)", + plugin_id, + consumer_result.get_error_message_or_default( + consumer_result.message or "unknown" + ), + correlation_id, + ) + continue + if consumer_result.unsubscribe_callbacks: plugin_unsubscribe_callbacks.extend( consumer_result.unsubscribe_callbacks ) @@ -3789,16 +4061,19 @@ async def _triage_on_message( # 9.8. Emit runtime manifest snapshot (OMN-11196). # Published once per startup after all phases complete. # Non-fatal: failures are logged and the kernel continues. + # + # OMN-15512: the snapshot now also carries the boot attach-readiness + # aggregate, so the NOT-READY blocker set (contract + the topics whose + # readiness confirm failed) lands in the runtime_manifests projection + # instead of only the log stream. Same event, same table, same row — + # no second producer. if ( auto_wiring_report is not None and auto_wiring_manifest_for_subscriptions is not None ): try: - from omnibase_core.models.events.model_event_envelope import ( - ModelEventEnvelope, - ) from omnibase_infra.runtime.manifest_builder import ( - build_runtime_manifest, + publish_runtime_manifest, ) from omnibase_infra.topics import SUFFIX_RUNTIME_MANIFEST_PUBLISHED @@ -3806,27 +4081,31 @@ async def _triage_on_message( SUFFIX_RUNTIME_MANIFEST_PUBLISHED, correlation_id=correlation_id, ) - _runtime_profile_for_manifest = os.getenv("RUNTIME_PROFILE", "main") - _image_digest = os.getenv("ONEX_IMAGE_DIGEST") - _runtime_manifest = build_runtime_manifest( + _published_manifest = await publish_runtime_manifest( + event_bus=event_bus, report=auto_wiring_report, manifest=auto_wiring_manifest_for_subscriptions, - runtime_profile=_runtime_profile_for_manifest, - image_digest=_image_digest, - ) - _manifest_envelope: ModelEventEnvelope[object] = ModelEventEnvelope( - payload=_runtime_manifest, - correlation_id=correlation_id, - event_type="runtime-manifest-published", - source_tool="service_kernel", - ) - await event_bus.publish_envelope( - envelope=_manifest_envelope, + runtime_profile=os.getenv("RUNTIME_PROFILE", "main"), topic=_manifest_topic, + correlation_id=correlation_id, + image_digest=os.getenv("ONEX_IMAGE_DIGEST"), + attach_readiness=attach_readiness, ) + _published_readiness = _published_manifest.attach_readiness logger.info( - "Runtime manifest published (topic=%s, correlation_id=%s)", + "Runtime manifest published (topic=%s, attach_state=%s, " + "not_ready_contracts=%d, correlation_id=%s)", _manifest_topic, + ( + _published_readiness.state.value + if _published_readiness is not None + else "unknown" + ), + ( + len(_published_readiness.results) + if _published_readiness is not None + else 0 + ), correlation_id, ) except ImportError: @@ -4003,6 +4282,19 @@ async def _triage_on_message( ) wiring_health_task = None + # Stop NOT_READY contract reconciliation loop (OMN-15215) + if not_ready_reconciliation_task is not None: + not_ready_reconciliation_task.cancel() + try: + await not_ready_reconciliation_task + except asyncio.CancelledError: + pass + logger.debug( + "NOT_READY reconciliation loop stopped (correlation_id=%s)", + correlation_id, + ) + not_ready_reconciliation_task = None + # Stop baselines batch compute loop and close its pool if baselines_task is not None: baselines_task.cancel() @@ -4260,6 +4552,14 @@ async def _triage_on_message( except (asyncio.CancelledError, Exception): # noqa: BLE001 pass + # Cleanup NOT_READY contract reconciliation loop (OMN-15215) + if not_ready_reconciliation_task is not None: + not_ready_reconciliation_task.cancel() + try: + await not_ready_reconciliation_task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + # Cleanup baselines batch compute loop and pool if baselines_task is not None: baselines_task.cancel() diff --git a/src/omnibase_infra/runtime/state_io/state_store_adapter.py b/src/omnibase_infra/runtime/state_io/state_store_adapter.py index dfec81dc73..17c15a0772 100644 --- a/src/omnibase_infra/runtime/state_io/state_store_adapter.py +++ b/src/omnibase_infra/runtime/state_io/state_store_adapter.py @@ -430,10 +430,10 @@ async def recover_stale_rows(self, ttl_seconds: int | None = None) -> int: class StateIoUnconfiguredError(RuntimeHostError): """Raised at wiring time when a contract declares state_io but its DSN env var is unset. - Unlike the optional db_io projection path (which logs and returns None - when its DSN is unset), state_io is a REQUIRED durability seam — a - contract that opts in without a working DSN is a startup-fatal - configuration error, not a degradable condition. + State IO is a REQUIRED durability seam: a contract that opts in without a + working DSN is a startup-fatal configuration error, not a degradable + condition. Projection db_io topology bindings use the same fail-closed + wiring-time configuration rule. """ diff --git a/src/omnibase_infra/services/contract_resolver/main.py b/src/omnibase_infra/services/contract_resolver/main.py index 8f9f31d1e0..6775672f19 100644 --- a/src/omnibase_infra/services/contract_resolver/main.py +++ b/src/omnibase_infra/services/contract_resolver/main.py @@ -11,7 +11,10 @@ # Create app with container (required) from omnibase_core.container import ModelONEXContainer container = ModelONEXContainer() - app = create_app(container=container, cors_origins=["http://localhost:3000"]) + app = create_app( + container=container, + cors_origins=["https://dashboard.example.invalid"], + ) # Run with uvicorn uvicorn.run(app, host="0.0.0.0", port=8091) @@ -116,7 +119,7 @@ def create_app( >>> from omnibase_infra.services.contract_resolver import create_app >>> from omnibase_core.container import ModelONEXContainer >>> container = ModelONEXContainer() - >>> app = create_app(container=container, cors_origins=["http://localhost:3000"]) + >>> app = create_app(container=container, cors_origins=["https://dashboard.example.invalid"]) >>> # Run with: uvicorn module:app --host 0.0.0.0 --port 8091 """ app = FastAPI( @@ -143,7 +146,7 @@ def create_app( "CORS_ORIGINS must be configured. " "Set the CORS_ORIGINS environment variable (comma-separated list of " "allowed origins) or pass cors_origins parameter to create_app(). " - "Example: CORS_ORIGINS=http://localhost:3000,https://dashboard.example.com", + "Example: CORS_ORIGINS=https://dashboard.example.invalid", context=context, ) origins = [o.strip() for o in env_origins.split(",") if o.strip()] @@ -204,7 +207,7 @@ async def root() -> dict[str, str]: # container = ModelONEXContainer() # app = create_app( # container=container, -# cors_origins=["http://localhost:3000"], +# cors_origins=["https://dashboard.example.invalid"], # ) # uvicorn.run(app, host="0.0.0.0", port=8091) app: FastAPI | None = None diff --git a/src/omnibase_infra/services/health_checker.py b/src/omnibase_infra/services/health_checker.py index 884f02fd8a..5d743a056f 100644 --- a/src/omnibase_infra/services/health_checker.py +++ b/src/omnibase_infra/services/health_checker.py @@ -83,6 +83,15 @@ ProtocolConfigurationError, RuntimeHostError, ) +from omnibase_infra.event_bus.enum_runtime_readiness_state import ( + EnumRuntimeReadinessState, +) +from omnibase_infra.runtime.health.runtime_health_block import ( + RUNTIME_HEALTH_DETAIL_KEY, + build_runtime_health_block, + fold_attach_readiness_into_status, + fold_runtime_verdict_into_status, +) from omnibase_infra.runtime.models.model_component_health import ModelComponentHealth from omnibase_infra.runtime.models.model_detailed_health_response import ( ModelDetailedHealthResponse, @@ -96,7 +105,15 @@ from omnibase_infra.utils.correlation import generate_correlation_id if TYPE_CHECKING: + from collections.abc import Callable + from omnibase_core.container import ModelONEXContainer + from omnibase_infra.event_bus.model_runtime_attach_readiness import ( + ModelRuntimeAttachReadiness, + ) + from omnibase_infra.models.health.model_runtime_health_check_event import ( + ModelRuntimeHealthCheckEvent, + ) from omnibase_infra.runtime.auto_wiring.models.model_auto_wiring_manifest import ( ModelAutoWiringManifest, ) @@ -145,6 +162,27 @@ def _get_port_from_env(default: int) -> int: return default +def _read_runtime_health_verdict( + provider: Callable[[], ModelRuntimeHealthCheckEvent | None] | None, +) -> ModelRuntimeHealthCheckEvent | None: + """Read the latest monitor verdict, never raising into a health probe. + + A best-effort verdict source must not be able to take liveness down with it: + a raising provider is reported as "no verdict", which consumers treat as + unknown (OMN-15217). + """ + if provider is None: + return None + try: + return provider() + except Exception: # noqa: BLE001 — boundary: verdict is best-effort + logger.warning( + "Runtime health verdict provider raised; reporting verdict as absent", + exc_info=True, + ) + return None + + class ServiceHealth: """Minimal HTTP server for liveness and readiness endpoints. @@ -311,6 +349,20 @@ def __init__( # OMN-11198: Manifest attached after startup for introspection endpoint self._manifest: ModelAutoWiringManifest | None = None + # OMN-15512: boot attach-readiness aggregate, attached by the kernel via + # attach_readiness(). Read ONLY to surface two counts on the EXISTING + # /health/detailed components map — the authoritative, queryable copy is + # the runtime_manifests projection. No new producer is created here. + self._attach_readiness: ModelRuntimeAttachReadiness | None = None + + # OMN-15217: provider for the ServiceRuntimeHealthMonitor verdict. Set by + # the kernel via set_runtime_health_provider(). Without it /health can + # only report process liveness, which is how a runtime logging + # status=DEGRADED served status=healthy for hours. + self._runtime_health_provider: ( + Callable[[], ModelRuntimeHealthCheckEvent | None] | None + ) = None + # OMN-519: Track last successful health check timestamps per component self._last_healthy_timestamps: dict[str, str] = {} # Track health phase transitions so frequent probes do not flood logs. @@ -437,6 +489,27 @@ def attach_manifest(self, manifest: ModelAutoWiringManifest) -> None: extra={"port": self._port, "host": self._host}, ) + def attach_readiness(self, readiness: ModelRuntimeAttachReadiness) -> None: + """Attach the boot attach-readiness aggregate (OMN-15512). + + Feeds the ``runtime_wiring`` entry on /health/detailed's existing + components map with ``not_ready_contract_count`` and + ``registered_handler_count``. Green liveness is provably not evidence + that consumers attached — on 2026-07-30 the dev lane served ``/health`` + 200 ``healthy:true`` while NOT-READY warnings were still firing — so + the counts sit next to the other components instead of staying + implicit. The topic-level detail is NOT served here; it is queried from + the ``runtime_manifests`` projection. + """ + self._attach_readiness = readiness + logger.info( + "ServiceHealth attached boot readiness (state=%s, attached=%d/%d)", + readiness.state.value, + readiness.attached_contracts, + readiness.required_contracts, + extra={"port": self._port, "host": self._host}, + ) + async def _try_attach_runtime_from_container(self) -> bool: """Resolve and attach runtime from the container when available.""" if self._runtime is not None or self._container is None: @@ -861,6 +934,24 @@ async def stop(self) -> None: correlation_id, ) + def set_runtime_health_provider( + self, + provider: Callable[[], ModelRuntimeHealthCheckEvent | None] | None, + ) -> None: + """Attach the runtime health monitor verdict source (OMN-15217). + + Args: + provider: Zero-arg callable returning the monitor's latest + ``ModelRuntimeHealthCheckEvent``, or ``None`` when no cycle has + completed. Pass ``None`` to detach. + + Notes: + The provider is called on every ``/health`` request, so it must be + cheap and non-blocking — it reads a cached event, it does not run a + health check. + """ + self._runtime_health_provider = provider + async def _handle_health(self, request: web.Request) -> web.Response: """Handle GET /health requests (liveness probe). @@ -987,6 +1078,11 @@ async def _handle_health(self, request: web.Request) -> web.Response: "is_running": False, "runtime_attached": False, "startup_phase": "runtime_pending", + # OMN-15217: the verdict key is always present in the + # payload — explicitly null while unknown — so a + # consumer can distinguish "no verdict yet" from + # "this build does not publish verdicts at all". + RUNTIME_HEALTH_DETAIL_KEY: None, }, ), ) @@ -1054,6 +1150,50 @@ async def _handle_health(self, request: web.Request) -> web.Response: status = "unhealthy" http_status = 503 + # OMN-15217: join the ServiceRuntimeHealthMonitor verdict (contract + # discovery, consumer-group coverage, topic coverage) onto this + # payload. RuntimeHostProcess.health_check() only sees process-local + # state, so without this fold a runtime with four contracts failing + # to load reports healthy=true/degraded=false — verified on the + # stability lane 2026-07-27T12:58Z while the monitor logged + # status=DEGRADED every five minutes. + # + # The HTTP status code is deliberately NOT folded. /health is also + # the liveness probe watched by autoheal, and a semantic degradation + # is typically restart-immune (a contract that fails to import will + # fail to import again), so flipping the code here would convert a + # visible degradation into a restart loop. The body carries the + # truth; strict container health is a per-lane opt-in via + # omnibase_infra.runtime.health.container_healthcheck. + verdict = _read_runtime_health_verdict(self._runtime_health_provider) + runtime_health_block = build_runtime_health_block(verdict) + status = fold_runtime_verdict_into_status( + status, verdict.status if verdict is not None else None + ) + # OMN-15642 (remediation): boot attach-readiness is DELIBERATELY NOT + # folded into this endpoint's gated `status`, unlike the runtime-verdict + # fold above. `/health` is the liveness probe that four real automated + # gates hard-fail on with no DEGRADED tolerance -- + # .github/workflows/reusable-runtime-boot.yml (`jq -e '.status == + # "healthy" ...' || exit 1`), scripts/deploy-agent/deploy_agent/executor.py + # (`_runtime_health_passed`), and the deploy-readiness checks in + # scripts/runtime_build/verify_stability_refresh.py and + # verify_dev_refresh.py. `ModelRuntimeAttachReadiness`'s own docstring + # (omnibase_infra.event_bus.model_runtime_attach_readiness) states "The + # readiness endpoint reports attach status ONLY -- it is not a source of + # truth for contract lifecycle", and OMN-13237 deliberately designed a + # NOT-READY/DEGRADED contract to be recorded and skipped, never fatal, + # never a restart/redeploy trigger (service_kernel.py's boot-walk + # comment: "it never recycles the process"). A single unprovisioned + # topic -- a documented live condition on onex-dev (OMN-15330) -- would + # therefore have turned a deliberately-non-fatal per-contract skip into a + # hard boot/deploy failure on every future boot. The aggregate is still + # fully visible without that risk: enriched_details["components"] + # ["runtime_wiring"] below (pre-existing, OMN-15512) and + # /health/detailed's own `status` (fold_attach_readiness_into_status, + # below in _handle_health_detailed -- that endpoint is not a probe + # target for any of the four consumers above). + self._log_health_transition( status=status, runtime_attached=runtime_attached, @@ -1063,9 +1203,16 @@ async def _handle_health(self, request: web.Request) -> web.Response: ) # OMN-519: Add component breakdown to health response details - components = self._build_component_health(health_details) + components = build_component_health( + health_details, + last_healthy_timestamps=self._last_healthy_timestamps, + attach_readiness=self._attach_readiness, + ) enriched_details = dict(health_details) - enriched_details["degraded"] = is_degraded + enriched_details["degraded"] = is_degraded or status == "degraded" + enriched_details[RUNTIME_HEALTH_DETAIL_KEY] = cast( + "JsonType", runtime_health_block + ) enriched_details["startup_in_progress"] = startup_in_progress enriched_details["runtime_attached"] = runtime_attached enriched_details["components"] = { @@ -1378,97 +1525,6 @@ async def _handle_skill(self, request: web.Request) -> web.Response: content_type="application/json", ) - def _build_component_health( - self, - health_details: dict[str, object], - ) -> dict[str, ModelComponentHealth]: - """Build per-component health status from runtime health details. - - Extracts component-level health information from the runtime's - health_check() response and constructs typed ModelComponentHealth - instances for each component. - - OMN-519: Component-level health diagnostics. - - Args: - health_details: The raw health check dict from RuntimeHostProcess. - - Returns: - Dictionary mapping component name to ModelComponentHealth. - """ - now = datetime.now(tz=UTC).isoformat() - components: dict[str, ModelComponentHealth] = {} - - # Event bus health - event_bus_healthy = bool(health_details.get("event_bus_healthy", False)) - event_bus_data = health_details.get("event_bus", {}) - if event_bus_healthy: - self._last_healthy_timestamps["event_bus"] = now - event_bus_details: dict[str, JsonType] | None = None - if isinstance(event_bus_data, dict): - event_bus_details = cast("dict[str, JsonType]", event_bus_data) - if event_bus_healthy: - components["event_bus"] = ModelComponentHealth.healthy( - name="event_bus", - last_healthy=self._last_healthy_timestamps.get("event_bus"), - details=event_bus_details, - ) - else: - error_msg = "" - if isinstance(event_bus_data, dict): - error_msg = str(event_bus_data.get("error", "unhealthy")) - else: - error_msg = "unhealthy" - components["event_bus"] = ModelComponentHealth.unhealthy( - name="event_bus", - error=error_msg, - last_healthy=self._last_healthy_timestamps.get("event_bus"), - details=event_bus_details, - ) - - # Per-handler health - handlers_data = health_details.get("handlers", {}) - if isinstance(handlers_data, dict): - for handler_type, handler_health in handlers_data.items(): - handler_healthy = False - handler_details: dict[str, JsonType] | None = None - handler_error: str | None = None - - if isinstance(handler_health, dict): - handler_healthy = bool(handler_health.get("healthy", False)) - handler_details = cast("dict[str, JsonType]", handler_health) - if not handler_healthy: - handler_error = str( - handler_health.get("error", "health check failed") - ) - - if handler_healthy: - self._last_healthy_timestamps[handler_type] = now - components[handler_type] = ModelComponentHealth.healthy( - name=handler_type, - last_healthy=self._last_healthy_timestamps.get(handler_type), - details=handler_details, - ) - else: - components[handler_type] = ModelComponentHealth.unhealthy( - name=handler_type, - error=handler_error or "health check failed", - last_healthy=self._last_healthy_timestamps.get(handler_type), - details=handler_details, - ) - - # Failed handlers (degraded components) - failed_handlers = health_details.get("failed_handlers", {}) - if isinstance(failed_handlers, dict): - for handler_type, error_msg_raw in failed_handlers.items(): - components[handler_type] = ModelComponentHealth.degraded( - name=handler_type, - error=str(error_msg_raw), - last_healthy=self._last_healthy_timestamps.get(handler_type), - ) - - return components - async def _handle_health_detailed(self, request: web.Request) -> web.Response: """Handle GET /health/detailed requests (verbose diagnostics). @@ -1533,8 +1589,37 @@ async def _handle_health_detailed(self, request: web.Request) -> web.Response: status = "unhealthy" http_status = 503 + # OMN-15642: fold the boot attach-readiness aggregate into THIS + # endpoint's status -- deliberately NOT into /health's (see the long + # comment in _handle_health for why). /health/detailed is not a probe + # target for any automated boot/deploy gate (verified: no k8s manifest, + # CI workflow, or deploy script under this repo or omninode_infra reads + # this path), so it is safe here and matches this endpoint's own + # documented contract (503 for "unhealthy"). Without this fold, + # /health/detailed's own status/http_status pair stayed green while + # attach_readiness.state != READY was visible only inside the nested + # components.runtime_wiring detail below. + status = fold_attach_readiness_into_status( + status, + self._attach_readiness.state + if self._attach_readiness is not None + else None, + ) + # Only "unhealthy" can change http_status here: fold_attach_readiness_ + # into_status() early-returns unchanged for an already-"unhealthy" + # payload_status, and every pre-fold path that yields "healthy" or + # "degraded" already set http_status=200 above -- so a post-fold + # "degraded" never needs (or gets) a different http_status than it + # already had. + if status == "unhealthy": + http_status = 503 + checked_at = datetime.now(tz=UTC).isoformat() - components = self._build_component_health(health_details) + components = build_component_health( + health_details, + last_healthy_timestamps=self._last_healthy_timestamps, + attach_readiness=self._attach_readiness, + ) # Add overall check latency to details enriched_details = dict(health_details) @@ -1606,4 +1691,149 @@ async def _handle_introspection_manifest( ) +def build_component_health( + health_details: dict[str, object], + *, + last_healthy_timestamps: dict[str, str], + attach_readiness: ModelRuntimeAttachReadiness | None = None, +) -> dict[str, ModelComponentHealth]: + """Build per-component health status from runtime health details. + + Extracts component-level health information from the runtime's + health_check() response and constructs typed ModelComponentHealth + instances for each component. + + OMN-519: Component-level health diagnostics. + OMN-15512: boot wiring counts as a ``runtime_wiring`` component. + + Args: + health_details: The raw health check dict from RuntimeHostProcess. + last_healthy_timestamps: Mutable per-component last-healthy map, updated + in place for every component observed healthy on this call. + attach_readiness: Boot attach-readiness aggregate, or None before the + kernel has attached it. + + Returns: + Dictionary mapping component name to ModelComponentHealth. + """ + now = datetime.now(tz=UTC).isoformat() + components: dict[str, ModelComponentHealth] = {} + + # Event bus health + event_bus_healthy = bool(health_details.get("event_bus_healthy", False)) + event_bus_data = health_details.get("event_bus", {}) + if event_bus_healthy: + last_healthy_timestamps["event_bus"] = now + event_bus_details: dict[str, JsonType] | None = None + if isinstance(event_bus_data, dict): + event_bus_details = cast("dict[str, JsonType]", event_bus_data) + if event_bus_healthy: + components["event_bus"] = ModelComponentHealth.healthy( + name="event_bus", + last_healthy=last_healthy_timestamps.get("event_bus"), + details=event_bus_details, + ) + else: + error_msg = "" + if isinstance(event_bus_data, dict): + error_msg = str(event_bus_data.get("error", "unhealthy")) + else: + error_msg = "unhealthy" + components["event_bus"] = ModelComponentHealth.unhealthy( + name="event_bus", + error=error_msg, + last_healthy=last_healthy_timestamps.get("event_bus"), + details=event_bus_details, + ) + + # Per-handler health + handlers_data = health_details.get("handlers", {}) + if isinstance(handlers_data, dict): + for handler_type, handler_health in handlers_data.items(): + handler_healthy = False + handler_details: dict[str, JsonType] | None = None + handler_error: str | None = None + + if isinstance(handler_health, dict): + handler_healthy = bool(handler_health.get("healthy", False)) + handler_details = cast("dict[str, JsonType]", handler_health) + if not handler_healthy: + handler_error = str( + handler_health.get("error", "health check failed") + ) + + if handler_healthy: + last_healthy_timestamps[handler_type] = now + components[handler_type] = ModelComponentHealth.healthy( + name=handler_type, + last_healthy=last_healthy_timestamps.get(handler_type), + details=handler_details, + ) + else: + components[handler_type] = ModelComponentHealth.unhealthy( + name=handler_type, + error=handler_error or "health check failed", + last_healthy=last_healthy_timestamps.get(handler_type), + details=handler_details, + ) + + # Failed handlers (degraded components) + failed_handlers = health_details.get("failed_handlers", {}) + if isinstance(failed_handlers, dict): + for handler_type, error_msg_raw in failed_handlers.items(): + components[handler_type] = ModelComponentHealth.degraded( + name=handler_type, + error=str(error_msg_raw), + last_healthy=last_healthy_timestamps.get(handler_type), + ) + + # Boot wiring counts (OMN-15512). Present only once the kernel has + # attached the aggregate — absent during early startup, exactly like + # the introspection manifest. + readiness = attach_readiness + if readiness is not None: + registered = health_details.get("registered_handlers", []) + registered_handler_count = ( + len(registered) if isinstance(registered, list | tuple) else 0 + ) + not_ready_contract_count = len(readiness.not_ready_results) + wiring_details: dict[str, JsonType] = { + "state": readiness.state.value, + "required_contracts": readiness.required_contracts, + "attached_contracts": readiness.attached_contracts, + "not_ready_contract_count": not_ready_contract_count, + "registered_handler_count": registered_handler_count, + } + if readiness.state is EnumRuntimeReadinessState.READY: + last_healthy_timestamps["runtime_wiring"] = now + components["runtime_wiring"] = ModelComponentHealth.healthy( + name="runtime_wiring", + last_healthy=last_healthy_timestamps.get("runtime_wiring"), + details=wiring_details, + ) + else: + # DEGRADED aggregate -> degraded component; FAILED -> unhealthy. + # Deliberately does NOT change the endpoint's HTTP status: a + # runtime stays live with NOT_READY contracts by design + # (OMN-13237). This only stops green liveness from implying + # that every consumer attached. + factory = ( + ModelComponentHealth.degraded + if readiness.state is EnumRuntimeReadinessState.DEGRADED + else ModelComponentHealth.unhealthy + ) + components["runtime_wiring"] = factory( + name="runtime_wiring", + error=( + f"{not_ready_contract_count} contract(s) did not attach " + f"(state={readiness.state.value}); query the " + f"runtime_manifests projection for the per-topic detail" + ), + last_healthy=last_healthy_timestamps.get("runtime_wiring"), + details=wiring_details, + ) + + return components + + __all__: list[str] = ["DEFAULT_HTTP_HOST", "DEFAULT_HTTP_PORT", "ServiceHealth"] diff --git a/src/omnibase_infra/services/observability/agent_actions/tests/test_hardening.py b/src/omnibase_infra/services/observability/agent_actions/tests/test_hardening.py index c334555526..c30db736b1 100644 --- a/src/omnibase_infra/services/observability/agent_actions/tests/test_hardening.py +++ b/src/omnibase_infra/services/observability/agent_actions/tests/test_hardening.py @@ -486,11 +486,23 @@ async def test_snapshot_with_no_latency(self) -> None: class TestDLQConfiguration: - """Test DLQ configuration defaults and overrides.""" + """Test DLQ configuration defaults and overrides. + + OMN-15410: ``kafka_bootstrap_servers`` is a REQUIRED field — OMN-7227 + (#1127, 2026-04-02) replaced its ``default="localhost:19092"`` with + ``Field(...)`` to kill the silent-fallback class. These constructions had + no such argument and have raised ``ValidationError`` ever since; the + failure was invisible because this whole ``tests/`` root was uncollected + by every CI job (OMN-15378 class). Supplied explicitly here — same literal + the already-passing ``_make_health_consumer`` helper in this module uses — + rather than relaxing the field, because fail-fast on missing Kafka wiring + is the behavior OMN-7227 deliberately bought. + """ def test_default_dlq_config(self) -> None: """Default DLQ configuration should have sensible defaults.""" config = ConfigAgentActionsConsumer( + kafka_bootstrap_servers="localhost:19092", postgres_dsn="postgresql://test:test@localhost:5432/test", ) assert config.dlq_enabled is True @@ -500,6 +512,7 @@ def test_default_dlq_config(self) -> None: def test_dlq_disabled(self) -> None: """DLQ can be disabled via config.""" config = ConfigAgentActionsConsumer( + kafka_bootstrap_servers="localhost:19092", postgres_dsn="postgresql://test:test@localhost:5432/test", dlq_enabled=False, ) @@ -512,11 +525,17 @@ def test_dlq_disabled(self) -> None: class TestHealthCheckHostDefault: - """Test health check host default changed from 0.0.0.0 to 127.0.0.1.""" + """Test health check host default changed from 0.0.0.0 to 127.0.0.1. + + OMN-15410: ``kafka_bootstrap_servers`` supplied for the same reason as in + :class:`TestDLQConfiguration` — the field became required in OMN-7227 and + these constructions have been RED-but-uncollected since 2026-04-02. + """ def test_default_health_check_host_is_localhost(self) -> None: """Default health check host should be 127.0.0.1 for security.""" config = ConfigAgentActionsConsumer( + kafka_bootstrap_servers="localhost:19092", postgres_dsn="postgresql://test:test@localhost:5432/test", ) assert config.health_check_host == "127.0.0.1" @@ -525,6 +544,7 @@ def test_health_check_host_override_for_containers(self) -> None: """Health check host can be overridden to 0.0.0.0 for containers.""" all_interfaces = "0.0.0.0" # noqa: S104 - test verifying container override config = ConfigAgentActionsConsumer( + kafka_bootstrap_servers="localhost:19092", postgres_dsn="postgresql://test:test@localhost:5432/test", health_check_host=all_interfaces, ) diff --git a/src/omnibase_infra/services/registry_api/main.py b/src/omnibase_infra/services/registry_api/main.py index cc4719e27f..f0089fad75 100644 --- a/src/omnibase_infra/services/registry_api/main.py +++ b/src/omnibase_infra/services/registry_api/main.py @@ -11,13 +11,12 @@ from omnibase_core.container import ModelONEXContainer container = ModelONEXContainer() - app = create_app(container=container, cors_origins=["http://localhost:3000"]) + app = create_app(container=container) # Create app with projection reader app = create_app( container=container, projection_reader=reader, - cors_origins=["http://localhost:3000"], ) # Run with uvicorn @@ -173,7 +172,7 @@ def create_app( "CORS_ORIGINS must be configured. " "Set the CORS_ORIGINS environment variable (comma-separated list of allowed origins) " "or pass cors_origins parameter to create_app(). " - "Example: CORS_ORIGINS=http://localhost:3000,https://dashboard.example.com", + "Example: CORS_ORIGINS=https://dashboard.example.invalid", context=context, ) origins = env_origins.split(",") @@ -240,7 +239,7 @@ async def root() -> dict[str, str]: # app = create_app( # container=container, # projection_reader=reader, -# cors_origins=["http://localhost:3000"], +# cors_origins=["https://dashboard.example.invalid"], # ) # uvicorn.run(app, host="0.0.0.0", port=8000) # diff --git a/src/omnibase_infra/services/service_runtime_health_monitor.py b/src/omnibase_infra/services/service_runtime_health_monitor.py index f03eba26fa..3146efa021 100644 --- a/src/omnibase_infra/services/service_runtime_health_monitor.py +++ b/src/omnibase_infra/services/service_runtime_health_monitor.py @@ -29,7 +29,7 @@ import math import os import time -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import UTC, datetime from typing import TYPE_CHECKING, Literal, NamedTuple @@ -296,6 +296,46 @@ def _topic_is_covered_by_legacy_group(topic: str, group_id: str) -> bool: _KAFKA_ADMIN_TIMEOUT_MS: int = 5_000 +# Cap on how many failing entry points are named in the discovery_errors detail. +# The detail string is served on every /health response and shipped in every +# health event; a runaway discovery failure must not turn it into a log dump. +_MAX_NAMED_DISCOVERY_ERRORS = 8 + + +def _describe_discovery_errors( + manifest: ProtocolAutoWiringManifestLike, error_count: int +) -> str: + """Build the discovery_errors detail, naming the failing entry points. + + OMN-15217: the pre-existing detail was ``"4 contract(s) failed to load"`` — + a count with no identities, which forced every investigation to go back to + raw container logs to learn *which* contracts failed (and the boot-time + ``Failed to load entry point`` lines roll out of the log buffer long before + anyone looks). Naming them here makes the health surface itself + self-diagnosing. + + ``ProtocolAutoWiringManifestLike`` only guarantees the counts, so the error + tuple is read defensively: any manifest that does not expose it degrades to + the original count-only detail rather than raising inside a health check. + """ + errors = getattr(manifest, "errors", None) + names: list[str] = [] + if isinstance(errors, Sequence) and not isinstance(errors, str | bytes): + for error in errors[:_MAX_NAMED_DISCOVERY_ERRORS]: + entry_point = getattr(error, "entry_point_name", None) + if entry_point: + names.append(str(entry_point)) + + base = f"{error_count} contract(s) failed to load" + if not names: + return base + listed = ", ".join(names) + remaining = error_count - len(names) + if remaining > 0: + listed = f"{listed}, +{remaining} more" + return f"{base}: {listed}" + + def _worst(statuses: list[_HealthStatus]) -> _HealthStatus: """Return the worst status from a list.""" if "CRITICAL" in statuses: @@ -370,6 +410,11 @@ def __init__( self._boot_grace_seconds = boot_grace_seconds self._started_at: float | None = None self._boot_grace_complete_logged = boot_grace_seconds == 0 + # OMN-15217: retain the latest verdict so the HTTP health surface can + # publish it. Before this the verdict existed only in logs and on the + # Kafka health topic, so /health reported healthy while the runtime was + # DEGRADED — see runtime_health_block for the full mask description. + self._latest_event: ModelRuntimeHealthCheckEvent | None = None async def start(self) -> None: """Start the background health check loop. Idempotent. @@ -411,6 +456,16 @@ async def stop(self) -> None: self._task = None logger.info("ServiceRuntimeHealthMonitor stopped") + @property + def latest_event(self) -> ModelRuntimeHealthCheckEvent | None: + """Return the most recent health-check event, or ``None`` before the first cycle. + + ``None`` means "no verdict computed yet" — a distinct state from + HEALTHY. Consumers that need proof of health must fail closed on + ``None`` rather than reading absence as green (OMN-15217). + """ + return self._latest_event + async def run_once(self) -> ModelRuntimeHealthCheckEvent: """Run a single health check cycle and return the event. @@ -448,7 +503,9 @@ async def run_once(self) -> ModelRuntimeHealthCheckEvent: ModelRuntimeHealthDimension( name="discovery_errors", status="DEGRADED", - detail=(f"{discovery_error_count} contract(s) failed to load"), + detail=_describe_discovery_errors( + manifest, discovery_error_count + ), ) ) else: @@ -656,6 +713,8 @@ async def run_once(self) -> ModelRuntimeHealthCheckEvent: dim.detail, ) + self._latest_event = event + await self._emit(event) return event diff --git a/src/omnibase_infra/tools/contract_topic_extractor.py b/src/omnibase_infra/tools/contract_topic_extractor.py index ad20c3bd77..8d84795c94 100644 --- a/src/omnibase_infra/tools/contract_topic_extractor.py +++ b/src/omnibase_infra/tools/contract_topic_extractor.py @@ -43,7 +43,16 @@ # Constants # --------------------------------------------------------------------------- -_VALID_KINDS: frozenset[str] = frozenset({"evt", "cmd", "intent", "dlq"}) +# OMN-15832: "snapshot" is a first-class kind, not a local addition — it +# mirrors omnibase_core.constants.constants_topic_taxonomy's canonical +# token-to-type mapping (get_valid_topic_suffix_kinds() already returns +# {"cmd", "dlq", "evt", "intent", "snapshot"}). This extractor's set had +# drifted behind that source of truth: every onex.snapshot.projection.* +# topic (declared under a contract's projection_api section, see +# _extract_raw_topics_from_contract below) was silently rejected here as an +# "invalid kind", which is why those topics were never in the provisioner's +# create-set despite auto-create being off on managed MSK. +_VALID_KINDS: frozenset[str] = frozenset({"evt", "cmd", "intent", "dlq", "snapshot"}) _VALID_DLQ_CATEGORIES: frozenset[str] = frozenset({"intents", "events", "commands"}) _RE_VERSION = re.compile(r"^v\d+$") _RE_EVENT_NAME = re.compile(r"^[a-z0-9._-]+$") @@ -97,7 +106,7 @@ class ModelContractTopicEntry(BaseModel): """ topic: str - kind: Literal["evt", "cmd", "intent", "dlq"] + kind: Literal["evt", "cmd", "intent", "dlq", "snapshot"] producer: str event_name: str version: str # e.g. "v1" @@ -251,7 +260,7 @@ def _parse_topic(raw: str, source: Path) -> ModelContractTopicEntry | None: return ModelContractTopicEntry( topic=raw, - kind=cast("Literal['evt', 'cmd', 'intent', 'dlq']", kind), + kind=cast("Literal['evt', 'cmd', 'intent', 'dlq', 'snapshot']", kind), producer=producer, event_name=event_name, version=version, @@ -401,9 +410,94 @@ def _extract_raw_topics_from_contract( ) ) + # --- projection_api.topic / projection_api.exposures[].topic (OMN-15832) --- + # onex.snapshot.projection.* topics are declared under `projection_api`, + # not under `event_bus.*` or the consumed/published/produced_events + # sections above — the only topic family with its own top-level contract + # key. Scoped to `expose: true` AND `bus_backed: true` exposures only: + # + # - `expose` is a SECTION-LEVEL gate (declared once under `projection_api`, + # never repeated per-exposure — see node_projection_delegation's + # contract.yaml for the exposures-list shape). It must be checked BEFORE + # iterating exposures, matching + # omnimarket.projection.discovery.build_projection_topic_map's own gate + # (`if not section.get("expose", False): continue`, checked before any + # bus_backed logic). Without this check, an `expose: false` + + # `bus_backed: true` contract would be provisioned here while + # build_projection_topic_map never serves it — a topic created but never + # read. + # - `bus_backed: true` is exactly the per-exposure set omnimarket's + # SnapshotCache blocks projection-api startup on (snapshot_cache.py's + # _wait_topics reads cfg.bus_backed the same way), so provisioning + # tracks the genuinely required set instead of eagerly creating topics + # for exposures nothing publishes to yet. + for proj_item in _projection_api_served_exposures(data.get("projection_api")): + topic_val = proj_item.get("topic") + if isinstance(topic_val, str) and topic_val: + partitions, rf, kc = _parse_topic_config(proj_item, source) + raw_topics.append( + RawTopicDecl( + topic=topic_val, + provisioning_priority=_topic_priority(proj_item), + partitions=partitions, + replication_factor=rf, + kafka_config=kc, + ) + ) + return raw_topics +def _projection_api_served_exposures( + projection_api: object, +) -> list[dict[object, object]]: + """Return the ``projection_api`` exposure dicts that are actually served. + + Single source of parsing truth for "is this projection_api exposure + genuinely reachable" — reused by both the contract.yaml scan above and + ``handler_wiring._contract_provision_topics`` (OMN-15832 Phase B + provisioning) via :func:`read_projection_api_topics`, so the boot-time + provision set and the extractor's static scan can never diverge. + + An exposure is served iff: + - the section itself declares ``expose: true`` (section-level gate, + never per-exposure — mirrors + ``omnimarket.projection.discovery.build_projection_topic_map``), AND + - the individual exposure declares ``bus_backed: true`` (the predicate + omnimarket's ``SnapshotCache`` gates startup on). + """ + if not isinstance(projection_api, dict): + return [] + if projection_api.get("expose") is not True: + return [] + return [ + item + for item in _iter_projection_api_exposures(projection_api) + if item.get("bus_backed") is True + ] + + +def _iter_projection_api_exposures( + section: dict[object, object], +) -> list[dict[object, object]]: + """Yield each exposure dict under a contract's ``projection_api`` section. + + Mirrors omnimarket.projection.discovery._parse_projection_api_sections's + legacy-singular vs. exposures-list duality: a contract declaring a single + ``projection_api.topic`` key (no ``exposures`` list) is one implicit + exposure; a contract declaring ``projection_api.exposures: [...]`` has one + exposure per list entry. A non-list, non-dict, or empty ``exposures`` + value yields nothing rather than falling back to the legacy shape — the + same ambiguity omnimarket's own parser treats as contract-invalid. + """ + exposures = section.get("exposures") + if exposures is not None: + if isinstance(exposures, list): + return [item for item in exposures if isinstance(item, dict)] + return [] + return [section] + + def _extract_topics_from_python_ast(source_path: Path) -> list[str]: """Extract ONEX topic string literals from a Python source file using AST. @@ -571,6 +665,41 @@ def _discover_installed_topic_packages() -> tuple[str, ...]: # --------------------------------------------------------------------------- +def read_projection_api_topics(contract_path: Path) -> tuple[str, ...]: + """Read one contract.yaml's served ``projection_api`` topics (OMN-15832). + + Single source of parsing truth for a contract's ``projection_api`` topics + — used by :meth:`ContractTopicExtractor.extract` (global scan, via + :func:`_projection_api_served_exposures`) AND by + ``handler_wiring._contract_provision_topics`` (per-contract Phase B boot + provisioning), so the two call sites can never disagree about which + ``projection_api`` topics are provisioned. + + Returns the raw topic strings (unvalidated — callers that need + :class:`ModelContractTopicEntry` validation should go through + :meth:`ContractTopicExtractor.extract`) for exposures where the section + declares ``expose: true`` AND the individual exposure declares + ``bus_backed: true``. Returns ``()`` if the file cannot be read, is not a + mapping, or has no served exposures — mirrors the degrade-not-abort + contract of ``_read_dlq_topics``. + """ + try: + with contract_path.open(encoding="utf-8") as fh: + raw_yaml = yaml.safe_load(fh) + except Exception: # noqa: BLE001 — boundary: degrade to no topics, never raise + return () + + if not isinstance(raw_yaml, dict): + return () + + topics: list[str] = [] + for item in _projection_api_served_exposures(raw_yaml.get("projection_api")): + topic_val = item.get("topic") + if isinstance(topic_val, str) and topic_val: + topics.append(topic_val) + return tuple(dict.fromkeys(topics)) + + class ContractTopicExtractor: """ Scan contract.yaml files and return validated ModelContractTopicEntry objects. diff --git a/src/omnibase_infra/topics/__init__.py b/src/omnibase_infra/topics/__init__.py index 46e7053f22..486e5a1196 100644 --- a/src/omnibase_infra/topics/__init__.py +++ b/src/omnibase_infra/topics/__init__.py @@ -19,7 +19,21 @@ TopicResolutionError: Error raised when topic resolution fails """ +from omnibase_infra.topics.broker_capacity_probe import ( + bind_policy_to_broker_capacity, + probe_broker_count, +) +from omnibase_infra.topics.enum_topic_provisioning_profile import ( + EnumTopicProvisioningProfile, +) from omnibase_infra.topics.model_bus_descriptor import ModelBusDescriptor +from omnibase_infra.topics.model_topic_provisioning_diff import ( + ModelTopicProvisioningDiff, + build_provisioning_diff, +) +from omnibase_infra.topics.model_topic_provisioning_policy import ( + ModelTopicProvisioningPolicy, +) from omnibase_infra.topics.model_topic_spec import ModelTopicSpec from omnibase_infra.topics.platform_topic_suffixes import ( ALL_INTELLIGENCE_TOPIC_SPECS, @@ -426,6 +440,13 @@ "ALL_PROVISIONED_TOPIC_SPECS", # Topic spec model "ModelTopicSpec", + # Topic provisioning policy + diff (OMN-15395) + "EnumTopicProvisioningProfile", + "ModelTopicProvisioningDiff", + "ModelTopicProvisioningPolicy", + "bind_policy_to_broker_capacity", + "build_provisioning_diff", + "probe_broker_count", # Bus descriptor model (Phase 5 - OMN-2894) "ModelBusDescriptor", # Topic resolution diff --git a/src/omnibase_infra/topics/broker_capacity_probe.py b/src/omnibase_infra/topics/broker_capacity_probe.py new file mode 100644 index 0000000000..80013c1091 --- /dev/null +++ b/src/omnibase_infra/topics/broker_capacity_probe.py @@ -0,0 +1,252 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Measure a live broker's node count and bind it to the provisioning policy. + +Why this exists (OMN-15395) +--------------------------- +:class:`~omnibase_infra.topics.model_topic_provisioning_policy.ModelTopicProvisioningPolicy` +may reduce a contract-declared replication factor down to what the target +broker can physically host. The first revision of that policy derived the +ceiling from the SASL mechanism — anything that was not ``AWS_MSK_IAM`` was +assumed to be a single node and had every declared RF clamped to 1. That +assumption is false for any multi-broker cluster reached over PLAIN or SCRAM, +and it silently recreated the exact ``AWS_KAFKA_HIGH_RISK_CONFIG_RF_EQUALS_ONE`` +condition the ticket exists to eliminate. + +This module replaces the assumption with a measurement: one ``describe_cluster`` +metadata request against the same admin client that is about to issue the +``CreateTopics``, taken **once per provisioner instance** — the attempt is +memoized, not just its success, so an unmeasurable cluster is probed once and +not re-probed on every entrypoint (OMN-15395 D4). It is a plain ``Metadata`` +API call — the same class of request the provisioner already makes for +``describe_topics`` — not the ``DescribeTopicDynamicConfiguration`` call MSK IAM +denies. + +Fail-open is not an option here, but neither is guessing: when the count cannot +be read the policy stays **unmeasured**, which means no ceiling and therefore no +reduction. A declared RF that the broker cannot host then fails loudly at +``CreateTopics``: the provisioner classifies the broker's +``INVALID_REPLICATION_FACTOR`` rejection and raises +:class:`~omnibase_infra.errors.TopicReplicationPolicyError` out of the +best-effort boundary rather than appending the topic to ``failed`` behind a +``logger.warning`` (OMN-15395 D5). Quietly downgrading is not an option and +quietly *not creating* is not one either — an unmeasurable probe must not leave +a topic silently absent. + +Two client shapes are supported, because there are two live ``CreateTopics`` +paths in this repository and both must resolve capacity the same way: + +* the async ``AIOKafkaAdminClient`` used by the runtime provisioner + (:func:`probe_broker_count` / :func:`bind_policy_to_broker_capacity`); and +* the synchronous ``confluent_kafka.admin.AdminClient`` used by the operator + CLI ``scripts/create_kafka_topics.py``, whose ``list_topics()`` already + returns a ``ClusterMetadata`` carrying the broker list + (:func:`broker_count_from_cluster_metadata`). + +Both funnel into :func:`bind_policy_to_broker_count`, so a ceiling installed by +either path obeys the identical invariants. +""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping, Sequence + +from omnibase_infra.topics.model_topic_provisioning_policy import ( + ModelTopicProvisioningPolicy, +) + +logger = logging.getLogger(__name__) + + +# Kafka error code 38. Matched by wire name rather than by importing +# ``aiokafka.errors.InvalidReplicationFactorError``: the classification must +# survive a driver that wraps or re-raises the broker error, and it must not +# make this module's fail-closed behaviour depend on an optional import that is +# absent in the ``aiokafka not available`` degradation path. +INVALID_REPLICATION_FACTOR_ERRNO = 38 +_INVALID_REPLICATION_FACTOR_MARKERS = ( + "INVALID_REPLICATION_FACTOR", + "InvalidReplicationFactor", +) + + +def is_invalid_replication_factor_error(exc: BaseException) -> bool: + """True when the broker rejected a ``CreateTopics`` for its replica count. + + OMN-15395 (D5): the capacity ceiling is a MEASUREMENT, so an unmeasurable + cluster gets no ceiling and a contract-declared RF the broker cannot host + reaches ``CreateTopics`` unreduced — deliberately, because guessing a + ceiling is a silent durability downgrade. That design is only honest if the + broker's refusal is then LOUD. Landing it in the generic + ``except Exception`` boundary made it a ``logger.warning`` plus a name in + ``failed``, indistinguishable from a transient connection blip, with the + pass returning ``status="partial"`` and the topic silently absent. + """ + if type(exc).__name__ == "InvalidReplicationFactorError": + return True + if getattr(exc, "errno", None) == INVALID_REPLICATION_FACTOR_ERRNO: + return True + text = f"{type(exc).__name__}: {exc}" + return any(marker in text for marker in _INVALID_REPLICATION_FACTOR_MARKERS) + + +async def probe_broker_count(admin: object) -> int | None: + """Return the live broker count, or ``None`` when it cannot be measured. + + Args: + admin: A started ``AIOKafkaAdminClient`` (or any object exposing an + awaitable ``describe_cluster()`` returning a mapping with a + ``brokers`` sequence). + + Returns: + The number of brokers the cluster reports, or ``None`` when the client + does not expose ``describe_cluster``, the call fails, or the response + carries no usable broker list. + """ + describe_cluster = getattr(admin, "describe_cluster", None) + if describe_cluster is None: + logger.warning( + "Admin client %s exposes no describe_cluster(); topic replication " + "will be resolved WITHOUT a capacity ceiling — a declared " + "replication factor above the broker's node count will fail at " + "CreateTopics rather than being silently reduced (OMN-15395)", + type(admin).__name__, + ) + return None + + try: + described = await describe_cluster() + except Exception as exc: # noqa: BLE001 — boundary: unmeasured beats guessed + logger.warning( + "describe_cluster() failed (%s); topic replication will be " + "resolved WITHOUT a capacity ceiling rather than assuming a node " + "count (OMN-15395)", + type(exc).__name__, + ) + return None + + if not isinstance(described, Mapping): + logger.warning( + "describe_cluster() returned %s, expected a mapping; leaving the " + "replication capacity ceiling unmeasured (OMN-15395)", + type(described).__name__, + ) + return None + + brokers = described.get("brokers") + if not isinstance(brokers, Sequence) or isinstance(brokers, (str, bytes)): + logger.warning( + "describe_cluster() carried no broker sequence; leaving the " + "replication capacity ceiling unmeasured (OMN-15395)", + ) + return None + + count = len(brokers) + if count < 1: + logger.warning( + "describe_cluster() reported zero brokers; leaving the replication " + "capacity ceiling unmeasured (OMN-15395)", + ) + return None + return count + + +def broker_count_from_cluster_metadata(cluster_metadata: object) -> int | None: + """Return the live broker count from a synchronous ``ClusterMetadata``. + + ``confluent_kafka.admin.AdminClient.list_topics()`` already carries the + cluster's broker map, so the operator CLI measures capacity from the SAME + metadata request it makes to diff topics — zero extra round trips, and no + second, differently-shaped notion of "how many brokers are there". + + Args: + cluster_metadata: A ``confluent_kafka.admin.ClusterMetadata`` (or any + object exposing a sized ``brokers`` attribute). + + Returns: + The number of brokers, or ``None`` when the count cannot be read. + """ + brokers = getattr(cluster_metadata, "brokers", None) + if brokers is None: + logger.warning( + "Cluster metadata %s carries no broker map; topic replication will " + "be resolved WITHOUT a capacity ceiling rather than assuming a node " + "count (OMN-15395)", + type(cluster_metadata).__name__, + ) + return None + try: + count = len(brokers) + except TypeError: + logger.warning( + "Cluster metadata broker map is not sized (%s); leaving the " + "replication capacity ceiling unmeasured (OMN-15395)", + type(brokers).__name__, + ) + return None + if count < 1: + logger.warning( + "Cluster metadata reported zero brokers; leaving the replication " + "capacity ceiling unmeasured (OMN-15395)", + ) + return None + return count + + +def bind_policy_to_broker_count( + policy: ModelTopicProvisioningPolicy, + broker_count: int | None, +) -> ModelTopicProvisioningPolicy: + """Return ``policy`` bound to ``broker_count``, or unchanged when ``None``. + + The single binding seam shared by the async runtime provisioner and the + synchronous operator CLI. The durability floor and the profile are never + altered by the measurement — see + :meth:`~omnibase_infra.topics.model_topic_provisioning_policy.ModelTopicProvisioningPolicy.with_broker_capacity`. + + Args: + policy: The configuration-derived (unmeasured) policy. + broker_count: A measured live node count, or ``None`` when unmeasurable. + + Returns: + The measured policy, or ``policy`` itself when no count was measured. + """ + if broker_count is None: + return policy + measured = policy.with_broker_capacity(broker_count) + logger.info( + "Broker capacity measured: %d node(s) on the %s cluster; replication " + "ceiling=%s, undeclared default=%s (OMN-15395)", + broker_count, + measured.profile.value, + measured.capacity_replication_factor, + measured.default_replication_factor, + ) + return measured + + +async def bind_policy_to_broker_capacity( + admin: object, + policy: ModelTopicProvisioningPolicy, +) -> ModelTopicProvisioningPolicy: + """Return ``policy`` bound to the live broker count, or unchanged. + + Args: + admin: A started admin client. + policy: The configuration-derived (unmeasured) policy. + + Returns: + The measured policy, or ``policy`` itself when no count could be read. + """ + return bind_policy_to_broker_count(policy, await probe_broker_count(admin)) + + +__all__: list[str] = [ + "INVALID_REPLICATION_FACTOR_ERRNO", + "bind_policy_to_broker_capacity", + "bind_policy_to_broker_count", + "broker_count_from_cluster_metadata", + "is_invalid_replication_factor_error", + "probe_broker_count", +] diff --git a/src/omnibase_infra/topics/enum_topic_provisioning_profile.py b/src/omnibase_infra/topics/enum_topic_provisioning_profile.py new file mode 100644 index 0000000000..4f1269373a --- /dev/null +++ b/src/omnibase_infra/topics/enum_topic_provisioning_profile.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Topic-provisioning environment profile (OMN-15395). + +The profile classifies the *broker* a provisioning pass is pointed at, not the +deployment lane name. It is derived from the live Kafka client configuration +(``ModelKafkaEventBusConfig``) rather than a caller-supplied label, so a caller +cannot claim ``SELF_HOSTED`` against a managed cluster to dodge the durability +floor. +""" + +from __future__ import annotations + +from enum import Enum + + +class EnumTopicProvisioningProfile(str, Enum): + """Durability class of the broker being provisioned. + + Attributes: + SELF_HOSTED: A self-hosted broker we own end to end (local Redpanda, + the ``.201`` lanes, CI sandboxes). Single-broker deployments are + normal here, so replication factor 1 is legitimate and an + explicitly declared environment default is allowed. + MANAGED: A managed cluster (AWS MSK — detected via ``AWS_MSK_IAM`` SASL + auth). RF1 means a single broker loss is unrecoverable data loss + and blocks broker updates, so RF1 is rejected fail-closed and an + undeclared replication factor is refused rather than defaulted. + """ + + SELF_HOSTED = "self_hosted" + MANAGED = "managed" + + +__all__: list[str] = ["EnumTopicProvisioningProfile"] diff --git a/src/omnibase_infra/topics/managed_staging_topic_checker.py b/src/omnibase_infra/topics/managed_staging_topic_checker.py new file mode 100644 index 0000000000..4c9121253b --- /dev/null +++ b/src/omnibase_infra/topics/managed_staging_topic_checker.py @@ -0,0 +1,411 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# Copyright (c) 2026 OmniNode Team +"""Managed-staging (mstg1) topic/group checker against a live broker (OMN-15283). + +This is the IAM-capable broker admin surface's counterpart to +``managed_staging_canary_catalog.verify_zero_collision``: instead of proving +the canary namespace is disjoint from *pre-existing* names, this module proves +the canary namespace is (or is not yet) fully *provisioned* on a live broker. + +Design +------ +1. The catalog is generated the same way every other Phase-1 canary consumer + generates it -- :func:`~omnibase_infra.topics.managed_staging_canary_catalog.build_canary_catalog_from_candidate`. + This module never re-implements prefixing or candidate topic extraction. +2. The broker connection uses the exact same admin-client construction as + ``TopicProvisioner`` (``AIOKafkaAdminClient`` + + ``build_aiokafka_auth_kwargs_from_env``) so MSK IAM auth behaves identically + to the real provisioning path. +3. :func:`build_topic_diff` is pure and transport-free (mirrors + ``verify_zero_collision``'s shape) -- it is testable without a live broker. +4. ``--create-missing`` is opt-in and, even when enabled, only ever creates + catalog-listed, prefix-scoped topics for names found missing by the diff -- + never a universe sweep, never an out-of-catalog name. + +Scope boundary (HARD): this module checks/diffs/creates only the +managed-staging canary namespace on whatever broker it is pointed at. Live +execution against the actual MSK cluster is the AWS lane's step, out of CI +scope here. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import sys +from collections.abc import Iterable, Sequence +from pathlib import Path +from typing import TYPE_CHECKING + +from omnibase_infra.event_bus.kafka_auth import build_aiokafka_auth_kwargs_from_env +from omnibase_infra.event_bus.models.config.model_kafka_event_bus_config import ( + ModelKafkaEventBusConfig, +) +from omnibase_infra.topics.broker_capacity_probe import ( + bind_policy_to_broker_capacity, + is_invalid_replication_factor_error, +) +from omnibase_infra.topics.managed_staging_canary_catalog import ( + build_canary_catalog_from_candidate, + load_canary_namespace, +) +from omnibase_infra.topics.model_canary_catalog import ModelCanaryCatalog +from omnibase_infra.topics.model_managed_staging_topic_diff import ( + ModelManagedStagingTopicDiff, +) +from omnibase_infra.topics.model_topic_provisioning_diff import ( + build_provisioning_diff, +) +from omnibase_infra.topics.model_topic_provisioning_policy import ( + ModelTopicProvisioningPolicy, +) + +if TYPE_CHECKING: + from aiokafka.admin import AIOKafkaAdminClient + +logger = logging.getLogger(__name__) + + +def build_topic_diff( + catalog: ModelCanaryCatalog, + *, + existing_topics: Iterable[str], + existing_groups: Iterable[str], +) -> ModelManagedStagingTopicDiff: + """Diff a generated canary catalog against a broker snapshot. + + Transport-free and pure -- mirrors + ``managed_staging_canary_catalog.verify_zero_collision``'s shape so it is + unit-testable without a live broker. + + Args: + catalog: The generated canary catalog (the desired state). + existing_topics: All topic names currently on the broker. + existing_groups: All consumer group names currently on the broker. + + Returns: + A :class:`ModelManagedStagingTopicDiff` classifying every catalog name + as missing/present, plus any out-of-catalog prefix-matching stray name. + """ + existing_topic_set = set(existing_topics) + existing_group_set = set(existing_groups) + catalog_topic_set = set(catalog.topic_names) + catalog_group_set = set(catalog.groups) + + # OMN-15395: the missing/present split is the SHARED provisioning diff every + # creation path runs — one diff engine, not a canary-only copy. This module + # keeps only what is genuinely canary-specific (prefix scoping + consumer + # groups). + topic_diff = build_provisioning_diff(sorted(catalog_topic_set), existing_topic_set) + missing_topics = topic_diff.missing_topics + present_topics = topic_diff.present_topics + out_of_catalog_topics = tuple( + sorted( + name + for name in existing_topic_set + if name.startswith(catalog.topic_prefix) and name not in catalog_topic_set + ) + ) + + missing_groups = tuple(sorted(catalog_group_set - existing_group_set)) + present_groups = tuple(sorted(catalog_group_set & existing_group_set)) + out_of_catalog_groups = tuple( + sorted( + name + for name in existing_group_set + if name.startswith(catalog.group_prefix) and name not in catalog_group_set + ) + ) + + return ModelManagedStagingTopicDiff( + topic_prefix=catalog.topic_prefix, + group_prefix=catalog.group_prefix, + missing_topics=missing_topics, + present_topics=present_topics, + out_of_catalog_topics=out_of_catalog_topics, + missing_groups=missing_groups, + present_groups=present_groups, + out_of_catalog_groups=out_of_catalog_groups, + ) + + +async def open_admin_client( + *, + bootstrap_servers: str, + request_timeout_ms: int = 30000, +) -> AIOKafkaAdminClient: + """Open + start an ``AIOKafkaAdminClient`` using the shared MSK IAM auth path. + + Mirrors ``TopicProvisioner``'s admin-client construction exactly (same + ``build_aiokafka_auth_kwargs_from_env`` call) so this checker authenticates + identically to the real provisioning path. Caller owns ``close()``. + """ + from aiokafka.admin import AIOKafkaAdminClient + + auth_kwargs = build_aiokafka_auth_kwargs_from_env() + admin = AIOKafkaAdminClient( + bootstrap_servers=bootstrap_servers, + request_timeout_ms=request_timeout_ms, + **auth_kwargs, + ) + await admin.start() + return admin + + +async def fetch_live_topics_and_groups( + admin: AIOKafkaAdminClient, +) -> tuple[tuple[str, ...], tuple[str, ...]]: + """List live topics + consumer group names from an open admin client. + + Args: + admin: An already-started ``AIOKafkaAdminClient``. + + Returns: + ``(topic_names, group_names)``. + """ + topics = await admin.list_topics() + groups_raw = await admin.list_consumer_groups() + # aiokafka returns a list of tuples; the first element is the group id. + groups = tuple(sorted({entry[0] for entry in groups_raw})) + return tuple(sorted(topics)), groups + + +async def create_missing_catalog_topics( + admin: AIOKafkaAdminClient, + catalog: ModelCanaryCatalog, + diff: ModelManagedStagingTopicDiff, + *, + policy: ModelTopicProvisioningPolicy | None = None, +) -> tuple[tuple[str, ...], tuple[str, ...]]: + """Create only catalog-listed, currently-missing topics (per-contract scoped). + + Never touches an out-of-catalog name and never sweeps the broker's full + topic universe -- only the names present in ``diff.missing_topics``, which + are guaranteed (by construction of :func:`build_topic_diff`) to be a + subset of the catalog's own topic set. + + Every spec is resolved through the environment replication policy BEFORE the + first ``CreateTopics`` (OMN-15395), so this path cannot mint an RF1 topic on + the managed cluster even if a catalog entry regressed to one. + + Args: + admin: An already-started ``AIOKafkaAdminClient``. + catalog: The generated canary catalog (source of namespace defaults -- + partitions/replication -- per topic). + diff: The diff previously computed by :func:`build_topic_diff`. + policy: Replication policy. Defaults to the policy derived from the live + Kafka client configuration. Whatever is supplied is bound to + ``admin``'s measured broker count before any spec is resolved. + + Returns: + ``(created, failed)`` topic name tuples. + + Raises: + TopicReplicationPolicyError: A catalog spec violates the policy. Raised + before any ``CreateTopics``. + """ + from aiokafka.admin import NewTopic + from aiokafka.errors import TopicAlreadyExistsError + + # Bind the (unmeasured) configuration policy to this cluster's live node + # count before resolving anything: a capacity ceiling that reduces a + # declared replication factor must be measured, never inferred from the + # SASL mechanism (OMN-15395). + resolved_policy = await bind_policy_to_broker_capacity( + admin, policy or ModelTopicProvisioningPolicy.from_env() + ) + specs_by_name = {spec.suffix: spec for spec in catalog.topics} + created: list[str] = [] + failed: list[str] = [] + + # Fail closed ahead of the create loop: a durability violation anywhere in + # the batch means nothing is created. + resolved_by_name = { + name: resolved_policy.resolve_spec(spec) + for name in diff.missing_topics + if (spec := specs_by_name.get(name)) is not None + } + + for name in diff.missing_topics: + spec = resolved_by_name.get(name) + if spec is None: + # Defensive: build_topic_diff guarantees missing_topics subset of + # catalog.topic_names, so this branch should be unreachable. + failed.append(name) + continue + new_topic = NewTopic( + name=spec.suffix, + num_partitions=spec.partitions, + replication_factor=spec.replication_factor, + topic_configs=dict(spec.kafka_config) if spec.kafka_config else {}, + ) + try: + await admin.create_topics([new_topic]) + created.append(name) + except TopicAlreadyExistsError: + created.append(name) + except Exception as exc: + # Boundary: report, do not raise. + # This surface is a REPORT generator, so it never raises — but a + # durability refusal must still be distinguishable from a transient + # miss in the log, the same rule the runtime provisioner enforces by + # re-raising (OMN-15395 D5). + if is_invalid_replication_factor_error(exc): + logger.exception( + "Broker REFUSED catalog topic %s with " + "INVALID_REPLICATION_FACTOR (requested " + "replication_factor=%s); the topic does not exist and this " + "is a durability failure, not a transient miss (OMN-15395)", + name, + spec.replication_factor, + ) + else: + logger.warning("Failed to create catalog topic %s", name) + failed.append(name) + + return tuple(created), tuple(failed) + + +def _render_report( + diff: ModelManagedStagingTopicDiff, + *, + created: Sequence[str] = (), + failed: Sequence[str] = (), +) -> str: + payload: dict[str, object] = { + "topic_prefix": diff.topic_prefix, + "group_prefix": diff.group_prefix, + "missing_topics": list(diff.missing_topics), + "present_topics": list(diff.present_topics), + "out_of_catalog_topics": list(diff.out_of_catalog_topics), + "missing_groups": list(diff.missing_groups), + "present_groups": list(diff.present_groups), + "out_of_catalog_groups": list(diff.out_of_catalog_groups), + "is_fully_present": diff.is_fully_present, + "created_topics": list(created), + "failed_topics": list(failed), + } + return json.dumps(payload, indent=2, sort_keys=False) + + +def _build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Check the managed-staging (mstg1) canary topic/group catalog " + "against a live broker. Check-only by default; --create-missing " + "opts into creating catalog-listed missing topics." + ) + ) + parser.add_argument( + "--namespace-path", + type=Path, + default=None, + help=( + "Path to the canary namespace YAML. Defaults to " + "managed_staging_canary_catalog.DEFAULT_CANARY_CATALOG_PATH." + ), + ) + parser.add_argument( + "--base-dir", + type=Path, + default=None, + help=( + "Base directory candidate_contract_roots resolve against. " + "Defaults to the repo root." + ), + ) + parser.add_argument( + "--bootstrap-servers", + type=str, + default=None, + help=( + "Kafka bootstrap servers. Defaults to the same resolution " + "ModelKafkaEventBusConfig.default() uses (KAFKA_BOOTSTRAP_SERVERS " + "env var)." + ), + ) + parser.add_argument( + "--create-missing", + action="store_true", + default=False, + help=( + "Create catalog-listed missing topics using the namespace's " + "declared partition/replication defaults. Default is CHECK-ONLY " + "(zero mutations)." + ), + ) + return parser + + +async def _run(args: argparse.Namespace) -> int: + namespace = load_canary_namespace(args.namespace_path) + catalog = build_canary_catalog_from_candidate(namespace, base_dir=args.base_dir) + + # Resolve bootstrap servers through the same boundary EventBusKafka/ + # TopicProvisioner use (ModelKafkaEventBusConfig.default() reads + # KAFKA_BOOTSTRAP_SERVERS via its own, already-declared env resolution) -- + # this module never reads os.environ directly. + bootstrap_servers = ( + args.bootstrap_servers or ModelKafkaEventBusConfig.default().bootstrap_servers + ) + admin = await open_admin_client(bootstrap_servers=bootstrap_servers) + try: + existing_topics, existing_groups = await fetch_live_topics_and_groups(admin) + diff = build_topic_diff( + catalog, + existing_topics=existing_topics, + existing_groups=existing_groups, + ) + + created: tuple[str, ...] = () + failed: tuple[str, ...] = () + if args.create_missing and diff.missing_topics: + created, failed = await create_missing_catalog_topics(admin, catalog, diff) + finally: + await admin.close() + + print(_render_report(diff, created=created, failed=failed)) # noqa: T201 + + if failed: + return 1 + if args.create_missing: + # After an opt-in create pass, success means every previously-missing + # topic is now accounted for (created or already existing). + return 0 + # Fail-closed gate semantics: required *topics* missing is a hard failure. + # Consumer groups are lazily created by their consumers and are not + # required to pre-exist, so they never fail the gate on their own. + return 0 if not diff.missing_topics else 1 + + +def _cli_main(argv: Sequence[str] | None = None) -> None: + """CLI entrypoint. + + Usage: + uv run python -m omnibase_infra.topics.managed_staging_topic_checker \\ + [--create-missing] + + Exits nonzero when required catalog topics/groups are missing (check-only + mode) or when a create-missing pass fails to create a catalog topic -- + fail-closed for use as a CI/ops gate. + """ + parser = _build_arg_parser() + args = parser.parse_args(argv) + exit_code = asyncio.run(_run(args)) + sys.exit(exit_code) + + +if __name__ == "__main__": + _cli_main() + + +__all__: list[str] = [ + "build_topic_diff", + "create_missing_catalog_topics", + "fetch_live_topics_and_groups", + "open_admin_client", +] diff --git a/src/omnibase_infra/topics/model_canary_namespace.py b/src/omnibase_infra/topics/model_canary_namespace.py index e036b9039c..0a1706ea7d 100644 --- a/src/omnibase_infra/topics/model_canary_namespace.py +++ b/src/omnibase_infra/topics/model_canary_namespace.py @@ -18,6 +18,10 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator +from omnibase_infra.topics.model_topic_provisioning_policy import ( + MANAGED_MINIMUM_REPLICATION_FACTOR, +) + def iam_pattern_authorizes(name: str, patterns: Iterable[str]) -> bool: """Return whether ``name`` is authorized by any MSK IAM resource pattern. @@ -63,7 +67,11 @@ class ModelCanaryNamespace(BaseModel): group_start_policy: Consumer group start/reset policy for the canary. default_partitions: Conservative per-topic partition count. Final sizing is the SS2.8 partition-pressure decision applied at Phase 3. - default_replication_factor: Per-topic replication factor. + default_replication_factor: Per-topic replication factor. Floored at 2 + (OMN-15395): this namespace only ever targets the managed cluster, + where RF1 is unrecoverable data loss on a single broker failure and + blocks broker updates. A regression to 1 fails model validation + rather than being caught downstream. candidate_contract_roots: Repo-relative roots scanned to extract the candidate's contract-owned topic suffixes + subscribing nodes. """ @@ -79,7 +87,10 @@ class ModelCanaryNamespace(BaseModel): iam_group_patterns: tuple[str, ...] = Field(..., min_length=1) group_start_policy: Literal["earliest", "latest", "none"] = "earliest" default_partitions: int = Field(default=1, ge=1) - default_replication_factor: int = Field(default=2, ge=1) + default_replication_factor: int = Field( + default=MANAGED_MINIMUM_REPLICATION_FACTOR, + ge=MANAGED_MINIMUM_REPLICATION_FACTOR, + ) candidate_contract_roots: tuple[str, ...] = Field(default_factory=tuple) @model_validator(mode="after") diff --git a/src/omnibase_infra/topics/model_managed_staging_topic_diff.py b/src/omnibase_infra/topics/model_managed_staging_topic_diff.py new file mode 100644 index 0000000000..c83df09128 --- /dev/null +++ b/src/omnibase_infra/topics/model_managed_staging_topic_diff.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# Copyright (c) 2026 OmniNode Team +"""Typed diff report for the managed-staging canary catalog vs. a live broker +(OMN-15283). + +:func:`~omnibase_infra.topics.managed_staging_topic_checker.build_topic_diff` +compares a generated :class:`~omnibase_infra.topics.model_canary_catalog.ModelCanaryCatalog` +against a snapshot of a broker's existing topics + consumer groups and reports +three disjoint buckets per resource kind: + +* **missing** -- catalog-listed names absent from the broker (the fail-closed + gate signal); +* **present** -- catalog-listed names already on the broker; +* **out_of_catalog** -- existing names that live under the catalog's prefix but + are not catalog-listed (a namespace conflict / stray name -- reported, never + mutated). +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelManagedStagingTopicDiff(BaseModel): + """Typed diff of a managed-staging canary catalog against a live broker. + + Attributes: + topic_prefix: Canary topic prefix the diff was computed against. + group_prefix: Canary group prefix the diff was computed against. + missing_topics: Catalog topics absent from the broker snapshot. + present_topics: Catalog topics already present on the broker. + out_of_catalog_topics: Existing topics under ``topic_prefix`` that are + not catalog-listed. + missing_groups: Catalog groups absent from the broker snapshot. + present_groups: Catalog groups already present on the broker. + out_of_catalog_groups: Existing groups under ``group_prefix`` that are + not catalog-listed. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + topic_prefix: str + group_prefix: str + missing_topics: tuple[str, ...] = Field(default_factory=tuple) + present_topics: tuple[str, ...] = Field(default_factory=tuple) + out_of_catalog_topics: tuple[str, ...] = Field(default_factory=tuple) + missing_groups: tuple[str, ...] = Field(default_factory=tuple) + present_groups: tuple[str, ...] = Field(default_factory=tuple) + out_of_catalog_groups: tuple[str, ...] = Field(default_factory=tuple) + + @property + def is_fully_present(self) -> bool: + """``True`` iff every catalog-listed topic and group is on the broker.""" + return not (self.missing_topics or self.missing_groups) + + @property + def has_out_of_catalog(self) -> bool: + """``True`` iff any stray prefix-matching name was found on the broker.""" + return bool(self.out_of_catalog_topics or self.out_of_catalog_groups) + + +__all__: list[str] = ["ModelManagedStagingTopicDiff"] diff --git a/src/omnibase_infra/topics/model_topic_provisioning_diff.py b/src/omnibase_infra/topics/model_topic_provisioning_diff.py new file mode 100644 index 0000000000..ebcd55624e --- /dev/null +++ b/src/omnibase_infra/topics/model_topic_provisioning_diff.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Desired-vs-live topic diff, shared by every provisioning path (OMN-15395). + +A full startup provisioning pass used to issue one ``CreateTopics`` call per +known topic — ~1,280 authorizations across ~1,026 topic ARNs on every boot, +whether or not anything was missing, which is what repeatedly tripped the AWS +unauthorized-API-call alarm. The provisioner never asked the broker what +already existed; it relied on ``TopicAlreadyExistsError`` as flow control. + +:func:`build_provisioning_diff` is the single, pure, transport-free set +comparison every provisioning path now runs *before* issuing creates. It is +also the topic-side engine behind +``managed_staging_topic_checker.build_topic_diff`` (which layers the canary +namespace's prefix/consumer-group semantics on top) so there is exactly one +diff implementation, not two. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelTopicProvisioningDiff(BaseModel): + """Desired topic set diffed against a live broker snapshot. + + Attributes: + desired_topics: Every topic name the provisioning pass wants to exist. + missing_topics: Desired names absent from the broker — the ONLY names a + provisioning pass may issue ``CreateTopics`` for. + present_topics: Desired names already on the broker. Never re-created + and never mutated; spec drift on these is reported, not repaired. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + desired_topics: tuple[str, ...] = Field(default_factory=tuple) + missing_topics: tuple[str, ...] = Field(default_factory=tuple) + present_topics: tuple[str, ...] = Field(default_factory=tuple) + + @property + def has_missing(self) -> bool: + """``True`` iff at least one desired topic is absent from the broker.""" + return bool(self.missing_topics) + + +def build_provisioning_diff( + desired_topics: Iterable[str], + existing_topics: Iterable[str], +) -> ModelTopicProvisioningDiff: + """Diff a desired topic set against a live broker snapshot. + + Pure and transport-free so provisioning behaviour is unit-testable without + a broker. + + Args: + desired_topics: Topic names the caller wants to exist. Order is + preserved (callers provision in contract-declared priority order); + duplicates are collapsed. + existing_topics: Topic names currently on the broker. + + Returns: + A :class:`ModelTopicProvisioningDiff` splitting the desired set into + missing (create these) and present (leave alone). + """ + desired = tuple(dict.fromkeys(desired_topics)) + existing = set(existing_topics) + return ModelTopicProvisioningDiff( + desired_topics=desired, + missing_topics=tuple(name for name in desired if name not in existing), + present_topics=tuple(name for name in desired if name in existing), + ) + + +__all__: list[str] = [ + "ModelTopicProvisioningDiff", + "build_provisioning_diff", +] diff --git a/src/omnibase_infra/topics/model_topic_provisioning_policy.py b/src/omnibase_infra/topics/model_topic_provisioning_policy.py new file mode 100644 index 0000000000..d808d86218 --- /dev/null +++ b/src/omnibase_infra/topics/model_topic_provisioning_policy.py @@ -0,0 +1,548 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Environment-resolved topic replication policy (OMN-15395). + +Why this exists +--------------- +AWS Health raised ``AWS_KAFKA_HIGH_RISK_CONFIG_RF_EQUALS_ONE`` against +``omninode-dev-msk``: 519 of 1,610 topics sat at replication factor 1. The +broker default is RF2 and broker-side auto-create is disabled, so every one of +those topics was created by an explicit ``CreateTopics`` call from this repo's +provisioner that *overrode the broker default down to 1* — a module-level +``DEFAULT_EVENT_TOPIC_REPLICATION_FACTOR = 1`` applied silently whenever the +owning contract declared nothing. + +This module removes the implicit default and replaces it with one explicit, +typed resolution seam: + +* ``ModelTopicSpec.replication_factor is None`` now means **the owning contract + declared nothing** — it is no longer silently 1. +* :class:`ModelTopicProvisioningPolicy` is the *only* place a ``None`` becomes + a concrete number, the only place a concrete number is checked against the + environment's durability floor, and the only place a value is reduced to what + the target broker can physically host. +* Against a managed cluster, a *declared* replication factor below the floor + (the RF1 case) is rejected before any ``CreateTopics`` is issued. An + *undeclared* replication factor resolves to the managed durability floor — + never to 1, and never below the MSK broker's own RF2 default. + +The capacity ceiling is MEASURED, never assumed +----------------------------------------------- +A policy constructed from configuration alone carries **no** capacity ceiling +(``capacity_replication_factor is None``, ``broker_count is None``). The ceiling +is installed only by :meth:`ModelTopicProvisioningPolicy.with_broker_capacity`, +from a live ``describe_cluster`` broker count read off the same admin client +that will issue the ``CreateTopics`` +(:mod:`omnibase_infra.topics.broker_capacity_probe`). + +This is load-bearing, not a refinement. The first revision of this policy set +``capacity_replication_factor = 1`` unconditionally for every cluster whose +``sasl_mechanism`` was not ``AWS_MSK_IAM``, and +:meth:`resolve_replication_factor` then silently reduced every declared value +down to it. ``ModelKafkaEventBusConfig`` accepts PLAIN / SCRAM-SHA-256 / +SCRAM-SHA-512 / OAUTHBEARER as well, so *any* multi-broker cluster not reached +over MSK IAM — including an MSK cluster fronted by SCRAM — had its +contract-declared RF2/RF3 clamped to RF1: the exact +``AWS_KAFKA_HIGH_RISK_CONFIG_RF_EQUALS_ONE`` condition this module exists to +eliminate, reintroduced by the mechanism meant to prevent it. A ceiling that is +an assumption about the broker rather than a measurement of it is a durability +downgrade wearing a capacity argument. + +Two invariants make the ceiling safe: + +* it may only ever *reduce* a value, never raise one; and +* it is never installed below the profile's durability floor — a measured + broker count under the floor leaves the ceiling unset so that resolution + *refuses* instead of silently creating an under-replicated topic. + +A durability requirement is expressed in the CONTRACT. The ceiling exists only +so a contract-declared RF2 does not make provisioning impossible on a broker +that physically has one node, and it says so out loud (``logger.warning``) every +time it fires. + +Why an undeclared RF resolves rather than refuses (deviation from a literal +reading of OMN-15395 acceptance criterion (a), recorded here because it is a +judgement call) +--------------------------------------------------------------------------- +A refuse-on-undeclared policy was implemented first and measured against the +real contract tree: **168 of 168** provisioned topics carry no contract-declared +replication factor, and **75 of those have no producing declaration anywhere in +this repository** — they appear only in ``event_bus.subscribe_topics``, i.e. +they are produced by omniclaude / omnimarket / CLI relays. Refusing every +undeclared topic therefore makes provisioning a permanent 100% no-op on MSK, +with a third of the topic universe having no in-repo contract that *could* be +fixed. That is strictly worse than the bug being repaired. + +What is actually forbidden by (a) is a *module-level constant of 1* silently +overriding the broker's own default. This policy is the opposite of that: the +value is profile-scoped, equal to the managed cluster's own RF2 default, and +identical to the number the managed-staging namespace catalog already declares +(``managed_staging_canary_catalog_namespace.yaml`` → +``default_replication_factor: 2``). Acceptance criterion (c) names the +*divergence* between that catalog's RF2 and the manager's RF1 as the defect; +converging both onto :data:`MANAGED_MINIMUM_REPLICATION_FACTOR` is the fix. A +contract that declares its own replication factor always wins, and a declared +value below the floor always fails closed. + +The policy is resolved from the live Kafka client configuration, not from a +lane label or a caller argument: ``sasl_mechanism == "AWS_MSK_IAM"`` is how +this codebase talks to MSK, so it is an un-forgeable discriminator for "this is +the managed cluster". +""" + +from __future__ import annotations + +import logging +from collections.abc import Sequence +from typing import TYPE_CHECKING, Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_infra.enums import EnumInfraTransportType +from omnibase_infra.errors import TopicReplicationPolicyError +from omnibase_infra.models.errors.model_infra_error_context import ( + ModelInfraErrorContext, +) +from omnibase_infra.topics.enum_topic_provisioning_profile import ( + EnumTopicProvisioningProfile, +) +from omnibase_infra.topics.model_topic_spec import ModelTopicSpec + +if TYPE_CHECKING: + from omnibase_infra.event_bus.models.config.model_kafka_event_bus_config import ( + ModelKafkaEventBusConfig, + ) + +logger = logging.getLogger(__name__) + +# The managed-staging durability floor, and the value an undeclared replication +# factor resolves to on the managed profile. RF1 on MSK is unrecoverable data +# loss on a single broker failure and blocks broker update operations; the MSK +# broker default is RF2, which is also what the managed-staging canary namespace +# declares (``managed_staging_canary_catalog_namespace.yaml`` → +# ``default_replication_factor``, bound to this constant in +# ``model_canary_namespace.py``). One constant, both paths — the RF2-here / +# RF1-there divergence is exactly what OMN-15395 (c) calls the defect. +MANAGED_MINIMUM_REPLICATION_FACTOR: int = 2 + +# What an UNDECLARED replication factor resolves to on a self-hosted broker +# whose node count has not been measured yet. It is a floor-of-last-resort for +# the unmeasured case only: once +# :meth:`ModelTopicProvisioningPolicy.with_broker_capacity` binds a live broker +# count, an undeclared RF on a multi-node self-hosted cluster resolves to +# ``MANAGED_MINIMUM_REPLICATION_FACTOR`` instead — a 3-broker Redpanda has no +# more business minting RF1 topics than MSK does. +# +# This constant is NOT a capacity ceiling. Nothing reduces a declared value to +# it; only a measured broker count can install a ceiling. +SELF_HOSTED_REPLICATION_FACTOR: int = 1 + +# The SASL mechanism this codebase uses to authenticate to AWS MSK. +MANAGED_SASL_MECHANISM: str = "AWS_MSK_IAM" + + +class ModelTopicProvisioningPolicy(BaseModel): + """Resolves and validates the replication factor for a topic being created. + + Attributes: + profile: Durability class of the target broker. + minimum_replication_factor: Hard floor. A spec resolving below this is + rejected fail-closed — never clamped, never warned-and-continued. + default_replication_factor: The value an *undeclared* replication + factor resolves to. ``None`` means there is no default and an + undeclared replication factor is refused. + capacity_replication_factor: The most replicas the target broker can + physically host, **as measured** from a live ``describe_cluster`` + broker count. A declared value ABOVE this is reduced to it (at + ``logger.warning`` — a durability downgrade is never emitted below + WARNING) because a ``CreateTopics`` carrying RF > broker count is + rejected outright with ``INVALID_REPLICATION_FACTOR``. ``None`` + means **unmeasured, therefore no ceiling** — nothing is reduced. + This only ever *reduces*, and never below + ``minimum_replication_factor``: the validator forbids a ceiling + under the floor, so a durability floor can never be silently + undercut by a capacity ceiling. + broker_count: The live broker count this policy was bound to, or + ``None`` when the cluster has not been probed. Provenance for + ``capacity_replication_factor`` — an unmeasured policy is + structurally incapable of reducing anything. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + profile: EnumTopicProvisioningProfile + minimum_replication_factor: int = Field(ge=1) + default_replication_factor: int | None = Field(default=None, ge=1) + capacity_replication_factor: int | None = Field(default=None, ge=1) + broker_count: int | None = Field(default=None, ge=1) + + @model_validator(mode="after") + def _bounds_are_coherent(self) -> Self: + """Floor <= ceiling, and the default must sit inside both bounds.""" + if ( + self.capacity_replication_factor is not None + and self.capacity_replication_factor < self.minimum_replication_factor + ): + raise ValueError( + f"capacity_replication_factor=" + f"{self.capacity_replication_factor} is below " + f"minimum_replication_factor=" + f"{self.minimum_replication_factor}; a capacity ceiling may " + f"never undercut a durability floor — that would silently " + f"create topics the policy is supposed to reject" + ) + if self.default_replication_factor is None: + return self + if self.default_replication_factor < self.minimum_replication_factor: + raise ValueError( + f"default_replication_factor=" + f"{self.default_replication_factor} is below " + f"minimum_replication_factor=" + f"{self.minimum_replication_factor}; a policy cannot default to " + f"a value it would itself reject" + ) + if ( + self.capacity_replication_factor is not None + and self.default_replication_factor > self.capacity_replication_factor + ): + raise ValueError( + f"default_replication_factor=" + f"{self.default_replication_factor} exceeds " + f"capacity_replication_factor=" + f"{self.capacity_replication_factor}; a policy cannot default to " + f"a value the broker cannot host" + ) + return self + + @property + def is_managed(self) -> bool: + """``True`` when the target broker is a managed (MSK) cluster.""" + return self.profile is EnumTopicProvisioningProfile.MANAGED + + @classmethod + def self_hosted( + cls, *, broker_count: int | None = None + ) -> ModelTopicProvisioningPolicy: + """Policy for a self-hosted broker (local Redpanda, ``.201``, CI). + + With ``broker_count`` unset the policy is **unmeasured**: there is no + capacity ceiling, so a contract-declared RF2/RF3 reaches + ``CreateTopics`` exactly as declared. Only a measured node count may + reduce it — see :meth:`with_broker_capacity`. + + An undeclared replication factor resolves to + :data:`SELF_HOSTED_REPLICATION_FACTOR` while unmeasured, and to the + durable :data:`MANAGED_MINIMUM_REPLICATION_FACTOR` once a measurement + proves the cluster has the nodes for it. + + Args: + broker_count: Live node count, when already known. + """ + policy = cls( + profile=EnumTopicProvisioningProfile.SELF_HOSTED, + minimum_replication_factor=SELF_HOSTED_REPLICATION_FACTOR, + default_replication_factor=SELF_HOSTED_REPLICATION_FACTOR, + capacity_replication_factor=None, + ) + if broker_count is None: + return policy + return policy.with_broker_capacity(broker_count) + + @classmethod + def managed( + cls, *, broker_count: int | None = None + ) -> ModelTopicProvisioningPolicy: + """Policy for a managed (MSK) cluster: RF1 rejected fail-closed. + + An undeclared replication factor resolves to + :data:`MANAGED_MINIMUM_REPLICATION_FACTOR` — the cluster's own broker + default and the value the managed-staging namespace catalog declares — + never to 1. See the module docstring for why this resolves rather than + refuses. + + Args: + broker_count: Live node count, when already known. A managed + cluster with FEWER nodes than the durability floor gets no + ceiling at all, so resolution refuses rather than clamping. + """ + policy = cls( + profile=EnumTopicProvisioningProfile.MANAGED, + minimum_replication_factor=MANAGED_MINIMUM_REPLICATION_FACTOR, + default_replication_factor=MANAGED_MINIMUM_REPLICATION_FACTOR, + capacity_replication_factor=None, + ) + if broker_count is None: + return policy + return policy.with_broker_capacity(broker_count) + + def with_broker_capacity(self, broker_count: int) -> ModelTopicProvisioningPolicy: + """Bind this policy to a MEASURED live broker count. + + This is the only way a capacity ceiling is ever installed. The + durability floor and the profile are carried through untouched; the + measurement may move exactly two things: + + * the ceiling, to ``broker_count`` — but only when the cluster has at + least ``minimum_replication_factor`` nodes. A measurement *below* the + floor leaves the ceiling unset (with a warning) so resolution refuses + rather than clamping a topic under the durability floor. + * the undeclared-RF default, UP toward + :data:`MANAGED_MINIMUM_REPLICATION_FACTOR` when the measured cluster + can host it. A 3-node self-hosted broker defaults undeclared topics + to RF2, not RF1 — the unmeasured RF1 default is a conservative + placeholder for "we have not looked", not a statement about the + cluster. + + Args: + broker_count: Live node count from ``describe_cluster``. + + Returns: + A new policy bound to the measurement. + + Raises: + ValueError: ``broker_count`` is not a positive node count. + """ + if broker_count < 1: + raise ValueError( + f"broker_count={broker_count} is not a live node count; a " + "capacity ceiling may only be installed from a real " + "measurement (OMN-15395)" + ) + + capacity: int | None = broker_count + if broker_count < self.minimum_replication_factor: + logger.warning( + "Cluster reports %d broker(s), below the %s durability floor " + "of %d; NOT installing a capacity ceiling that would undercut " + "the floor — under-replicated specs will be refused rather " + "than silently created (OMN-15395)", + broker_count, + self.profile.value, + self.minimum_replication_factor, + ) + capacity = None + + default = self.default_replication_factor + if default is not None: + # Raise a conservative unmeasured default up to the durable value + # the measured cluster can actually host, then hold it under the + # ceiling. Never below the floor: `capacity` is either None or + # >= minimum_replication_factor by the branch above. + default = max( + default, min(MANAGED_MINIMUM_REPLICATION_FACTOR, broker_count) + ) + if capacity is not None: + default = min(default, capacity) + default = max(default, self.minimum_replication_factor) + + return ModelTopicProvisioningPolicy( + profile=self.profile, + minimum_replication_factor=self.minimum_replication_factor, + default_replication_factor=default, + capacity_replication_factor=capacity, + broker_count=broker_count, + ) + + @classmethod + def from_kafka_config( + cls, config: ModelKafkaEventBusConfig + ) -> ModelTopicProvisioningPolicy: + """Derive the policy from the live Kafka client configuration. + + MSK IAM auth (``sasl_mechanism == "AWS_MSK_IAM"``) is how this codebase + reaches the managed cluster, so it is the discriminator. It is read from + the same config object the admin client authenticates with, which is why + a caller cannot declare itself self-hosted while pointed at MSK. + + The discriminator selects the *durability floor*, and nothing else. It + deliberately does NOT imply a node count: ``ModelKafkaEventBusConfig`` + also accepts PLAIN / SCRAM-SHA-256 / SCRAM-SHA-512 / OAUTHBEARER, so a + non-IAM cluster may well be multi-broker. The returned policy is + therefore **unmeasured** — no capacity ceiling — until + :meth:`with_broker_capacity` binds a live ``describe_cluster`` count. + """ + if config.sasl_mechanism == MANAGED_SASL_MECHANISM: + return cls.managed() + return cls.self_hosted() + + @classmethod + def from_env(cls) -> ModelTopicProvisioningPolicy: + """Derive the policy from the standard runtime Kafka environment.""" + from omnibase_infra.event_bus.models.config.model_kafka_event_bus_config import ( + ModelKafkaEventBusConfig, + ) + + return cls.from_kafka_config(ModelKafkaEventBusConfig.default()) + + def resolve_replication_factor(self, *, topic: str, declared: int | None) -> int: + """Return the explicit replication factor to create ``topic`` with. + + Args: + topic: Topic name (for the error message). + declared: The owning contract's declared replication factor, or + ``None`` when the contract declared none. + + Returns: + The resolved, explicit replication factor. + + Raises: + TopicReplicationPolicyError: When ``declared`` is ``None`` and the + policy has no default, or when the resolved value is below the + policy's durability floor (the RF1-on-MSK case). + """ + if declared is None: + if self.default_replication_factor is None: + raise self._violation( + topic=topic, + detail=( + "no replication_factor declared by the owning contract " + f"and the {self.profile.value} policy has no default. " + "Declare topic_config.replication_factor >= " + f"{self.minimum_replication_factor} in the contract that " + "owns this topic (OMN-13238 seam) — refusing to create a " + "topic whose durability nobody declared" + ), + declared=declared, + ) + resolved = self.default_replication_factor + else: + resolved = declared + + # Capacity ceiling: only ever reduces, only ever from a MEASURED broker + # count, and the validator guarantees it cannot reduce below the + # durability floor. A declared RF2 on a broker measured at one node + # becomes RF1 here rather than failing CreateTopics with + # INVALID_REPLICATION_FACTOR. + # + # WARNING, not INFO: this is a durability downgrade of a value some + # contract explicitly asked for. Emitting it below WARNING makes it + # invisible under normal log filtering, which is how a silent RF + # reduction stops being auditable. + if ( + self.capacity_replication_factor is not None + and resolved > self.capacity_replication_factor + ): + logger.warning( + "Reducing replication_factor %d -> %d for topic %s: the %s " + "cluster measured %s broker(s) and cannot host more replicas " + "than it has nodes (OMN-15395)", + resolved, + self.capacity_replication_factor, + topic, + self.profile.value, + self.broker_count, + ) + resolved = self.capacity_replication_factor + + if resolved < self.minimum_replication_factor: + raise self._violation( + topic=topic, + detail=( + f"replication_factor={resolved} is below the " + f"{self.profile.value} floor of " + f"{self.minimum_replication_factor}. RF1 on a managed " + "cluster is unrecoverable data loss on a single broker " + "failure and blocks broker updates " + "(AWS_KAFKA_HIGH_RISK_CONFIG_RF_EQUALS_ONE). Not clamping, " + "not warning — refusing to create" + ), + declared=declared, + ) + return resolved + + def resolve_spec(self, spec: ModelTopicSpec) -> ModelTopicSpec: + """Return ``spec`` with an explicit, policy-approved replication factor. + + Partitions and ``kafka_config`` are carried through untouched — this + seam only ever *adds* the resolved replication factor, it never + re-defaults a value the contract declared. + """ + resolved = self.resolve_replication_factor( + topic=spec.suffix, declared=spec.replication_factor + ) + if spec.replication_factor == resolved: + return spec + return spec.model_copy(update={"replication_factor": resolved}) + + def _violation( + self, *, topic: str, detail: str, declared: int | None + ) -> TopicReplicationPolicyError: + context = ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.KAFKA, + operation="resolve_topic_replication_factor", + target_name=topic, + ) + return TopicReplicationPolicyError( + f"Refusing to provision topic {topic!r}: {detail}", + context=context, + topic=topic, + declared_replication_factor=declared, + profile=self.profile.value, + minimum_replication_factor=self.minimum_replication_factor, + ) + + +def resolve_specs_for_creation( + policy: ModelTopicProvisioningPolicy, + specs: Sequence[ModelTopicSpec], +) -> tuple[ModelTopicSpec, ...]: + """Resolve EVERY spec before any ``CreateTopics`` is issued. + + Fail-closed and batch-scoped: a single spec that violates the environment + replication policy — the RF1-on-MSK case — aborts the whole batch with ZERO + creates issued. Not a warning, not a clamp-and-continue, and not a per-topic + skip that lets the rest of the pass proceed while a durability defect sits + unfixed in a contract we own. Every violation is collected first so one run + surfaces every offending contract instead of one per redeploy. + + Module-level rather than a method (matching ``build_provisioning_diff``'s + shape in the sibling diff module) because there is more than one live + ``CreateTopics`` path in this repository: the runtime provisioner + (:class:`~omnibase_infra.event_bus.service_topic_manager.TopicProvisioner`), + the managed-staging canary checker, and the operator CLI + ``scripts/create_kafka_topics.py``. A batch resolver owned by one of them is + a resolver the others silently do without — which is exactly how the CLI + shipped a flat ``--replication-factor 1`` default that discarded every + contract's declared ``topic_config.replication_factor`` (OMN-15395 D2). + + Args: + policy: The environment policy to resolve against. + specs: The specs about to be created. + + Returns: + The resolved specs, each carrying an explicit replication factor, in + input order. + + Raises: + TopicReplicationPolicyError: Any spec violates the policy. The message + enumerates up to ten violations and counts the rest. + """ + resolved: list[ModelTopicSpec] = [] + violations: list[str] = [] + for spec in specs: + try: + resolved.append(policy.resolve_spec(spec)) + except TopicReplicationPolicyError as exc: + violations.append(str(exc)) + if violations: + shown = violations[:10] + suffix = ( + f" (+{len(violations) - len(shown)} more)" + if len(violations) > len(shown) + else "" + ) + raise TopicReplicationPolicyError( + f"Refusing to provision {len(violations)} topic(s) under the " + f"{policy.profile.value} replication policy; no CreateTopics was " + "issued. Violations: " + " | ".join(shown) + suffix + ) + return tuple(resolved) + + +__all__: list[str] = [ + "MANAGED_MINIMUM_REPLICATION_FACTOR", + "MANAGED_SASL_MECHANISM", + "SELF_HOSTED_REPLICATION_FACTOR", + "ModelTopicProvisioningPolicy", + "resolve_specs_for_creation", +] diff --git a/src/omnibase_infra/topics/model_topic_spec.py b/src/omnibase_infra/topics/model_topic_spec.py index fb85f4be2a..caef128cf1 100644 --- a/src/omnibase_infra/topics/model_topic_spec.py +++ b/src/omnibase_infra/topics/model_topic_spec.py @@ -27,12 +27,17 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator -# Canonical defaults for platform topic creation. -# These live here (not in service_topic_manager) to avoid a circular import: +# Canonical partition default for platform topic creation. +# This lives here (not in service_topic_manager) to avoid a circular import: # topics/__init__ -> model_topic_spec -> service_topic_manager -> topics/__init__ -# service_topic_manager re-imports these constants for its own fallback path. +# +# OMN-15395: there is deliberately NO replication-factor constant here any more. +# ``DEFAULT_EVENT_TOPIC_REPLICATION_FACTOR = 1`` used to be applied silently +# whenever the owning contract declared nothing, which is how 519 RF1 topics +# were created on MSK against a broker whose own default is RF2. Replication is +# now resolved exclusively by ``ModelTopicProvisioningPolicy``, which fails +# closed on a managed cluster instead of defaulting. DEFAULT_EVENT_TOPIC_PARTITIONS: int = 6 -DEFAULT_EVENT_TOPIC_REPLICATION_FACTOR: int = 1 class ModelTopicSpec(BaseModel): @@ -41,7 +46,11 @@ class ModelTopicSpec(BaseModel): Attributes: suffix: Full ONEX 5-segment topic name (e.g., "onex.evt.platform.node-registration.v1"). # onex-topic-allow: pending contract auto-wiring partitions: Number of partitions for the topic. - replication_factor: Replication factor for the topic. + replication_factor: Replication factor declared by the owning contract. + ``None`` means **the contract declared none** — it does NOT mean 1. + Resolution to an explicit value (or a fail-closed refusal) is + :class:`~omnibase_infra.topics.model_topic_provisioning_policy.ModelTopicProvisioningPolicy`'s + job and happens on the creation path, never here (OMN-15395). kafka_config: Optional Kafka topic config overrides (e.g., {"cleanup.policy": "compact"}). provisioning_priority: Lower values are provisioned first. """ @@ -50,7 +59,7 @@ class ModelTopicSpec(BaseModel): suffix: str partitions: int = DEFAULT_EVENT_TOPIC_PARTITIONS - replication_factor: int = DEFAULT_EVENT_TOPIC_REPLICATION_FACTOR + replication_factor: int | None = Field(default=None, ge=1) kafka_config: Mapping[str, str] | None = Field(default=None) provisioning_priority: int = 100 @@ -67,6 +76,5 @@ def freeze_kafka_config( __all__: list[str] = [ "DEFAULT_EVENT_TOPIC_PARTITIONS", - "DEFAULT_EVENT_TOPIC_REPLICATION_FACTOR", "ModelTopicSpec", ] diff --git a/src/omnibase_infra/topics/platform_topic_suffixes.py b/src/omnibase_infra/topics/platform_topic_suffixes.py index ceee10d80b..67a915e740 100644 --- a/src/omnibase_infra/topics/platform_topic_suffixes.py +++ b/src/omnibase_infra/topics/platform_topic_suffixes.py @@ -60,6 +60,9 @@ from omnibase_core.errors import OnexError from omnibase_core.validation import validate_topic_suffix +from omnibase_infra.topics.model_topic_provisioning_policy import ( + MANAGED_MINIMUM_REPLICATION_FACTOR, +) from omnibase_infra.topics.model_topic_spec import ModelTopicSpec from omnibase_infra.utils.util_runtime_packages import is_runtime_package_active @@ -1041,6 +1044,7 @@ ModelTopicSpec( suffix=SUFFIX_RUNNER_HEALTH_SNAPSHOT, partitions=1, + replication_factor=MANAGED_MINIMUM_REPLICATION_FACTOR, kafka_config={ "retention.ms": "604800000", "cleanup.policy": "delete", @@ -1050,6 +1054,7 @@ ModelTopicSpec( suffix=SUFFIX_NETWORK_POOL_STATUS, partitions=1, + replication_factor=MANAGED_MINIMUM_REPLICATION_FACTOR, kafka_config={ "retention.ms": "604800000", "cleanup.policy": "delete", @@ -1059,6 +1064,7 @@ ModelTopicSpec( suffix=SUFFIX_ROW_COUNT_DIAGNOSTIC, partitions=1, + replication_factor=MANAGED_MINIMUM_REPLICATION_FACTOR, kafka_config={ "retention.ms": "604800000", "cleanup.policy": "delete", @@ -1113,6 +1119,7 @@ ModelTopicSpec( suffix=SUFFIX_LLM_CALL_COMPLETED_INFRA, partitions=3, + replication_factor=MANAGED_MINIMUM_REPLICATION_FACTOR, kafka_config={ "retention.ms": "604800000", "cleanup.policy": "delete", @@ -1326,11 +1333,13 @@ ModelTopicSpec( suffix=SUFFIX_DELEGATION_AGENT_TASK_LIFECYCLE, partitions=3, + replication_factor=MANAGED_MINIMUM_REPLICATION_FACTOR, kafka_config={"retention.ms": "604800000", "cleanup.policy": "delete"}, ), ModelTopicSpec( suffix=SUFFIX_DELEGATION_INFERENCE_RESPONSE, partitions=3, + replication_factor=MANAGED_MINIMUM_REPLICATION_FACTOR, kafka_config={"retention.ms": "604800000", "cleanup.policy": "delete"}, ), ModelTopicSpec( @@ -2134,11 +2143,13 @@ def _snapshot_kafka_config() -> dict[str, str]: ModelTopicSpec( suffix=SUFFIX_TOPIC_CATALOG_RESPONSE, partitions=1, + replication_factor=MANAGED_MINIMUM_REPLICATION_FACTOR, kafka_config={"retention.ms": "3600000", "cleanup.policy": "delete"}, ), ModelTopicSpec( suffix=SUFFIX_TOPIC_CATALOG_CHANGED, partitions=1, + replication_factor=MANAGED_MINIMUM_REPLICATION_FACTOR, kafka_config={"retention.ms": "604800000", "cleanup.policy": "delete"}, ), # DLQ aggregation topic (OMN-6136) — consumed by omnidash /dlq dashboard @@ -2171,7 +2182,11 @@ def _snapshot_kafka_config() -> dict[str, str]: suffix=SUFFIX_INTELLIGENCE_PATTERN_LIFECYCLE_TRANSITIONED, partitions=3 ), ModelTopicSpec(suffix=SUFFIX_INTELLIGENCE_PATTERN_DISCOVERED, partitions=3), - ModelTopicSpec(suffix=SUFFIX_INTELLIGENCE_LLM_CALL_COMPLETED, partitions=3), + ModelTopicSpec( + suffix=SUFFIX_INTELLIGENCE_LLM_CALL_COMPLETED, + partitions=3, + replication_factor=MANAGED_MINIMUM_REPLICATION_FACTOR, + ), ModelTopicSpec(suffix=SUFFIX_INTELLIGENCE_DISPATCH_OUTCOME_EVALUATED, partitions=3), # Decision recording topics (OMN-2943 — previously unprovisioned gap) ModelTopicSpec(suffix=SUFFIX_INTELLIGENCE_DECISION_RECORDED_EVT, partitions=3), diff --git a/src/omnibase_infra/topology/__init__.py b/src/omnibase_infra/topology/__init__.py new file mode 100644 index 0000000000..6649952812 --- /dev/null +++ b/src/omnibase_infra/topology/__init__.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Checked-in deployment-topology authority and projection helpers.""" + +from omnibase_infra.topology.application_database import ( + APPLICATION_DATABASE_REF, + SUPPORTED_ENVIRONMENTS, + SUPPORTED_TOPOLOGY_PROFILES, + TOPOLOGY_PROFILE_INSTANCE_MAP, + load_environment_topology, + load_topology_profile, + render_database_projection, + validate_application_database_invariants, + validate_database_projection, + validate_docker_catalog_parity, + validate_docker_topology_profile_injections, +) + +__all__ = [ + "APPLICATION_DATABASE_REF", + "SUPPORTED_ENVIRONMENTS", + "SUPPORTED_TOPOLOGY_PROFILES", + "TOPOLOGY_PROFILE_INSTANCE_MAP", + "load_environment_topology", + "load_topology_profile", + "render_database_projection", + "validate_application_database_invariants", + "validate_database_projection", + "validate_docker_catalog_parity", + "validate_docker_topology_profile_injections", +] diff --git a/src/omnibase_infra/topology/application_database.py b/src/omnibase_infra/topology/application_database.py new file mode 100644 index 0000000000..5ca6c2f74e --- /dev/null +++ b/src/omnibase_infra/topology/application_database.py @@ -0,0 +1,720 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Authoritative application-database topology loading and parity validation. + +The checked-in ``instances/*.yaml`` files are the only platform authority used by +this module. Host-local ``~/.omnibase/topology.yaml`` is deliberately not searched: +local setup may render a projection there, but it cannot override platform truth. +""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path +from types import MappingProxyType +from typing import cast + +import yaml + +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_core.models.core import ModelDeploymentTopology +from omnibase_infra.docker.catalog.resolver import _load_manifest +from omnibase_infra.runtime.models.model_runtime_policy_contract import ( + ModelRuntimePolicyContract, +) +from omnibase_infra.topology.models import ( + ModelApplicationDatabaseTopologyProfile, + ModelApplicationDatabaseTopologyProfileCatalog, + ModelDockerDatabaseConsumerCatalog, +) + +APPLICATION_DATABASE_REF = "application" +APPLICATION_DATABASE_PHYSICAL_NAME = "omnidash_analytics" + +# OMN-15655 AC-2. ADR-0027 unified the tenant/internal *application* pair into +# one physical database and explicitly left "identity-plane and independently +# service-owned databases ... separate". omniintelligence is one of those: its +# own physical database, its own migration runner, its own DSN key +# (docs/patterns/db_url_contract.md). It is declared here so a node contract +# that names ``database_ref: omniintelligence`` resolves through the same typed +# authority as an application relation instead of hard-failing strict-mode +# auto-wiring at boot. +OMNIINTELLIGENCE_DATABASE_REF = "omniintelligence" +OMNIINTELLIGENCE_DATABASE_PHYSICAL_NAME = "omniintelligence" +_OMNIINTELLIGENCE_SCHEMAS = {"public": EnumDatabaseSchemaDomain.OMNINODE_INTERNAL} +_OMNIINTELLIGENCE_SCHEMA_OWNERS = {"public": "owner_omniintelligence"} +_OMNIINTELLIGENCE_BINDING_PRINCIPALS = { + "omninode_runtime_service": "role_omniintelligence" +} +# One DSN key across every instance: unlike the application pair, no lane +# rebinds the omniintelligence DSN to a different variable. +_OMNIINTELLIGENCE_BINDING_DSN_ENVS = { + "omninode_runtime_service": "OMNIINTELLIGENCE_DB_URL" +} + +# OMN-15337 (operator ruling R-q, 2026-08-05): the flat forward-migration set +# (POSTGRES_DB=omnibase_infra in docker-compose.infra.yml's forward-migration +# service, docs/patterns/db_url_contract.md's OMNIBASE_INFRA_DB_URL row) is +# omnibase_infra's own runtime database -- distinct from the "application" +# pair and from omniintelligence. delegation_workflow_state +# (docker/migrations/forward/090_create_delegation_workflow_state.sql) is +# ruled OMNINODE_INTERNAL: its tenant_id column is retained as denormalized +# provenance only (never an authorization key), and FORCE-RLS on the +# runtime's own FSM state store would break non-tenant-context access +# (recovery/retries/sweeps). Declared here so the physical database this +# table already lives in has a typed home, the same way OMN-15655 AC-2 gave +# omniintelligence one. +OMNIBASE_INFRA_DATABASE_REF = "omnibase_infra" +OMNIBASE_INFRA_DATABASE_PHYSICAL_NAME = "omnibase_infra" +_OMNIBASE_INFRA_SCHEMAS = {"public": EnumDatabaseSchemaDomain.OMNINODE_INTERNAL} +_OMNIBASE_INFRA_SCHEMA_OWNERS = {"public": "owner_omnibase_infra"} +_OMNIBASE_INFRA_BINDING_PRINCIPALS = {"omninode_runtime_service": "role_omnibase_infra"} +# One DSN key across every instance, per docs/patterns/db_url_contract.md. +_OMNIBASE_INFRA_BINDING_DSN_ENVS = {"omninode_runtime_service": "OMNIBASE_INFRA_DB_URL"} + +_EXPECTED_PROFILE_INSTANCE_MAP = { + "local": "local", + "test": "local", + "stability-test": "local", + "judge": "local", + "prod": "local", + "onex-dev": "onex-dev", + "onex-prod": "onex-prod", +} +_EXPECTED_PROFILE_INJECTION_SURFACES = { + "local": ("OmniNode-ai/omnibase_infra", "docker/docker-compose.infra.yml"), + "test": ("OmniNode-ai/omnibase_infra", "docker/docker-compose.e2e.yml"), + "stability-test": ( + "OmniNode-ai/omnibase_infra", + "docker/docker-compose.stability-test.yml", + ), + "judge": ("OmniNode-ai/omnibase_infra", "docker/docker-compose.judge.yml"), + "prod": ("OmniNode-ai/omnibase_infra", "docker/docker-compose.prod.yml"), + "onex-dev": ( + "OmniNode-ai/omninode_infra", + "k8s/onex-dev/runtime/configmap.yaml", + ), + "onex-prod": ( + "OmniNode-ai/omninode_infra", + "k8s/onex-prod/runtime/configmap.yaml", + ), +} +_EXPECTED_RUNTIME_POLICY_PROFILE_MAP = { + "dev": "local", + "stability-test": "stability-test", + "judge": "judge", + "prod": "prod", +} +SUPPORTED_TOPOLOGY_PROFILES = frozenset(_EXPECTED_PROFILE_INSTANCE_MAP) +TOPOLOGY_PROFILE_INSTANCE_MAP = MappingProxyType(_EXPECTED_PROFILE_INSTANCE_MAP) +# Compatibility name for the draft API. New consumers must use the explicit +# profile terminology because ONEX_ENVIRONMENT is a separate event namespace. +SUPPORTED_ENVIRONMENTS = SUPPORTED_TOPOLOGY_PROFILES + +_TOPOLOGY_INSTANCE_ROOT = Path(__file__).resolve().parent / "instances" +_PROFILE_CATALOG_PATH = ( + Path(__file__).resolve().parent / "application_database_profiles.yaml" +) +_REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +_TOPOLOGY_REPOSITORY = "OmniNode-ai/omnibase_infra" +_TOPOLOGY_SOURCE_PREFIX = "src/omnibase_infra/topology/instances" +_PROFILE_CATALOG_SOURCE_PATH = ( + "src/omnibase_infra/topology/application_database_profiles.yaml" +) +_TOPOLOGY_PROFILE_ENV_VAR = "ONEX_DATABASE_TOPOLOGY_PROFILE" +_TOPOLOGY_PROFILE_LINE = re.compile( + rf"^\s*{_TOPOLOGY_PROFILE_ENV_VAR}:\s*[\"']?(?P[a-z0-9-]+)" + r"[\"']?\s*(?:#.*)?$", + re.MULTILINE, +) + +_EXPECTED_SCHEMAS = { + "public": EnumDatabaseSchemaDomain.TENANT, + "tenant": EnumDatabaseSchemaDomain.TENANT, + "omninode_internal": EnumDatabaseSchemaDomain.OMNINODE_INTERNAL, + "platform_catalog": EnumDatabaseSchemaDomain.PLATFORM_CATALOG, +} +_EXPECTED_SCHEMA_OWNERS = { + "public": "owner_onex_tenant", + "tenant": "owner_onex_tenant", + "omninode_internal": "owner_omninode_internal", + "platform_catalog": "owner_platform_catalog", +} +_EXPECTED_BINDING_PRINCIPALS = { + "onex_api": "onex_api", + "tenant_projection": "tenant_projection_writer", + "app_dashboard": "app_dashboard", + "omninode_runtime_service": "omninode_runtime", +} +_EXPECTED_BINDING_DSN_ENVS = { + "local": { + "onex_api": "OMNINODE_CLOUD_DB_URL", + "tenant_projection": "OMNIDASH_ANALYTICS_DB_URL", + "app_dashboard": "OMNIDASH_ANALYTICS_DB_URL", + "omninode_runtime_service": "OMNINODE_INTERNAL_DB_URL", + }, + "onex-dev": { + "onex_api": "OMNINODE_CLOUD_DB_URL", + "tenant_projection": "OMNIDASH_ANALYTICS_DB_URL", + "app_dashboard": "DATABASE_URL", + "omninode_runtime_service": "OMNINODE_INTERNAL_DB_URL", + }, + "onex-prod": { + "onex_api": "OMNINODE_CLOUD_DB_URL", + "tenant_projection": "OMNIDASH_ANALYTICS_DB_URL", + "app_dashboard": "DATABASE_URL", + "omninode_runtime_service": "OMNINODE_INTERNAL_DB_URL", + }, +} + + +def _load_profile_catalog( + profile_catalog_path: Path | None = None, +) -> ModelApplicationDatabaseTopologyProfileCatalog: + """Load and validate the exact checked-in profile-to-instance contract.""" + path = profile_catalog_path or _PROFILE_CATALOG_PATH + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + catalog = ModelApplicationDatabaseTopologyProfileCatalog.model_validate(raw) + if set(catalog.profiles) != SUPPORTED_TOPOLOGY_PROFILES: + raise ValueError( + "Database topology profile set drift: expected " + f"{sorted(SUPPORTED_TOPOLOGY_PROFILES)}, got " + f"{sorted(catalog.profiles)}" + ) + actual_instance_map = { + profile: binding.instance for profile, binding in catalog.profiles.items() + } + if actual_instance_map != _EXPECTED_PROFILE_INSTANCE_MAP: + raise ValueError( + "Database topology profile/instance drift: expected " + f"{_EXPECTED_PROFILE_INSTANCE_MAP}, got {actual_instance_map}" + ) + actual_surfaces = { + profile: (binding.deployment_repository, binding.injection_path) + for profile, binding in catalog.profiles.items() + } + if actual_surfaces != _EXPECTED_PROFILE_INJECTION_SURFACES: + raise ValueError("Database topology profile injection surface drift") + actual_runtime_profiles = { + binding.runtime_policy_profile: profile + for profile, binding in catalog.profiles.items() + if binding.runtime_policy_profile is not None + } + if actual_runtime_profiles != _EXPECTED_RUNTIME_POLICY_PROFILE_MAP: + raise ValueError("Database topology runtime-policy profile drift") + return catalog + + +def _resolve_profile( + profile: str, + profile_catalog_path: Path | None = None, +) -> ModelApplicationDatabaseTopologyProfile: + """Resolve one exact profile without environment inference or fallback.""" + catalog = _load_profile_catalog(profile_catalog_path) + binding = catalog.profiles.get(profile) + if binding is None: + raise ValueError( + f"Unsupported database topology profile '{profile}'; expected one of " + f"{sorted(SUPPORTED_TOPOLOGY_PROFILES)}" + ) + return binding + + +def _topology_instance_path( + instance: str, + topology_root: Path | None = None, +) -> Path: + """Return one allowlisted checked-in topology instance without fallback.""" + if instance not in _EXPECTED_BINDING_DSN_ENVS: + raise ValueError(f"Unsupported database topology instance '{instance}'") + root = topology_root if topology_root is not None else _TOPOLOGY_INSTANCE_ROOT + path = root / f"{instance}.yaml" + if not path.is_file(): + raise FileNotFoundError( + f"Required checked-in deployment topology does not exist: {path}" + ) + return path + + +def load_topology_profile( + profile: str, + topology_root: Path | None = None, + *, + profile_catalog_path: Path | None = None, +) -> ModelDeploymentTopology: + """Load a topology by its independent database-topology profile.""" + binding = _resolve_profile(profile, profile_catalog_path) + topology = ModelDeploymentTopology.from_yaml( + _topology_instance_path(binding.instance, topology_root) + ) + validate_application_database_invariants(topology, binding.instance) + validate_omniintelligence_database_invariants(topology) + validate_omnibase_infra_database_invariants(topology) + return topology + + +def load_environment_topology( + environment: str, + topology_root: Path | None = None, + *, + profile_catalog_path: Path | None = None, +) -> ModelDeploymentTopology: + """Compatibility wrapper for the explicit database-topology profile API.""" + return load_topology_profile( + environment, + topology_root, + profile_catalog_path=profile_catalog_path, + ) + + +def validate_application_database_invariants( + topology: ModelDeploymentTopology, + topology_instance: str, +) -> None: + """Fail on physical database, schema, role, or binding drift.""" + if topology_instance not in _EXPECTED_BINDING_DSN_ENVS: + raise ValueError( + f"Unsupported database topology instance '{topology_instance}'" + ) + + database = topology.databases.get(APPLICATION_DATABASE_REF) + if database is None: + raise ValueError("Topology must declare the 'application' database resource") + if database.physical_name != APPLICATION_DATABASE_PHYSICAL_NAME: + raise ValueError( + "application database must resolve to " + f"'{APPLICATION_DATABASE_PHYSICAL_NAME}', got '{database.physical_name}'" + ) + + actual_schemas = {name: schema.domain for name, schema in database.schemas.items()} + if actual_schemas != _EXPECTED_SCHEMAS: + raise ValueError( + f"application schema/domain drift: expected {_EXPECTED_SCHEMAS}, " + f"got {actual_schemas}" + ) + actual_owners = {name: schema.owner for name, schema in database.schemas.items()} + if actual_owners != _EXPECTED_SCHEMA_OWNERS: + raise ValueError( + f"application schema-owner drift: expected {_EXPECTED_SCHEMA_OWNERS}, " + f"got {actual_owners}" + ) + + missing_principals = sorted( + set(_EXPECTED_BINDING_PRINCIPALS.values()) - database.principals.keys() + ) + if missing_principals: + raise ValueError(f"application principals missing: {missing_principals}") + + if set(database.bindings) != set(_EXPECTED_BINDING_PRINCIPALS): + raise ValueError( + "application binding drift: expected " + f"{sorted(_EXPECTED_BINDING_PRINCIPALS)}, got " + f"{sorted(database.bindings)}" + ) + for binding_name, expected_principal in _EXPECTED_BINDING_PRINCIPALS.items(): + binding = database.bindings[binding_name] + if binding.database_ref != APPLICATION_DATABASE_REF: + raise ValueError( + f"Binding '{binding_name}' must resolve to database_ref " + f"'{APPLICATION_DATABASE_REF}'" + ) + if binding.principal != expected_principal: + raise ValueError( + f"Binding '{binding_name}' principal drift: expected " + f"'{expected_principal}', got '{binding.principal}'" + ) + expected_dsn_env = _EXPECTED_BINDING_DSN_ENVS[topology_instance][binding_name] + if binding.dsn_env != expected_dsn_env: + raise ValueError( + f"Binding '{binding_name}' dsn_env drift: expected " + f"'{expected_dsn_env}', got '{binding.dsn_env}'" + ) + + bound_principals = {binding.principal for binding in database.bindings.values()} + if len(bound_principals) != len(database.bindings): + raise ValueError( + "Every application-domain pool must resolve to a distinct PostgreSQL " + "principal" + ) + if "omninode_runtime" not in topology.services: + raise ValueError("Topology must declare the omninode_runtime service") + if "omninode_runtime" in database.bindings: + raise ValueError( + "The omninode_runtime service name cannot double as a database binding; " + "use omninode_runtime_service for the PostgreSQL principal namespace" + ) + + +def validate_omniintelligence_database_invariants( + topology: ModelDeploymentTopology, +) -> None: + """Fail on drift in the omniintelligence service-database declaration. + + ``node_dispatch_outcome_bridge_effect`` runs on the ``effects`` runtime + profile, where ``ONEX_WIRING_STRICT_MODE`` makes an unresolved + ``database_ref`` a boot-fatal error rather than a skipped handler. Pinning + the declaration here means a silent edit to the instance YAML fails the + topology loader in CI instead of the pod at rollout. + """ + database = topology.databases.get(OMNIINTELLIGENCE_DATABASE_REF) + if database is None: + raise ValueError( + "Topology must declare the 'omniintelligence' service database; " + "node contracts declare database_ref: omniintelligence and " + "strict-mode auto-wiring fails closed without it" + ) + if database.physical_name != OMNIINTELLIGENCE_DATABASE_PHYSICAL_NAME: + raise ValueError( + "omniintelligence database must resolve to " + f"'{OMNIINTELLIGENCE_DATABASE_PHYSICAL_NAME}', got " + f"'{database.physical_name}'" + ) + + actual_schemas = {name: schema.domain for name, schema in database.schemas.items()} + if actual_schemas != _OMNIINTELLIGENCE_SCHEMAS: + raise ValueError( + "omniintelligence schema/domain drift: expected " + f"{_OMNIINTELLIGENCE_SCHEMAS}, got {actual_schemas}" + ) + actual_owners = {name: schema.owner for name, schema in database.schemas.items()} + if actual_owners != _OMNIINTELLIGENCE_SCHEMA_OWNERS: + raise ValueError( + "omniintelligence schema-owner drift: expected " + f"{_OMNIINTELLIGENCE_SCHEMA_OWNERS}, got {actual_owners}" + ) + + if set(database.bindings) != set(_OMNIINTELLIGENCE_BINDING_PRINCIPALS): + raise ValueError( + "omniintelligence binding drift: expected " + f"{sorted(_OMNIINTELLIGENCE_BINDING_PRINCIPALS)}, got " + f"{sorted(database.bindings)}" + ) + for ( + binding_name, + expected_principal, + ) in _OMNIINTELLIGENCE_BINDING_PRINCIPALS.items(): + binding = database.bindings[binding_name] + if binding.database_ref != OMNIINTELLIGENCE_DATABASE_REF: + raise ValueError( + f"Binding '{binding_name}' must resolve to database_ref " + f"'{OMNIINTELLIGENCE_DATABASE_REF}'" + ) + if binding.principal != expected_principal: + raise ValueError( + f"omniintelligence binding '{binding_name}' principal drift: " + f"expected '{expected_principal}', got '{binding.principal}'" + ) + expected_dsn_env = _OMNIINTELLIGENCE_BINDING_DSN_ENVS[binding_name] + if binding.dsn_env != expected_dsn_env: + raise ValueError( + f"omniintelligence binding '{binding_name}' dsn_env drift: " + f"expected '{expected_dsn_env}', got '{binding.dsn_env}'" + ) + + +def validate_omnibase_infra_database_invariants( + topology: ModelDeploymentTopology, +) -> None: + """Fail on drift in the omnibase_infra runtime-database declaration. + + ``delegation_workflow_state`` (OMN-14208 durable FSM state) is read and + written through the legacy ``state_io`` contract subcontract, which + resolves its DSN directly from ``_DB_URL_ENV_MAP`` rather than through a + ``db_io.db_tables`` binding. Pinning the declaration here means a silent + edit to the instance YAML fails the topology loader in CI, the same + protection ``validate_omniintelligence_database_invariants`` gives the + contract-driven path. + """ + database = topology.databases.get(OMNIBASE_INFRA_DATABASE_REF) + if database is None: + raise ValueError( + "Topology must declare the 'omnibase_infra' runtime database; " + "delegation_workflow_state and the flat forward-migration set " + "live there and OMNIBASE_INFRA_DB_URL resolves against it" + ) + if database.physical_name != OMNIBASE_INFRA_DATABASE_PHYSICAL_NAME: + raise ValueError( + "omnibase_infra database must resolve to " + f"'{OMNIBASE_INFRA_DATABASE_PHYSICAL_NAME}', got " + f"'{database.physical_name}'" + ) + + actual_schemas = {name: schema.domain for name, schema in database.schemas.items()} + if actual_schemas != _OMNIBASE_INFRA_SCHEMAS: + raise ValueError( + "omnibase_infra schema/domain drift: expected " + f"{_OMNIBASE_INFRA_SCHEMAS}, got {actual_schemas}" + ) + actual_owners = {name: schema.owner for name, schema in database.schemas.items()} + if actual_owners != _OMNIBASE_INFRA_SCHEMA_OWNERS: + raise ValueError( + "omnibase_infra schema-owner drift: expected " + f"{_OMNIBASE_INFRA_SCHEMA_OWNERS}, got {actual_owners}" + ) + + if set(database.bindings) != set(_OMNIBASE_INFRA_BINDING_PRINCIPALS): + raise ValueError( + "omnibase_infra binding drift: expected " + f"{sorted(_OMNIBASE_INFRA_BINDING_PRINCIPALS)}, got " + f"{sorted(database.bindings)}" + ) + for ( + binding_name, + expected_principal, + ) in _OMNIBASE_INFRA_BINDING_PRINCIPALS.items(): + binding = database.bindings[binding_name] + if binding.database_ref != OMNIBASE_INFRA_DATABASE_REF: + raise ValueError( + f"Binding '{binding_name}' must resolve to database_ref " + f"'{OMNIBASE_INFRA_DATABASE_REF}'" + ) + if binding.principal != expected_principal: + raise ValueError( + f"omnibase_infra binding '{binding_name}' principal drift: " + f"expected '{expected_principal}', got '{binding.principal}'" + ) + expected_dsn_env = _OMNIBASE_INFRA_BINDING_DSN_ENVS[binding_name] + if binding.dsn_env != expected_dsn_env: + raise ValueError( + f"omnibase_infra binding '{binding_name}' dsn_env drift: " + f"expected '{expected_dsn_env}', got '{binding.dsn_env}'" + ) + + +def render_database_projection( + environment: str, + topology_root: Path | None = None, + *, + profile_catalog_path: Path | None = None, +) -> dict[str, object]: + """Render the stable, secret-free database subset for downstream consumers.""" + binding = _resolve_profile(environment, profile_catalog_path) + source_path = _topology_instance_path(binding.instance, topology_root) + catalog_path = profile_catalog_path or _PROFILE_CATALOG_PATH + topology = load_topology_profile( + environment, + topology_root, + profile_catalog_path=profile_catalog_path, + ) + dumped = topology.model_dump(mode="json") + databases = cast("dict[str, object]", dumped["databases"]) + return { + "schema_version": "1.0", + "environment": environment, + "topology_instance": binding.instance, + "source": { + "repository": _TOPOLOGY_REPOSITORY, + "path": f"{_TOPOLOGY_SOURCE_PREFIX}/{binding.instance}.yaml", + "sha256": hashlib.sha256(source_path.read_bytes()).hexdigest(), + "profile_catalog_path": _PROFILE_CATALOG_SOURCE_PATH, + "profile_catalog_sha256": hashlib.sha256( + catalog_path.read_bytes() + ).hexdigest(), + }, + "databases": databases, + } + + +class ProjectionDumper(yaml.SafeDumper): + """Dumper that indents sequence items under their parent mapping key. + + PyYAML's default ``SafeDumper`` emits block sequences at the *same* + indentation as the mapping key that owns them, but every checked-in + catalog under ``docker/catalog/database-topology/`` is written in the + indented form. Without this override ``--output`` reformats all seven + catalogs on every regeneration: measured on ``bf070a94e`` with no + semantic change at all, a bare re-render produced 819 insertions / 819 + deletions of pure indentation churn. ``validate_database_projection`` + compares parsed YAML, so the drift never failed a gate — it just made + the documented "regenerate it from the checked-in typed topology" + instruction unusable, because the real change was buried in style noise + and collided with every other open PR touching these files. + + ``ModelDeploymentTopology``'s instance writer + (``scripts/generate_application_database_table_grants.py::_InstanceDumper``) + already carries the identical override for the same reason; this keeps + the source instances and their rendered projections on one style + contract instead of two. + """ + + def increase_indent(self, flow: bool = False, indentless: bool = False) -> None: + return super().increase_indent(flow, False) + + +def write_database_projection( + environment: str, + output: Path, + topology_root: Path | None = None, + *, + profile_catalog_path: Path | None = None, +) -> None: + """Write a deterministic database projection without secret material.""" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + yaml.dump( + render_database_projection( + environment, + topology_root, + profile_catalog_path=profile_catalog_path, + ), + Dumper=ProjectionDumper, + sort_keys=True, + default_flow_style=False, + ), + encoding="utf-8", + ) + + +def validate_database_projection( + environment: str, + projection_path: Path, + topology_root: Path | None = None, + *, + profile_catalog_path: Path | None = None, +) -> None: + """Require a checked-in projection to exactly match its typed source.""" + actual_raw = yaml.safe_load(projection_path.read_text(encoding="utf-8")) + if not isinstance(actual_raw, dict): + raise ValueError(f"Database projection must be a mapping: {projection_path}") + actual = cast("dict[str, object]", actual_raw) + expected = render_database_projection( + environment, + topology_root, + profile_catalog_path=profile_catalog_path, + ) + if actual != expected: + raise ValueError( + f"Database projection drift for '{environment}': {projection_path}; " + "regenerate it from the checked-in typed topology" + ) + + +def _load_docker_consumer_catalog(path: Path) -> ModelDockerDatabaseConsumerCatalog: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + return ModelDockerDatabaseConsumerCatalog.model_validate(raw) + + +def validate_docker_topology_profile_injections( + repo_root: Path = _REPOSITORY_ROOT, + *, + profile_catalog_path: Path | None = None, +) -> None: + """Prove every checked-in Docker lane injects its exact DB profile.""" + catalog = _load_profile_catalog(profile_catalog_path) + for profile, binding in catalog.profiles.items(): + if binding.deployment_repository != _TOPOLOGY_REPOSITORY: + continue + path = repo_root / binding.injection_path + declared = set(_TOPOLOGY_PROFILE_LINE.findall(path.read_text(encoding="utf-8"))) + if profile not in declared: + raise ValueError( + f"Docker topology profile injection drift for '{profile}': " + f"{binding.injection_path} must declare " + f"{_TOPOLOGY_PROFILE_ENV_VAR}: {profile}" + ) + + runtime_policy_path = ( + repo_root / "contracts" / "services" / "runtime_policy.contract.yaml" + ) + runtime_policy = ModelRuntimePolicyContract.model_validate( + yaml.safe_load(runtime_policy_path.read_text(encoding="utf-8")) + ) + mapped_runtime_profiles = { + binding.runtime_policy_profile + for binding in catalog.profiles.values() + if binding.runtime_policy_profile is not None + } + if mapped_runtime_profiles != set(runtime_policy.profiles): + raise ValueError("Docker topology runtime-policy profile coverage drift") + + +def validate_docker_catalog_parity( + repo_root: Path = _REPOSITORY_ROOT, + topology_root: Path | None = None, +) -> None: + """Prove Docker DSN/database bindings consume the local typed topology.""" + environment = "local" + topology = load_environment_topology(environment, topology_root) + database = topology.databases[APPLICATION_DATABASE_REF] + catalog = _load_docker_consumer_catalog( + repo_root / "docker" / "catalog" / "database-consumers.yaml" + ) + if catalog.environment != environment: + raise ValueError( + f"Docker database catalog targets '{catalog.environment}', expected 'local'" + ) + + services_dir = repo_root / "docker" / "catalog" / "services" + application_dsn_envs = {binding.dsn_env for binding in database.bindings.values()} + discovered_dsn_consumers: set[str] = set() + for manifest_path in services_dir.glob("*.yaml"): + manifest = _load_manifest(manifest_path) + env_names = ( + set(manifest.required_env) + | manifest.hardcoded_env.keys() + | manifest.operational_defaults.keys() + | manifest.catalog_env.keys() + ) + if env_names & application_dsn_envs: + discovered_dsn_consumers.add(manifest.name) + + declared_dsn_consumers = { + service_name + for service_name, consumer in catalog.consumers.items() + if consumer.bindings + } | set(catalog.deferred_consumers) + if discovered_dsn_consumers != declared_dsn_consumers: + raise ValueError( + "Docker application-DSN inventory drift: discovered " + f"{sorted(discovered_dsn_consumers)}, declared " + f"{sorted(declared_dsn_consumers)}" + ) + + for service_name, consumer in catalog.consumers.items(): + manifest = _load_manifest(services_dir / f"{service_name}.yaml") + env_values = { + **manifest.hardcoded_env, + **manifest.operational_defaults, + **manifest.catalog_env, + } + env_names = set(manifest.required_env) | env_values.keys() + for binding_name in consumer.bindings: + binding = database.bindings.get(binding_name) + if binding is None: + raise ValueError( + f"Docker service '{service_name}' references unknown topology " + f"binding '{binding_name}'" + ) + if binding.dsn_env not in env_names: + raise ValueError( + f"Docker service '{service_name}' must consume " + f"{binding.dsn_env} for binding '{binding_name}'" + ) + for env_name in consumer.physical_database_envs: + if env_values.get(env_name) != database.physical_name: + raise ValueError( + f"Docker service '{service_name}' {env_name} must be " + f"'{database.physical_name}', got {env_values.get(env_name)!r}" + ) + + +__all__ = [ + "APPLICATION_DATABASE_REF", + "OMNIBASE_INFRA_DATABASE_REF", + "OMNIINTELLIGENCE_DATABASE_REF", + "SUPPORTED_ENVIRONMENTS", + "SUPPORTED_TOPOLOGY_PROFILES", + "TOPOLOGY_PROFILE_INSTANCE_MAP", + "load_environment_topology", + "load_topology_profile", + "render_database_projection", + "validate_application_database_invariants", + "validate_database_projection", + "validate_docker_catalog_parity", + "validate_docker_topology_profile_injections", + "validate_omnibase_infra_database_invariants", + "validate_omniintelligence_database_invariants", + "write_database_projection", +] diff --git a/src/omnibase_infra/topology/application_database_profiles.yaml b/src/omnibase_infra/topology/application_database_profiles.yaml new file mode 100644 index 0000000000..e1d702332e --- /dev/null +++ b/src/omnibase_infra/topology/application_database_profiles.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +schema_version: "1.0" +# This catalog binds database-topology profiles, not Kafka/event namespaces. +# ONEX_ENVIRONMENT and KAFKA_ENVIRONMENT may legitimately contain other values +# such as test-env; they never select a topology instance. +profiles: + local: + instance: local + deployment_repository: OmniNode-ai/omnibase_infra + injection_path: docker/docker-compose.infra.yml + # The runtime-policy dev lane uses the base omnibase-infra compose project, + # whose database topology is explicitly local. + runtime_policy_profile: dev + test: + instance: local + deployment_repository: OmniNode-ai/omnibase_infra + injection_path: docker/docker-compose.e2e.yml + stability-test: + instance: local + deployment_repository: OmniNode-ai/omnibase_infra + injection_path: docker/docker-compose.stability-test.yml + runtime_policy_profile: stability-test + judge: + instance: local + deployment_repository: OmniNode-ai/omnibase_infra + injection_path: docker/docker-compose.judge.yml + runtime_policy_profile: judge + prod: + instance: local + deployment_repository: OmniNode-ai/omnibase_infra + injection_path: docker/docker-compose.prod.yml + runtime_policy_profile: prod + onex-dev: + instance: onex-dev + deployment_repository: OmniNode-ai/omninode_infra + injection_path: k8s/onex-dev/runtime/configmap.yaml + onex-prod: + instance: onex-prod + deployment_repository: OmniNode-ai/omninode_infra + injection_path: k8s/onex-prod/runtime/configmap.yaml diff --git a/src/omnibase_infra/topology/instances/README.md b/src/omnibase_infra/topology/instances/README.md new file mode 100644 index 0000000000..16e815f49f --- /dev/null +++ b/src/omnibase_infra/topology/instances/README.md @@ -0,0 +1,30 @@ +# Environment topology instances + +These files are the authoritative, secret-free deployment topology for database +semantics. They are parsed as +`omnibase_core.models.core.ModelDeploymentTopology`; unknown fields and unresolved +schema, owner, principal, binding, or ledger references fail closed. + +`omnibase_infra` owns these instances because it already owns the typed Docker +service catalog and ships the topology consumer library. Docker projections under +`docker/catalog/database-topology/` and Kubernetes projections in +`omninode_infra/topology/kubernetes/` are generated or parity-validated consumers. +The host-local `~/.omnibase/topology.yaml` file is never read here and is only a +projection for local setup. + +`../application_database_profiles.yaml` is the typed, fail-closed mapping from +deployment profiles to these instances. The exact supported database profiles are +`local`, `test`, `stability-test`, `judge`, `prod`, `onex-dev`, and `onex-prod`. +The five Docker profiles intentionally share the `local` database instance because +their checked-in Compose surfaces use the same internal database name and DSN +environment contract; the mapping and each deployment injection are validated. +The two Kubernetes profiles use their matching cloud instances. + +`ONEX_DATABASE_TOPOLOGY_PROFILE` is independent of `ONEX_ENVIRONMENT` and +`KAFKA_ENVIRONMENT`. The latter two are event namespaces and may carry values such +as `test-env`; they must never select or silently fall back to a database topology. + +The files contain environment-variable names and Kubernetes/Docker service names, +never passwords, tokens, or DSN values. The `omninode_runtime` service and the +`omninode_runtime` PostgreSQL principal are separate namespaces: the service consumes +the `omninode_runtime_service` binding. diff --git a/src/omnibase_infra/topology/instances/local.yaml b/src/omnibase_infra/topology/instances/local.yaml new file mode 100644 index 0000000000..d9d1111950 --- /dev/null +++ b/src/omnibase_infra/topology/instances/local.yaml @@ -0,0 +1,274 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +active_preset: application-local +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: omninode_internal + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - live_events + - log_entries + privileges: [INSERT, SELECT, UPDATE] + schema: omninode_internal + - object_type: TABLE + objects: + - baselines_breakdown + - baselines_comparisons + - baselines_quality_snapshots + - baselines_roi_snapshots + - baselines_snapshots + - baselines_trend + - capsule_store + - contract_registry + - cost_by_repo_snapshots + - deployment_evidence_projection + - deployment_readiness_projection + - event_chain + - evidence_correlation_trace_projection + - evidence_dashboard_projection + - evidence_readiness_aggregate_projection + - gate_activity + - gate_metrics + - generation_events + - intent_classification_events + - llm_call_metrics + - llm_delegation_daily_projection + - llm_routing_decisions + - mcp_tools + - merge_state_transitions + - nightly_loop_decisions + - nightly_loop_iterations + - node_service_registry + - overnight_session_phases + - overnight_sessions + - pr_lifecycle_ledger_entries + - pr_merged_events + - receipt_gate_rows + - renderer_capability_projection + - sandbox_decisions + - session_outcomes + - session_replay_snapshots + - swarm_runs + - traces + - voice_sessions + privileges: [INSERT, SELECT, UPDATE] + schema: public + - object_type: TABLE + objects: + - nightly_loop_configs + privileges: [SELECT] + schema: public + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - agent_routing_decisions + - capability_scores + - context_roi_scores + - delegation_budget_state + - delegation_events + - delegation_judge_verdict_events + - delegation_shadow_comparisons + - dep_health_findings + - instruction_eval_aggregate_snapshots + - llm_cost_aggregates + - pattern_learning_artifacts + - projection_delegation_inference_response_text + - savings_estimates + - skill_execution_snapshots + privileges: [INSERT, SELECT, UPDATE] + schema: public + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omnibase_infra: + bindings: + omninode_runtime_service: + database_ref: omnibase_infra + dsn_env: OMNIBASE_INFRA_DB_URL + principal: role_omnibase_infra + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.service + owners: + owner_omnibase_infra: + login: false + physical_name: omnibase_infra + principals: + role_omnibase_infra: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - delegation_workflow_state + privileges: [INSERT, SELECT, UPDATE] + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omnibase_infra + omniintelligence: + bindings: + omninode_runtime_service: + database_ref: omniintelligence + dsn_env: OMNIINTELLIGENCE_DB_URL + principal: role_omniintelligence + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omniintelligence.service + owners: + owner_omniintelligence: + login: false + physical_name: omniintelligence + principals: + role_omniintelligence: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - dispatch_eval_results + privileges: [INSERT, SELECT, UPDATE] + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omniintelligence +presets: + application-local: + - postgres + - omnidash + - omninode_runtime +schema_version: "2.0" +services: + omnidash: + local: + compose_service: omnidash + host_port: 3000 + mode: LOCAL + omninode_runtime: + local: + compose_service: omninode-runtime + host_port: 8085 + mode: LOCAL + postgres: + local: + compose_service: omnibase-infra-postgres + host_port: 5436 + mode: LOCAL diff --git a/src/omnibase_infra/topology/instances/onex-dev.yaml b/src/omnibase_infra/topology/instances/onex-dev.yaml new file mode 100644 index 0000000000..ab3ac71c39 --- /dev/null +++ b/src/omnibase_infra/topology/instances/onex-dev.yaml @@ -0,0 +1,268 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +active_preset: application-cloud +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: DATABASE_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: omninode_internal + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - live_events + - log_entries + privileges: [INSERT, SELECT, UPDATE] + schema: omninode_internal + - object_type: TABLE + objects: + - baselines_breakdown + - baselines_comparisons + - baselines_quality_snapshots + - baselines_roi_snapshots + - baselines_snapshots + - baselines_trend + - capsule_store + - contract_registry + - cost_by_repo_snapshots + - deployment_evidence_projection + - deployment_readiness_projection + - event_chain + - evidence_correlation_trace_projection + - evidence_dashboard_projection + - evidence_readiness_aggregate_projection + - gate_activity + - gate_metrics + - generation_events + - intent_classification_events + - llm_call_metrics + - llm_delegation_daily_projection + - llm_routing_decisions + - mcp_tools + - merge_state_transitions + - nightly_loop_decisions + - nightly_loop_iterations + - node_service_registry + - overnight_session_phases + - overnight_sessions + - pr_lifecycle_ledger_entries + - pr_merged_events + - receipt_gate_rows + - renderer_capability_projection + - sandbox_decisions + - session_outcomes + - session_replay_snapshots + - swarm_runs + - traces + - voice_sessions + privileges: [INSERT, SELECT, UPDATE] + schema: public + - object_type: TABLE + objects: + - nightly_loop_configs + privileges: [SELECT] + schema: public + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - agent_routing_decisions + - capability_scores + - context_roi_scores + - delegation_budget_state + - delegation_events + - delegation_judge_verdict_events + - delegation_shadow_comparisons + - dep_health_findings + - instruction_eval_aggregate_snapshots + - llm_cost_aggregates + - pattern_learning_artifacts + - projection_delegation_inference_response_text + - savings_estimates + - skill_execution_snapshots + privileges: [INSERT, SELECT, UPDATE] + schema: public + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omnibase_infra: + bindings: + omninode_runtime_service: + database_ref: omnibase_infra + dsn_env: OMNIBASE_INFRA_DB_URL + principal: role_omnibase_infra + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.service + owners: + owner_omnibase_infra: + login: false + physical_name: omnibase_infra + principals: + role_omnibase_infra: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - delegation_workflow_state + privileges: [INSERT, SELECT, UPDATE] + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omnibase_infra + omniintelligence: + bindings: + omninode_runtime_service: + database_ref: omniintelligence + dsn_env: OMNIINTELLIGENCE_DB_URL + principal: role_omniintelligence + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omniintelligence.service + owners: + owner_omniintelligence: + login: false + physical_name: omniintelligence + principals: + role_omniintelligence: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - dispatch_eval_results + privileges: [INSERT, SELECT, UPDATE] + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omniintelligence +presets: + application-cloud: + - postgres + - onex_api + - omnidash + - omninode_runtime +schema_version: "2.0" +services: + omnidash: + mode: CLOUD + omninode_runtime: + mode: CLOUD + onex_api: + mode: CLOUD + postgres: + mode: CLOUD diff --git a/src/omnibase_infra/topology/instances/onex-prod.yaml b/src/omnibase_infra/topology/instances/onex-prod.yaml new file mode 100644 index 0000000000..ab3ac71c39 --- /dev/null +++ b/src/omnibase_infra/topology/instances/onex-prod.yaml @@ -0,0 +1,268 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +active_preset: application-cloud +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: DATABASE_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: omninode_internal + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - live_events + - log_entries + privileges: [INSERT, SELECT, UPDATE] + schema: omninode_internal + - object_type: TABLE + objects: + - baselines_breakdown + - baselines_comparisons + - baselines_quality_snapshots + - baselines_roi_snapshots + - baselines_snapshots + - baselines_trend + - capsule_store + - contract_registry + - cost_by_repo_snapshots + - deployment_evidence_projection + - deployment_readiness_projection + - event_chain + - evidence_correlation_trace_projection + - evidence_dashboard_projection + - evidence_readiness_aggregate_projection + - gate_activity + - gate_metrics + - generation_events + - intent_classification_events + - llm_call_metrics + - llm_delegation_daily_projection + - llm_routing_decisions + - mcp_tools + - merge_state_transitions + - nightly_loop_decisions + - nightly_loop_iterations + - node_service_registry + - overnight_session_phases + - overnight_sessions + - pr_lifecycle_ledger_entries + - pr_merged_events + - receipt_gate_rows + - renderer_capability_projection + - sandbox_decisions + - session_outcomes + - session_replay_snapshots + - swarm_runs + - traces + - voice_sessions + privileges: [INSERT, SELECT, UPDATE] + schema: public + - object_type: TABLE + objects: + - nightly_loop_configs + privileges: [SELECT] + schema: public + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - agent_routing_decisions + - capability_scores + - context_roi_scores + - delegation_budget_state + - delegation_events + - delegation_judge_verdict_events + - delegation_shadow_comparisons + - dep_health_findings + - instruction_eval_aggregate_snapshots + - llm_cost_aggregates + - pattern_learning_artifacts + - projection_delegation_inference_response_text + - savings_estimates + - skill_execution_snapshots + privileges: [INSERT, SELECT, UPDATE] + schema: public + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omnibase_infra: + bindings: + omninode_runtime_service: + database_ref: omnibase_infra + dsn_env: OMNIBASE_INFRA_DB_URL + principal: role_omnibase_infra + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.service + owners: + owner_omnibase_infra: + login: false + physical_name: omnibase_infra + principals: + role_omnibase_infra: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - delegation_workflow_state + privileges: [INSERT, SELECT, UPDATE] + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omnibase_infra + omniintelligence: + bindings: + omninode_runtime_service: + database_ref: omniintelligence + dsn_env: OMNIINTELLIGENCE_DB_URL + principal: role_omniintelligence + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omniintelligence.service + owners: + owner_omniintelligence: + login: false + physical_name: omniintelligence + principals: + role_omniintelligence: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - dispatch_eval_results + privileges: [INSERT, SELECT, UPDATE] + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omniintelligence +presets: + application-cloud: + - postgres + - onex_api + - omnidash + - omninode_runtime +schema_version: "2.0" +services: + omnidash: + mode: CLOUD + omninode_runtime: + mode: CLOUD + onex_api: + mode: CLOUD + postgres: + mode: CLOUD diff --git a/src/omnibase_infra/topology/models/__init__.py b/src/omnibase_infra/topology/models/__init__.py new file mode 100644 index 0000000000..46f43bde56 --- /dev/null +++ b/src/omnibase_infra/topology/models/__init__.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed Docker projection adapter models.""" + +from omnibase_infra.topology.models.model_application_database_topology_profile import ( + ModelApplicationDatabaseTopologyProfile, +) +from omnibase_infra.topology.models.model_application_database_topology_profile_catalog import ( + ModelApplicationDatabaseTopologyProfileCatalog, +) +from omnibase_infra.topology.models.model_deferred_docker_database_consumer import ( + ModelDeferredDockerDatabaseConsumer, +) +from omnibase_infra.topology.models.model_docker_database_consumer import ( + ModelDockerDatabaseConsumer, +) +from omnibase_infra.topology.models.model_docker_database_consumer_catalog import ( + ModelDockerDatabaseConsumerCatalog, +) + +__all__ = [ + "ModelApplicationDatabaseTopologyProfile", + "ModelApplicationDatabaseTopologyProfileCatalog", + "ModelDeferredDockerDatabaseConsumer", + "ModelDockerDatabaseConsumer", + "ModelDockerDatabaseConsumerCatalog", +] diff --git a/src/omnibase_infra/topology/models/model_application_database_topology_profile.py b/src/omnibase_infra/topology/models/model_application_database_topology_profile.py new file mode 100644 index 0000000000..278623a21d --- /dev/null +++ b/src/omnibase_infra/topology/models/model_application_database_topology_profile.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed application-database topology profile binding.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +__all__ = ["ModelApplicationDatabaseTopologyProfile"] + + +class ModelApplicationDatabaseTopologyProfile(BaseModel): + """One explicit deployment-profile to topology-instance binding.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + instance: str = Field(pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + deployment_repository: Literal[ + "OmniNode-ai/omnibase_infra", + "OmniNode-ai/omninode_infra", + ] + injection_path: str = Field(min_length=1) + runtime_policy_profile: str | None = Field( + default=None, + pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$", + ) + + @field_validator("injection_path") + @classmethod + def injection_path_is_repository_relative(cls, value: str) -> str: + """Reject absolute and parent-traversing evidence paths.""" + parts = value.split("/") + if value.startswith("/") or ".." in parts or any(not part for part in parts): + raise ValueError("injection_path must be a repository-relative path") + return value diff --git a/src/omnibase_infra/topology/models/model_application_database_topology_profile_catalog.py b/src/omnibase_infra/topology/models/model_application_database_topology_profile_catalog.py new file mode 100644 index 0000000000..0b669445a4 --- /dev/null +++ b/src/omnibase_infra/topology/models/model_application_database_topology_profile_catalog.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed application-database topology profile catalog.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from omnibase_infra.topology.models.model_application_database_topology_profile import ( + ModelApplicationDatabaseTopologyProfile, +) + +__all__ = ["ModelApplicationDatabaseTopologyProfileCatalog"] + + +class ModelApplicationDatabaseTopologyProfileCatalog(BaseModel): + """Secret-free profile bindings owned by ``omnibase_infra``.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: Literal["1.0"] + profiles: dict[str, ModelApplicationDatabaseTopologyProfile] = Field(min_length=1) + + @field_validator("profiles") + @classmethod + def profile_names_are_canonical( + cls, + value: dict[str, ModelApplicationDatabaseTopologyProfile], + ) -> dict[str, ModelApplicationDatabaseTopologyProfile]: + """Keep profile identifiers exact and shell/YAML safe.""" + for profile in value: + if not profile or any( + not (character.islower() or character.isdigit() or character == "-") + for character in profile + ): + raise ValueError(f"invalid database topology profile {profile!r}") + return value diff --git a/src/omnibase_infra/topology/models/model_deferred_docker_database_consumer.py b/src/omnibase_infra/topology/models/model_deferred_docker_database_consumer.py new file mode 100644 index 0000000000..f75f86ea32 --- /dev/null +++ b/src/omnibase_infra/topology/models/model_deferred_docker_database_consumer.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed fail-closed hold for an unclassified Docker database consumer.""" + +from pydantic import BaseModel, ConfigDict, Field + +__all__ = ["ModelDeferredDockerDatabaseConsumer"] + + +class ModelDeferredDockerDatabaseConsumer(BaseModel): + """Explicit hold for a consumer awaiting semantic classification.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + tracking_ticket: str = Field(pattern=r"^OMN-[0-9]+$") + reason: str = Field(min_length=1) diff --git a/src/omnibase_infra/topology/models/model_docker_database_consumer.py b/src/omnibase_infra/topology/models/model_docker_database_consumer.py new file mode 100644 index 0000000000..53cc1352e0 --- /dev/null +++ b/src/omnibase_infra/topology/models/model_docker_database_consumer.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed Docker consumer of topology-owned database semantics.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +__all__ = ["ModelDockerDatabaseConsumer"] + + +class ModelDockerDatabaseConsumer(BaseModel): + """One Docker catalog consumer of topology-owned database semantics.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + bindings: tuple[str, ...] = Field(default=()) + physical_database_envs: tuple[str, ...] = Field(default=()) + + @model_validator(mode="after") + def has_a_database_projection(self) -> ModelDockerDatabaseConsumer: + """Reject inert entries which validate no binding or physical name.""" + if not self.bindings and not self.physical_database_envs: + raise ValueError( + "Docker database consumers require a binding or physical_database_env" + ) + return self diff --git a/src/omnibase_infra/topology/models/model_docker_database_consumer_catalog.py b/src/omnibase_infra/topology/models/model_docker_database_consumer_catalog.py new file mode 100644 index 0000000000..c8c4455221 --- /dev/null +++ b/src/omnibase_infra/topology/models/model_docker_database_consumer_catalog.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed Docker database-consumer catalog projection.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_infra.topology.models.model_deferred_docker_database_consumer import ( + ModelDeferredDockerDatabaseConsumer, +) +from omnibase_infra.topology.models.model_docker_database_consumer import ( + ModelDockerDatabaseConsumer, +) + +__all__ = ["ModelDockerDatabaseConsumerCatalog"] + + +class ModelDockerDatabaseConsumerCatalog(BaseModel): + """Adapter from Docker service names to topology binding references.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: str = Field(pattern=r"^[0-9]+\.[0-9]+$") + environment: str + consumers: dict[str, ModelDockerDatabaseConsumer] + deferred_consumers: dict[str, ModelDeferredDockerDatabaseConsumer] = Field( + default_factory=dict + ) + + @model_validator(mode="after") + def consumer_sets_do_not_overlap(self) -> ModelDockerDatabaseConsumerCatalog: + """A service cannot be both classified and held.""" + overlap = sorted(self.consumers.keys() & self.deferred_consumers.keys()) + if overlap: + raise ValueError( + f"Docker consumers cannot be mapped and deferred: {overlap}" + ) + return self diff --git a/src/omnibase_infra/topology/physical_schema_mapping.py b/src/omnibase_infra/topology/physical_schema_mapping.py new file mode 100644 index 0000000000..b6a0f91822 --- /dev/null +++ b/src/omnibase_infra/topology/physical_schema_mapping.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Temporary logical-to-physical schema mapping for application table grants.""" + +from __future__ import annotations + +TENANT_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359: frozenset[str] = frozenset( + { + "agent_routing_decisions", + "capability_scores", + "context_roi_scores", + "delegation_budget_state", + "delegation_events", + "delegation_judge_verdict_events", + "delegation_shadow_comparisons", + "dep_health_findings", + "instruction_eval_aggregate_snapshots", + "llm_cost_aggregates", + "pattern_learning_artifacts", + "projection_delegation_inference_response_text", + "savings_estimates", + "skill_execution_snapshots", + } +) + +# OMN-15359 (P2-P4 build). The `omninode_internal` schema now physically exists +# (docker/migrations/forward/098_create_omninode_internal_schema.sql), but the +# tables that are logically OMNINODE_INTERNAL-domain per the shipped topology +# (`omninode_runtime` principal's TABLE grants, identical across all 7 shipped +# database-topology profiles as of this ticket) have not been copied into it — +# every one of them is still physically created, unqualified, in `public` by +# its node migration. This is the exact gap OMN-15426's live readback named: +# handler_wiring issues schema-qualified SQL against `omninode_internal` for +# these relations while they resolve nowhere, producing `relation does not +# exist` rather than a permission error. Enumerated (not a blanket public-> +# internal rule) so a *new* table declared against `omninode_internal` after +# this landed resolves to its real target schema, matching the precedent set +# by TENANT_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359 above. +# +# `live_events` is REMOVED from this set as of +# docker/migrations/forward/099_create_omninode_internal_live_events.sql -- +# it is the first family individually transform-copied out of the bridge. +# Post-099, physical_grant_schema_for_table('omninode_internal', 'live_events') +# must return 'omninode_internal' (no override) so it agrees with the real +# INSERT-target schema handler_wiring._resolve_projection_database_target +# already resolves from the node contract's literal db_io.db_tables[0].schema. +# Leaving it enumerated here after the physical table exists would silently +# reintroduce the drift 099 exists to close: the grant-privilege check would +# keep asserting against `public` while every write already lands in +# `omninode_internal`. +INTERNAL_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359: frozenset[str] = frozenset( + { + "baselines_breakdown", + "baselines_comparisons", + "baselines_quality_snapshots", + "baselines_roi_snapshots", + "baselines_snapshots", + "baselines_trend", + "capsule_store", + "contract_registry", + "cost_by_repo_snapshots", + "deployment_evidence_projection", + "deployment_readiness_projection", + "event_chain", + "evidence_correlation_trace_projection", + "evidence_dashboard_projection", + "evidence_readiness_aggregate_projection", + "gate_activity", + "gate_metrics", + "generation_events", + "intent_classification_events", + "llm_call_metrics", + "llm_delegation_daily_projection", + "llm_routing_decisions", + "mcp_tools", + "merge_state_transitions", + "nightly_loop_configs", + "nightly_loop_decisions", + "nightly_loop_iterations", + "node_service_registry", + "overnight_session_phases", + "overnight_sessions", + "pr_lifecycle_ledger_entries", + "pr_merged_events", + "receipt_gate_rows", + "renderer_capability_projection", + "sandbox_decisions", + "session_outcomes", + "session_replay_snapshots", + "swarm_runs", + "traces", + "voice_sessions", + } +) + + +def physical_grant_schema_for_table(schema: str, table_name: str) -> str: + """Return the schema where PostgreSQL ACLs currently apply for a table. + + OMN-15359 owns the physical ``public`` -> ``tenant``/``omninode_internal`` + moves. Until each relation family is individually transform-copied and + proven, these tables remain physically in ``public`` even though their + contracts and runtime routing are logically tenant- or + omninode_internal-domain. + """ + if ( + schema == "tenant" + and table_name in TENANT_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359 + ): + return "public" + if ( + schema == "omninode_internal" + and table_name in INTERNAL_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359 + ): + return "public" + return schema diff --git a/src/omnibase_infra/topology/table_grant_derivation.py b/src/omnibase_infra/topology/table_grant_derivation.py new file mode 100644 index 0000000000..2ac4d10fce --- /dev/null +++ b/src/omnibase_infra/topology/table_grant_derivation.py @@ -0,0 +1,393 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Derive ``object_type: TABLE`` grants from node contract ``db_io.db_tables``. + +OMN-15656. The OMN-15418 privilege validator +(:func:`omnibase_infra.runtime.auto_wiring.handler_wiring._require_projection_binding_privileges`) +requires an explicit per-table grant before a projection handler may wire. The +checked-in topology instances declared none, so every contract-declared table +failed on every profile. + +This module is the *authoring* side of that contract: the grants are a +projection of the node contracts that consume them, never a hand-maintained +list. Contracts own ``schema``/``name``/``access``; the deployment topology owns +which principal serves which schema domain. Nothing here invents a relation, and +nothing here widens a privilege beyond what the validator demands. + +The derivation is deliberately total and fail-closed: a declaration this module +cannot map to a topology principal is returned as a typed residual with a +reason, never silently dropped. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path + +import yaml + +from omnibase_core.enums.enum_database_grant_object_type import ( + EnumDatabaseGrantObjectType, +) +from omnibase_core.enums.enum_database_privilege import EnumDatabasePrivilege +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_core.models.contracts.subcontracts.model_db_table_declaration import ( + ModelDbTableDeclaration, +) +from omnibase_core.models.core import ModelDeploymentTopology +from omnibase_core.models.core.model_deployment_topology_database_grant import ( + ModelDeploymentTopologyDatabaseGrant, +) +from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _INTERNAL_PROJECTION_BINDING, + _TENANT_PROJECTION_BINDING, +) +from omnibase_infra.topology.physical_schema_mapping import ( + INTERNAL_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359, + TENANT_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359, + physical_grant_schema_for_table, +) + +__all__ = [ + "ContractTableDeclaration", + "DerivedTableGrants", + "TopologyTableGrants", + "UnmappableDeclaration", + "READ_PRIVILEGES", + "WRITE_PRIVILEGES", + "DOMAIN_PROJECTION_BINDINGS", + "STATE_IO_TABLE_DECLARATIONS", + "TENANT_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359", + "INTERNAL_TABLES_PHYSICALLY_IN_PUBLIC_UNTIL_OMN15359", + "physical_grant_schema_for_table", + "derive_table_grants", + "derive_topology_table_grants", + "load_contract_declarations", +] + +# Mirrors ``_require_projection_binding_privileges``: PostgreSQL needs SELECT +# alongside INSERT/UPDATE because the projection adapter issues +# ``INSERT ... ON CONFLICT DO UPDATE``. +READ_PRIVILEGES: frozenset[EnumDatabasePrivilege] = frozenset( + {EnumDatabasePrivilege.SELECT} +) +WRITE_PRIVILEGES: frozenset[EnumDatabasePrivilege] = frozenset( + { + EnumDatabasePrivilege.SELECT, + EnumDatabasePrivilege.INSERT, + EnumDatabasePrivilege.UPDATE, + } +) + +# Mirrors ``_projection_operation_bindings``. PLATFORM_CATALOG is absent by +# design: that domain requires a caller-supplied read/write binding, so a +# contract declaration alone cannot name the principal. Such declarations are +# returned as residuals instead of being guessed at. +DOMAIN_PROJECTION_BINDINGS: Mapping[EnumDatabaseSchemaDomain, str] = { + EnumDatabaseSchemaDomain.TENANT: _TENANT_PROJECTION_BINDING, + EnumDatabaseSchemaDomain.OMNINODE_INTERNAL: _INTERNAL_PROJECTION_BINDING, +} + + +@dataclass(frozen=True, slots=True) +class ContractTableDeclaration: + """One ``db_io.db_tables`` entry tied back to the contract that declared it.""" + + node: str + contract_path: Path + table: ModelDbTableDeclaration + + +# ``state_io`` (OMN-14208, ``handler_wiring._read_state_io``) is a legacy +# contract subcontract that pre-dates ``ModelDbTableDeclaration`` and has no +# core model of its own -- ``load_contract_declarations`` only scans +# ``db_io.db_tables`` and can never discover a state_io-owned relation. +# Without an entry here, ``delegation_workflow_state`` would derive zero TABLE +# grants for every profile and stay permanently ungranted while ``--check`` +# reported green, the exact "declared but never granted" shape OMN-15656 +# exists to prevent. This is the sanctioned, checked-in way a state_io table +# still participates in derivation (OMN-15337 / operator ruling R-q). +STATE_IO_TABLE_DECLARATIONS: tuple[ContractTableDeclaration, ...] = ( + ContractTableDeclaration( + node="state_io:delegation_workflow_state", + contract_path=Path( + "docker/migrations/forward/090_create_delegation_workflow_state.sql" + ), + table=ModelDbTableDeclaration( + name="delegation_workflow_state", + database_ref="omnibase_infra", + schema="public", + migration=( + "docker/migrations/forward/090_create_delegation_workflow_state.sql" + ), + access="read_write", + role="state", + ), + ), +) + + +@dataclass(frozen=True, slots=True) +class UnmappableDeclaration: + """A declaration that cannot be projected onto a topology principal.""" + + node: str + database_ref: str + schema: str + name: str + reason: str + + @property + def key(self) -> tuple[str, str, str]: + """Stable identity used by the shrink-only residual ratchet.""" + return (self.database_ref, self.schema, self.name) + + +@dataclass(frozen=True, slots=True) +class DerivedTableGrants: + """TABLE grants per principal plus the residuals that could not be derived.""" + + grants: Mapping[str, tuple[ModelDeploymentTopologyDatabaseGrant, ...]] + unmappable: tuple[UnmappableDeclaration, ...] + + +@dataclass(frozen=True, slots=True) +class TopologyTableGrants: + """Per-logical-database derivation for a whole topology instance. + + The topology stopped being single-database when the omniintelligence + service database was declared (OMN-15655 AC-2). Deriving only the + ``application`` database would have silently classified every + service-database declaration as an undeliverable residual, which is the + exact "declared but never granted" shape OMN-15656 exists to prevent. + """ + + per_database: Mapping[str, DerivedTableGrants] + unmappable: tuple[UnmappableDeclaration, ...] + + +def _privileges_for_access(access: str) -> frozenset[EnumDatabasePrivilege]: + """Map a declared access mode onto the privileges the validator demands.""" + if access == "read": + return READ_PRIVILEGES + if access in {"write", "read_write"}: + # read_write is the union, and WRITE_PRIVILEGES already contains SELECT. + return WRITE_PRIVILEGES + raise ValueError(f"Unsupported db_tables access mode {access!r}") + + +def load_contract_declarations( + contracts_root: Path, +) -> tuple[ContractTableDeclaration, ...]: + """Read every ``contract.yaml`` under ``contracts_root`` for ``db_io.db_tables``. + + Raises when the root does not exist so a missing cross-repo checkout fails + the gate loudly instead of degrading into a vacuous zero-declaration pass. + """ + if not contracts_root.is_dir(): + raise FileNotFoundError( + f"contracts root {contracts_root} does not exist; the cross-repo " + "checkout that provides node contracts is required for derivation" + ) + declarations: list[ContractTableDeclaration] = [] + for contract_path in sorted(contracts_root.rglob("contract.yaml")): + document = yaml.safe_load(contract_path.read_text(encoding="utf-8")) + if not isinstance(document, dict): + continue + db_io = document.get("db_io") + if not isinstance(db_io, dict): + continue + entries = db_io.get("db_tables") + if not isinstance(entries, Sequence): + continue + for entry in entries: + declarations.append( + ContractTableDeclaration( + node=contract_path.parent.name, + contract_path=contract_path, + table=ModelDbTableDeclaration(**entry), + ) + ) + if not declarations: + raise ValueError( + f"no db_io.db_tables declarations found under {contracts_root}; " + "refusing to derive an empty grant set" + ) + return tuple(declarations) + + +def derive_table_grants( + topology: ModelDeploymentTopology, + declarations: Iterable[ContractTableDeclaration], + *, + database_ref: str = "application", +) -> DerivedTableGrants: + """Project contract table declarations onto per-principal TABLE grants.""" + database = topology.databases.get(database_ref) + if database is None: + raise ValueError(f"topology declares no database {database_ref!r}") + + # (principal, schema, table) -> required privileges, unioned across every + # contract that declares the same relation. + required: dict[tuple[str, str, str], set[EnumDatabasePrivilege]] = {} + unmappable: dict[tuple[str, str, str], UnmappableDeclaration] = {} + + for declaration in declarations: + table = declaration.table + key = (table.database_ref, table.schema, table.name) + if table.database_ref != database_ref: + unmappable.setdefault( + key, + UnmappableDeclaration( + node=declaration.node, + database_ref=table.database_ref, + schema=table.schema, + name=table.name, + reason=( + f"database_ref {table.database_ref!r} is not declared in " + "the application topology" + ), + ), + ) + continue + schema = database.schemas.get(table.schema) + if schema is None: + unmappable.setdefault( + key, + UnmappableDeclaration( + node=declaration.node, + database_ref=table.database_ref, + schema=table.schema, + name=table.name, + reason=f"schema {table.schema!r} is not declared in the topology", + ), + ) + continue + binding_ref = DOMAIN_PROJECTION_BINDINGS.get(schema.domain) + if binding_ref is None: + unmappable.setdefault( + key, + UnmappableDeclaration( + node=declaration.node, + database_ref=table.database_ref, + schema=table.schema, + name=table.name, + reason=( + f"domain {schema.domain.value} requires an explicit " + "caller-supplied binding and cannot be derived from a " + "contract declaration alone" + ), + ), + ) + continue + binding = database.bindings.get(binding_ref) + if binding is None: + unmappable.setdefault( + key, + UnmappableDeclaration( + node=declaration.node, + database_ref=table.database_ref, + schema=table.schema, + name=table.name, + reason=f"topology declares no binding {binding_ref!r}", + ), + ) + continue + grant_schema = physical_grant_schema_for_table(table.schema, table.name) + entry = required.setdefault( + (binding.principal, grant_schema, table.name), set() + ) + entry.update(_privileges_for_access(table.access)) + + grants: dict[str, tuple[ModelDeploymentTopologyDatabaseGrant, ...]] = {} + # Group by (principal, schema, privilege set) so each principal carries at + # most one grant per schema per distinct privilege shape. + grouped: dict[str, dict[tuple[str, tuple[str, ...]], list[str]]] = {} + for (principal, schema_name, table_name), privileges in required.items(): + privilege_key = tuple(sorted(privilege.value for privilege in privileges)) + grouped.setdefault(principal, {}).setdefault( + (schema_name, privilege_key), [] + ).append(table_name) + + for principal in sorted(grouped): + principal_grants: list[ModelDeploymentTopologyDatabaseGrant] = [] + for schema_name, privilege_key in sorted(grouped[principal]): + names = sorted(grouped[principal][(schema_name, privilege_key)]) + principal_grants.append( + ModelDeploymentTopologyDatabaseGrant( + object_type=EnumDatabaseGrantObjectType.TABLE, + schema=schema_name, + objects=tuple(names), + privileges=tuple( + EnumDatabasePrivilege(value) for value in privilege_key + ), + ) + ) + grants[principal] = tuple(principal_grants) + + return DerivedTableGrants( + grants=grants, + unmappable=tuple( + unmappable[key] for key in sorted(unmappable, key=lambda item: item) + ), + ) + + +def derive_topology_table_grants( + topology: ModelDeploymentTopology, + declarations: Iterable[ContractTableDeclaration], +) -> TopologyTableGrants: + """Derive TABLE grants for every logical database the topology declares. + + Each declaration is routed to the database its contract names, so a + service-database relation is derived against that database's own + principals rather than being reported as an ``application`` residual. A + declaration naming a database no instance declares stays a typed residual + — routing must never invent a database to make a contract resolvable. + """ + materialized = tuple(declarations) + by_database_ref: dict[str, list[ContractTableDeclaration]] = {} + for declaration in materialized: + by_database_ref.setdefault(declaration.table.database_ref, []).append( + declaration + ) + + per_database: dict[str, DerivedTableGrants] = {} + residuals: dict[tuple[str, str, str], UnmappableDeclaration] = {} + for database_ref in topology.databases: + # Databases with no declarations still get an entry so the renderer + # clears any stale TABLE grant no contract backs any more. + derived = derive_table_grants( + topology, + by_database_ref.get(database_ref, ()), + database_ref=database_ref, + ) + per_database[database_ref] = derived + for residual in derived.unmappable: + residuals.setdefault(residual.key, residual) + + for database_ref, database_declarations in by_database_ref.items(): + if database_ref in topology.databases: + continue + for declaration in database_declarations: + table = declaration.table + residuals.setdefault( + (table.database_ref, table.schema, table.name), + UnmappableDeclaration( + node=declaration.node, + database_ref=table.database_ref, + schema=table.schema, + name=table.name, + reason=( + f"database_ref {table.database_ref!r} is not declared in " + "the topology" + ), + ), + ) + + return TopologyTableGrants( + per_database=per_database, + unmappable=tuple(residuals[key] for key in sorted(residuals)), + ) diff --git a/src/omnibase_infra/utils/util_onex_topic_format.py b/src/omnibase_infra/utils/util_onex_topic_format.py index 28ab414757..df184baba1 100644 --- a/src/omnibase_infra/utils/util_onex_topic_format.py +++ b/src/omnibase_infra/utils/util_onex_topic_format.py @@ -8,10 +8,22 @@ where kind is one of: evt, cmd, intent, dlq. -Legacy DLQ topics matching ``.dlq..v`` are accepted with a -distinct result code so callers can distinguish them from fully canonical names. +Tenant gateway wire topics matching ``tenant-.`` +and legacy DLQ topics matching ``.dlq..v`` are accepted with +distinct result codes so callers can distinguish them from bare canonical names. Kafka-internal topics (prefixed with ``__``) are silently skipped. + +OMN-15792: the ``tenant-.`` wire-prefix branch delegates to +``resolve_tenant_from_wire_topic`` -- THE single runtime topic resolver +(``service_gateway_topic_transform``) also consulted by the subscribe/dispatch +path (``handler_wiring.py``). This module previously hand-rolled its own +``tenant-.`` regex with no ``RESERVED_TENANT_SLUGS`` awareness -- a +THIRD independent resolver on the live Kafka publish path +(``event_bus_kafka.py``'s ``_enforce_onex_topic_format``) that could +publish-ALLOW a topic the subscribe side rejects (or vice versa). Delegating +here closes that divergence: this module and the subscribe path now share the +exact same slug-validation primitive and cannot disagree. """ from __future__ import annotations @@ -19,6 +31,10 @@ import re from enum import StrEnum +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( + resolve_tenant_from_wire_topic, +) + _RE_ONEX_TOPIC = re.compile( r"^onex\.(evt|cmd|intent|dlq)\.[a-z0-9-]+\.[a-z0-9._-]+\.v[1-9]\d*$" ) @@ -32,6 +48,7 @@ class TopicValidationResult(StrEnum): """Outcome of validating a topic name against the ONEX format.""" VALID = "valid" + VALID_TENANT_WIRE = "valid_tenant_wire" VALID_LEGACY_DLQ = "valid_legacy_dlq" INVALID = "invalid" SKIPPED_INTERNAL = "skipped_internal" @@ -47,10 +64,24 @@ def validate_onex_topic_format(topic: str) -> tuple[TopicValidationResult, str]: return (TopicValidationResult.SKIPPED_INTERNAL, "") if _RE_ONEX_TOPIC.match(topic): return (TopicValidationResult.VALID, "") + try: + tenant_slug, _canonical_topic = resolve_tenant_from_wire_topic(topic) + except ValueError as exc: + # A structurally tenant-prefixed topic (``tenant-.``) whose slug + # the shared resolver rejects (reserved, e.g. ``tenant-system.``, or + # malformed) -- publish-enforce and subscribe-resolve must agree, so + # this raises INVALID here exactly as the subscribe path raises. + return ( + TopicValidationResult.INVALID, + f"Topic '{topic}' has an invalid tenant wire prefix: {exc}", + ) + if tenant_slug is not None: + return (TopicValidationResult.VALID_TENANT_WIRE, "") if _RE_LEGACY_DLQ.match(topic): return (TopicValidationResult.VALID_LEGACY_DLQ, "legacy DLQ format") return ( TopicValidationResult.INVALID, f"Topic '{topic}' does not match ONEX format: " - "onex.(evt|cmd|intent|dlq)...v", + "onex.(evt|cmd|intent|dlq)...v or " + "tenant-.", ) diff --git a/src/omnibase_infra/utils/util_pydantic_validators.py b/src/omnibase_infra/utils/util_pydantic_validators.py index 8d28e770fb..5d8e4c5847 100644 --- a/src/omnibase_infra/utils/util_pydantic_validators.py +++ b/src/omnibase_infra/utils/util_pydantic_validators.py @@ -188,12 +188,12 @@ def _sanitize_url_for_logging(url: str, parsed: ParseResult | None = None) -> st URL with password redacted (if present), or original URL if no password. Example: - >>> _sanitize_url_for_logging("http://user:secret@localhost:8080/path") - 'http://user:****@localhost:8080/path' - >>> _sanitize_url_for_logging("http://localhost:8080/path") - 'http://localhost:8080/path' - >>> _sanitize_url_for_logging("user:secret@localhost:5432") - 'user:****@localhost:5432' + >>> _sanitize_url_for_logging("https://user:secret@api.example.invalid/path") + 'https://user:****@api.example.invalid/path' + >>> _sanitize_url_for_logging("https://api.example.invalid/path") + 'https://api.example.invalid/path' + >>> _sanitize_url_for_logging("user:secret@db.example.invalid:5432") + 'user:****@db.example.invalid:5432' """ if parsed is None: parsed = urlparse(url) @@ -256,15 +256,15 @@ def validate_endpoint_urls_dict(endpoints: dict[str, str]) -> dict[str, str]: >>> from omnibase_infra.utils import validate_endpoint_urls_dict >>> >>> # Valid endpoints - >>> endpoints = {"api": "http://localhost:8080", "grpc": "grpc://localhost:9090"} + >>> endpoints = {"api": "https://api.example.invalid", "grpc": "grpc://grpc.example.invalid"} >>> validate_endpoint_urls_dict(endpoints) == endpoints True >>> >>> # Invalid: missing scheme - >>> validate_endpoint_urls_dict({"api": "localhost:8080"}) # doctest: +ELLIPSIS + >>> validate_endpoint_urls_dict({"api": "api.example.invalid:8080"}) # doctest: +ELLIPSIS Traceback (most recent call last): ... - ValueError: Invalid URL for endpoint 'api': localhost:8080 + ValueError: Invalid URL for endpoint 'api': api.example.invalid:8080 Usage in Pydantic model: @field_validator("endpoints") diff --git a/src/omnibase_infra/validation/application_database_acl.py b/src/omnibase_infra/validation/application_database_acl.py new file mode 100644 index 0000000000..c2b0d502fa --- /dev/null +++ b/src/omnibase_infra/validation/application_database_acl.py @@ -0,0 +1,3771 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Generate and validate explicit application-database ACL matrices. + +The matrix is a projection. Deployment topology owns role/database/schema and +explicit object grants; typed relation inventories and service manifests own the +object set and access evidence. No table name or PostgreSQL role allowlist is +maintained here. +""" + +from __future__ import annotations + +import re +from collections import Counter, defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Protocol + +from omnibase_core.enums.enum_database_grant_object_type import ( + EnumDatabaseGrantObjectType, +) +from omnibase_core.enums.enum_database_privilege import EnumDatabasePrivilege +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_core.models.contracts.subcontracts.model_db_table_declaration import ( + ModelDbTableDeclaration, +) +from omnibase_core.models.core.model_deployment_topology import ModelDeploymentTopology +from omnibase_infra.validation.enums.enum_application_database_acl_authorization_scope import ( + EnumApplicationDatabaseAclAuthorizationScope, +) +from omnibase_infra.validation.enums.enum_application_database_acl_policy_source_kind import ( + EnumApplicationDatabaseAclPolicySourceKind, +) +from omnibase_infra.validation.enums.enum_application_database_acl_render_phase import ( + EnumApplicationDatabaseAclRenderPhase, +) +from omnibase_infra.validation.enums.enum_application_database_object_kind import ( + EnumApplicationDatabaseObjectKind, +) +from omnibase_infra.validation.enums.enum_application_database_principal_inventory_source_kind import ( + EnumApplicationDatabasePrincipalInventorySourceKind, +) +from omnibase_infra.validation.enums.enum_application_inventory_object_kind import ( + EnumApplicationInventoryObjectKind, +) +from omnibase_infra.validation.enums.enum_application_relation_kind import ( + EnumApplicationRelationKind, +) +from omnibase_infra.validation.models.model_application_database_acl_matrix import ( + ModelApplicationDatabaseAclMatrix, +) +from omnibase_infra.validation.models.model_application_database_acl_object import ( + ModelApplicationDatabaseAclObject, +) +from omnibase_infra.validation.models.model_application_database_acl_policy import ( + ModelApplicationDatabaseAclPolicy, +) +from omnibase_infra.validation.models.model_application_database_acl_row import ( + ModelApplicationDatabaseAclRow, +) +from omnibase_infra.validation.models.model_application_database_acl_source import ( + ModelApplicationDatabaseAclSource, +) +from omnibase_infra.validation.models.model_application_database_activity_result_evidence import ( + ModelApplicationDatabaseActivityResultEvidence, +) +from omnibase_infra.validation.models.model_application_database_catalog_object_evidence import ( + ModelApplicationDatabaseCatalogObjectEvidence, +) +from omnibase_infra.validation.models.model_application_database_catalog_result_evidence import ( + ModelApplicationDatabaseCatalogResultEvidence, +) +from omnibase_infra.validation.models.model_application_database_connection_policy import ( + ModelApplicationDatabaseConnectionPolicy, +) +from omnibase_infra.validation.models.model_application_database_default_acl_row import ( + ModelApplicationDatabaseDefaultAclRow, +) +from omnibase_infra.validation.models.model_application_database_observed_role_state import ( + ModelApplicationDatabaseObservedRoleState, +) +from omnibase_infra.validation.models.model_application_database_principal_inventory import ( + ModelApplicationDatabasePrincipalInventory, +) +from omnibase_infra.validation.models.model_application_database_role_membership import ( + ModelApplicationDatabaseRoleMembership, +) +from omnibase_infra.validation.models.model_application_database_role_state import ( + ModelApplicationDatabaseRoleState, +) +from omnibase_infra.validation.models.model_application_relation_evidence_inventory import ( + ModelApplicationRelationEvidenceInventory, +) +from omnibase_infra.validation.models.model_migration_ownership_manifest import ( + ModelMigrationOwnershipManifest, +) + +PUBLIC_PRINCIPAL = "PUBLIC" +_SQL_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]*$") +_READY_STATUSES = frozenset({"complete", "pass", "passed", "verified"}) +_OBJECT_TYPES = ( + EnumDatabaseGrantObjectType.TABLE, + EnumDatabaseGrantObjectType.SEQUENCE, + EnumDatabaseGrantObjectType.FUNCTION, + EnumDatabaseGrantObjectType.TYPE, +) +_DEPLOYMENT_CONNECT_DATABASES = frozenset( + { + "keycloak", + "omnibase_infra", + "omnidash_analytics", + "omninode_cloud", + "omniclaude", + "omniintelligence", + "omnimemory", + "umami", + } +) + + +@dataclass +class ObjectEvidence: + obj: ModelApplicationDatabaseAclObject + + +class ProtocolApplicationDatabaseRoleAttributeState(Protocol): + """Structural shape shared by observed and governed PostgreSQL role state.""" + + @property + def login(self) -> bool: ... + + @property + def superuser(self) -> bool: ... + + @property + def bypass_rls(self) -> bool: ... + + @property + def create_database(self) -> bool: ... + + @property + def create_role(self) -> bool: ... + + @property + def replication(self) -> bool: ... + + @property + def inherit(self) -> bool: ... + + +def _status_is_ready(status: str | None) -> bool: + return status is not None and status.strip().lower() in _READY_STATUSES + + +def _role_attribute_values( + state: ProtocolApplicationDatabaseRoleAttributeState, +) -> tuple[bool, ...]: + """Return the security-relevant PostgreSQL role attributes in stable order.""" + return ( + state.login, + state.superuser, + state.bypass_rls, + state.create_database, + state.create_role, + state.replication, + state.inherit, + ) + + +def validate_application_database_principal_evidence( + inventory: ModelApplicationDatabasePrincipalInventory, + catalog_result: ModelApplicationDatabaseCatalogResultEvidence, + activity_result: ModelApplicationDatabaseActivityResultEvidence, +) -> tuple[str, ...]: + """Bind inventory authorization fields to parsed immutable result content.""" + violations: list[str] = [] + for field_name in ( + "database_ref", + "physical_database", + "completion_status", + "catalog_parity_status", + "catalog_query_sha256", + "database_owner_role", + ): + inventory_field = ( + inventory.catalog_query_sha256 + if field_name == "catalog_query_sha256" + else getattr(inventory, field_name) + ) + if getattr(catalog_result, field_name) != inventory_field: + violations.append(f"catalog result {field_name} disagrees with inventory") + for field_name in ( + "principal_refs", + "absent_principal_refs", + "owner_refs", + "absent_owner_refs", + "absent_schema_refs", + ): + if set(getattr(catalog_result, field_name)) != set( + getattr(inventory, field_name) + ): + violations.append(f"catalog result {field_name} disagrees with inventory") + inventory_role_states = { + state.role: state.model_dump(mode="json") + for state in inventory.observed_role_states + } + result_role_states = { + state.role: state.model_dump(mode="json") + for state in catalog_result.observed_role_states + } + if result_role_states != inventory_role_states: + violations.append( + "catalog result observed_role_states disagrees with inventory" + ) + if catalog_result.observed_schema_owners != inventory.observed_schema_owners: + violations.append( + "catalog result observed_schema_owners disagrees with inventory" + ) + inventory_objects = { + obj.identity: obj.model_dump(mode="json") for obj in inventory.observed_objects + } + result_objects = { + obj.identity: obj.model_dump(mode="json") + for obj in catalog_result.observed_objects + } + if result_objects != inventory_objects: + violations.append("catalog result observed_objects disagrees with inventory") + activity = inventory.activity_evidence + if activity is None: + violations.append("inventory lacks activity evidence") + else: + for field_name, expected in ( + ("database_ref", inventory.database_ref), + ("physical_database", inventory.physical_database), + ("window_started_at", activity.window_started_at), + ("window_ended_at", activity.window_ended_at), + ("activity_query_sha256", activity.query_sha256), + ("observation_count", activity.observation_count), + ): + if getattr(activity_result, field_name) != expected: + violations.append( + f"activity result {field_name} disagrees with inventory" + ) + result_activity_principals = { + row.principal for row in activity_result.active_principals + } + if result_activity_principals != set(inventory.activity_principal_refs): + violations.append( + "activity result active principals disagree with inventory" + ) + return tuple(sorted(set(violations))) + + +def _schema_for_domain( + *, + topology: ModelDeploymentTopology, + database_ref: str, + domain: EnumDatabaseSchemaDomain, +) -> str: + database = topology.databases[database_ref] + matches = [ + name for name, schema in database.schemas.items() if schema.domain is domain + ] + if len(matches) != 1: + raise ValueError( + f"database_ref {database_ref!r} must resolve domain {domain.value!r} " + f"to exactly one schema, got {sorted(matches)!r}" + ) + return matches[0] + + +def _object_type_from_inventory( + kind: EnumApplicationInventoryObjectKind, +) -> EnumDatabaseGrantObjectType | None: + if kind in { + EnumApplicationInventoryObjectKind.TABLE, + EnumApplicationInventoryObjectKind.VIEW, + EnumApplicationInventoryObjectKind.MATERIALIZED_VIEW, + EnumApplicationInventoryObjectKind.FOREIGN_TABLE, + }: + return EnumDatabaseGrantObjectType.TABLE + if kind is EnumApplicationInventoryObjectKind.SEQUENCE: + return EnumDatabaseGrantObjectType.SEQUENCE + if kind in { + EnumApplicationInventoryObjectKind.FUNCTION, + EnumApplicationInventoryObjectKind.AGGREGATE, + EnumApplicationInventoryObjectKind.WINDOW_FUNCTION, + EnumApplicationInventoryObjectKind.PROCEDURE, + }: + return EnumDatabaseGrantObjectType.FUNCTION + if kind in { + EnumApplicationInventoryObjectKind.TYPE, + EnumApplicationInventoryObjectKind.BASE_TYPE, + EnumApplicationInventoryObjectKind.RANGE_TYPE, + EnumApplicationInventoryObjectKind.MULTIRANGE_TYPE, + }: + return EnumDatabaseGrantObjectType.TYPE + return None + + +def _catalog_kind_from_inventory( + kind: EnumApplicationInventoryObjectKind, +) -> str: + return kind.value + + +def _object_type_from_service( + kind: EnumApplicationDatabaseObjectKind, +) -> EnumDatabaseGrantObjectType | None: + if kind is EnumApplicationDatabaseObjectKind.SEQUENCE: + return EnumDatabaseGrantObjectType.SEQUENCE + if kind in { + EnumApplicationDatabaseObjectKind.FUNCTION, + EnumApplicationDatabaseObjectKind.AGGREGATE, + EnumApplicationDatabaseObjectKind.WINDOW_FUNCTION, + EnumApplicationDatabaseObjectKind.PROCEDURE, + }: + return EnumDatabaseGrantObjectType.FUNCTION + if kind in { + EnumApplicationDatabaseObjectKind.TYPE, + EnumApplicationDatabaseObjectKind.BASE_TYPE, + EnumApplicationDatabaseObjectKind.RANGE_TYPE, + EnumApplicationDatabaseObjectKind.MULTIRANGE_TYPE, + }: + return EnumDatabaseGrantObjectType.TYPE + return None + + +def _unsupported_object_acl_blocker( + *, + source_id: str, + schema_name: str | None, + object_name: str, + kind: str, +) -> str: + """Explain why an owned object cannot enter PostgreSQL's object ACL matrix.""" + location = f"{schema_name}.{object_name}" if schema_name else object_name + return ( + f"{source_id}:{location}:{kind} has no PostgreSQL object ACL or " + "default-privilege class; explicit extension security proof is required" + ) + + +def _source_status_blockers( + source_id: str, + inventory: ModelApplicationRelationEvidenceInventory, +) -> list[str]: + blockers: list[str] = [] + if inventory.relation_counts.type is None: + blockers.append( + f"{source_id}: relation_counts.type is not inventoried; " + "zero user-defined types cannot be inferred" + ) + if inventory.relation_counts.procedure is None: + blockers.append( + f"{source_id}: relation_counts.procedure is not inventoried; " + "zero procedures cannot be inferred" + ) + if not _status_is_ready(inventory.completion_status): + blockers.append( + f"{source_id}: completion_status={inventory.completion_status!r}" + ) + census = inventory.retained_live_census + typed_kind_counts = Counter(relation.kind.value for relation in inventory.relations) + expected_census_counts = { + "observed_base_tables": typed_kind_counts.get("table", 0), + "observed_views_and_materialized_views": ( + typed_kind_counts.get("view", 0) + + typed_kind_counts.get("materialized_view", 0) + ), + "observed_sequences": typed_kind_counts.get("sequence", 0), + "observed_functions": sum( + typed_kind_counts.get(kind, 0) + for kind in ("function", "aggregate", "window_function") + ), + "observed_procedures": typed_kind_counts.get("procedure", 0), + "observed_types": sum( + typed_kind_counts.get(kind, 0) + for kind in ("type", "base_type", "range_type", "multirange_type") + ), + "observed_extensions": typed_kind_counts.get("extension", 0), + } + for field_name in ( + "observed_base_tables", + "observed_views_and_materialized_views", + "observed_sequences", + "observed_functions", + "observed_procedures", + "observed_types", + "observed_extensions", + ): + live_count = getattr(census, field_name) + if live_count is None: + blockers.append( + f"{source_id}: retained_live_census.{field_name} is not inventoried" + ) + elif live_count != expected_census_counts[field_name]: + blockers.append( + f"{source_id}: retained_live_census.{field_name}={live_count} " + "does not match exact typed rows=" + f"{expected_census_counts[field_name]}" + ) + if not _status_is_ready(census.parity_status): + blockers.append( + f"{source_id}: retained_live_census={census.parity_status!r}: " + f"{census.reason or 'no reason supplied'}" + ) + for evidence_name in ( + "full_day_datname_usename_activity", + "live_catalog_parity", + ): + status = getattr(inventory.runtime_evidence, evidence_name) + if not _status_is_ready(status.status): + blockers.append( + f"{source_id}: runtime_evidence.{evidence_name}=" + f"{status.status.value!r}: " + f"{status.reason or 'no reason supplied'}" + ) + blockers.extend( + f"{source_id}: blocked {blocked.kind} {blocked.name!r}: {blocked.reason}" + for blocked in inventory.blocked_relations + ) + return blockers + + +def _service_status_blockers( + source_id: str, + manifest: ModelMigrationOwnershipManifest, +) -> list[str]: + blockers: list[str] = [] + if not manifest.completion_status: + blockers.append(f"{source_id}: completion_status is missing") + elif not _status_is_ready(manifest.completion_status): + blockers.append( + f"{source_id}: completion_status={manifest.completion_status!r}" + ) + if manifest.retained_live_census is None: + blockers.append(f"{source_id}: retained_live_census is missing") + else: + census = manifest.retained_live_census + relation_counts = Counter( + relation.kind.value for relation in manifest.relation_evidence + ) + object_counts = Counter( + database_object.kind.value for database_object in manifest.database_objects + ) + expected_counts = { + "observed_base_tables": relation_counts.get("table", 0), + "observed_views_and_materialized_views": ( + relation_counts.get("view", 0) + + relation_counts.get("materialized_view", 0) + ), + "observed_sequences": object_counts.get("sequence", 0), + "observed_functions": sum( + object_counts.get(kind, 0) + for kind in ("function", "aggregate", "window_function") + ), + "observed_procedures": object_counts.get("procedure", 0), + "observed_types": sum( + object_counts.get(kind, 0) + for kind in ("type", "base_type", "range_type", "multirange_type") + ), + "observed_extensions": object_counts.get("extension", 0), + } + for field_name, typed_count in expected_counts.items(): + live_count = getattr(census, field_name) + if live_count is None: + blockers.append( + f"{source_id}: retained_live_census.{field_name} is missing" + ) + elif live_count != typed_count: + blockers.append( + f"{source_id}: retained_live_census.{field_name}={live_count} " + f"does not match exact typed rows={typed_count}" + ) + census_status = census.parity_status + if not _status_is_ready(census_status): + blockers.append( + f"{source_id}: retained_live_census={census_status!r}: " + f"{census.reason or 'no reason supplied'}" + ) + required_runtime_evidence = { + "full_day_datname_usename_activity", + "live_catalog_parity", + } + missing_runtime_evidence = sorted( + required_runtime_evidence - manifest.runtime_evidence.keys() + ) + if missing_runtime_evidence: + blockers.append( + f"{source_id}: required runtime_evidence is missing " + f"{missing_runtime_evidence!r}" + ) + for evidence_name, runtime_status in sorted(manifest.runtime_evidence.items()): + if not _status_is_ready(runtime_status.status): + blockers.append( + f"{source_id}: runtime_evidence.{evidence_name}=" + f"{runtime_status.status.value!r}: " + f"{runtime_status.reason or 'no reason supplied'}" + ) + blockers.extend( + f"{source_id}: blocked {blocked.kind} {blocked.name!r}: {blocked.reason}" + for blocked in manifest.blocked_relations + ) + return blockers + + +def _merge_object( + objects: dict[ + tuple[str, str, EnumDatabaseGrantObjectType, str, str | None], + ObjectEvidence, + ], + evidence: ObjectEvidence, + blockers: list[str], +) -> None: + identity = evidence.obj.identity + existing = objects.get(identity) + if existing is None: + objects[identity] = evidence + return + blockers.append( + f"duplicate authoritative ownership declaration for {identity!r}: " + f"{existing.obj.owner_declaration!r} from {existing.obj.source_keys!r} vs " + f"{evidence.obj.owner_declaration!r} from {evidence.obj.source_keys!r}" + ) + + +def _append_inventory_objects( + *, + topology: ModelDeploymentTopology, + source_id: str, + inventory: ModelApplicationRelationEvidenceInventory, + objects: dict[ + tuple[str, str, EnumDatabaseGrantObjectType, str, str | None], + ObjectEvidence, + ], + blockers: list[str], + excluded: list[str], +) -> None: + database = topology.databases.get(inventory.database_ref) + if database is None: + blockers.append(f"{source_id}: unknown database_ref {inventory.database_ref!r}") + return + if database.physical_name != inventory.physical_seed_database: + blockers.append( + f"{source_id}: physical database drift: topology=" + f"{database.physical_name!r}, inventory=" + f"{inventory.physical_seed_database!r}" + ) + for relation in inventory.relations: + object_type = _object_type_from_inventory(relation.kind) + if object_type is None: + blockers.append( + _unsupported_object_acl_blocker( + source_id=source_id, + schema_name=relation.target_schema, + object_name=relation.name, + kind=relation.kind.value, + ) + ) + continue + if relation.target_schema not in database.schemas: + blockers.append( + f"{source_id}: object {relation.name!r} uses unknown schema " + f"{relation.target_schema!r}" + ) + continue + domain = database.schemas[relation.target_schema].domain + if relation.domain is None or relation.domain is not domain: + blockers.append( + f"{source_id}: object {relation.name!r} domain does not match " + f"topology schema {relation.target_schema!r}" + ) + if relation.classification_status.strip().lower() != "classified": + blockers.append( + f"{source_id}: object {relation.name!r} classification_status=" + f"{relation.classification_status!r}" + ) + if relation.target_schema not in relation.current_schema: + blockers.append( + f"{source_id}: object {relation.name!r} is currently in schemas " + f"{sorted(relation.current_schema)!r}, not the target schema " + f"{relation.target_schema!r}; full object ACL rendering is gated " + "until the additive target object is materialized and inventoried" + ) + for reason in relation.blocked_reasons: + blockers.append(f"{source_id}: object {relation.name!r} blocked: {reason}") + if relation.owner_declaration is None: + blockers.append( + f"{source_id}: object {relation.name!r} lacks an exact " + "owner_declaration" + ) + continue + owner = database.schemas[relation.target_schema].owner + _merge_object( + objects, + ObjectEvidence( + obj=ModelApplicationDatabaseAclObject( + database_ref=inventory.database_ref, + physical_database=database.physical_name, + schema_ref=relation.target_schema, + domain=domain, + object_type=object_type, + object_ref=relation.name, + catalog_kind=_catalog_kind_from_inventory(relation.kind), + owner=owner, + owner_declaration=relation.owner_declaration, + target_materialized=( + inventory.physical_seed_database == database.physical_name + and relation.target_schema in relation.current_schema + ), + function_signature=relation.function_signature, + source_keys=(source_id,), + ), + ), + blockers, + ) + + +def _append_service_objects( + *, + topology: ModelDeploymentTopology, + source_id: str, + manifest: ModelMigrationOwnershipManifest, + objects: dict[ + tuple[str, str, EnumDatabaseGrantObjectType, str, str | None], + ObjectEvidence, + ], + blockers: list[str], + excluded: list[str], +) -> None: + database = topology.databases.get(manifest.target_database_ref) + if database is None: + blockers.append( + f"{source_id}: unknown database_ref {manifest.target_database_ref!r}" + ) + return + if manifest.current_physical_database is None: + blockers.append(f"{source_id}: current_physical_database is not inventoried") + if database.physical_name not in manifest.materialized_physical_databases: + blockers.append( + f"{source_id}: target physical database {database.physical_name!r} is " + "absent from materialized_physical_databases; full object ACL rendering " + "is gated until the additive target is inventoried" + ) + tables_by_name: dict[str, list[ModelDbTableDeclaration]] = defaultdict(list) + for table in manifest.db_io.db_tables: + tables_by_name[table.name].append(table) + covered_table_identities: set[tuple[str, str, str]] = set() + for relation in manifest.relation_evidence: + if relation.kind is EnumApplicationRelationKind.FUNCTION: + blockers.append( + f"{source_id}: function relation_evidence {relation.name!r} requires " + "exact database_objects evidence including a routine signature" + ) + continue + schema_name: str | None + database_ref: str + if relation.kind is EnumApplicationRelationKind.TABLE: + matching_tables = tables_by_name.get(relation.name, []) + if relation.database_ref is not None: + matching_tables = [ + table + for table in matching_tables + if table.database_ref == relation.database_ref + ] + if relation.schema is not None: + matching_tables = [ + table + for table in matching_tables + if table.schema == relation.schema + ] + if len(matching_tables) != 1: + blockers.append( + f"{source_id}: table relation_evidence {relation.name!r} must " + "match exactly one db_io.db_tables declaration, got " + f"{[(table.database_ref, table.schema, table.name) for table in matching_tables]!r}" + ) + continue + table = matching_tables[0] + schema_name = relation.schema or table.schema + database_ref = relation.database_ref or table.database_ref + covered_table_identities.add((table.database_ref, table.schema, table.name)) + else: + schema_name = relation.schema + database_ref = relation.database_ref or manifest.target_database_ref + if schema_name is None: + try: + schema_name = _schema_for_domain( + topology=topology, + database_ref=database_ref, + domain=relation.domain, + ) + except (KeyError, ValueError) as exc: + blockers.append(f"{source_id}: {relation.name!r}: {exc}") + continue + if database_ref != manifest.target_database_ref: + blockers.append( + f"{source_id}: object {relation.name!r} database_ref " + f"{database_ref!r} conflicts with manifest target" + ) + continue + if schema_name not in database.schemas: + blockers.append( + f"{source_id}: object {relation.name!r} uses unknown schema " + f"{schema_name!r}" + ) + continue + if schema_name not in relation.current_schemas: + blockers.append( + f"{source_id}: object {relation.name!r} current_schemas=" + f"{relation.current_schemas!r} do not prove target schema " + f"{schema_name!r}; full object ACL rendering is gated" + ) + topology_domain = database.schemas[schema_name].domain + if relation.domain is not topology_domain: + blockers.append( + f"{source_id}: object {relation.name!r} domain does not match " + f"topology schema {schema_name!r}" + ) + _merge_object( + objects, + ObjectEvidence( + obj=ModelApplicationDatabaseAclObject( + database_ref=database_ref, + physical_database=database.physical_name, + schema_ref=schema_name, + domain=topology_domain, + object_type=EnumDatabaseGrantObjectType.TABLE, + object_ref=relation.name, + catalog_kind=relation.kind.value, + owner=database.schemas[schema_name].owner, + owner_declaration=relation.owner_declaration, + target_materialized=( + database.physical_name + in manifest.materialized_physical_databases + and schema_name in relation.current_schemas + ), + source_keys=(source_id,), + ), + ), + blockers, + ) + + for table in manifest.db_io.db_tables: + table_identity = (table.database_ref, table.schema, table.name) + if table_identity not in covered_table_identities: + blockers.append( + f"{source_id}: db_io table {table.database_ref}." + f"{table.schema}.{table.name} lacks exact relation_evidence" + ) + + for database_object in manifest.database_objects: + object_type = _object_type_from_service(database_object.kind) + if object_type is None: + blockers.append( + _unsupported_object_acl_blocker( + source_id=source_id, + schema_name=database_object.schema, + object_name=database_object.name, + kind=database_object.kind.value, + ) + ) + continue + database_ref = database_object.database_ref or manifest.target_database_ref + try: + schema_name = database_object.schema or _schema_for_domain( + topology=topology, + database_ref=database_ref, + domain=database_object.domain, + ) + except (KeyError, ValueError) as exc: + blockers.append(f"{source_id}: {database_object.name!r}: {exc}") + continue + if database_ref != manifest.target_database_ref: + blockers.append( + f"{source_id}: object {database_object.name!r} database_ref " + f"{database_ref!r} conflicts with manifest target" + ) + continue + topology_domain = database.schemas[schema_name].domain + if database_object.domain is not topology_domain: + blockers.append( + f"{source_id}: object {database_object.name!r} domain does not " + f"match topology schema {schema_name!r}" + ) + if schema_name not in database_object.current_schemas: + blockers.append( + f"{source_id}: object {database_object.name!r} current_schemas=" + f"{database_object.current_schemas!r} do not prove target schema " + f"{schema_name!r}; full object ACL rendering is gated" + ) + _merge_object( + objects, + ObjectEvidence( + obj=ModelApplicationDatabaseAclObject( + database_ref=database_ref, + physical_database=database.physical_name, + schema_ref=schema_name, + domain=topology_domain, + object_type=object_type, + object_ref=database_object.name, + catalog_kind=database_object.kind.value, + owner=database.schemas[schema_name].owner, + owner_declaration=database_object.owner_declaration, + target_materialized=( + database.physical_name + in manifest.materialized_physical_databases + and schema_name in database_object.current_schemas + ), + function_signature=database_object.function_signature, + source_keys=(source_id,), + ), + ), + blockers, + ) + + +def _topology_privileges( + topology: ModelDeploymentTopology, +) -> dict[ + tuple[ + str, + str, + EnumDatabaseGrantObjectType, + str | None, + str | None, + ], + set[EnumDatabasePrivilege], +]: + result: dict[ + tuple[ + str, + str, + EnumDatabaseGrantObjectType, + str | None, + str | None, + ], + set[EnumDatabasePrivilege], + ] = defaultdict(set) + for database_ref, database in topology.databases.items(): + for principal_name, principal in database.principals.items(): + for grant in principal.grants: + if grant.object_type in { + EnumDatabaseGrantObjectType.DATABASE, + EnumDatabaseGrantObjectType.SCHEMA, + }: + scope_key: tuple[ + str, + str, + EnumDatabaseGrantObjectType, + str | None, + str | None, + ] = ( + principal_name, + database_ref, + grant.object_type, + grant.schema, + None, + ) + result[scope_key].update(grant.privileges) + continue + for object_name in grant.objects: + object_key: tuple[ + str, + str, + EnumDatabaseGrantObjectType, + str | None, + str | None, + ] = ( + principal_name, + database_ref, + grant.object_type, + grant.schema, + object_name, + ) + result[object_key].update(grant.privileges) + return result + + +def build_application_database_acl_matrix( + *, + topology: ModelDeploymentTopology, + sources: Sequence[ModelApplicationDatabaseAclSource], + relation_inventories: Mapping[str, ModelApplicationRelationEvidenceInventory], + service_manifests: Mapping[str, ModelMigrationOwnershipManifest], + principal_inventories: Mapping[str, ModelApplicationDatabasePrincipalInventory], + acl_policies: Mapping[str, ModelApplicationDatabaseAclPolicy], + authorization_scope: EnumApplicationDatabaseAclAuthorizationScope, + required_connect_databases: Sequence[str] | None = None, + catalog_results: Mapping[ + str, + ModelApplicationDatabaseCatalogResultEvidence, + ] + | None = None, + activity_results: Mapping[ + str, + ModelApplicationDatabaseActivityResultEvidence, + ] + | None = None, +) -> ModelApplicationDatabaseAclMatrix: + """Build a complete cell matrix without inventing missing grants.""" + source_ids = {source.source_key for source in sources} + if len(source_ids) != len(sources): + raise ValueError("Matrix source keys must be unique") + source_by_key = {source.source_key: source for source in sources} + parsed_catalog_results = dict(catalog_results or {}) + parsed_activity_results = dict(activity_results or {}) + unknown_result_sources = sorted( + set(parsed_catalog_results).union(parsed_activity_results) - source_ids + ) + if unknown_result_sources: + raise ValueError( + "Parsed evidence keys are absent from source records: " + f"{unknown_result_sources!r}" + ) + invalid_catalog_sources = sorted( + source_key + for source_key in parsed_catalog_results + if source_by_key[source_key].purpose != "catalog_result_evidence" + ) + invalid_activity_sources = sorted( + source_key + for source_key in parsed_activity_results + if source_by_key[source_key].purpose != "activity_result_evidence" + ) + if invalid_catalog_sources or invalid_activity_sources: + raise ValueError( + "Parsed evidence source purposes disagree: " + f"catalog={invalid_catalog_sources!r} activity={invalid_activity_sources!r}" + ) + verified_evidence: set[str] = set() + missing_source_ids = sorted( + ( + set(relation_inventories) + | set(service_manifests) + | set(principal_inventories) + | set(acl_policies) + ) + - source_ids + ) + if missing_source_ids: + raise ValueError(f"Matrix inputs lack source records: {missing_source_ids!r}") + + blockers: list[str] = [] + object_blockers: list[str] = [] + if required_connect_databases is None: + if ( + authorization_scope + is EnumApplicationDatabaseAclAuthorizationScope.DEPLOYMENT + ): + blockers.append( + "deployment authorization requires the explicit eight-database " + "CONNECT inventory" + ) + required_connect_values: Sequence[str] = () + else: + required_connect_values = tuple( + database.physical_name for database in topology.databases.values() + ) + else: + required_connect_values = required_connect_databases + required_connect = tuple(sorted(required_connect_values)) + if ( + authorization_scope is EnumApplicationDatabaseAclAuthorizationScope.DEPLOYMENT + and set(required_connect) != _DEPLOYMENT_CONNECT_DATABASES + ): + blockers.append( + "deployment CONNECT database universe disagrees with the approved " + f"eight-database scope: missing=" + f"{sorted(_DEPLOYMENT_CONNECT_DATABASES - set(required_connect))!r} " + f"extra={sorted(set(required_connect) - _DEPLOYMENT_CONNECT_DATABASES)!r}" + ) + if len(set(required_connect)) != len(required_connect): + blockers.append("required CONNECT database names must be unique") + topology_sources = { + source.source_key for source in sources if source.purpose == "topology" + } + if len(topology_sources) != 1: + blockers.append( + "matrix requires exactly one topology source, got " + f"{sorted(topology_sources)!r}" + ) + declared_relation_sources = { + source.source_key + for source in sources + if source.purpose == "relation_inventory" + } + declared_service_sources = { + source.source_key for source in sources if source.purpose == "service_ownership" + } + declared_principal_sources = { + source.source_key + for source in sources + if source.purpose == "principal_inventory" + } + declared_policy_sources = { + source.source_key for source in sources if source.purpose == "acl_policy" + } + declared_ownership_sources = declared_relation_sources.union( + declared_service_sources + ) + if not declared_ownership_sources: + object_blockers.append( + "matrix requires at least one typed ownership evidence source" + ) + missing_relation_inputs = sorted( + declared_relation_sources - relation_inventories.keys() + ) + if missing_relation_inputs: + object_blockers.append( + "declared relation_inventory sources were not parsed: " + f"{missing_relation_inputs!r}" + ) + missing_service_inputs = sorted(declared_service_sources - service_manifests.keys()) + if missing_service_inputs: + object_blockers.append( + "declared service_ownership sources were not parsed: " + f"{missing_service_inputs!r}" + ) + if not declared_principal_sources: + blockers.append( + "matrix requires a typed principal_inventory source for every database" + ) + missing_principal_inputs = sorted( + declared_principal_sources - principal_inventories.keys() + ) + if missing_principal_inputs: + blockers.append( + "declared principal_inventory sources were not parsed: " + f"{missing_principal_inputs!r}" + ) + if not declared_policy_sources: + blockers.append( + "matrix requires an independent typed acl_policy source for every database" + ) + missing_policy_inputs = sorted(declared_policy_sources - acl_policies.keys()) + if missing_policy_inputs: + blockers.append( + f"declared acl_policy sources were not parsed: {missing_policy_inputs!r}" + ) + excluded: list[str] = [] + objects: dict[ + tuple[str, str, EnumDatabaseGrantObjectType, str, str | None], + ObjectEvidence, + ] = {} + for source_id, inventory in sorted(relation_inventories.items()): + object_blockers.extend(_source_status_blockers(source_id, inventory)) + _append_inventory_objects( + topology=topology, + source_id=source_id, + inventory=inventory, + objects=objects, + blockers=object_blockers, + excluded=excluded, + ) + for source_id, manifest in sorted(service_manifests.items()): + object_blockers.extend(_service_status_blockers(source_id, manifest)) + _append_service_objects( + topology=topology, + source_id=source_id, + manifest=manifest, + objects=objects, + blockers=object_blockers, + excluded=excluded, + ) + topology_privileges = _topology_privileges(topology) + object_identities_by_topology_target: dict[ + tuple[str, str, EnumDatabaseGrantObjectType, str], + list[tuple[str, str, EnumDatabaseGrantObjectType, str, str | None]], + ] = defaultdict(list) + for identity in objects: + object_identities_by_topology_target[identity[:4]].append(identity) + resolved_object_privileges: dict[ + tuple[ + str, + str, + EnumDatabaseGrantObjectType, + str, + str, + str | None, + ], + set[EnumDatabasePrivilege], + ] = defaultdict(set) + explicit_object_grants = { + key: privileges + for key, privileges in topology_privileges.items() + if key[2] in _OBJECT_TYPES and privileges + } + if objects and not explicit_object_grants: + object_blockers.append( + "topology declares zero explicit object grants for " + f"{len(objects)} typed ownership objects" + ) + for key, privileges in sorted( + topology_privileges.items(), key=lambda item: tuple(str(x) for x in item[0]) + ): + principal, database_ref, object_type, schema_name, object_name = key + if object_type not in _OBJECT_TYPES: + continue + topology_target = ( + database_ref, + schema_name or "", + object_type, + object_name or "", + ) + matching_identities = object_identities_by_topology_target.get( + topology_target, [] + ) + if not matching_identities: + object_blockers.append( + f"topology grant {principal}:{object_type.value}:" + f"{schema_name}.{object_name} targets no typed ownership object " + f"({sorted(privilege.value for privilege in privileges)!r})" + ) + elif len(matching_identities) > 1: + signatures = sorted( + identity[4] or "" for identity in matching_identities + ) + object_blockers.append( + f"topology grant {principal}:{object_type.value}:" + f"{schema_name}.{object_name} is ambiguous across exact object " + f"identities {signatures!r}; overload-specific targeting is required" + ) + else: + exact_identity = matching_identities[0] + resolved_object_privileges[ + ( + principal, + exact_identity[0], + exact_identity[2], + exact_identity[1], + exact_identity[3], + exact_identity[4], + ) + ].update(privileges) + + for evidence in objects.values(): + obj = evidence.obj + if ( + obj.object_type is EnumDatabaseGrantObjectType.FUNCTION + and not obj.function_signature + ): + object_blockers.append( + f"{obj.schema_ref}.{obj.object_ref}: ownership and ACL rendering " + "require an explicit function_signature to avoid overload widening" + ) + + if not objects: + object_blockers.append( + "typed ownership evidence resolved to zero database objects" + ) + object_databases = {identity[0] for identity in objects} + for database_ref in sorted(set(topology.databases) - object_databases): + object_blockers.append( + f"{database_ref}: no typed ownership objects cover this topology database" + ) + + principal_inputs_by_database: dict[ + str, + list[tuple[str, ModelApplicationDatabasePrincipalInventory]], + ] = defaultdict(list) + observed_owner_roles = tuple( + sorted( + { + owner + for inventory in principal_inventories.values() + for owner in inventory.owner_refs + } + ) + ) + absent_owner_roles = tuple( + sorted( + { + owner + for inventory in principal_inventories.values() + for owner in inventory.absent_owner_refs + } + ) + ) + observed_role_state_by_name: dict[ + str, + ModelApplicationDatabaseObservedRoleState, + ] = {} + for source_id, principal_inventory in sorted(principal_inventories.items()): + for observed_state in principal_inventory.observed_role_states: + existing_state = observed_role_state_by_name.get(observed_state.role) + if existing_state is not None and existing_state != observed_state: + blockers.append( + f"{source_id}: observed role attributes for " + f"{observed_state.role!r} conflict across cluster inventories" + ) + observed_role_state_by_name[observed_state.role] = observed_state + if not _status_is_ready(principal_inventory.completion_status): + blockers.append( + f"{source_id}: completion_status=" + f"{principal_inventory.completion_status!r}" + ) + if not _status_is_ready(principal_inventory.catalog_parity_status): + blockers.append( + f"{source_id}: catalog_parity_status=" + f"{principal_inventory.catalog_parity_status!r}: " + f"{principal_inventory.reason}" + ) + if ( + authorization_scope + is EnumApplicationDatabaseAclAuthorizationScope.DEPLOYMENT + ): + if principal_inventory.source_kind != ( + EnumApplicationDatabasePrincipalInventorySourceKind.AUTHORIZED_CATALOG + ): + blockers.append( + f"{source_id}: deployment authorization requires " + "source_kind='authorized_catalog'" + ) + if ( + principal_inventory.activity_evidence is None + or principal_inventory.catalog_query_sha256 is None + or principal_inventory.catalog_result_sha256 is None + or principal_inventory.catalog_query_source_key is None + or principal_inventory.catalog_result_source_key is None + ): + blockers.append( + f"{source_id}: deployment authorization lacks durable full-day " + "activity and catalog query/result provenance" + ) + else: + catalog_result_key = principal_inventory.catalog_result_source_key + activity_result_key = ( + principal_inventory.activity_evidence.result_source_key + ) + catalog_result = parsed_catalog_results.get(catalog_result_key) + activity_result = parsed_activity_results.get(activity_result_key) + if catalog_result is None: + blockers.append( + f"{source_id}: catalog result {catalog_result_key!r} was " + "not supplied as parsed typed content" + ) + if activity_result is None: + blockers.append( + f"{source_id}: activity result {activity_result_key!r} was " + "not supplied as parsed typed content" + ) + if catalog_result is not None and activity_result is not None: + semantic_violations = ( + validate_application_database_principal_evidence( + principal_inventory, + catalog_result, + activity_result, + ) + ) + if semantic_violations: + blockers.extend( + f"{source_id}: typed evidence mismatch: {violation}" + for violation in semantic_violations + ) + else: + verified_evidence.update( + {catalog_result_key, activity_result_key} + ) + evidence_bindings = ( + ( + principal_inventory.catalog_query_source_key, + principal_inventory.catalog_query_sha256, + "catalog_query_evidence", + ), + ( + principal_inventory.catalog_result_source_key, + principal_inventory.catalog_result_sha256, + "catalog_result_evidence", + ), + ( + principal_inventory.activity_evidence.query_source_key, + principal_inventory.activity_evidence.query_sha256, + "activity_query_evidence", + ), + ( + principal_inventory.activity_evidence.result_source_key, + principal_inventory.activity_evidence.result_sha256, + "activity_result_evidence", + ), + ) + for ( + evidence_key, + evidence_digest, + evidence_purpose, + ) in evidence_bindings: + evidence_source = source_by_key.get(evidence_key) + if evidence_source is None: + blockers.append( + f"{source_id}: evidence source {evidence_key!r} is absent " + "from the immutable source lock" + ) + elif evidence_source.purpose != evidence_purpose: + blockers.append( + f"{source_id}: evidence source {evidence_key!r} has purpose " + f"{evidence_source.purpose!r}, expected {evidence_purpose!r}" + ) + elif evidence_source.sha256 != evidence_digest: + blockers.append( + f"{source_id}: evidence digest {evidence_digest!r} does not " + f"match locked source {evidence_key!r} " + f"({evidence_source.sha256!r})" + ) + elif ( + evidence_purpose + in { + "catalog_result_evidence", + "activity_result_evidence", + } + and evidence_key not in verified_evidence + ): + blockers.append( + f"{source_id}: evidence source {evidence_key!r} was " + "hash-bound but its typed result content was not " + "semantically verified" + ) + elif principal_inventory.source_kind != ( + EnumApplicationDatabasePrincipalInventorySourceKind.SYNTHETIC_FIXTURE + ): + blockers.append( + f"{source_id}: synthetic proof requires source_kind='synthetic_fixture'" + ) + database = topology.databases.get(principal_inventory.database_ref) + if ( + database is not None + and database.physical_name != principal_inventory.physical_database + ): + blockers.append( + f"{source_id}: physical database drift: topology=" + f"{database.physical_name!r} inventory=" + f"{principal_inventory.physical_database!r}" + ) + principal_inputs_by_database[principal_inventory.database_ref].append( + (source_id, principal_inventory) + ) + + policy_inputs_by_database: dict[ + str, + list[tuple[str, ModelApplicationDatabaseAclPolicy]], + ] = defaultdict(list) + for source_id, policy in sorted(acl_policies.items()): + if not _status_is_ready(policy.completion_status): + blockers.append( + f"{source_id}: completion_status={policy.completion_status!r}" + ) + required_policy_kind = ( + EnumApplicationDatabaseAclPolicySourceKind.TOPOLOGY_CONTRACT + if authorization_scope + is EnumApplicationDatabaseAclAuthorizationScope.DEPLOYMENT + else EnumApplicationDatabaseAclPolicySourceKind.SYNTHETIC_FIXTURE + ) + if policy.source_kind is not required_policy_kind: + blockers.append( + f"{source_id}: {authorization_scope.value} authorization requires " + f"ACL policy source_kind={required_policy_kind.value!r}" + ) + database = topology.databases.get(policy.database_ref) + if database is None: + blockers.append( + f"{source_id}: unknown database_ref {policy.database_ref!r}" + ) + continue + if database.physical_name != policy.physical_database: + blockers.append( + f"{source_id}: physical database drift: topology=" + f"{database.physical_name!r} policy={policy.physical_database!r}" + ) + policy_inputs_by_database[policy.database_ref].append((source_id, policy)) + + connection_policy_inputs: dict[ + str, + list[tuple[str, ModelApplicationDatabaseConnectionPolicy]], + ] = defaultdict(list) + policy_governed_roles = { + state.role + for policy in acl_policies.values() + for state in policy.governed_role_states + } + noncreatable_policy_roles = { + state.role + for policy in acl_policies.values() + for state in policy.governed_role_states + if not state.manage_attributes + } + for source_id, policy in sorted(acl_policies.items()): + for connection_policy in policy.connection_policies: + connection_policy_inputs[connection_policy.physical_database].append( + (source_id, connection_policy) + ) + + allowed_connect_principals: dict[str, tuple[str, ...]] = {} + observed_connect_principals: dict[str, tuple[str, ...]] = {} + absent_connect_principals: dict[str, tuple[str, ...]] = {} + observed_connect_database_owners: dict[str, str] = {} + connection_database_refs: set[str] = set() + for physical_database in required_connect: + policies = connection_policy_inputs.get(physical_database, []) + if len(policies) != 1: + blockers.append( + f"{physical_database}: requires exactly one CONNECT policy, got " + f"{[source_id for source_id, _ in policies]!r}" + ) + continue + source_id, connection_policy = policies[0] + database_ref = connection_policy.database_ref + connection_database_refs.add(database_ref) + inventory_inputs = principal_inputs_by_database.get(database_ref, []) + if len(inventory_inputs) != 1: + blockers.append( + f"{physical_database}: requires exactly one principal inventory " + f"for CONNECT policy {database_ref!r}, got " + f"{[item[0] for item in inventory_inputs]!r}" + ) + continue + connection_inventory = inventory_inputs[0][1] + if connection_inventory.physical_database != physical_database: + blockers.append( + f"{physical_database}: CONNECT inventory physical database is " + f"{connection_inventory.physical_database!r}" + ) + continue + allowed = tuple(sorted(connection_policy.allowed_principals)) + connection_observed = tuple(sorted(connection_inventory.principal_refs)) + connection_absent = tuple(sorted(connection_inventory.absent_principal_refs)) + missing_allowed = sorted( + set(allowed) - set(connection_observed) - set(connection_absent) + ) + if missing_allowed: + blockers.append( + f"{physical_database}: CONNECT inventory lacks presence/absence " + "evidence for allowed principals " + f"{missing_allowed!r}" + ) + unmanaged_absent = sorted( + set(allowed) + .intersection(connection_absent) + .intersection(noncreatable_policy_roles) + ) + if unmanaged_absent: + blockers.append( + f"{physical_database}: CONNECT-only policy cannot create absent " + "external principals without governed role-state authorization: " + f"{unmanaged_absent!r}" + ) + parent_policy = acl_policies[source_id] + classified_activity_principals = ( + set(allowed) + .union(parent_policy.retained_administrative_principals) + .union({parent_policy.migration_principal}) + .union({connection_inventory.database_owner_role}) + .union(connection_inventory.owner_refs) + ) + unclassified_activity = sorted( + set(connection_inventory.activity_principal_refs) + - classified_activity_principals + ) + if unclassified_activity: + object_blockers.append( + f"{physical_database}: full-day activity contains principals not " + "classified as allowed, migration, administrative, or owner: " + f"{unclassified_activity!r}" + ) + if ( + connection_policy.observed_database_owner_role + != connection_inventory.database_owner_role + ): + blockers.append( + f"{physical_database}: CONNECT policy observed database owner " + f"{connection_policy.observed_database_owner_role!r} disagrees " + f"with catalog inventory {connection_inventory.database_owner_role!r}" + ) + topology_database = topology.databases.get(database_ref) + if topology_database is not None: + topology_allowed = { + principal + for principal, principal_contract in topology_database.principals.items() + if any( + grant.object_type is EnumDatabaseGrantObjectType.DATABASE + and EnumDatabasePrivilege.CONNECT in grant.privileges + for grant in principal_contract.grants + ) + } + if set(allowed) != topology_allowed: + blockers.append( + f"{physical_database}: CONNECT policy disagrees with topology: " + f"missing={sorted(topology_allowed - set(allowed))!r} " + f"extra={sorted(set(allowed) - topology_allowed)!r}" + ) + observed_connect_database_owners[physical_database] = ( + connection_policy.observed_database_owner_role + ) + allowed_connect_principals[physical_database] = allowed + observed_connect_principals[physical_database] = connection_observed + absent_connect_principals[physical_database] = connection_absent + _ = source_id + + unexpected_connection_policies = sorted( + set(connection_policy_inputs) - set(required_connect) + ) + if unexpected_connection_policies: + blockers.append( + "CONNECT policies include databases outside the required inventory: " + f"{unexpected_connection_policies!r}" + ) + consumed_inventory_refs = set(topology.databases).union(connection_database_refs) + unexpected_inventory_refs = sorted( + set(principal_inputs_by_database) - consumed_inventory_refs + ) + if unexpected_inventory_refs: + blockers.append( + "principal inventories are not bound to topology or CONNECT policy: " + f"{unexpected_inventory_refs!r}" + ) + + declared_principals: dict[str, tuple[str, ...]] = {} + observed_principals: dict[str, tuple[str, ...]] = {} + absent_principals: dict[str, tuple[str, ...]] = {} + database_owners: dict[str, str] = {} + observed_schema_owners: dict[str, dict[str, str]] = {} + absent_schemas: dict[str, tuple[str, ...]] = {} + observed_catalog_objects: list[ModelApplicationDatabaseCatalogObjectEvidence] = [] + principal_domains: dict[str, set[EnumDatabaseSchemaDomain]] = defaultdict(set) + allowed_memberships: list[ModelApplicationDatabaseRoleMembership] = [] + governed_role_states: list[ModelApplicationDatabaseRoleState] = [] + retained_administrative_principals: set[str] = set() + for database_ref, database in sorted(topology.databases.items()): + declared = tuple(sorted(database.principals)) + declared_principals[database_ref] = declared + + inventory_inputs = principal_inputs_by_database.get(database_ref, []) + if len(inventory_inputs) != 1: + blockers.append( + f"{database_ref}: requires exactly one principal inventory, got " + f"{[source_id for source_id, _ in inventory_inputs]!r}" + ) + observed: tuple[str, ...] = () + absent: tuple[str, ...] = () + else: + observed = tuple(sorted(inventory_inputs[0][1].principal_refs)) + absent = tuple(sorted(inventory_inputs[0][1].absent_principal_refs)) + missing_declared = sorted(set(declared) - set(observed) - set(absent)) + if missing_declared: + blockers.append( + f"{database_ref}: principal inventory lacks presence/absence " + "evidence for topology principals " + f"{missing_declared!r}" + ) + observed_principals[database_ref] = observed + absent_principals[database_ref] = absent + if len(inventory_inputs) == 1: + observed_schema_owner_values = dict( + sorted(inventory_inputs[0][1].observed_schema_owners.items()) + ) + absent_schema_refs = tuple( + sorted(inventory_inputs[0][1].absent_schema_refs) + ) + missing_schema_evidence = sorted( + set(database.schemas) + - set(observed_schema_owner_values) + - set(absent_schema_refs) + ) + extra_schema_evidence = sorted( + set(observed_schema_owner_values).union(absent_schema_refs) + - set(database.schemas) + ) + if missing_schema_evidence or extra_schema_evidence: + blockers.append( + f"{database_ref}: target schema presence/absence evidence " + f"differs from topology: missing={missing_schema_evidence!r} " + f"extra={extra_schema_evidence!r}" + ) + observed_catalog_objects.extend(inventory_inputs[0][1].observed_objects) + else: + observed_schema_owner_values = {} + absent_schema_refs = () + observed_schema_owners[database_ref] = observed_schema_owner_values + absent_schemas[database_ref] = absent_schema_refs + + policy_inputs = policy_inputs_by_database.get(database_ref, []) + if len(policy_inputs) != 1: + blockers.append( + f"{database_ref}: requires exactly one independent ACL policy, got " + f"{[source_id for source_id, _ in policy_inputs]!r}" + ) + continue + policy = policy_inputs[0][1] + policy_principals = set(policy.principal_domains) + declared_set = set(declared) + if policy_principals != declared_set: + blockers.append( + f"{database_ref}: ACL policy principals differ from topology: " + f"missing={sorted(declared_set - policy_principals)!r} " + f"extra={sorted(policy_principals - declared_set)!r}" + ) + expected_owners = set(database.owners) + policy_owners = set(policy.migration_owner_roles) + if policy_owners != expected_owners: + blockers.append( + f"{database_ref}: migration owner roles differ from topology: " + f"missing={sorted(expected_owners - policy_owners)!r} " + f"extra={sorted(policy_owners - expected_owners)!r}" + ) + if policy.database_owner_role not in expected_owners: + blockers.append( + f"{database_ref}: database owner role " + f"{policy.database_owner_role!r} is not a topology owner" + ) + else: + database_owners[database_ref] = policy.database_owner_role + if len(inventory_inputs) == 1: + principal_inventory = inventory_inputs[0][1] + owner_evidence = set(principal_inventory.owner_refs).union( + principal_inventory.absent_owner_refs + ) + missing_owner_evidence = sorted(expected_owners - owner_evidence) + if missing_owner_evidence: + blockers.append( + f"{database_ref}: managed owner roles lack presence/absence " + f"evidence {missing_owner_evidence!r}" + ) + if policy.migration_principal not in observed: + blockers.append( + f"{database_ref}: principal inventory omits migration principal " + f"{policy.migration_principal!r}" + ) + if policy.migration_principal in declared_set or ( + policy.migration_principal in expected_owners + ): + blockers.append( + f"{database_ref}: migration principal must be distinct from " + "workload and owner roles" + ) + external_connect_roles = { + principal + for connection_policy in policy.connection_policies + for principal in connection_policy.allowed_principals + } - declared_set + expected_governed_roles = ( + expected_owners.union(declared_set) + .union({policy.migration_principal}) + .union(external_connect_roles) + ) + actual_governed_roles = {state.role for state in policy.governed_role_states} + if actual_governed_roles != expected_governed_roles: + blockers.append( + f"{database_ref}: governed role-state policy differs from topology " + f"and migration roles: missing=" + f"{sorted(expected_governed_roles - actual_governed_roles)!r} " + f"extra={sorted(actual_governed_roles - expected_governed_roles)!r}" + ) + retained_admins = set(policy.retained_administrative_principals) + retained_administrative_principals.update(retained_admins) + if retained_admins & actual_governed_roles: + blockers.append( + f"{database_ref}: retained administrative principals overlap " + "governed role-state policy" + ) + missing_retained_admins = sorted(retained_admins - set(observed)) + if missing_retained_admins: + blockers.append( + f"{database_ref}: retained administrative principals are absent " + f"from the observed census: {missing_retained_admins!r}" + ) + for state in policy.governed_role_states: + expected_kind = ( + "owner" + if state.role in expected_owners + else "workload" + if state.role in declared_set + else "migration" + if state.role == policy.migration_principal + else "external_connect" + ) + expected_login = state.role in declared_set.union(external_connect_roles) + expected_manage_attributes = state.role not in external_connect_roles + if ( + state.role_kind != expected_kind + or state.login is not expected_login + or state.manage_attributes is not expected_manage_attributes + or not state.manage_memberships + ): + blockers.append( + f"{database_ref}: governed role state for {state.role!r} must " + f"be kind={expected_kind!r} login={expected_login!r} " + f"manage_attributes={expected_manage_attributes!r} " + "manage_memberships=True" + ) + if not state.manage_attributes: + actual_role_state = observed_role_state_by_name.get(state.role) + if actual_role_state is None: + blockers.append( + f"{database_ref}: non-mutating governed role {state.role!r} " + "lacks observed attribute evidence" + ) + elif any( + getattr(actual_role_state, field_name) != getattr(state, field_name) + for field_name in ( + "login", + "superuser", + "bypass_rls", + "create_database", + "create_role", + "replication", + "inherit", + ) + ): + blockers.append( + f"{database_ref}: non-mutating governed role {state.role!r} " + "does not already match the safe desired attribute state" + ) + governed_role_states.extend(policy.governed_role_states) + for owner in sorted(policy_owners & expected_owners): + allowed_memberships.append( + ModelApplicationDatabaseRoleMembership( + database_ref=database_ref, + role=owner, + member=policy.migration_principal, + admin_option=False, + inherit_option=False, + set_option=True, + ) + ) + for principal_name, domains in policy.principal_domains.items(): + if principal_name in declared_set: + principal_domains[principal_name].update(domains) + + rows: list[ModelApplicationDatabaseAclRow] = [] + for database_ref, database in sorted(topology.databases.items()): + principal_universe = set(database.principals) | set( + observed_principals[database_ref] + ) + grantees = (PUBLIC_PRINCIPAL, *sorted(principal_universe)) + for principal in grantees: + rows.append( + ModelApplicationDatabaseAclRow( + principal=principal, + database_ref=database_ref, + physical_database=database.physical_name, + object_type=EnumDatabaseGrantObjectType.DATABASE, + privileges=tuple( + sorted( + topology_privileges.get( + ( + principal, + database_ref, + EnumDatabaseGrantObjectType.DATABASE, + None, + None, + ), + set(), + ), + key=lambda privilege: privilege.value, + ) + ), + ) + ) + for schema_name in sorted(database.schemas): + rows.append( + ModelApplicationDatabaseAclRow( + principal=principal, + database_ref=database_ref, + physical_database=database.physical_name, + object_type=EnumDatabaseGrantObjectType.SCHEMA, + schema_ref=schema_name, + privileges=tuple( + sorted( + topology_privileges.get( + ( + principal, + database_ref, + EnumDatabaseGrantObjectType.SCHEMA, + schema_name, + None, + ), + set(), + ), + key=lambda privilege: privilege.value, + ) + ), + ) + ) + + ordered_objects = tuple( + sorted( + (evidence.obj for evidence in objects.values()), + key=lambda obj: ( + obj.database_ref, + obj.schema_ref, + obj.object_type.value, + obj.object_ref, + obj.function_signature or "", + ), + ) + ) + expected_catalog_object_ids: set[tuple[str, str, str, str]] = { + ( + obj.catalog_kind, + obj.schema_ref, + obj.object_ref, + obj.function_signature or "", + ) + for obj in ordered_objects + } + observed_catalog_object_ids: set[tuple[str, str, str, str]] = { + obj.identity for obj in observed_catalog_objects + } + if expected_catalog_object_ids != observed_catalog_object_ids: + object_blockers.append( + "live catalog object identities differ from the exact typed ownership " + f"projection: missing=" + f"{sorted(expected_catalog_object_ids - observed_catalog_object_ids)!r} " + f"extra={sorted(observed_catalog_object_ids - expected_catalog_object_ids)!r}" + ) + for obj in ordered_objects: + database = topology.databases[obj.database_ref] + principal_universe = set(database.principals) | set( + observed_principals[obj.database_ref] + ) + for principal in (PUBLIC_PRINCIPAL, *sorted(principal_universe)): + rows.append( + ModelApplicationDatabaseAclRow( + principal=principal, + database_ref=obj.database_ref, + physical_database=obj.physical_database, + object_type=obj.object_type, + schema_ref=obj.schema_ref, + object_ref=obj.object_ref, + function_signature=obj.function_signature, + privileges=tuple( + sorted( + resolved_object_privileges.get( + ( + principal, + obj.database_ref, + obj.object_type, + obj.schema_ref, + obj.object_ref, + obj.function_signature, + ), + set(), + ), + key=lambda privilege: privilege.value, + ) + ), + ) + ) + + default_rows: list[ModelApplicationDatabaseDefaultAclRow] = [] + for database_ref, database in sorted(topology.databases.items()): + principal_universe = set(database.principals) | set( + observed_principals[database_ref] + ) + grantees = (PUBLIC_PRINCIPAL, *sorted(principal_universe)) + for schema_name, schema in sorted(database.schemas.items()): + for object_type in _OBJECT_TYPES: + for grantee in grantees: + default_rows.append( + ModelApplicationDatabaseDefaultAclRow( + owner=schema.owner, + database_ref=database_ref, + physical_database=database.physical_name, + schema_ref=schema_name, + object_type=object_type, + grantee=grantee, + ) + ) + + scaffold_blockers = tuple(sorted(set(blockers))) + unique_blockers = tuple(sorted({*blockers, *object_blockers})) + matrix = ModelApplicationDatabaseAclMatrix( + authorization_scope=authorization_scope, + scaffold_status="BLOCKED" if scaffold_blockers else "READY", + scaffold_blockers=scaffold_blockers, + status="BLOCKED" if unique_blockers else "READY", + sources=tuple(sorted(sources, key=lambda source: source.source_key)), + verified_evidence_source_keys=tuple(sorted(verified_evidence)), + declared_principals=declared_principals, + observed_principals=observed_principals, + absent_principals=absent_principals, + observed_owner_roles=observed_owner_roles, + absent_owner_roles=absent_owner_roles, + observed_role_states=tuple( + sorted(observed_role_state_by_name.values(), key=lambda state: state.role) + ), + governed_role_states=tuple( + sorted(governed_role_states, key=lambda state: state.role) + ), + retained_administrative_principals=tuple( + sorted(retained_administrative_principals) + ), + database_owners=database_owners, + required_connect_databases=required_connect, + observed_connect_database_owners=observed_connect_database_owners, + allowed_connect_principals=allowed_connect_principals, + observed_connect_principals=observed_connect_principals, + absent_connect_principals=absent_connect_principals, + schema_domains={ + database_ref: { + schema_name: schema.domain + for schema_name, schema in sorted(database.schemas.items()) + } + for database_ref, database in sorted(topology.databases.items()) + }, + observed_schema_owners=observed_schema_owners, + absent_schemas=absent_schemas, + principal_domains={ + principal: tuple(sorted(domains, key=lambda domain: domain.value)) + for principal, domains in sorted(principal_domains.items()) + }, + allowed_memberships=tuple( + sorted( + allowed_memberships, + key=lambda item: item.identity, + ) + ), + observed_objects=tuple( + sorted(observed_catalog_objects, key=lambda obj: obj.identity) + ), + objects=ordered_objects, + rows=tuple( + sorted( + rows, + key=lambda row: tuple(str(item) for item in row.identity), + ) + ), + default_privileges=tuple( + sorted( + default_rows, + key=lambda row: ( + row.database_ref, + row.schema_ref, + row.owner, + row.object_type.value, + row.grantee, + ), + ) + ), + blockers=unique_blockers, + excluded_objects=tuple(sorted(set(excluded))), + ) + shared_scaffold_violations = validate_application_database_acl_scaffold( + matrix, + require_safe_existing_roles=False, + ) + scaffold_violations = validate_application_database_acl_scaffold(matrix) + if scaffold_violations: + rendered_scaffold_blockers = tuple( + sorted( + { + *matrix.scaffold_blockers, + *( + f"ACL scaffold policy violation: {violation}" + for violation in scaffold_violations + ), + } + ) + ) + matrix = matrix.model_copy( + update={ + "scaffold_status": "BLOCKED", + "scaffold_blockers": rendered_scaffold_blockers, + } + ) + if shared_scaffold_violations: + matrix = matrix.model_copy( + update={ + "status": "BLOCKED", + "blockers": tuple( + sorted( + { + *matrix.blockers, + *( + f"ACL scaffold policy violation: {violation}" + for violation in shared_scaffold_violations + ), + } + ) + ), + } + ) + structural_violations = validate_application_database_acl_matrix(matrix) + if structural_violations: + matrix = matrix.model_copy( + update={ + "status": "BLOCKED", + "blockers": tuple( + sorted( + { + *matrix.blockers, + *( + f"ACL policy violation: {violation}" + for violation in structural_violations + ), + } + ) + ), + } + ) + return ModelApplicationDatabaseAclMatrix.model_validate( + matrix.model_dump(mode="json") + ) + + +def validate_application_database_acl_matrix( + matrix: ModelApplicationDatabaseAclMatrix, +) -> tuple[str, ...]: + """Validate deny-by-default, ownership, completeness, and domain separation.""" + violations: list[str] = [] + declared_principals = { + principal + for principals in matrix.declared_principals.values() + for principal in principals + } + matrix_principals = declared_principals.union( + principal + for principals in ( + *matrix.observed_principals.values(), + *matrix.observed_connect_principals.values(), + *matrix.absent_principals.values(), + *matrix.absent_connect_principals.values(), + *matrix.allowed_connect_principals.values(), + ) + for principal in principals + ) + for database_ref, owner in matrix.database_owners.items(): + if owner in matrix_principals: + violations.append( + f"runtime principal {owner!r} owns database {database_ref!r}" + ) + for obj in matrix.objects: + if not obj.target_materialized: + violations.append( + f"target object {obj.physical_database}.{obj.schema_ref}." + f"{obj.object_ref} is not materialized at the rendered location" + ) + if ( + obj.object_type is EnumDatabaseGrantObjectType.FUNCTION + and obj.function_signature is None + ): + violations.append( + f"routine {obj.schema_ref}.{obj.object_ref} lacks an exact signature" + ) + if obj.owner in matrix_principals: + violations.append( + f"runtime principal {obj.owner!r} owns " + f"{obj.schema_ref}.{obj.object_ref}" + ) + for row in matrix.rows: + if row.principal == PUBLIC_PRINCIPAL and row.privileges: + violations.append( + f"PUBLIC has {sorted(p.value for p in row.privileges)!r} on " + f"{row.object_type.value}:{row.schema_ref}:{row.object_ref}" + ) + + domains_by_object = {obj.identity: obj.domain for obj in matrix.objects} + for row in matrix.rows: + if not row.privileges: + continue + if row.object_type is EnumDatabaseGrantObjectType.SCHEMA: + domain = matrix.schema_domains.get(row.database_ref, {}).get( + row.schema_ref or "" + ) + elif row.object_type in _OBJECT_TYPES: + object_identity = ( + row.database_ref, + row.schema_ref or "", + row.object_type, + row.object_ref or "", + row.function_signature, + ) + domain = domains_by_object.get(object_identity) + else: + domain = None + if domain is None: + continue + allowed = matrix.principal_domains.get(row.principal, ()) + if domain not in allowed: + violations.append( + f"principal {row.principal!r} has cross-domain privileges on " + f"{row.schema_ref}.{row.object_ref} ({domain.value})" + ) + prohibited = { + EnumDatabasePrivilege.CREATE, + EnumDatabasePrivilege.TEMPORARY, + EnumDatabasePrivilege.TRIGGER, + EnumDatabasePrivilege.REFERENCES, + EnumDatabasePrivilege.TRUNCATE, + } + for acl_row in matrix.rows: + if acl_row.principal in matrix_principals and prohibited & set( + acl_row.privileges + ): + violations.append( + f"runtime principal {acl_row.principal!r} has DDL privilege on " + f"{acl_row.object_type.value}:{acl_row.schema_ref}:" + f"{acl_row.object_ref}" + ) + declared_by_database = { + database_ref: set(principals) + for database_ref, principals in matrix.declared_principals.items() + } + observed_by_database = { + database_ref: set(principals) + for database_ref, principals in matrix.observed_principals.items() + } + owner_roles = ( + set(matrix.database_owners.values()) + .union(obj.owner for obj in matrix.objects) + .union(row.owner for row in matrix.default_privileges) + ) + for membership in matrix.allowed_memberships: + if membership.member in declared_by_database.get( + membership.database_ref, set() + ): + violations.append( + f"workload principal {membership.member!r} has permitted role " + f"membership {membership.role!r}" + ) + if membership.member not in observed_by_database.get( + membership.database_ref, set() + ): + violations.append( + f"membership principal {membership.member!r} is absent from the " + "observed principal census" + ) + if membership.role not in owner_roles: + violations.append( + f"permitted membership role {membership.role!r} is not an owner role" + ) + if ( + membership.admin_option + or membership.inherit_option + or not membership.set_option + ): + violations.append( + f"migration membership {membership.role!r} -> " + f"{membership.member!r} must be SET-only" + ) + for default_row in matrix.default_privileges: + if default_row.grantee == PUBLIC_PRINCIPAL and default_row.privileges: + violations.append( + f"PUBLIC has future {default_row.object_type.value} privileges in " + f"{default_row.schema_ref!r}" + ) + if default_row.grantee in matrix_principals and default_row.privileges: + violations.append( + f"runtime principal {default_row.grantee!r} has broad future " + f"{default_row.object_type.value} privileges in " + f"{default_row.schema_ref!r}" + ) + + schema_usage = { + (row.principal, row.database_ref, row.schema_ref) + for row in matrix.rows + if row.object_type is EnumDatabaseGrantObjectType.SCHEMA + and EnumDatabasePrivilege.USAGE in row.privileges + } + for row in matrix.rows: + if row.object_type not in _OBJECT_TYPES or not row.privileges: + continue + if (row.principal, row.database_ref, row.schema_ref) not in schema_usage: + violations.append( + f"principal {row.principal!r} has object privilege without schema " + f"USAGE on {row.schema_ref!r}" + ) + + object_ids = {obj.identity for obj in matrix.objects} + object_row_ids = { + ( + row.database_ref, + row.schema_ref, + row.object_type, + row.object_ref, + row.function_signature, + ) + for row in matrix.rows + if row.object_type in _OBJECT_TYPES + } + if object_ids != object_row_ids: + violations.append("object rows do not cover the exact ownership object set") + expected_grantees_by_database = { + database_ref: { + row.principal + for row in matrix.rows + if row.database_ref == database_ref + and row.object_type is EnumDatabaseGrantObjectType.DATABASE + } + for database_ref in {obj.database_ref for obj in matrix.objects} + } + for obj in matrix.objects: + actual_grantees = { + row.principal + for row in matrix.rows + if ( + row.database_ref, + row.schema_ref, + row.object_type, + row.object_ref, + row.function_signature, + ) + == obj.identity + } + if actual_grantees != expected_grantees_by_database[obj.database_ref]: + violations.append( + f"{obj.schema_ref}.{obj.object_ref} lacks a complete principal row set" + ) + return tuple(sorted(set(violations))) + + +def validate_application_database_acl_scaffold( + matrix: ModelApplicationDatabaseAclMatrix, + *, + require_safe_existing_roles: bool = True, +) -> tuple[str, ...]: + """Validate the additive P1 roles/schemas/CONNECT/default-ACL stage only.""" + violations: list[str] = [] + database_refs = set(matrix.declared_principals) + if not database_refs: + violations.append("scaffold declares zero topology databases") + elif len(database_refs) != 1: + violations.append("scaffold requires exactly one topology application database") + if ( + set(matrix.observed_principals) != database_refs + or set(matrix.absent_principals) != database_refs + ): + violations.append( + "scaffold principal presence/absence database keys are incomplete" + ) + if set(matrix.database_owners) != database_refs: + violations.append("scaffold database owner map is incomplete") + if set(matrix.schema_domains) != database_refs: + violations.append("scaffold schema domain map is incomplete") + if ( + set(matrix.observed_schema_owners) != database_refs + or set(matrix.absent_schemas) != database_refs + ): + violations.append("scaffold schema presence/absence maps are incomplete") + application_physical_databases = { + row.physical_database for row in matrix.default_privileges + } + if len(application_physical_databases) != 1: + violations.append( + "scaffold requires exactly one application physical database target" + ) + if any( + _SQL_IDENTIFIER.fullmatch(database) is None + for database in application_physical_databases + ): + violations.append("scaffold application database target is not canonical") + row_physical_databases = { + row.physical_database + for row in matrix.rows + if row.database_ref in database_refs + } + object_physical_databases = { + obj.physical_database + for obj in matrix.objects + if obj.database_ref in database_refs + } + if row_physical_databases != application_physical_databases or ( + object_physical_databases + and object_physical_databases != application_physical_databases + ): + violations.append( + "scaffold application physical database target disagrees across rows, " + "objects, and future-ACL cells" + ) + if not application_physical_databases <= set(matrix.required_connect_databases): + violations.append( + "scaffold application physical database is absent from CONNECT scope" + ) + + required_connect = set(matrix.required_connect_databases) + for name, values in ( + ("allowed CONNECT", matrix.allowed_connect_principals), + ("observed CONNECT", matrix.observed_connect_principals), + ("observed CONNECT owner", matrix.observed_connect_database_owners), + ): + if set(values) != required_connect: + violations.append(f"{name} database keys do not cover required CONNECT set") + + declared = { + principal + for principals in matrix.declared_principals.values() + for principal in principals + } + observed = { + principal + for principals in matrix.observed_principals.values() + for principal in principals + } + connection_principals = { + principal + for principals in matrix.observed_connect_principals.values() + for principal in principals + } + desired_connect = { + principal + for principals in matrix.allowed_connect_principals.values() + for principal in principals + } + runtime_principals = declared.union( + observed, + connection_principals, + desired_connect, + ) + owner_roles = set(matrix.database_owners.values()) | { + row.owner for row in matrix.default_privileges + } + if owner_roles & runtime_principals: + violations.append("scaffold owner roles overlap runtime principal census") + + for database_ref in sorted(database_refs): + expected_grantees = { + PUBLIC_PRINCIPAL, + *matrix.declared_principals[database_ref], + *matrix.observed_principals[database_ref], + } + existing = set(matrix.observed_principals[database_ref]) + absent = set(matrix.absent_principals[database_ref]) + if existing & absent: + violations.append( + f"{database_ref}: principal presence/absence evidence overlaps" + ) + if not set(matrix.declared_principals[database_ref]) <= existing.union(absent): + violations.append( + f"{database_ref}: scaffold lacks presence/absence evidence for " + "declared principals" + ) + schemas = set(matrix.schema_domains[database_ref]) + if not schemas: + violations.append(f"{database_ref}: scaffold declares zero schemas") + observed_schemas = set(matrix.observed_schema_owners.get(database_ref, {})) + absent_schemas = set(matrix.absent_schemas.get(database_ref, ())) + if observed_schemas & absent_schemas: + violations.append( + f"{database_ref}: schema presence/absence evidence overlaps" + ) + if schemas != observed_schemas.union(absent_schemas): + violations.append( + f"{database_ref}: scaffold lacks exact target schema evidence" + ) + database_rows = { + row.principal: row + for row in matrix.rows + if row.database_ref == database_ref + and row.object_type is EnumDatabaseGrantObjectType.DATABASE + } + if set(database_rows) != expected_grantees: + violations.append( + f"{database_ref}: database ACL cells do not cover exact principal census" + ) + for schema_name in sorted(schemas): + schema_rows = { + row.principal: row + for row in matrix.rows + if row.database_ref == database_ref + and row.object_type is EnumDatabaseGrantObjectType.SCHEMA + and row.schema_ref == schema_name + } + if set(schema_rows) != expected_grantees: + violations.append( + f"{database_ref}.{schema_name}: schema ACL cells do not cover " + "exact principal census" + ) + owners = { + row.owner + for row in matrix.default_privileges + if row.database_ref == database_ref and row.schema_ref == schema_name + } + if len(owners) != 1: + violations.append( + f"{database_ref}.{schema_name}: requires exactly one schema owner" + ) + expected_defaults = { + (object_type, grantee) + for object_type in _OBJECT_TYPES + for grantee in expected_grantees + } + actual_defaults = { + (row.object_type, row.grantee) + for row in matrix.default_privileges + if row.database_ref == database_ref and row.schema_ref == schema_name + } + if actual_defaults != expected_defaults: + violations.append( + f"{database_ref}.{schema_name}: default ACL cells are incomplete" + ) + + for physical_database in sorted(required_connect): + allowed = set(matrix.allowed_connect_principals.get(physical_database, ())) + observed_connect = set( + matrix.observed_connect_principals.get(physical_database, ()) + ) + absent_connect = set( + matrix.absent_connect_principals.get(physical_database, ()) + ) + if not allowed: + violations.append(f"{physical_database}: CONNECT allowlist is empty") + if observed_connect & absent_connect: + violations.append( + f"{physical_database}: CONNECT presence/absence evidence overlaps" + ) + if not allowed <= observed_connect.union(absent_connect): + violations.append( + f"{physical_database}: CONNECT allowlist lacks presence/absence evidence" + ) + managed_owner_evidence = set(matrix.observed_owner_roles).union( + matrix.absent_owner_roles + ) + if not owner_roles <= managed_owner_evidence: + violations.append("scaffold managed owners lack presence/absence evidence") + governed_states = {state.role: state for state in matrix.governed_role_states} + if not owner_roles <= { + role for role, state in governed_states.items() if state.role_kind == "owner" + }: + violations.append("scaffold managed owners lack governed owner-role state") + if not declared <= { + role + for role, state in governed_states.items() + if state.role_kind == "workload" and state.login + }: + violations.append("scaffold workloads lack governed LOGIN role state") + observed_states = {state.role: state for state in matrix.observed_role_states} + globally_absent_roles = { + principal + for principals in ( + *matrix.absent_principals.values(), + *matrix.absent_connect_principals.values(), + matrix.absent_owner_roles, + ) + for principal in principals + } + scaffold_safe_roles = set(globally_absent_roles) + scaffold_safe_roles.update( + role + for role, governed_state in governed_states.items() + if role in observed_states + and _role_attribute_values(observed_states[role]) + == _role_attribute_values(governed_state) + ) + intended_scaffold_roles = set(owner_roles) + intended_scaffold_roles.update( + principal + for principals in matrix.allowed_connect_principals.values() + for principal in principals + ) + intended_scaffold_roles.update( + row.principal + for row in matrix.rows + if row.principal != PUBLIC_PRINCIPAL + and row.object_type + in { + EnumDatabaseGrantObjectType.DATABASE, + EnumDatabaseGrantObjectType.SCHEMA, + } + and row.privileges + ) + intended_scaffold_roles.update( + principal + for membership in matrix.allowed_memberships + for principal in (membership.role, membership.member) + ) + unsafe_scaffold_roles = sorted(intended_scaffold_roles - scaffold_safe_roles) + if require_safe_existing_roles and unsafe_scaffold_roles: + violations.append( + "additive scaffold intended roles are not already safe or proven absent: " + f"{unsafe_scaffold_roles!r}" + ) + + prohibited = { + EnumDatabasePrivilege.CREATE, + EnumDatabasePrivilege.TEMPORARY, + EnumDatabasePrivilege.TRIGGER, + EnumDatabasePrivilege.REFERENCES, + EnumDatabasePrivilege.TRUNCATE, + } + for row in matrix.rows: + if row.object_type not in { + EnumDatabaseGrantObjectType.DATABASE, + EnumDatabaseGrantObjectType.SCHEMA, + }: + continue + if row.principal == PUBLIC_PRINCIPAL and row.privileges: + violations.append("PUBLIC has privileges in the additive scaffold") + if row.principal in runtime_principals and prohibited & set(row.privileges): + violations.append( + f"runtime principal {row.principal!r} has scaffold DDL privileges" + ) + if row.object_type is EnumDatabaseGrantObjectType.SCHEMA and row.privileges: + domain = matrix.schema_domains.get(row.database_ref, {}).get( + row.schema_ref or "" + ) + if domain not in matrix.principal_domains.get(row.principal, ()): + violations.append( + f"principal {row.principal!r} has cross-domain scaffold access" + ) + if any(row.privileges for row in matrix.default_privileges): + violations.append("scaffold future-object defaults must be deny-by-default") + + for membership in matrix.allowed_memberships: + if membership.role not in owner_roles: + violations.append("scaffold membership targets a non-owner role") + if membership.member in declared: + violations.append("runtime workload has scaffold owner membership") + if ( + membership.admin_option + or membership.inherit_option + or not membership.set_option + ): + violations.append("scaffold owner membership must be SET-only") + return tuple(sorted(set(violations))) + + +def _quote_identifier(identifier: str) -> str: + if _SQL_IDENTIFIER.fullmatch(identifier) is None: + raise ValueError(f"Unsafe or non-canonical SQL identifier {identifier!r}") + return f'"{identifier}"' + + +def _quote_sql_literal(value: str) -> str: + """Quote one trusted typed value as a PostgreSQL string literal.""" + return "'" + value.replace("'", "''") + "'" + + +def _managed_provenance_marker( + matrix: ModelApplicationDatabaseAclMatrix, + *, + kind: str, + name: str, +) -> str: + """Return a deterministic source-lock marker for additive managed objects.""" + _ = matrix + return f"omnibase_application_acl:v1:managed:{kind}:{name}" + + +def _sql_object_target( + obj: ModelApplicationDatabaseAclObject, +) -> tuple[str, str, str]: + """Return ownership keyword, ACL keyword, and exact SQL target.""" + qualified = ( + f"{_quote_identifier(obj.schema_ref)}.{_quote_identifier(obj.object_ref)}" + ) + if obj.object_type is EnumDatabaseGrantObjectType.TABLE: + owner_keyword = { + "table": "TABLE", + "view": "VIEW", + "materialized_view": "MATERIALIZED VIEW", + "foreign_table": "FOREIGN TABLE", + }[obj.catalog_kind] + return owner_keyword, "TABLE", qualified + if obj.object_type is EnumDatabaseGrantObjectType.SEQUENCE: + return "SEQUENCE", "SEQUENCE", qualified + if obj.object_type is EnumDatabaseGrantObjectType.TYPE: + return "TYPE", "TYPE", qualified + if obj.function_signature is None: + raise ValueError( + f"Function {obj.schema_ref}.{obj.object_ref} lacks function_signature" + ) + if obj.catalog_kind == "procedure": + return "PROCEDURE", "PROCEDURE", f"{qualified}{obj.function_signature}" + owner_keyword = "AGGREGATE" if obj.catalog_kind == "aggregate" else "FUNCTION" + return owner_keyword, "FUNCTION", f"{qualified}{obj.function_signature}" + + +def render_application_database_acl_sql( + matrix: ModelApplicationDatabaseAclMatrix, + *, + allow_synthetic_proof: bool = False, + phase: EnumApplicationDatabaseAclRenderPhase = ( + EnumApplicationDatabaseAclRenderPhase.FULL + ), +) -> str: + """Render an atomic additive scaffold or the materialized-object full phase.""" + if phase is EnumApplicationDatabaseAclRenderPhase.FULL and matrix.status != "READY": + raise ValueError( + f"Cannot render blocked ACL matrix ({len(matrix.blockers)} blocker(s))" + ) + if ( + phase is EnumApplicationDatabaseAclRenderPhase.SCAFFOLD + and matrix.scaffold_status != "READY" + ): + raise ValueError( + "Cannot render blocked ACL scaffold " + f"({len(matrix.scaffold_blockers)} blocker(s))" + ) + if ( + matrix.authorization_scope + is EnumApplicationDatabaseAclAuthorizationScope.SYNTHETIC_PROOF + and not allow_synthetic_proof + ): + raise ValueError( + "Synthetic proof matrices require explicit allow_synthetic_proof=true" + ) + scaffold_violations = validate_application_database_acl_scaffold( + matrix, + require_safe_existing_roles=( + phase is EnumApplicationDatabaseAclRenderPhase.SCAFFOLD + ), + ) + violations = ( + tuple( + sorted( + set(scaffold_violations).union( + validate_application_database_acl_matrix(matrix) + ) + ) + ) + if phase is EnumApplicationDatabaseAclRenderPhase.FULL + else scaffold_violations + ) + if violations: + raise ValueError("Cannot render invalid ACL matrix: " + "; ".join(violations)) + + declared_principals = sorted( + { + principal + for principals in matrix.declared_principals.values() + for principal in principals + } + ) + allowed_connect_principals = sorted( + { + principal + for principals in matrix.allowed_connect_principals.values() + for principal in principals + } + ) + governed_states = {state.role: state for state in matrix.governed_role_states} + revocation_principals = sorted( + set(declared_principals) + .union(allowed_connect_principals) + .union( + principal + for principals in ( + *matrix.observed_principals.values(), + *matrix.observed_connect_principals.values(), + ) + for principal in principals + ) + ) + owners = sorted( + set(matrix.database_owners.values()) + .union(obj.owner for obj in matrix.objects) + .union(row.owner for row in matrix.default_privileges) + ) + globally_observed_roles = sorted( + { + principal + for principals in ( + *matrix.observed_principals.values(), + *matrix.observed_connect_principals.values(), + matrix.observed_owner_roles, + tuple(matrix.observed_connect_database_owners.values()), + ) + for principal in principals + } + ) + globally_absent_roles = { + principal + for principals in ( + *matrix.absent_principals.values(), + *matrix.absent_connect_principals.values(), + matrix.absent_owner_roles, + ) + for principal in principals + } + application_physical_database = next( + iter({row.physical_database for row in matrix.default_privileges}) + ) + application_database_ref = next(iter(matrix.declared_principals)) + application_desired_owner = matrix.database_owners[application_database_ref] + schema_owners = { + (row.database_ref, row.schema_ref): row.owner + for row in matrix.default_privileges + } + observed_states = {state.role: state for state in matrix.observed_role_states} + desired_object_owners: dict[tuple[str, str, str, str], str] = { + ( + obj.catalog_kind, + obj.schema_ref, + obj.object_ref, + obj.function_signature or "", + ): obj.owner + for obj in matrix.objects + } + + def role_attribute_sql(role_name: str) -> tuple[str, ...]: + state = governed_states[role_name] + return ( + f"role.rolcanlogin IS DISTINCT FROM {'TRUE' if state.login else 'FALSE'}", + "role.rolsuper IS DISTINCT FROM FALSE", + "role.rolbypassrls IS DISTINCT FROM FALSE", + "role.rolcreatedb IS DISTINCT FROM FALSE", + "role.rolcreaterole IS DISTINCT FROM FALSE", + "role.rolreplication IS DISTINCT FROM FALSE", + "role.rolinherit IS DISTINCT FROM FALSE", + ) + + def role_attribute_clause(role_name: str) -> str: + state = governed_states[role_name] + return ( + f"{'LOGIN' if state.login else 'NOLOGIN'} NOSUPERUSER NOBYPASSRLS " + "NOCREATEDB NOCREATEROLE NOREPLICATION NOINHERIT" + ) + + def role_state_match_clause( + state: ProtocolApplicationDatabaseRoleAttributeState, + ) -> str: + attribute_columns = ( + ("login", "rolcanlogin"), + ("superuser", "rolsuper"), + ("bypass_rls", "rolbypassrls"), + ("create_database", "rolcreatedb"), + ("create_role", "rolcreaterole"), + ("replication", "rolreplication"), + ("inherit", "rolinherit"), + ) + return ( + "(" + + " AND ".join( + f"role.{column} IS {'TRUE' if getattr(state, field) else 'FALSE'}" + for field, column in attribute_columns + ) + + ")" + ) + + lines = [ + "-- Generated application-database ACL; do not hand edit.", + f"-- Render phase: {phase.value}", + "-- Source revisions:", + *[ + f"-- {source.source_key}: {source.repository}@{source.revision} " + f"{source.path} sha256:{source.sha256}" + for source in matrix.sources + ], + "\\set ON_ERROR_STOP on", + "BEGIN;", + "DO $acl_database_guard$ BEGIN", + f" IF current_database() <> '{application_physical_database}' THEN", + " RAISE EXCEPTION 'application ACL connected to unexpected database %', " + "current_database();", + " END IF;", + "END $acl_database_guard$;", + "LOCK TABLE pg_catalog.pg_authid IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_auth_members IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_database IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_namespace IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_class IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_attribute IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_proc IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_type IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_extension IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_depend IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_default_acl IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_description IN SHARE MODE;", + "LOCK TABLE pg_catalog.pg_shdescription IN SHARE MODE;", + "", + ] + lines.extend( + [ + "DO $acl_evidence_guard$", + "DECLARE actual_owner text;", + "DECLARE actual_objects text[];", + "BEGIN", + ] + ) + for role_name in globally_observed_roles: + lines.extend( + [ + " IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_authid role", + f" WHERE role.rolname = '{role_name}') THEN", + f" RAISE EXCEPTION 'locked role census is stale: expected present role {role_name}';", + " END IF;", + ] + ) + for role_name, observed_state in sorted(observed_states.items()): + accepted_state_clauses = [role_state_match_clause(observed_state)] + governed_state = governed_states.get(role_name) + if governed_state is not None and governed_state.manage_attributes: + accepted_state_clauses.append(role_state_match_clause(governed_state)) + accepted_state_sql = " OR ".join(sorted(set(accepted_state_clauses))) + lines.extend( + [ + " IF EXISTS (SELECT 1 FROM pg_catalog.pg_authid role", + f" WHERE role.rolname = '{role_name}'", + f" AND NOT ({accepted_state_sql})) THEN", + f" RAISE EXCEPTION 'locked role attribute census is stale for {role_name}';", + " END IF;", + ] + ) + for role_name, state in sorted(governed_states.items()): + if state.manage_attributes: + continue + mismatch = " OR ".join(role_attribute_sql(role_name)) + lines.extend( + [ + " IF EXISTS (SELECT 1 FROM pg_catalog.pg_authid role", + f" WHERE role.rolname = '{role_name}'", + f" AND ({mismatch})) THEN", + f" RAISE EXCEPTION 'non-mutating governed role {role_name} has unsafe attributes';", + " END IF;", + ] + ) + allowed_membership_conditions = [ + f"(parent.rolname = '{membership.role}' AND member.rolname = '{membership.member}')" + for membership in matrix.allowed_memberships + ] + allowed_membership_sql = ( + " AND NOT (" + " OR ".join(allowed_membership_conditions) + ")" + if allowed_membership_conditions + else "" + ) + for role_name in sorted(globally_absent_roles.intersection(governed_states)): + state = governed_states[role_name] + if not state.manage_attributes: + continue + marker = _managed_provenance_marker(matrix, kind="role", name=role_name) + mismatch = " OR ".join( + ( + f"pg_catalog.shobj_description(role.oid, 'pg_authid') IS DISTINCT FROM '{marker}'", + "role.rolpassword IS NOT NULL", + *role_attribute_sql(role_name), + ) + ) + lines.extend( + [ + " IF EXISTS (SELECT 1 FROM pg_catalog.pg_authid role", + f" WHERE role.rolname = '{role_name}'", + f" AND ({mismatch}))", + " OR EXISTS (", + " SELECT 1 FROM pg_catalog.pg_auth_members membership", + " JOIN pg_catalog.pg_authid parent ON parent.oid = membership.roleid", + " JOIN pg_catalog.pg_authid member ON member.oid = membership.member", + f" WHERE (parent.rolname = '{role_name}' OR member.rolname = '{role_name}')", + f" {allowed_membership_sql}", + " ) THEN", + f" RAISE EXCEPTION 'expected-absent role collision for {role_name}';", + " END IF;", + ] + ) + for physical_database, observed_owner in sorted( + matrix.observed_connect_database_owners.items() + ): + accepted_owners = {observed_owner} + if physical_database == application_physical_database: + accepted_owners.add(application_desired_owner) + owner_literals = ", ".join(f"'{owner}'" for owner in sorted(accepted_owners)) + lines.extend( + [ + " SELECT owner.rolname INTO actual_owner", + " FROM pg_catalog.pg_database database", + " JOIN pg_catalog.pg_authid owner ON owner.oid = database.datdba", + f" WHERE database.datname = '{physical_database}';", + f" IF actual_owner IS NULL OR actual_owner NOT IN ({owner_literals}) THEN", + f" RAISE EXCEPTION 'locked database owner census is stale for {physical_database}: %', actual_owner;", + " END IF;", + ] + ) + for (database_ref, schema_name), owner in sorted(schema_owners.items()): + if schema_name is None: + continue + if schema_name in matrix.observed_schema_owners[database_ref]: + observed_schema_owner = matrix.observed_schema_owners[database_ref][ + schema_name + ] + accepted_schema_owners = ", ".join( + f"'{role}'" for role in sorted({observed_schema_owner, owner}) + ) + lines.extend( + [ + " IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace namespace", + f" WHERE namespace.nspname = '{schema_name}') THEN", + f" RAISE EXCEPTION 'locked schema census is stale: expected present schema {schema_name}';", + " END IF;", + " IF EXISTS (SELECT 1 FROM pg_catalog.pg_namespace namespace", + " JOIN pg_catalog.pg_authid schema_owner ON schema_owner.oid = namespace.nspowner", + f" WHERE namespace.nspname = '{schema_name}'", + f" AND schema_owner.rolname NOT IN ({accepted_schema_owners})) THEN", + f" RAISE EXCEPTION 'locked schema owner census is stale for {schema_name}';", + " END IF;", + ] + ) + else: + marker = _managed_provenance_marker( + matrix, + kind="schema", + name=schema_name, + ) + lines.extend( + [ + " IF EXISTS (", + " SELECT 1 FROM pg_catalog.pg_namespace namespace", + " JOIN pg_catalog.pg_authid owner ON owner.oid = namespace.nspowner", + f" WHERE namespace.nspname = '{schema_name}'", + " AND (", + f" pg_catalog.obj_description(namespace.oid, 'pg_namespace') IS DISTINCT FROM '{marker}'", + f" OR owner.rolname IS DISTINCT FROM '{owner}'", + " )", + " ) OR EXISTS (", + " SELECT 1 FROM pg_catalog.pg_depend dependency", + " JOIN pg_catalog.pg_namespace namespace", + " ON namespace.oid = dependency.refobjid", + " WHERE dependency.refclassid = 'pg_catalog.pg_namespace'::regclass", + f" AND namespace.nspname = '{schema_name}'", + " ) THEN", + f" RAISE EXCEPTION 'expected-absent schema collision for {schema_name}';", + " END IF;", + ] + ) + if phase is EnumApplicationDatabaseAclRenderPhase.FULL: + managed_schema_literals = ", ".join( + repr(schema_name) + for schema_name in sorted( + { + schema_name + for schemas in matrix.schema_domains.values() + for schema_name in schemas + } + ) + ) + expected_object_identities = tuple( + sorted( + "|".join( + ( + obj.catalog_kind, + obj.schema_ref, + obj.object_ref, + obj.function_signature or "", + ) + ) + for obj in matrix.observed_objects + ) + ) + expected_object_array = ( + "ARRAY[" + + ", ".join( + _quote_sql_literal(identity) for identity in expected_object_identities + ) + + "]::text[]" + if expected_object_identities + else "ARRAY[]::text[]" + ) + lines.extend( + [ + " SELECT COALESCE(array_agg(catalog_identity), ARRAY[]::text[])", + " INTO actual_objects", + " FROM (", + " SELECT CASE relation.relkind", + " WHEN 'r' THEN 'table'", + " WHEN 'p' THEN 'table'", + " WHEN 'v' THEN 'view'", + " WHEN 'm' THEN 'materialized_view'", + " WHEN 'f' THEN 'foreign_table'", + " WHEN 'S' THEN 'sequence'", + " END || '|' || namespace.nspname || '|' ||", + " relation.relname || '|' AS catalog_identity", + " FROM pg_catalog.pg_class relation", + " JOIN pg_catalog.pg_namespace namespace ON namespace.oid = relation.relnamespace", + f" WHERE namespace.nspname IN ({managed_schema_literals})", + " AND relation.relkind IN ('r', 'p', 'v', 'm', 'f', 'S')", + " UNION ALL", + " SELECT CASE procedure.prokind", + " WHEN 'p' THEN 'procedure'", + " WHEN 'a' THEN 'aggregate'", + " WHEN 'w' THEN 'window_function'", + " ELSE 'function'", + " END || '|' ||", + " namespace.nspname || '|' || procedure.proname || '|' ||", + " '(' || pg_catalog.pg_get_function_identity_arguments(procedure.oid) || ')'", + " FROM pg_catalog.pg_proc procedure", + " JOIN pg_catalog.pg_namespace namespace ON namespace.oid = procedure.pronamespace", + f" WHERE namespace.nspname IN ({managed_schema_literals})", + " AND procedure.prokind IN ('f', 'p', 'a', 'w')", + " UNION ALL", + " SELECT CASE type.typtype", + " WHEN 'b' THEN 'base_type'", + " WHEN 'r' THEN 'range_type'", + " WHEN 'm' THEN 'multirange_type'", + " ELSE 'type'", + " END || '|' || namespace.nspname || '|' || type.typname || '|'", + " FROM pg_catalog.pg_type type", + " JOIN pg_catalog.pg_namespace namespace ON namespace.oid = type.typnamespace", + " LEFT JOIN pg_catalog.pg_class relation ON relation.oid = type.typrelid", + f" WHERE namespace.nspname IN ({managed_schema_literals})", + " AND type.typisdefined", + " AND ((type.typtype IN ('b', 'd', 'e', 'r', 'm') AND type.typelem = 0)", + " OR (type.typtype = 'c' AND relation.relkind = 'c'))", + " ) locked_object_census;", + f" IF cardinality(actual_objects) <> {len(expected_object_identities)}", + f" OR NOT (actual_objects @> {expected_object_array}", + f" AND actual_objects <@ {expected_object_array}) THEN", + " RAISE EXCEPTION 'locked object census is stale: %', actual_objects;", + " END IF;", + ] + ) + for observed_object in sorted( + matrix.observed_objects, + key=lambda obj: obj.identity, + ): + identity = observed_object.identity + desired_owner = desired_object_owners[identity] + accepted_object_owners = ", ".join( + repr(owner) for owner in sorted({observed_object.owner, desired_owner}) + ) + if observed_object.catalog_kind in { + "table", + "view", + "materialized_view", + "foreign_table", + "sequence", + }: + relkind_condition = { + "table": "relation.relkind IN ('r', 'p')", + "view": "relation.relkind = 'v'", + "materialized_view": "relation.relkind = 'm'", + "foreign_table": "relation.relkind = 'f'", + "sequence": "relation.relkind = 'S'", + }[observed_object.catalog_kind] + owner_query = [ + " SELECT owner.rolname INTO actual_owner", + " FROM pg_catalog.pg_class relation", + " JOIN pg_catalog.pg_namespace namespace ON namespace.oid = relation.relnamespace", + " JOIN pg_catalog.pg_authid owner ON owner.oid = relation.relowner", + f" WHERE namespace.nspname = '{observed_object.schema_ref}'", + f" AND relation.relname = '{observed_object.object_ref}'", + f" AND {relkind_condition};", + ] + elif observed_object.catalog_kind in { + "function", + "aggregate", + "window_function", + "procedure", + }: + prokind = { + "function": "f", + "aggregate": "a", + "window_function": "w", + "procedure": "p", + }[observed_object.catalog_kind] + owner_query = [ + " SELECT owner.rolname INTO actual_owner", + " FROM pg_catalog.pg_proc procedure", + " JOIN pg_catalog.pg_namespace namespace ON namespace.oid = procedure.pronamespace", + " JOIN pg_catalog.pg_authid owner ON owner.oid = procedure.proowner", + f" WHERE namespace.nspname = '{observed_object.schema_ref}'", + f" AND procedure.proname = '{observed_object.object_ref}'", + f" AND procedure.prokind = '{prokind}'", + " AND '(' || pg_catalog.pg_get_function_identity_arguments(procedure.oid) || ')' = " + f"{_quote_sql_literal(observed_object.function_signature or '')};", + ] + else: + type_condition = { + "type": ( + "((type.typtype IN ('d', 'e') AND type.typelem = 0) " + "OR (type.typtype = 'c' AND relation.relkind = 'c'))" + ), + "base_type": "type.typtype = 'b' AND type.typelem = 0", + "range_type": "type.typtype = 'r' AND type.typelem = 0", + "multirange_type": "type.typtype = 'm' AND type.typelem = 0", + }[observed_object.catalog_kind] + owner_query = [ + " SELECT owner.rolname INTO actual_owner", + " FROM pg_catalog.pg_type type", + " JOIN pg_catalog.pg_namespace namespace ON namespace.oid = type.typnamespace", + " JOIN pg_catalog.pg_authid owner ON owner.oid = type.typowner", + " LEFT JOIN pg_catalog.pg_class relation ON relation.oid = type.typrelid", + f" WHERE namespace.nspname = '{observed_object.schema_ref}'", + f" AND type.typname = '{observed_object.object_ref}'", + " AND type.typisdefined", + f" AND ({type_condition});", + ] + lines.extend( + [ + *owner_query, + f" IF actual_owner IS NULL OR actual_owner NOT IN ({accepted_object_owners}) THEN", + " RAISE EXCEPTION 'locked object owner census is stale for " + f"{observed_object.schema_ref}.{observed_object.object_ref}: %', actual_owner;", + " END IF;", + ] + ) + lines.extend(["END", "$acl_evidence_guard$;", ""]) + + for role_name, state in sorted(governed_states.items()): + if not state.manage_attributes: + continue + quoted = _quote_identifier(role_name) + attributes = role_attribute_clause(role_name) + if role_name in globally_absent_roles: + marker = _managed_provenance_marker(matrix, kind="role", name=role_name) + lines.extend( + [ + "DO $acl_create_role$ BEGIN", + f" IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_authid WHERE rolname = '{role_name}') THEN", # noqa: S608 -- strict identifier + f" CREATE ROLE {quoted} {attributes};", + f" COMMENT ON ROLE {quoted} IS '{marker}';", + " END IF;", + "END $acl_create_role$;", + ] + ) + if phase is EnumApplicationDatabaseAclRenderPhase.FULL: + lines.append(f"ALTER ROLE {quoted} {attributes};") + membership_members = sorted( + set(revocation_principals).union(owners) + - set(matrix.retained_administrative_principals) + ) + governed_parents = sorted( + state.role for state in matrix.governed_role_states if state.manage_memberships + ) + member_literals = ", ".join(repr(name) for name in membership_members) + parent_literals = ", ".join(repr(name) for name in governed_parents) + if phase is EnumApplicationDatabaseAclRenderPhase.FULL: + lines.extend( + [ + "DO $acl_membership$", + "DECLARE membership_record record;", + "BEGIN", + " FOR membership_record IN", + " SELECT parent.rolname AS parent_role,", + " member.rolname AS member_role,", + " grantor.rolname AS grantor_role", + " FROM pg_auth_members membership", + " JOIN pg_roles parent ON parent.oid = membership.roleid", + " JOIN pg_roles member ON member.oid = membership.member", + " JOIN pg_roles grantor ON grantor.oid = membership.grantor", + f" WHERE member.rolname IN ({member_literals})", + f" OR parent.rolname IN ({parent_literals})", + " LOOP", + " IF EXISTS (", + " SELECT 1 FROM pg_catalog.pg_auth_members current_membership", + " JOIN pg_catalog.pg_authid parent ON parent.oid = current_membership.roleid", + " JOIN pg_catalog.pg_authid member ON member.oid = current_membership.member", + " JOIN pg_catalog.pg_authid grantor ON grantor.oid = current_membership.grantor", + " WHERE parent.rolname = membership_record.parent_role", + " AND member.rolname = membership_record.member_role", + " AND grantor.rolname = membership_record.grantor_role", + " ) THEN", + " EXECUTE format('SET LOCAL ROLE %I', membership_record.grantor_role);", + " EXECUTE format(", + " 'REVOKE %I FROM %I GRANTED BY %I CASCADE',", + " membership_record.parent_role,", + " membership_record.member_role,", + " membership_record.grantor_role", + " );", + " EXECUTE 'RESET ROLE';", + " END IF;", + " END LOOP;", + "END", + "$acl_membership$;", + ] + ) + for membership in matrix.allowed_memberships: + lines.append( + f"GRANT {_quote_identifier(membership.role)} TO " + f"{_quote_identifier(membership.member)} WITH ADMIN " + f"{'TRUE' if membership.admin_option else 'FALSE'}, INHERIT " + f"{'TRUE' if membership.inherit_option else 'FALSE'}, SET " + f"{'TRUE' if membership.set_option else 'FALSE'};" + ) + + database_literals = ", ".join( + repr(database) for database in matrix.required_connect_databases + ) + if phase is EnumApplicationDatabaseAclRenderPhase.FULL: + lines.extend( + [ + "DO $acl_database_grantors$", + "DECLARE acl_record record;", + "DECLARE grantee_sql text;", + "BEGIN", + " FOR acl_record IN", + " SELECT database.datname AS database_name,", + " COALESCE(grantee.rolname, 'PUBLIC') AS grantee,", + " grantor.rolname AS grantor, acl.privilege_type", + " FROM pg_catalog.pg_database database", + " CROSS JOIN LATERAL pg_catalog.aclexplode(", + " COALESCE(database.datacl, pg_catalog.acldefault('d', database.datdba))", + " ) acl", + " LEFT JOIN pg_catalog.pg_authid grantee ON grantee.oid = acl.grantee", + " JOIN pg_catalog.pg_authid grantor ON grantor.oid = acl.grantor", + f" WHERE database.datname IN ({database_literals})", + " AND acl.grantee <> database.datdba", + " LOOP", + " grantee_sql := CASE WHEN acl_record.grantee = 'PUBLIC'", + " THEN 'PUBLIC'", + " ELSE format('%I', acl_record.grantee) END;", + " IF EXISTS (", + " SELECT 1 FROM pg_catalog.pg_database current_database_acl", + " CROSS JOIN LATERAL pg_catalog.aclexplode(", + " COALESCE(current_database_acl.datacl,", + " pg_catalog.acldefault('d', current_database_acl.datdba))", + " ) current_acl", + " LEFT JOIN pg_catalog.pg_authid current_grantee ON current_grantee.oid = current_acl.grantee", + " JOIN pg_catalog.pg_authid current_grantor ON current_grantor.oid = current_acl.grantor", + " WHERE current_database_acl.datname = acl_record.database_name", + " AND COALESCE(current_grantee.rolname, 'PUBLIC') = acl_record.grantee", + " AND current_grantor.rolname = acl_record.grantor", + " AND current_acl.privilege_type = acl_record.privilege_type", + " ) THEN", + " EXECUTE format('SET LOCAL ROLE %I', acl_record.grantor);", + " EXECUTE format(", + " 'REVOKE %s ON DATABASE %I FROM %s GRANTED BY %I CASCADE',", + " acl_record.privilege_type, acl_record.database_name,", + " grantee_sql, acl_record.grantor", + " );", + " EXECUTE 'RESET ROLE';", + " END IF;", + " END LOOP;", + "END", + "$acl_database_grantors$;", + ] + ) + + database_rows = [ + row + for row in matrix.rows + if row.object_type is EnumDatabaseGrantObjectType.DATABASE + ] + for physical_database in matrix.required_connect_databases: + connection_grantees = ", ".join( + ["PUBLIC", *(_quote_identifier(name) for name in revocation_principals)] + ) + quoted_database = _quote_identifier(physical_database) + if phase is EnumApplicationDatabaseAclRenderPhase.FULL: + lines.append( + f"REVOKE ALL PRIVILEGES ON DATABASE {quoted_database} FROM " + f"{connection_grantees} CASCADE;" + ) + for principal in matrix.allowed_connect_principals[physical_database]: + lines.append( + f"GRANT CONNECT ON DATABASE {quoted_database} TO " + f"{_quote_identifier(principal)};" + ) + database_grantees = ", ".join( + ["PUBLIC", *(_quote_identifier(name) for name in revocation_principals)] + ) + for physical_database in ( + sorted({row.physical_database for row in database_rows}) + if phase is EnumApplicationDatabaseAclRenderPhase.FULL + else () + ): + quoted_database = _quote_identifier(physical_database) + database_ref = next( + row.database_ref + for row in database_rows + if row.physical_database == physical_database + ) + lines.append( + f"ALTER DATABASE {quoted_database} OWNER TO " + f"{_quote_identifier(matrix.database_owners[database_ref])};" + ) + lines.append( + f"REVOKE ALL PRIVILEGES ON DATABASE {quoted_database} FROM " + f"{database_grantees} CASCADE;" + ) + for row in database_rows: + if row.physical_database != physical_database or not row.privileges: + continue + privileges = ", ".join(p.value for p in row.privileges) + lines.append( + f"GRANT {privileges} ON DATABASE {quoted_database} TO " + f"{_quote_identifier(row.principal)};" + ) + + schema_rows = [ + row + for row in matrix.rows + if row.object_type is EnumDatabaseGrantObjectType.SCHEMA + ] + schema_literals = ", ".join( + repr(schema_name) + for schema_name in sorted( + {"public", *(schema_name for _, schema_name in schema_owners)} + ) + ) + if phase is EnumApplicationDatabaseAclRenderPhase.FULL: + lines.extend( + [ + "DO $acl_schema_grantors$", + "DECLARE acl_record record;", + "DECLARE grantee_sql text;", + "BEGIN", + " FOR acl_record IN", + " SELECT namespace.nspname AS schema_name,", + " COALESCE(grantee.rolname, 'PUBLIC') AS grantee,", + " grantor.rolname AS grantor, acl.privilege_type", + " FROM pg_catalog.pg_namespace namespace", + " CROSS JOIN LATERAL pg_catalog.aclexplode(", + " COALESCE(namespace.nspacl, pg_catalog.acldefault('n', namespace.nspowner))", + " ) acl", + " LEFT JOIN pg_catalog.pg_authid grantee ON grantee.oid = acl.grantee", + " JOIN pg_catalog.pg_authid grantor ON grantor.oid = acl.grantor", + f" WHERE namespace.nspname IN ({schema_literals})", + " AND acl.grantee <> namespace.nspowner", + " LOOP", + " grantee_sql := CASE WHEN acl_record.grantee = 'PUBLIC'", + " THEN 'PUBLIC'", + " ELSE format('%I', acl_record.grantee) END;", + " IF EXISTS (", + " SELECT 1 FROM pg_catalog.pg_namespace current_namespace", + " CROSS JOIN LATERAL pg_catalog.aclexplode(", + " COALESCE(current_namespace.nspacl,", + " pg_catalog.acldefault('n', current_namespace.nspowner))", + " ) current_acl", + " LEFT JOIN pg_catalog.pg_authid current_grantee ON current_grantee.oid = current_acl.grantee", + " JOIN pg_catalog.pg_authid current_grantor ON current_grantor.oid = current_acl.grantor", + " WHERE current_namespace.nspname = acl_record.schema_name", + " AND COALESCE(current_grantee.rolname, 'PUBLIC') = acl_record.grantee", + " AND current_grantor.rolname = acl_record.grantor", + " AND current_acl.privilege_type = acl_record.privilege_type", + " ) THEN", + " EXECUTE format('SET LOCAL ROLE %I', acl_record.grantor);", + " EXECUTE format(", + " 'REVOKE %s ON SCHEMA %I FROM %s GRANTED BY %I CASCADE',", + " acl_record.privilege_type, acl_record.schema_name,", + " grantee_sql, acl_record.grantor", + " );", + " EXECUTE 'RESET ROLE';", + " END IF;", + " END LOOP;", + "END", + "$acl_schema_grantors$;", + ] + ) + for (database_ref, schema_name), owner in sorted(schema_owners.items()): + if schema_name is None: + continue + quoted_schema = _quote_identifier(schema_name) + quoted_owner = _quote_identifier(owner) + if schema_name in matrix.absent_schemas[database_ref]: + marker = _managed_provenance_marker( + matrix, + kind="schema", + name=schema_name, + ) + lines.extend( + [ + "DO $acl_create_schema$ BEGIN", + " IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace", + f" WHERE nspname = '{schema_name}') THEN", + f" CREATE SCHEMA {quoted_schema} AUTHORIZATION {quoted_owner};", + f" COMMENT ON SCHEMA {quoted_schema} IS '{marker}';", + " END IF;", + "END $acl_create_schema$;", + ] + ) + if phase is EnumApplicationDatabaseAclRenderPhase.FULL: + lines.extend( + [ + f"ALTER SCHEMA {quoted_schema} OWNER TO {quoted_owner};", + f"REVOKE ALL PRIVILEGES ON SCHEMA {quoted_schema} FROM " + f"{database_grantees} CASCADE;", + ] + ) + for row in schema_rows: + if ( + row.database_ref != database_ref + or row.schema_ref != schema_name + or not row.privileges + ): + continue + privileges = ", ".join(p.value for p in row.privileges) + lines.append( + f"GRANT {privileges} ON SCHEMA {quoted_schema} TO " + f"{_quote_identifier(row.principal)};" + ) + if phase is EnumApplicationDatabaseAclRenderPhase.FULL: + lines.append( + f'REVOKE ALL PRIVILEGES ON SCHEMA "public" FROM {database_grantees} CASCADE;' + ) + lines.append("") + + rows_by_object: dict[ + tuple[str, str, EnumDatabaseGrantObjectType, str, str | None], + list[ModelApplicationDatabaseAclRow], + ] = defaultdict(list) + for row in matrix.rows: + if row.object_type in _OBJECT_TYPES: + rows_by_object[ + ( + row.database_ref, + row.schema_ref or "", + row.object_type, + row.object_ref or "", + row.function_signature, + ) + ].append(row) + all_grantees = ", ".join( + ["PUBLIC", *(_quote_identifier(name) for name in revocation_principals)] + ) + ownership_order = { + EnumDatabaseGrantObjectType.TABLE: 0, + EnumDatabaseGrantObjectType.SEQUENCE: 1, + EnumDatabaseGrantObjectType.TYPE: 2, + EnumDatabaseGrantObjectType.FUNCTION: 3, + } + for obj in sorted( + (matrix.objects if phase is EnumApplicationDatabaseAclRenderPhase.FULL else ()), + key=lambda item: ( + ownership_order[item.object_type], + item.database_ref, + item.schema_ref, + item.object_ref, + item.function_signature or "", + ), + ): + owner_keyword, acl_keyword, target = _sql_object_target(obj) + dynamic_target = target.replace("%", "%%") + dynamic_revoke_pattern = ( + f"REVOKE %s ON {acl_keyword} {dynamic_target} FROM %s GRANTED BY %I CASCADE" + ) + if obj.object_type in { + EnumDatabaseGrantObjectType.TABLE, + EnumDatabaseGrantObjectType.SEQUENCE, + }: + default_acl_kind = ( + "s" if obj.object_type is EnumDatabaseGrantObjectType.SEQUENCE else "r" + ) + relkind_condition = { + "table": "object.relkind IN ('r', 'p')", + "view": "object.relkind = 'v'", + "materialized_view": "object.relkind = 'm'", + "foreign_table": "object.relkind = 'f'", + "sequence": "object.relkind = 'S'", + }[obj.catalog_kind] + catalog_acl_lines = [ + " FROM pg_catalog.pg_class object", + " JOIN pg_catalog.pg_namespace namespace ON namespace.oid = object.relnamespace", + " CROSS JOIN LATERAL pg_catalog.aclexplode(", + f" COALESCE(object.relacl, pg_catalog.acldefault('{default_acl_kind}', object.relowner))", + " ) acl", + " LEFT JOIN pg_catalog.pg_authid grantee ON grantee.oid = acl.grantee", + " JOIN pg_catalog.pg_authid grantor ON grantor.oid = acl.grantor", + f" WHERE namespace.nspname = '{obj.schema_ref}'", + f" AND object.relname = '{obj.object_ref}'", + f" AND {relkind_condition}", + " AND acl.grantee <> object.relowner", + ] + elif obj.object_type is EnumDatabaseGrantObjectType.FUNCTION: + prokind = { + "function": "f", + "aggregate": "a", + "window_function": "w", + "procedure": "p", + }[obj.catalog_kind] + catalog_acl_lines = [ + " FROM pg_catalog.pg_proc object", + " JOIN pg_catalog.pg_namespace namespace ON namespace.oid = object.pronamespace", + " CROSS JOIN LATERAL pg_catalog.aclexplode(", + " COALESCE(object.proacl, pg_catalog.acldefault('f', object.proowner))", + " ) acl", + " LEFT JOIN pg_catalog.pg_authid grantee ON grantee.oid = acl.grantee", + " JOIN pg_catalog.pg_authid grantor ON grantor.oid = acl.grantor", + f" WHERE namespace.nspname = '{obj.schema_ref}'", + f" AND object.proname = '{obj.object_ref}'", + f" AND object.prokind = '{prokind}'", + " AND '(' || pg_catalog.pg_get_function_identity_arguments(object.oid) || ')' = " + f"{_quote_sql_literal(obj.function_signature or '')}", + " AND acl.grantee <> object.proowner", + ] + else: + type_condition = { + "type": ( + "((object.typtype IN ('d', 'e') AND object.typelem = 0) " + "OR (object.typtype = 'c' AND relation.relkind = 'c'))" + ), + "base_type": "object.typtype = 'b' AND object.typelem = 0", + "range_type": "object.typtype = 'r' AND object.typelem = 0", + "multirange_type": "object.typtype = 'm' AND object.typelem = 0", + }[obj.catalog_kind] + catalog_acl_lines = [ + " FROM pg_catalog.pg_type object", + " JOIN pg_catalog.pg_namespace namespace ON namespace.oid = object.typnamespace", + " LEFT JOIN pg_catalog.pg_class relation ON relation.oid = object.typrelid", + " CROSS JOIN LATERAL pg_catalog.aclexplode(", + " COALESCE(object.typacl, pg_catalog.acldefault('T', object.typowner))", + " ) acl", + " LEFT JOIN pg_catalog.pg_authid grantee ON grantee.oid = acl.grantee", + " JOIN pg_catalog.pg_authid grantor ON grantor.oid = acl.grantor", + f" WHERE namespace.nspname = '{obj.schema_ref}'", + f" AND object.typname = '{obj.object_ref}'", + " AND object.typisdefined", + f" AND ({type_condition})", + " AND acl.grantee <> object.typowner", + ] + lines.extend( + [ + "DO $acl_object_grantors$", + "DECLARE acl_record record;", + "DECLARE grantee_sql text;", + "DECLARE temporary_schema_usage boolean;", + "BEGIN", + " FOR acl_record IN", + " SELECT COALESCE(grantee.rolname, 'PUBLIC') AS grantee,", + " grantor.rolname AS grantor, acl.privilege_type", + *catalog_acl_lines, + " LOOP", + " grantee_sql := CASE WHEN acl_record.grantee = 'PUBLIC'", + " THEN 'PUBLIC'", + " ELSE format('%I', acl_record.grantee) END;", + " IF EXISTS (", + " SELECT 1", + *catalog_acl_lines, + " AND COALESCE(grantee.rolname, 'PUBLIC') = acl_record.grantee", + " AND grantor.rolname = acl_record.grantor", + " AND acl.privilege_type = acl_record.privilege_type", + " ) THEN", + " SELECT NOT pg_catalog.has_schema_privilege(", + f" acl_record.grantor, '{obj.schema_ref}', 'USAGE'", + " ) INTO temporary_schema_usage;", + " IF temporary_schema_usage THEN", + " EXECUTE format(", + f" 'GRANT USAGE ON SCHEMA %I TO %I', '{obj.schema_ref}',", + " acl_record.grantor", + " );", + " END IF;", + " EXECUTE format('SET LOCAL ROLE %I', acl_record.grantor);", + " EXECUTE format(", + f" {_quote_sql_literal(dynamic_revoke_pattern)},", + " acl_record.privilege_type, grantee_sql, acl_record.grantor", + " );", + " EXECUTE 'RESET ROLE';", + " IF temporary_schema_usage THEN", + " EXECUTE format(", + f" 'REVOKE USAGE ON SCHEMA %I FROM %I CASCADE', '{obj.schema_ref}',", + " acl_record.grantor", + " );", + " END IF;", + " END IF;", + " END LOOP;", + "END", + "$acl_object_grantors$;", + ] + ) + lines.append( + f"ALTER {owner_keyword} {target} OWNER TO {_quote_identifier(obj.owner)};" + ) + lines.append( + f"REVOKE ALL PRIVILEGES ON {acl_keyword} {target} FROM " + f"{all_grantees} CASCADE;" + ) + if obj.object_type is EnumDatabaseGrantObjectType.TABLE: + lines.extend( + [ + "DO $acl_column_grantors$", + "DECLARE acl_record record;", + "DECLARE grantee_sql text;", + "DECLARE temporary_schema_usage boolean;", + "BEGIN", + " FOR acl_record IN", + " SELECT attribute.attname AS column_name,", + " COALESCE(grantee.rolname, 'PUBLIC') AS grantee,", + " grantor.rolname AS grantor, acl.privilege_type", + " FROM pg_catalog.pg_attribute attribute", + " JOIN pg_catalog.pg_class relation ON relation.oid = attribute.attrelid", + " JOIN pg_catalog.pg_namespace namespace ON namespace.oid = relation.relnamespace", + " CROSS JOIN LATERAL pg_catalog.aclexplode(attribute.attacl) acl", + " LEFT JOIN pg_catalog.pg_authid grantee ON grantee.oid = acl.grantee", + " JOIN pg_catalog.pg_authid grantor ON grantor.oid = acl.grantor", + f" WHERE namespace.nspname = '{obj.schema_ref}'", + f" AND relation.relname = '{obj.object_ref}'", + " AND attribute.attnum > 0 AND NOT attribute.attisdropped", + " AND attribute.attacl IS NOT NULL", + " LOOP", + " grantee_sql := CASE WHEN acl_record.grantee = 'PUBLIC'", + " THEN 'PUBLIC'", + " ELSE format('%I', acl_record.grantee) END;", + " IF EXISTS (", + " SELECT 1 FROM pg_catalog.pg_attribute current_attribute", + " JOIN pg_catalog.pg_class current_relation ON current_relation.oid = current_attribute.attrelid", + " JOIN pg_catalog.pg_namespace current_namespace ON current_namespace.oid = current_relation.relnamespace", + " CROSS JOIN LATERAL pg_catalog.aclexplode(current_attribute.attacl) current_acl", + " LEFT JOIN pg_catalog.pg_authid current_grantee ON current_grantee.oid = current_acl.grantee", + " JOIN pg_catalog.pg_authid current_grantor ON current_grantor.oid = current_acl.grantor", + f" WHERE current_namespace.nspname = '{obj.schema_ref}'", + f" AND current_relation.relname = '{obj.object_ref}'", + " AND current_attribute.attname = acl_record.column_name", + " AND COALESCE(current_grantee.rolname, 'PUBLIC') = acl_record.grantee", + " AND current_grantor.rolname = acl_record.grantor", + " AND current_acl.privilege_type = acl_record.privilege_type", + " ) THEN", + " SELECT NOT pg_catalog.has_schema_privilege(", + f" acl_record.grantor, '{obj.schema_ref}', 'USAGE'", + " ) INTO temporary_schema_usage;", + " IF temporary_schema_usage THEN", + " EXECUTE format(", + f" 'GRANT USAGE ON SCHEMA %I TO %I', '{obj.schema_ref}',", + " acl_record.grantor", + " );", + " END IF;", + " EXECUTE format('SET LOCAL ROLE %I', acl_record.grantor);", + " EXECUTE format(", + " 'REVOKE %s (%I) ON TABLE %I.%I FROM %s GRANTED BY %I CASCADE',", + " acl_record.privilege_type, acl_record.column_name,", + f" '{obj.schema_ref}', '{obj.object_ref}', grantee_sql, acl_record.grantor", + " );", + " EXECUTE 'RESET ROLE';", + " IF temporary_schema_usage THEN", + " EXECUTE format(", + f" 'REVOKE USAGE ON SCHEMA %I FROM %I CASCADE', '{obj.schema_ref}',", + " acl_record.grantor", + " );", + " END IF;", + " END IF;", + " END LOOP;", + "END", + "$acl_column_grantors$;", + "DO $acl_columns$", + "DECLARE column_name text;", + "BEGIN", + " FOR column_name IN", + " SELECT attribute.attname", + " FROM pg_attribute attribute", + " JOIN pg_class relation ON relation.oid = attribute.attrelid", + " JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace", + f" WHERE namespace.nspname = '{obj.schema_ref}'", + f" AND relation.relname = '{obj.object_ref}'", + " AND attribute.attnum > 0 AND NOT attribute.attisdropped", + " LOOP", + " EXECUTE format(", + f" 'REVOKE ALL PRIVILEGES (%I) ON TABLE %I.%I FROM {all_grantees} CASCADE',", + f" column_name, '{obj.schema_ref}', '{obj.object_ref}'", + " );", + " END LOOP;", + "END", + "$acl_columns$;", + ] + ) + for row in rows_by_object[obj.identity]: + if not row.privileges: + continue + privileges = ", ".join(p.value for p in row.privileges) + lines.append( + f"GRANT {privileges} ON {acl_keyword} {target} TO " + f"{_quote_identifier(row.principal)};" + ) + + default_keyword = { + EnumDatabaseGrantObjectType.TABLE: "TABLES", + EnumDatabaseGrantObjectType.SEQUENCE: "SEQUENCES", + EnumDatabaseGrantObjectType.FUNCTION: "FUNCTIONS", + EnumDatabaseGrantObjectType.TYPE: "TYPES", + } + default_owner_scope = {row.owner for row in matrix.default_privileges} + if phase is EnumApplicationDatabaseAclRenderPhase.SCAFFOLD: + # P1 may establish deny-by-default only for roles it creates from exact + # absence evidence. Existing owners' legacy defaults remain untouched. + default_owner_scope.intersection_update(globally_absent_roles) + phase_default_rows = tuple( + row for row in matrix.default_privileges if row.owner in default_owner_scope + ) + global_default_identities = sorted( + {(row.owner, row.object_type) for row in phase_default_rows}, + key=lambda item: (item[0], item[1].value), + ) + default_owner_literals = ", ".join( + _quote_sql_literal(owner) for owner in sorted(default_owner_scope) + ) + managed_default_schema_literals = ", ".join( + _quote_sql_literal(schema_name) + for schema_name in sorted({row.schema_ref for row in phase_default_rows}) + ) + if phase_default_rows: + lines.extend( + [ + "DO $acl_default_grantor_guard$ BEGIN", + " IF EXISTS (", + " SELECT 1 FROM pg_catalog.pg_default_acl defaults", + " JOIN pg_catalog.pg_authid owner ON owner.oid = defaults.defaclrole", + " LEFT JOIN pg_catalog.pg_namespace namespace ON namespace.oid = defaults.defaclnamespace", + " CROSS JOIN LATERAL pg_catalog.aclexplode(defaults.defaclacl) acl", + " LEFT JOIN pg_catalog.pg_authid grantee ON grantee.oid = acl.grantee", + " JOIN pg_catalog.pg_authid grantor ON grantor.oid = acl.grantor", + f" WHERE owner.rolname IN ({default_owner_literals})", + " AND (defaults.defaclnamespace = 0", + f" OR namespace.nspname IN ({managed_default_schema_literals}))", + " AND grantor.oid <> defaults.defaclrole", + " ) THEN", + " RAISE EXCEPTION 'default ACL contains an unsupported alternate grantor';", + " END IF;", + "END $acl_default_grantor_guard$;", + "DO $acl_default_actual_rows$", + "DECLARE acl_record record;", + "DECLARE grantee_sql text;", + "BEGIN", + " FOR acl_record IN", + " SELECT owner.rolname AS owner, namespace.nspname AS schema_name,", + " CASE defaults.defaclobjtype", + " WHEN 'r' THEN 'TABLES'", + " WHEN 'S' THEN 'SEQUENCES'", + " WHEN 'f' THEN 'FUNCTIONS'", + " WHEN 'T' THEN 'TYPES'", + " END AS object_keyword,", + " COALESCE(grantee.rolname, 'PUBLIC') AS grantee,", + " acl.privilege_type", + " FROM pg_catalog.pg_default_acl defaults", + " JOIN pg_catalog.pg_authid owner ON owner.oid = defaults.defaclrole", + " LEFT JOIN pg_catalog.pg_namespace namespace ON namespace.oid = defaults.defaclnamespace", + " CROSS JOIN LATERAL pg_catalog.aclexplode(defaults.defaclacl) acl", + " LEFT JOIN pg_catalog.pg_authid grantee ON grantee.oid = acl.grantee", + f" WHERE owner.rolname IN ({default_owner_literals})", + " AND (defaults.defaclnamespace = 0", + f" OR namespace.nspname IN ({managed_default_schema_literals}))", + " AND acl.grantee <> defaults.defaclrole", + " LOOP", + " grantee_sql := CASE WHEN acl_record.grantee = 'PUBLIC'", + " THEN 'PUBLIC'", + " ELSE format('%I', acl_record.grantee) END;", + " EXECUTE format('SET LOCAL ROLE %I', acl_record.owner);", + " IF acl_record.schema_name IS NULL THEN", + " EXECUTE format(", + " 'ALTER DEFAULT PRIVILEGES FOR ROLE %I REVOKE %s ON %s FROM %s CASCADE',", + " acl_record.owner, acl_record.privilege_type,", + " acl_record.object_keyword, grantee_sql", + " );", + " ELSE", + " EXECUTE format(", + " 'ALTER DEFAULT PRIVILEGES FOR ROLE %I IN SCHEMA %I REVOKE %s ON %s FROM %s CASCADE',", + " acl_record.owner, acl_record.schema_name,", + " acl_record.privilege_type, acl_record.object_keyword, grantee_sql", + " );", + " END IF;", + " EXECUTE 'RESET ROLE';", + " END LOOP;", + "END", + "$acl_default_actual_rows$;", + ] + ) + default_revocation_principals = (PUBLIC_PRINCIPAL, *revocation_principals) + for owner, object_type in global_default_identities: + for principal in default_revocation_principals: + grantee = ( + "PUBLIC" + if principal == PUBLIC_PRINCIPAL + else _quote_identifier(principal) + ) + lines.append( + "ALTER DEFAULT PRIVILEGES FOR ROLE " + f"{_quote_identifier(owner)} REVOKE ALL PRIVILEGES ON " + f"{default_keyword[object_type]} FROM {grantee} CASCADE;" + ) + schema_default_identities = sorted( + {(row.owner, row.schema_ref, row.object_type) for row in phase_default_rows}, + key=lambda item: (item[0], item[1], item[2].value), + ) + for owner, schema_ref, object_type in schema_default_identities: + for principal in default_revocation_principals: + grantee = ( + "PUBLIC" + if principal == PUBLIC_PRINCIPAL + else _quote_identifier(principal) + ) + lines.append( + "ALTER DEFAULT PRIVILEGES FOR ROLE " + f"{_quote_identifier(owner)} IN SCHEMA " + f"{_quote_identifier(schema_ref)} REVOKE ALL PRIVILEGES ON " + f"{default_keyword[object_type]} FROM {grantee} CASCADE;" + ) + lines.append("COMMIT;") + return "\n".join(lines) + "\n" + + +__all__ = [ + "PUBLIC_PRINCIPAL", + "build_application_database_acl_matrix", + "render_application_database_acl_sql", + "validate_application_database_acl_scaffold", + "validate_application_database_acl_matrix", +] diff --git a/src/omnibase_infra/validation/application_database_domain_enforcement.py b/src/omnibase_infra/validation/application_database_domain_enforcement.py new file mode 100644 index 0000000000..025f6b0b55 --- /dev/null +++ b/src/omnibase_infra/validation/application_database_domain_enforcement.py @@ -0,0 +1,2465 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Fail-closed application database domain enforcement (OMN-15361).""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +import sqlparse +from sqlparse.tokens import DML + +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_core.models.core.model_deployment_topology import ModelDeploymentTopology +from omnibase_core.models.core.model_deployment_topology_database import ( + ModelDeploymentTopologyDatabase, +) +from omnibase_infra.validation.application_relation_ownership import ( + load_service_ownership_manifest, +) +from omnibase_infra.validation.enums.enum_application_database_identity_root import ( + EnumApplicationDatabaseIdentityRoot, +) +from omnibase_infra.validation.enums.enum_application_database_identity_root_operation import ( + EnumApplicationDatabaseIdentityRootOperation, +) +from omnibase_infra.validation.enums.enum_application_inventory_object_kind import ( + EnumApplicationInventoryObjectKind, +) +from omnibase_infra.validation.enums.enum_application_relation_kind import ( + EnumApplicationRelationKind, +) +from omnibase_infra.validation.models.model_application_database_catalog_identity import ( + ModelApplicationDatabaseCatalogIdentity, +) +from omnibase_infra.validation.models.model_application_database_function_state import ( + ModelApplicationDatabaseFunctionState, +) +from omnibase_infra.validation.models.model_application_database_pool_identity import ( + ModelApplicationDatabasePoolIdentity, +) +from omnibase_infra.validation.models.model_application_database_relation_state import ( + ModelApplicationDatabaseRelationState, +) +from omnibase_infra.validation.models.model_application_database_tenant_isolation_evidence import ( + ModelApplicationDatabaseTenantIsolationEvidence, +) + +CANONICAL_TENANT_PREDICATE = "tenant_id = current_setting('app.tenant_id', true)::uuid" + +_EXPECTED_IDENTITY_ROOT_RELATIONS = { + EnumApplicationDatabaseIdentityRoot.CANONICAL_TENANT: "tenants", + EnumApplicationDatabaseIdentityRoot.PRE_TENANT_BOOTSTRAP: "bootstrap_tokens", +} +_EXPECTED_IDENTITY_ROOT_OPERATIONS = frozenset( + EnumApplicationDatabaseIdentityRootOperation +) +_EXPECTED_CANONICAL_POLICY_ROLES = ("PUBLIC",) +_SQL_IDENTIFIER = ( + r'(?:"(?:[^"]|"")+"|' + r"(?:[a-z_]|[^\x00-\x7f])(?:[a-z0-9_$]|[^\x00-\x7f])*)" +) +_OPTIONAL_ONLY_TARGET = r"(?:only\s+(?:\(\s*)?)?" +_SYSTEM_READ_SCHEMAS = frozenset({"information_schema", "pg_catalog"}) +_RELATION_OBJECT_KINDS = frozenset( + { + EnumApplicationInventoryObjectKind.TABLE, + EnumApplicationInventoryObjectKind.FOREIGN_TABLE, + EnumApplicationInventoryObjectKind.VIEW, + EnumApplicationInventoryObjectKind.MATERIALIZED_VIEW, + } +) +_ROUTINE_OBJECT_KINDS = frozenset( + { + EnumApplicationInventoryObjectKind.FUNCTION, + EnumApplicationInventoryObjectKind.WINDOW_FUNCTION, + EnumApplicationInventoryObjectKind.PROCEDURE, + EnumApplicationInventoryObjectKind.AGGREGATE, + } +) + + +@dataclass(frozen=True, slots=True) +class ApplicationDatabaseSqlTargetRequirement: + """One kind- and overload-aware ownership requirement found in SQL.""" + + schema: str + name: str + allowed_kinds: frozenset[EnumApplicationInventoryObjectKind] + function_signature: str | None = None + + @property + def location(self) -> tuple[str, str]: + """Return the schema-qualified target location.""" + return (self.schema, self.name) + + +def application_database_function_definition_sha256( + *, + schema: str, + name: str, + signature: str, + language: str, + source_body: str, + parsed_sql_body: str | None, + security_definer: bool, + leakproof: bool, + volatility: str, + parallel: str, + config: Sequence[str], + kind: str, + strict: bool, + returns_set: bool, + result_type: str, +) -> str: + """Fingerprint the complete security-relevant routine catalog definition.""" + payload = { + "config": sorted(config), + "kind": kind, + "language": language, + "leakproof": leakproof, + "name": name, + "parallel": parallel, + "parsed_sql_body": parsed_sql_body, + "result_type": result_type, + "returns_set": returns_set, + "schema": schema, + "security_definer": security_definer, + "signature": signature, + "source_body": source_body, + "strict": strict, + "volatility": volatility, + } + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _compile_relation_target(prefix: str) -> re.Pattern[str]: + return re.compile( + rf"{prefix}(?:(?P{_SQL_IDENTIFIER})\s*\.\s*)?" + rf"(?P{_SQL_IDENTIFIER})", + re.IGNORECASE + re.DOTALL, + ) + + +# pattern, permits a CTE/table-function target, permits a system read schema +_RELATION_TARGETS: tuple[tuple[re.Pattern[str], bool, bool], ...] = ( + ( + _compile_relation_target( + r"^\s*create\s+(?:(?:global|local)\s+)?" + r"(?:(?:temporary|temp|unlogged)\s+)?(?:foreign\s+)?table\s+" + r"(?:if\s+not\s+exists\s+)?" + ), + False, + False, + ), + ( + _compile_relation_target( + r"^\s*create\s+(?:or\s+replace\s+)?" + r"(?:view|materialized\s+view|function|procedure|sequence|type|domain|aggregate)\s+" + r"(?:if\s+not\s+exists\s+)?" + ), + False, + False, + ), + ( + _compile_relation_target( + r"^\s*(?:alter|drop)\s+" + r"(?:foreign\s+table|table|view|materialized\s+view|function|" + r"procedure|sequence|type|domain|aggregate)\s+" + r"(?:if\s+exists\s+)?(?:only\s+)?" + ), + False, + False, + ), + ( + _compile_relation_target( + r"^\s*(?:insert\s+into|update|delete\s+from|merge\s+into|copy|" + r"truncate(?:\s+table)?)\s+(?:only\s+)?" + ), + False, + False, + ), + (_compile_relation_target(r"^\s*call\s+"), False, False), + ( + _compile_relation_target( + rf"^\s*create\s+(?:unique\s+)?index\s+(?:concurrently\s+)?" + rf"(?:(?:if\s+not\s+exists\s+)?{_SQL_IDENTIFIER}" + rf"(?:\s*\.\s*{_SQL_IDENTIFIER})?\s+)?on\s+" + rf"{_OPTIONAL_ONLY_TARGET}" + ), + False, + False, + ), + ( + _compile_relation_target(rf"^\s*table\s+{_OPTIONAL_ONLY_TARGET}"), + False, + False, + ), + ( + _compile_relation_target( + r"^\s*comment\s+on\s+" + r"(?:foreign\s+table|materialized\s+view|table|view|column|" + r"function|procedure|aggregate|sequence|type|domain)\s+" + ), + False, + False, + ), + ( + _compile_relation_target(r"\bexecute\s+(?:function|procedure)\s+"), + False, + False, + ), + ( + _compile_relation_target( + r"^\s*(?:grant|revoke)\s+[\s\S]*?\s+on\s+" + r"(?:(?:table|sequence|function|procedure|type|domain)\s+)?" + r"(?!(?:database|schema)\b)" + ), + False, + False, + ), + (_compile_relation_target(r"\breferences\s+"), False, False), + ( + _compile_relation_target( + rf"\b(?:from|join)\s+(?:lateral\s+)?{_OPTIONAL_ONLY_TARGET}" + ), + True, + True, + ), + ( + _compile_relation_target(r"^\s*merge\b[\s\S]*?\busing\s+(?:only\s+)?"), + True, + True, + ), + ( + _compile_relation_target(r"^\s*delete\b[\s\S]*?\busing\s+(?:only\s+)?"), + False, + True, + ), + ( + _compile_relation_target( + rf"^\s*(?:(?:create|alter)\s+policy\s+{_SQL_IDENTIFIER}|" + rf"drop\s+policy\s+(?:if\s+exists\s+)?{_SQL_IDENTIFIER})\s+on\s+" + ), + False, + False, + ), + ( + _compile_relation_target( + rf"^\s*(?:(?:create\s+(?:or\s+replace\s+)?(?:constraint\s+)?|alter\s+)" + rf"trigger\s+{_SQL_IDENTIFIER}[\s\S]*?\bon\s+|" + rf"drop\s+trigger\s+(?:if\s+exists\s+)?{_SQL_IDENTIFIER}\s+on\s+)" + ), + False, + False, + ), + ( + _compile_relation_target( + r"^\s*refresh\s+materialized\s+view\s+(?:concurrently\s+)?" + ), + False, + False, + ), + ( + _compile_relation_target( + r"^\s*reindex\s+(?:\([^)]*\)\s*)?table\s+(?:concurrently\s+)?" + ), + False, + False, + ), + ( + _compile_relation_target(r"^\s*cluster\s+(?:(?:\([^)]*\)|verbose)\s+)?"), + False, + False, + ), + (_compile_relation_target(r"\bpartition\s+of\s+"), False, False), + ( + _compile_relation_target(r"\b(?:attach|detach)\s+partition\s+"), + False, + False, + ), + (_compile_relation_target(r"\binherits\s*\(\s*"), False, False), + (_compile_relation_target(r"\binherit\s+"), False, False), + (_compile_relation_target(r"\bas\s+table\s+"), False, False), + (_compile_relation_target(r"\blike\s+"), False, False), +) +_LIST_TARGET = re.compile( + rf"^\s*(?:lateral\s+)?{_OPTIONAL_ONLY_TARGET}" + rf"(?:(?P{_SQL_IDENTIFIER})\s*\.\s*)?" + rf"(?P{_SQL_IDENTIFIER})", + re.IGNORECASE, +) +_RELATION_LISTS: tuple[tuple[re.Pattern[str], bool, bool], ...] = ( + ( + re.compile( + r"\bfrom\s+(?P[\s\S]*?)" + r"(?=\bwhere\b|\bjoin\b|\bgroup\s+by\b|\border\s+by\b|" + r"\blimit\b|\bunion\b|\breturning\b|\bfor\s+(?:update|share)\b|;|$)", + re.IGNORECASE, + ), + True, + True, + ), + ( + re.compile( + r"^\s*(?:drop\s+(?:foreign\s+table|table|view|materialized\s+view|" + r"function|procedure|aggregate|sequence|type)|" + r"truncate(?:\s+table)?|lock(?:\s+table)?)\s+" + r"(?:if\s+exists\s+)?(?P[\s\S]*?)" + r"(?=\s+(?:cascade|restrict|nowait)\b|;|$)", + re.IGNORECASE, + ), + False, + False, + ), + ( + re.compile( + r"^\s*(?:grant|revoke)\s+[\s\S]*?\s+on\s+" + r"(?!(?:database|schema)\b)" + r"(?:(?:table|sequence|function|procedure|type)\s+)?" + r"(?P[\s\S]*?)\s+(?:to|from)\b", + re.IGNORECASE, + ), + False, + False, + ), + ( + re.compile( + r"\binherits\s*\((?P[^)]*)\)", + re.IGNORECASE, + ), + False, + False, + ), + ( + re.compile( + r"^\s*vacuum\s+(?:\([^)]*\)\s*)?" + r"(?:(?:full|freeze|verbose|analyze)\s+)*" + r"(?P[\s\S]*?)(?=;|$)", + re.IGNORECASE, + ), + False, + False, + ), + ( + re.compile( + r"^\s*analyze\s+(?:\([^)]*\)\s*)?(?:verbose\s+)?" + r"(?P[\s\S]*?)(?=;|$)", + re.IGNORECASE, + ), + False, + False, + ), +) +_CREATED_APPLICATION_OBJECT = re.compile( + rf"^\s*create\s+(?:or\s+replace\s+)?" + rf"(?:(?:(?:global|local)\s+)?(?:temporary|temp|unlogged)\s+)?" + rf"(?Pforeign\s+table|materialized\s+view|table|view|function|" + rf"procedure|sequence|type|domain|aggregate|extension)\s+" + rf"(?:if\s+not\s+exists\s+)?" + rf"(?:(?P{_SQL_IDENTIFIER})\s*\.\s*)?" + rf"(?P{_SQL_IDENTIFIER})", + re.IGNORECASE, +) +_SELECT_INTO_TARGET = _compile_relation_target( + r"^\s*(?:(?:temporary|temp|unlogged)\s+)?(?:table\s+)?" +) +_CREATED_KIND_MAP = { + "foreign table": EnumApplicationInventoryObjectKind.FOREIGN_TABLE, + "materialized view": EnumApplicationInventoryObjectKind.MATERIALIZED_VIEW, + "table": EnumApplicationInventoryObjectKind.TABLE, + "view": EnumApplicationInventoryObjectKind.VIEW, + "function": EnumApplicationInventoryObjectKind.FUNCTION, + "procedure": EnumApplicationInventoryObjectKind.PROCEDURE, + "sequence": EnumApplicationInventoryObjectKind.SEQUENCE, + "type": EnumApplicationInventoryObjectKind.TYPE, + "domain": EnumApplicationInventoryObjectKind.TYPE, + "aggregate": EnumApplicationInventoryObjectKind.AGGREGATE, + "extension": EnumApplicationInventoryObjectKind.EXTENSION, +} +_NON_EXACT_MAINTENANCE_OPERATIONS: tuple[re.Pattern[str], ...] = ( + re.compile( + r"^\s*vacuum\b\s*(?:\([^)]*\)\s*)?" + r"(?:(?:full|freeze|verbose|analyze)\b\s*)*;?\s*$", + re.IGNORECASE, + ), + re.compile( + r"^\s*analyze\b\s*(?:\([^)]*\)\s*)?(?:verbose\b\s*)?;?\s*$", + re.IGNORECASE, + ), + re.compile( + r"^\s*cluster\b\s*(?:(?:\([^)]*\)|verbose\b)\s*)?;?\s*$", + re.IGNORECASE, + ), + re.compile( + r"^\s*reindex\s+(?:\([^)]*\)\s*)?" + r"(?:index|schema|database|system)\b[\s\S]*$", + re.IGNORECASE, + ), +) + + +def _unquote_identifier(token: str) -> str: + token = token.strip() + if token.startswith('"') and token.endswith('"'): + return token[1:-1].replace('""', '"') + return token.lower() + + +def _main_dml_tail(statement: str) -> str: + """Return the top-level DML tail so CTE-prefixed writes cannot evade anchors.""" + parsed = sqlparse.parse(statement) + if not parsed: + return statement + parsed_statement = parsed[0] + # sqlparse's bundled typing does not annotate Statement.get_type(). + statement_type = parsed_statement.get_type().upper() # type: ignore[no-untyped-call] + if statement_type not in {"INSERT", "UPDATE", "DELETE", "MERGE"}: + return statement + for index, token in enumerate(parsed_statement.tokens): + if token.ttype is DML and token.normalized.upper() == statement_type: + return "".join(str(item) for item in parsed_statement.tokens[index:]) + return statement + + +def _split_top_level_commas(value: str) -> tuple[str, ...]: + """Split a SQL target list while preserving commas in calls and quoted text.""" + segments: list[str] = [] + start = 0 + depth = 0 + quote: str | None = None + index = 0 + while index < len(value): + character = value[index] + if quote is not None: + if character == quote: + if index + 1 < len(value) and value[index + 1] == quote: + index += 2 + continue + quote = None + index += 1 + continue + if character in {"'", '"'}: + quote = character + elif character == "(": + depth += 1 + elif character == ")": + depth = max(0, depth - 1) + elif character == "," and depth == 0: + segments.append(value[start:index]) + start = index + 1 + index += 1 + segments.append(value[start:]) + return tuple(segment.strip() for segment in segments if segment.strip()) + + +def _skip_whitespace(value: str, index: int) -> int: + while index < len(value) and value[index].isspace(): + index += 1 + return index + + +def _balanced_parenthesized( + value: str, + start: int, +) -> tuple[str, int] | None: + """Return one balanced SQL parenthesis body and the first following offset.""" + if start >= len(value) or value[start] != "(": + return None + depth = 0 + index = start + while index < len(value): + character = value[index] + if character in {"'", '"'}: + span_end = _quoted_sql_span_end( + value, + index, + escape_backslashes=( + character == "'" + and ( + _has_sql_prefix(value, index, "e") + or _has_sql_prefix(value, index, "u&") + ) + ), + ) + if span_end is None: + return None + index = span_end + continue + if character == "$": + dollar_quote = _dollar_quote_at(value, index) + if dollar_quote is not None: + end = value.find(dollar_quote, index + len(dollar_quote)) + if end < 0: + return None + index = end + len(dollar_quote) + continue + if character == "(": + depth += 1 + elif character == ")": + depth -= 1 + if depth == 0: + return value[start + 1 : index], index + 1 + index += 1 + return None + + +def _identifier_at(value: str, index: int) -> tuple[str, int] | None: + match = re.match(_SQL_IDENTIFIER, value[index:], re.IGNORECASE) + if match is None: + return None + return match.group(0), index + match.end() + + +def _keyword_at(value: str, index: int, keyword: str) -> int | None: + match = re.match(rf"{keyword}\b", value[index:], re.IGNORECASE) + if match is None: + return None + return index + match.end() + + +def _leading_ctes( + statement: str, +) -> tuple[bool, tuple[tuple[str, str], ...], str] | None: + """Parse a leading WITH clause into ordered, independently scoped bodies.""" + index = _skip_whitespace(statement, 0) + next_index = _keyword_at(statement, index, "with") + if next_index is None: + return None + index = _skip_whitespace(statement, next_index) + recursive_index = _keyword_at(statement, index, "recursive") + recursive = recursive_index is not None + if recursive_index is not None: + index = _skip_whitespace(statement, recursive_index) + + definitions: list[tuple[str, str]] = [] + while True: + identifier = _identifier_at(statement, index) + if identifier is None: + return None + name_token, index = identifier + index = _skip_whitespace(statement, index) + if index < len(statement) and statement[index] == "(": + columns = _balanced_parenthesized(statement, index) + if columns is None: + return None + _, index = columns + index = _skip_whitespace(statement, index) + as_index = _keyword_at(statement, index, "as") + if as_index is None: + return None + index = _skip_whitespace(statement, as_index) + not_index = _keyword_at(statement, index, "not") + if not_index is not None: + index = _skip_whitespace(statement, not_index) + materialized_index = _keyword_at(statement, index, "materialized") + if materialized_index is not None: + index = _skip_whitespace(statement, materialized_index) + body = _balanced_parenthesized(statement, index) + if body is None: + return None + body_sql, index = body + definitions.append((_unquote_identifier(name_token), body_sql)) + index = _skip_whitespace(statement, index) + if index >= len(statement) or statement[index] != ",": + break + index = _skip_whitespace(statement, index + 1) + return recursive, tuple(definitions), statement[index:] + + +_VIEW_HEAD_MODIFIERS: tuple[str, ...] = ( + "or", + "replace", + "temp", + "temporary", + "unlogged", + "recursive", + "materialized", +) + + +def _view_query_offset(statement: str) -> int | None: + """Return the offset of the query body in a ``CREATE ... VIEW ... AS`` statement. + + A view body may open with its own WITH clause, in which case the CTE names are + only reachable past the ``AS``. ``_leading_ctes`` matches a WITH at offset zero, + so without this the CTE names of a ``CREATE VIEW ... AS WITH ...`` statement are + never collected and every later reference to one is misread as an unqualified + application relation (OMN-15361). + """ + index = _skip_whitespace(statement, 0) + create_index = _keyword_at(statement, index, "create") + if create_index is None: + return None + index = _skip_whitespace(statement, create_index) + + while True: + for modifier in _VIEW_HEAD_MODIFIERS: + candidate = _keyword_at(statement, index, modifier) + if candidate is not None: + index = _skip_whitespace(statement, candidate) + break + else: + break + + view_index = _keyword_at(statement, index, "view") + if view_index is None: + return None + index = _skip_whitespace(statement, view_index) + + for guard in ("if", "not", "exists"): + candidate = _keyword_at(statement, index, guard) + if candidate is not None: + index = _skip_whitespace(statement, candidate) + + identifier = _identifier_at(statement, index) + if identifier is None: + return None + _, index = identifier + index = _skip_whitespace(statement, index) + if index < len(statement) and statement[index] == ".": + qualified = _identifier_at(statement, _skip_whitespace(statement, index + 1)) + if qualified is None: + return None + _, index = qualified + index = _skip_whitespace(statement, index) + + if index < len(statement) and statement[index] == "(": + columns = _balanced_parenthesized(statement, index) + if columns is None: + return None + _, index = columns + index = _skip_whitespace(statement, index) + + # ``WITH (security_invoker = true)`` is a view option list, not a CTE. It is only + # consumed when a parenthesis actually follows, so a CTE WITH is never eaten here. + options_index = _keyword_at(statement, index, "with") + if options_index is not None: + options_start = _skip_whitespace(statement, options_index) + if options_start < len(statement) and statement[options_start] == "(": + options = _balanced_parenthesized(statement, options_start) + if options is None: + return None + _, index = options + index = _skip_whitespace(statement, index) + + as_index = _keyword_at(statement, index, "as") + if as_index is None: + return None + return _skip_whitespace(statement, as_index) + + +def _is_sql_identifier_character(character: str) -> bool: + """Return whether a character can continue an unquoted PostgreSQL name.""" + return ( + character == "_" + or character == "$" + or character.isalnum() + or ord(character) >= 128 + ) + + +def _has_sql_prefix(value: str, quote_index: int, prefix: str) -> bool: + start = quote_index - len(prefix) + return ( + start >= 0 + and value[start:quote_index].lower() == prefix.lower() + and (start == 0 or not _is_sql_identifier_character(value[start - 1])) + ) + + +def _dollar_quote_at(value: str, index: int) -> str | None: + match = re.match( + r"\$(?:(?:[a-z_]|[^\x00-\x7f])" + r"(?:[a-z0-9_]|[^\x00-\x7f])*)?\$", + value[index:], + re.IGNORECASE, + ) + return None if match is None else match.group(0) + + +def _quoted_sql_span_end( + value: str, + start: int, + *, + escape_backslashes: bool, +) -> int | None: + quote = value[start] + index = start + 1 + while index < len(value): + character = value[index] + if escape_backslashes and character == "\\": + index += 2 + continue + if character != quote: + index += 1 + continue + if index + 1 < len(value) and value[index + 1] == quote: + index += 2 + continue + return index + 1 + return None + + +def _postgresql_lexical_violations(value: str) -> tuple[str, ...]: + """Reject lexical forms whose boundaries cannot be proven before sqlparse.""" + violations: list[str] = [] + index = 0 + while index < len(value): + if value.startswith("--", index): + newline = value.find("\n", index + 2) + index = len(value) if newline < 0 else newline + 1 + continue + if value.startswith("/*", index): + depth = 1 + index += 2 + while index < len(value) and depth: + if value.startswith("/*", index): + depth += 1 + violations.append( + "nested block comments cannot be proven statically" + ) + index += 2 + elif value.startswith("*/", index): + depth -= 1 + index += 2 + else: + index += 1 + if depth: + violations.append( + "unterminated block comment cannot be proven statically" + ) + continue + if ( + value[index : index + 2].lower() == "u&" + and index + 2 < len(value) + and value[index + 2] in {"'", '"'} + and (index == 0 or not _is_sql_identifier_character(value[index - 1])) + ): + violations.append( + "Unicode-escaped identifiers and literals cannot be proven statically" + ) + index += 2 + continue + character = value[index] + if character in {"'", '"'}: + span_end = _quoted_sql_span_end( + value, + index, + escape_backslashes=( + character == "'" + and ( + _has_sql_prefix(value, index, "e") + or _has_sql_prefix(value, index, "u&") + ) + ), + ) + if span_end is None: + violations.append( + "unterminated SQL literal or delimited identifier cannot be " + "proven statically" + ) + break + index = span_end + continue + if character == "$": + delimiter = _dollar_quote_at(value, index) + if delimiter is not None: + end = value.find(delimiter, index + len(delimiter)) + if end < 0: + violations.append( + "unterminated dollar-quoted SQL body cannot be proven statically" + ) + break + index = end + len(delimiter) + continue + index += 1 + return tuple(dict.fromkeys(violations)) + + +def _mask_sql_string_bodies( + value: str, + *, + mask_quoted_identifiers: bool = False, +) -> str: + """Blank quoted SQL bodies without changing offsets or target identifiers.""" + masked = list(value) + index = 0 + while index < len(value): + if value.startswith("--", index): + newline = value.find("\n", index + 2) + stop = len(value) if newline < 0 else newline + for offset in range(index, stop): + masked[offset] = " " + index = stop + continue + if value.startswith("/*", index): + end = value.find("*/", index + 2) + stop = len(value) if end < 0 else end + 2 + for offset in range(index, stop): + masked[offset] = " " + index = stop + continue + character = value[index] + if character == '"': + span_end = _quoted_sql_span_end( + value, + index, + escape_backslashes=False, + ) + stop = len(value) if span_end is None else span_end + if mask_quoted_identifiers: + for offset in range(index, stop): + masked[offset] = " " + index = stop + continue + if character == "'": + span_end = _quoted_sql_span_end( + value, + index, + escape_backslashes=( + _has_sql_prefix(value, index, "e") + or _has_sql_prefix(value, index, "u&") + ), + ) + stop = len(value) if span_end is None else span_end + for offset in range(index, stop): + masked[offset] = " " + index = stop + continue + if character == "$": + delimiter = _dollar_quote_at(value, index) + if delimiter is not None: + end = value.find(delimiter, index + len(delimiter)) + stop = len(value) if end < 0 else end + len(delimiter) + for offset in range(index, stop): + masked[offset] = " " + index = stop + continue + index += 1 + return "".join(masked) + + +def _contains_unquoted_keyword(value: str, keyword: str) -> bool: + """Recognize a SQL keyword outside literals and delimited identifiers.""" + masked = _mask_sql_string_bodies(value, mask_quoted_identifiers=True) + return re.search(rf"\b{keyword}\b", masked, re.IGNORECASE) is not None + + +def _without_leading_maintenance_options(value: str) -> str: + """Blank one balanced VACUUM/ANALYZE/CLUSTER/REINDEX option group.""" + masked = _mask_sql_string_bodies(value, mask_quoted_identifiers=True) + operation = re.match( + r"^\s*(?:vacuum|analyze|cluster|reindex)\b", + masked, + re.IGNORECASE, + ) + if operation is None: + return value + option_start = _skip_whitespace(value, operation.end()) + if option_start >= len(value) or value[option_start] != "(": + return value + options = _balanced_parenthesized(value, option_start) + if options is None: + return value + _, option_end = options + return ( + value[:option_start] + (" " * (option_end - option_start)) + value[option_end:] + ) + + +def _select_into_target_match(statement: str) -> re.Match[str] | None: + """Find the top-level SELECT INTO target outside aliases and subqueries.""" + masked = _mask_sql_string_bodies(statement, mask_quoted_identifiers=True) + if re.match(r"^\s*select\b", masked, re.IGNORECASE) is None: + return None + into_offsets = { + match.start(): match.end() + for match in re.finditer(r"\binto\b", masked, re.IGNORECASE) + } + depth = 0 + index = 0 + while index < len(masked): + character = masked[index] + if character == "(": + depth += 1 + elif character == ")": + depth = max(0, depth - 1) + elif depth == 0 and index in into_offsets: + return _SELECT_INTO_TARGET.match(statement[into_offsets[index] :]) + index += 1 + return None + + +def _sql_literal_at(value: str, index: int) -> tuple[str, int] | None: + """Decode one single- or dollar-quoted PostgreSQL literal at ``index``.""" + index = _skip_whitespace(value, index) + if index >= len(value): + return None + if value[index] == "'": + body: list[str] = [] + index += 1 + while index < len(value): + character = value[index] + if character != "'": + body.append(character) + index += 1 + continue + if index + 1 < len(value) and value[index + 1] == "'": + body.append("'") + index += 2 + continue + return "".join(body), index + 1 + return None + if value[index] != "$": + return None + tag = re.match(r"\$(?:[a-z_][a-z0-9_]*)?\$", value[index:], re.IGNORECASE) + if tag is None: + return None + delimiter = tag.group(0) + body_start = index + len(delimiter) + body_end = value.find(delimiter, body_start) + if body_end < 0: + return None + return value[body_start:body_end], body_end + len(delimiter) + + +def _routine_language(statement: str) -> str | None: + """Return a supported procedural language for one CREATE routine.""" + masked = _mask_sql_string_bodies(statement) + if ( + re.match( + r"^\s*create\s+(?:or\s+replace\s+)?(?:function|procedure)\b", + masked, + re.IGNORECASE, + ) + is None + ): + return None + language_match = re.search( + rf"\blanguage\s+(?P{_SQL_IDENTIFIER})", + masked, + re.IGNORECASE, + ) + if language_match is None: + return None + language = _unquote_identifier(language_match.group("language")) + return language if language in {"sql", "plpgsql"} else None + + +def _routine_sql_bodies(statement: str) -> tuple[tuple[str, str], ...]: + """Extract statically inspectable SQL/PLpgSQL routine definition bodies.""" + language = _routine_language(statement) + if language is None: + return () + masked = _mask_sql_string_bodies(statement) + language_match = re.search( + rf"\blanguage\s+{_SQL_IDENTIFIER}", + masked, + re.IGNORECASE, + ) + if language_match is None: # pragma: no cover - guaranteed by helper + return () + + as_matches = tuple(re.finditer(r"\bas\b", masked, re.IGNORECASE)) + if as_matches: + literal = _sql_literal_at(statement, as_matches[-1].end()) + if literal is not None: + body, _ = literal + return ((language, body),) + + if language == "sql": + sql_body = re.search( + r"\b(?:return\b|begin\s+atomic\b)", + masked[language_match.end() :], + re.IGNORECASE, + ) + if sql_body is not None: + start = language_match.end() + sql_body.start() + return ((language, statement[start:]),) + return () + + +def _routine_body_is_uninspectable(statement: str) -> bool: + """Fail closed when a supported routine body has no static representation.""" + return _routine_language(statement) is not None and not _routine_sql_bodies( + statement + ) + + +def _explained_statement(value: str) -> str: + """Return the statement wrapped by EXPLAIN, retaining original SQL offsets.""" + masked = _mask_sql_string_bodies(value) + prefix = re.match(r"^\s*explain\b", masked, re.IGNORECASE) + if prefix is None: + return value + wrapped = re.search( + r"\b(?:select|insert|update|delete|merge|copy)\b", + masked[prefix.end() :], + re.IGNORECASE, + ) + if wrapped is None: + return value + return value[prefix.end() + wrapped.start() :] + + +def _requires_dynamic_sql_rejection(statement: str) -> bool: + """Reject procedural execution whose relation targets are runtime strings.""" + masked = _mask_sql_string_bodies(statement) + if re.match(r"^\s*do\b", masked, re.IGNORECASE) is not None: + return True + return any( + language == "plpgsql" and _contains_unquoted_keyword(body, "execute") + for language, body in _routine_sql_bodies(statement) + ) + + +def _record_sql_target( + *, + schema_token: str | None, + name_token: str, + remaining: str, + permits_ephemeral: bool, + permits_system_read: bool, + cte_names: set[str], + application_schemas: set[str], + violations: list[str], + target_locations: list[tuple[str, str]], +) -> None: + """Apply one topology-derived qualification verdict to a parsed target.""" + name = _unquote_identifier(name_token) + if schema_token is None: + if permits_ephemeral and ( + name in cte_names or remaining.lstrip().startswith("(") + ): + return + violations.append( + f"application relation target {name!r} must be schema-qualified" + ) + return + schema = _unquote_identifier(schema_token) + target = f"{schema}.{name}" + if schema == "public": + violations.append( + f"application relation target {target!r} is prohibited in public" + ) + elif permits_system_read and schema in _SYSTEM_READ_SCHEMAS: + return + elif schema not in application_schemas: + violations.append( + f"application relation target {target!r} uses unknown topology schema" + ) + else: + target_locations.append((schema, name)) + + +def _normalize_predicate(expression: str | None) -> str: + if expression is None: + return "" + normalized = sqlparse.format( + expression, + keyword_case="lower", + strip_comments=True, + use_space_around_operators=True, + ).lower() + normalized = re.sub( + r"'app\.tenant_id'\s*::\s*text", + "'app.tenant_id'", + normalized, + ) + return re.sub(r"[\s()\"]+", "", normalized) + + +_NORMALIZED_TENANT_PREDICATE = _normalize_predicate(CANONICAL_TENANT_PREDICATE) + + +def _tenant_predicate(identity_column: str) -> str: + return f"{identity_column} = current_setting('app.tenant_id', true)::uuid" + + +def _relation_label(state: ModelApplicationDatabaseRelationState) -> str: + declaration = state.declaration + return f"{declaration.schema}.{declaration.name}" + + +def _database_for( + topology: ModelDeploymentTopology, + database_ref: str, + *, + label: str, + violations: list[str], +) -> ModelDeploymentTopologyDatabase | None: + database = topology.databases.get(database_ref) + if database is None: + violations.append( + f"{label}: database_ref {database_ref!r} is absent from typed topology" + ) + return database + + +def _schema_owner_for( + state: ModelApplicationDatabaseRelationState, + topology: ModelDeploymentTopology, + violations: list[str], +) -> str | None: + declaration = state.declaration + label = _relation_label(state) + database = _database_for( + topology, + declaration.database_ref, + label=label, + violations=violations, + ) + if database is None: + return None + schema = database.schemas.get(declaration.schema) + if schema is None: + violations.append( + f"{label}: schema is absent from database_ref {declaration.database_ref!r} " + "in typed topology" + ) + return None + if declaration.domain is None: + violations.append(f"{label}: relation lacks one topology schema domain") + elif schema.domain is not declaration.domain: + violations.append( + f"{label}: declared domain {declaration.domain.value!r} differs from " + f"typed topology domain {schema.domain.value!r}" + ) + return schema.owner + + +def _validate_tenant_isolation_evidence( + evidence: ModelApplicationDatabaseTenantIsolationEvidence | None, + *, + label: str, + surface: str, + violations: list[str], +) -> None: + if evidence is None: + violations.append(f"{label}: {surface} requires behavioral evidence") + return + expected = evidence.expected_rows_by_tenant + observed = evidence.observed_rows_by_tenant + if set(expected) != set(observed): + violations.append( + f"{label}: {surface} tenant evidence does not cover the same tenants" + ) + if expected != observed: + violations.append( + f"{label}: {surface} observed tenant results differ from expected results" + ) + if any(count <= 0 for count in expected.values()): + violations.append( + f"{label}: {surface} evidence requires nonzero rows for every tenant" + ) + if len(set(expected.values())) < 2: + violations.append( + f"{label}: {surface} evidence must discriminate tenant result sets" + ) + if evidence.unset_context_rows != 0: + violations.append(f"{label}: {surface} exposes rows with tenant context unset") + if not evidence.malformed_context_denied: + violations.append(f"{label}: {surface} does not deny malformed tenant context") + + +def _validate_function_state( + state: ModelApplicationDatabaseRelationState, + schema_owner: str | None, + violations: list[str], +) -> None: + label = _relation_label(state) + function = state.function_state + if function is None: + violations.append(f"{label}: application function lacks catalog state") + return + if function.public_execute: + violations.append(f"{label}: application function retains PUBLIC EXECUTE") + if state.declaration.domain is EnumDatabaseSchemaDomain.TENANT: + _validate_tenant_isolation_evidence( + function.tenant_isolation_evidence, + label=label, + surface="tenant function", + violations=violations, + ) + if not function.security_definer: + return + if schema_owner is None or function.owner != schema_owner: + violations.append( + f"{label}: SECURITY DEFINER owner must equal typed topology schema owner " + f"{schema_owner!r}; observed {function.owner!r}" + ) + if function.audit_id is None: + violations.append(f"{label}: SECURITY DEFINER requires an audit identifier") + if function.definition_sha256 is None: + violations.append(f"{label}: SECURITY DEFINER audit requires a definition hash") + if function.audited_definition_sha256 is None: + violations.append( + f"{label}: SECURITY DEFINER requires an independently audited definition hash" + ) + elif ( + function.definition_sha256 is not None + and function.definition_sha256 != function.audited_definition_sha256 + ): + violations.append( + f"{label}: live SECURITY DEFINER definition does not match the audited " + "definition hash" + ) + if function.audited_definition_sha256 is not None and ( + function.audit_id is None + or function.audited_definition_sha256 not in function.audit_id + ): + violations.append( + f"{label}: SECURITY DEFINER audit identifier must bind its definition hash" + ) + allowed_paths = { + ("pg_catalog", "pg_temp"), + ("pg_catalog", state.declaration.schema, "pg_temp"), + } + if function.search_path not in allowed_paths: + violations.append( + f"{label}: SECURITY DEFINER requires a fixed safe search_path of " + "pg_catalog optionally followed by its topology schema, with pg_temp last" + ) + + +def _validate_tenant_table( + state: ModelApplicationDatabaseRelationState, + expected_runtime_principals: tuple[str, ...], + violations: list[str], +) -> None: + label = _relation_label(state) + identity_column = state.tenant_identity_column + if identity_column is None: + violations.append( + f"{label}: tenant table requires an explicit tenant_identity_column" + ) + identity_column = "tenant_id" + matching_columns = [ + column for column in state.columns if column.name == identity_column + ] + if len(matching_columns) != 1: + violations.append( + f"{label}: tenant identity column {identity_column!r} must exist exactly once" + ) + else: + column = matching_columns[0] + if column.data_type.strip().lower() != "uuid": + violations.append(f"{label}: tenant identity column must be UUID") + if column.nullable: + violations.append(f"{label}: tenant identity column must be NOT NULL") + if column.default_expression is not None: + violations.append(f"{label}: tenant identity column cannot have a default") + + root_contract = state.identity_root_contract + if identity_column != "tenant_id" and root_contract is None: + violations.append( + f"{label}: non-canonical tenant identity requires an identity-root contract" + ) + if root_contract is not None: + expected_relation = _EXPECTED_IDENTITY_ROOT_RELATIONS[root_contract] + if state.declaration.name != expected_relation: + violations.append( + f"{label}: identity-root contract {root_contract.value!r} is reserved " + f"for relation {expected_relation!r}" + ) + if state.primary_key_columns != (identity_column,): + violations.append( + f"{label}: identity-root column {identity_column!r} must be the exact " + "primary key" + ) + control = state.identity_root_control_state + if control is None: + violations.append( + f"{label}: identity-root exception requires live control-operation " + "evidence" + ) + else: + declared_root_operations = set(control.declared_operations) + observed_root_operations = set(control.observed_operations) + if declared_root_operations != _EXPECTED_IDENTITY_ROOT_OPERATIONS: + violations.append( + f"{label}: identity-root control authority must declare the exact " + "tenant-creation and cross-tenant-enumeration operation set" + ) + if observed_root_operations != declared_root_operations: + violations.append( + f"{label}: identity-root observed control operations differ from " + "the declared operation set" + ) + if len(control.behavioral_proof_ids) != len(control.observed_operations): + violations.append( + f"{label}: identity-root control operations lack behavioral proof" + ) + if control.role_can_login: + violations.append( + f"{label}: identity-root control role must be non-runtime NOLOGIN" + ) + if control.role_superuser: + violations.append( + f"{label}: identity-root control role must not be superuser" + ) + if not control.role_bypass_rls: + violations.append( + f"{label}: FORCE RLS identity-root operations require an audited " + "BYPASSRLS control role, never a widened policy" + ) + if control.runtime_membership_principals: + violations.append( + f"{label}: runtime principals must have no direct or transitive " + "membership path to the identity-root control role; observed " + f"{control.runtime_membership_principals!r}" + ) + if ( + control.runtime_set_role_denied_principals + != expected_runtime_principals + ): + violations.append( + f"{label}: identity-root control authority requires SET ROLE " + "denial for the exact topology runtime-principal set " + f"{expected_runtime_principals!r}; observed " + f"{control.runtime_set_role_denied_principals!r}" + ) + elif state.identity_root_control_state is not None: + violations.append( + f"{label}: identity-root control evidence requires an identity-root contract" + ) + + if not state.rls_enabled: + violations.append(f"{label}: missing ENABLE ROW LEVEL SECURITY") + if not state.rls_forced: + violations.append(f"{label}: missing FORCE ROW LEVEL SECURITY") + + canonical_name = state.canonical_policy_name + if canonical_name is None: + violations.append( + f"{label}: tenant table requires an explicit canonical_policy_name" + ) + canonical_name = "tenant_isolation" + canonical = [policy for policy in state.policies if policy.name == canonical_name] + expected_predicate = ( + _NORMALIZED_TENANT_PREDICATE + if identity_column == "tenant_id" + else _normalize_predicate(_tenant_predicate(identity_column)) + ) + if len(canonical) != 1: + violations.append( + f"{label}: canonical policy {canonical_name!r} must exist exactly once" + ) + else: + policy = canonical[0] + if not policy.permissive or policy.command != "ALL": + violations.append( + f"{label}: canonical policy must be the sole permissive ALL policy" + ) + if policy.roles != _EXPECTED_CANONICAL_POLICY_ROLES: + violations.append( + f"{label}: canonical policy role scope must be exactly " + f"{_EXPECTED_CANONICAL_POLICY_ROLES!r}; observed {policy.roles!r}" + ) + if _normalize_predicate(policy.using_expression) != expected_predicate: + violations.append( + f"{label}: canonical policy USING predicate drift from declared " + f"identity column {identity_column!r}" + ) + if _normalize_predicate(policy.with_check_expression) != expected_predicate: + violations.append( + f"{label}: canonical policy WITH CHECK predicate drift from declared " + f"identity column {identity_column!r}" + ) + + extras = [policy for policy in state.policies if policy.name != canonical_name] + declared = set(state.declared_restrictive_policy_names) + proofs = state.restrictive_policy_proofs + for policy in extras: + if policy.permissive: + violations.append( + f"{label}: extra permissive policy {policy.name!r} widens access" + ) + if policy.name not in declared: + violations.append( + f"{label}: restrictive policy {policy.name!r} is not contract-declared" + ) + if not proofs.get(policy.name): + violations.append( + f"{label}: restrictive policy {policy.name!r} lacks behavioral proof" + ) + missing_policy_rows = declared.difference(policy.name for policy in extras) + if missing_policy_rows: + violations.append( + f"{label}: declared restrictive policies are absent: " + f"{sorted(missing_policy_rows)!r}" + ) + undeclared_proofs = set(proofs).difference(declared) + if undeclared_proofs: + violations.append( + f"{label}: behavioral proofs name undeclared restrictive policies: " + f"{sorted(undeclared_proofs)!r}" + ) + + +def _validate_tenant_state( + state: ModelApplicationDatabaseRelationState, + schema_owner: str | None, + expected_runtime_principals: tuple[str, ...], + violations: list[str], +) -> None: + kind = state.declaration.kind + label = _relation_label(state) + if kind is EnumApplicationRelationKind.TABLE: + _validate_tenant_table(state, expected_runtime_principals, violations) + elif kind is EnumApplicationRelationKind.VIEW: + if state.security_invoker is not True: + violations.append(f"{label}: tenant view must set security_invoker=true") + _validate_tenant_isolation_evidence( + state.view_tenant_isolation_evidence, + label=label, + surface="tenant view", + violations=violations, + ) + elif kind is EnumApplicationRelationKind.MATERIALIZED_VIEW: + violations.append( + f"{label}: tenant materialized view is prohibited; use a tenant table" + ) + elif kind is EnumApplicationRelationKind.FOREIGN_TABLE: + violations.append( + f"{label}: tenant foreign table is prohibited; use a tenant table" + ) + elif kind is EnumApplicationRelationKind.FUNCTION: + _validate_function_state(state, schema_owner, violations) + + +def _validate_non_tenant_state( + state: ModelApplicationDatabaseRelationState, + schema_owner: str | None, + violations: list[str], +) -> None: + label = _relation_label(state) + if any(column.name == "tenant_id" for column in state.columns): + violations.append( + f"{label}: internal/catalog relation cannot declare tenant_id" + ) + if state.rls_enabled or state.rls_forced: + violations.append( + f"{label}: internal/catalog relation cannot enable tenant RLS" + ) + if state.policies: + violations.append( + f"{label}: internal/catalog relation cannot declare tenant policy" + ) + if state.tenant_identity_column is not None: + violations.append( + f"{label}: internal/catalog relation cannot declare tenant identity" + ) + source_columns = [ + column for column in state.columns if column.name == "source_tenant_id" + ] + if source_columns: + source = source_columns[0] + if source.data_type.strip().lower() != "uuid": + violations.append(f"{label}: source_tenant_id provenance must be UUID") + if not source.nullable: + violations.append(f"{label}: source_tenant_id provenance must be nullable") + if source.default_expression is not None: + violations.append( + f"{label}: source_tenant_id provenance cannot have a default" + ) + if state.source_tenant_provenance_contract != "non_authoritative_provenance": + violations.append( + f"{label}: source_tenant_id requires a non-authoritative provenance contract" + ) + if "source_tenant_id" in state.primary_key_columns: + violations.append( + f"{label}: source_tenant_id provenance cannot be authoritative primary-key state" + ) + + generated_dependencies = { + column.name: tuple( + candidate.name + for candidate in state.columns + if column.generated_expression is not None + and re.search( + rf"(? tuple[str, ...]: + expanded: list[str] = [] + pending = list(columns) + while pending: + column_name = pending.pop(0) + if column_name in expanded: + continue + expanded.append(column_name) + pending.extend(generated_dependencies.get(column_name, ())) + return tuple(expanded) + + authoritative_uses: tuple[tuple[str, Sequence[str]], ...] = ( + ( + "uniqueness", + expand_generated_authority( + tuple( + column + for column_set in state.unique_index_column_sets + for column in column_set + ) + ), + ), + ( + "foreign key", + expand_generated_authority( + tuple( + column + for column_set in state.foreign_key_column_sets + for column in column_set + ) + ), + ), + ("partition", expand_generated_authority(state.partition_key_columns)), + ( + "deduplication", + expand_generated_authority(state.deduplication_key_columns), + ), + ( + "authorization", + expand_generated_authority(state.authorization_dependency_columns), + ), + ( + "write eligibility", + expand_generated_authority(state.write_eligibility_dependency_columns), + ), + ) + for authority_kind, columns in authoritative_uses: + if "source_tenant_id" in columns: + violations.append( + f"{label}: source_tenant_id provenance cannot drive " + f"{authority_kind}" + ) + elif state.source_tenant_provenance_contract is not None: + violations.append( + f"{label}: source-tenant provenance contract requires source_tenant_id" + ) + if state.declaration.kind is EnumApplicationRelationKind.FUNCTION: + _validate_function_state(state, schema_owner, violations) + + +def validate_application_database_relation_states( + states: Sequence[ModelApplicationDatabaseRelationState], + topology: ModelDeploymentTopology, +) -> tuple[str, ...]: + """Validate classification, topology placement, RLS, views, and functions.""" + violations: list[str] = [] + if not states: + violations.append("application relation state set cannot be empty") + identities = [state.declaration.identity for state in states] + for identity, count in Counter(identities).items(): + if count != 1: + violations.append( + f"application relation {identity!r} has {count} catalog states; expected 1" + ) + + for state in states: + declaration = state.declaration + label = _relation_label(state) + if declaration.owner_declaration is None: + violations.append(f"{label}: relation lacks exactly one owner declaration") + if declaration.schema == "public": + violations.append( + f"{label}: application relations are prohibited in public" + ) + schema_owner = _schema_owner_for(state, topology, violations) + domain = declaration.domain + if domain is None: + continue + if domain is EnumDatabaseSchemaDomain.TENANT: + database = topology.databases.get(declaration.database_ref) + expected_runtime_principals = ( + () + if database is None + else tuple( + sorted(binding.principal for binding in database.bindings.values()) + ) + ) + _validate_tenant_state( + state, + schema_owner, + expected_runtime_principals, + violations, + ) + else: + _validate_non_tenant_state(state, schema_owner, violations) + return tuple(sorted(set(violations))) + + +def validate_application_database_catalog_census( + states: Sequence[ModelApplicationDatabaseRelationState], + observed: Sequence[ModelApplicationDatabaseCatalogIdentity], + topology: ModelDeploymentTopology, + *, + authoritative_identities: Sequence[ModelApplicationDatabaseCatalogIdentity] + | None = None, +) -> tuple[str, ...]: + """Require an exact contract-to-catalog census, including absence of extras.""" + violations = list(validate_application_database_relation_states(states, topology)) + if not observed: + violations.append("application catalog census cannot be empty") + expected_identities = ( + [identity.identity for identity in authoritative_identities] + if authoritative_identities is not None + else [ + ( + state.declaration.schema, + state.declaration.name, + EnumApplicationInventoryObjectKind(state.declaration.kind.value), + state.declaration.function_signature, + ) + for state in states + ] + ) + if not expected_identities: + violations.append("authoritative application catalog cannot be empty") + observed_identities = [identity.identity for identity in observed] + for identity, count in Counter(observed_identities).items(): + if count != 1: + violations.append( + f"application catalog identity {identity!r} appears {count} times" + ) + missing = set(expected_identities).difference(observed_identities) + extra = set(observed_identities).difference(expected_identities) + + def identity_sort_key( + identity: tuple[ + str, + str, + EnumApplicationInventoryObjectKind, + str | None, + ], + ) -> tuple[str, ...]: + return tuple("" if item is None else str(item) for item in identity) + + if missing: + violations.append( + "application catalog census is missing declared objects: " + f"{sorted(missing, key=identity_sort_key)!r}" + ) + if extra: + violations.append( + "application catalog census contains undeclared objects: " + f"{sorted(extra, key=identity_sort_key)!r}" + ) + return tuple(sorted(set(violations))) + + +def application_database_created_catalog_identities( + sql: str, +) -> tuple[ModelApplicationDatabaseCatalogIdentity, ...]: + """Project application objects created by SQL into typed catalog identities.""" + if _postgresql_lexical_violations(sql): + return () + identities: list[ModelApplicationDatabaseCatalogIdentity] = [] + cleaned = sqlparse.format(sql, strip_comments=True) + for statement in sqlparse.split(cleaned): + match = _CREATED_APPLICATION_OBJECT.match(statement) + if match is None: + select_statement = statement + leading_ctes = _leading_ctes(statement) + if leading_ctes is not None: + _, _, select_statement = leading_ctes + select_into = _select_into_target_match(select_statement) + if select_into is None or select_into.group("schema") is None: + continue + identities.append( + ModelApplicationDatabaseCatalogIdentity( + schema=_unquote_identifier(select_into.group("schema")), + name=_unquote_identifier(select_into.group("name")), + kind=EnumApplicationInventoryObjectKind.TABLE, + ) + ) + continue + kind_token = " ".join(match.group("kind").lower().split()) + schema_token = match.group("schema") + if kind_token == "extension": + schema_match = re.search( + rf"\bschema\s+(?P{_SQL_IDENTIFIER})", + statement[match.end() :], + re.IGNORECASE, + ) + if schema_match is None: + continue + schema_token = schema_match.group("schema") + if schema_token is None: + continue + + schema = _unquote_identifier(schema_token) + name = _unquote_identifier(match.group("name")) + kind = _CREATED_KIND_MAP[kind_token] + function_signature: str | None = None + remainder = statement[match.end() :] + remainder_offset = _skip_whitespace(remainder, 0) + if kind in { + EnumApplicationInventoryObjectKind.FUNCTION, + EnumApplicationInventoryObjectKind.PROCEDURE, + EnumApplicationInventoryObjectKind.AGGREGATE, + }: + signature = _balanced_parenthesized(remainder, remainder_offset) + if signature is not None: + signature_body, _ = signature + function_signature = f"({signature_body.strip()})" + if ( + kind is EnumApplicationInventoryObjectKind.FUNCTION + and _contains_unquoted_keyword(remainder, "window") + ): + kind = EnumApplicationInventoryObjectKind.WINDOW_FUNCTION + elif kind is EnumApplicationInventoryObjectKind.TYPE: + if remainder[remainder_offset:].startswith("("): + kind = EnumApplicationInventoryObjectKind.BASE_TYPE + elif re.match( + r"as\s+range\b", + remainder[remainder_offset:], + re.IGNORECASE, + ): + kind = EnumApplicationInventoryObjectKind.RANGE_TYPE + + identities.append( + ModelApplicationDatabaseCatalogIdentity( + schema=schema, + name=name, + kind=kind, + function_signature=function_signature, + ) + ) + if kind is EnumApplicationInventoryObjectKind.RANGE_TYPE: + multirange_match = re.search( + rf"\bmultirange_type_name\s*=\s*" + rf"(?:(?P{_SQL_IDENTIFIER})\s*\.\s*)?" + rf"(?P{_SQL_IDENTIFIER})", + remainder, + re.IGNORECASE, + ) + if multirange_match is not None: + multirange_schema = multirange_match.group("schema") + identities.append( + ModelApplicationDatabaseCatalogIdentity( + schema=( + schema + if multirange_schema is None + else _unquote_identifier(multirange_schema) + ), + name=_unquote_identifier(multirange_match.group("name")), + kind=EnumApplicationInventoryObjectKind.MULTIRANGE_TYPE, + ) + ) + return tuple(identities) + + +def load_application_database_ownership_identities( + manifest_paths: Sequence[Path], +) -> tuple[ModelApplicationDatabaseCatalogIdentity, ...]: + """Load one exact owner-backed catalog identity from every manifest row.""" + declarations: dict[ + tuple[ + str, + str, + EnumApplicationInventoryObjectKind, + str | None, + ], + list[tuple[str, str]], + ] = {} + + def record( + *, + schema: str, + name: str, + kind: EnumApplicationInventoryObjectKind, + function_signature: str | None, + owner: str, + source_path: Path, + ) -> None: + identity = (schema, name, kind, function_signature) + declarations.setdefault(identity, []).append((owner, str(source_path))) + + routine_kinds = { + EnumApplicationInventoryObjectKind.FUNCTION, + EnumApplicationInventoryObjectKind.AGGREGATE, + EnumApplicationInventoryObjectKind.WINDOW_FUNCTION, + EnumApplicationInventoryObjectKind.PROCEDURE, + } + for path in sorted(manifest_paths): + manifest = load_service_ownership_manifest(path) + authority = manifest.owner_declaration or f"service:{manifest.service}" + table_identities = { + (table.database_ref, table.schema, table.name) + for table in manifest.db_io.db_tables + } + for table in manifest.db_io.db_tables: + record( + schema=table.schema, + name=table.name, + kind=EnumApplicationInventoryObjectKind.TABLE, + function_signature=None, + owner=authority, + source_path=path, + ) + for relation in manifest.relation_evidence: + if relation.database_ref is None or relation.schema is None: + continue + if relation.owner_declaration != authority: + raise ValueError( + f"{path}: catalog relation {relation.name!r} owner " + f"{relation.owner_declaration!r} conflicts with manifest " + f"authority {authority!r}" + ) + if relation.kind is EnumApplicationRelationKind.TABLE: + typed_identity = ( + relation.database_ref, + relation.schema, + relation.name, + ) + if typed_identity not in table_identities: + raise ValueError( + f"{path}: table evidence {relation.name!r} lacks one typed " + "db_io ownership declaration" + ) + continue + kind = EnumApplicationInventoryObjectKind(relation.kind.value) + if kind in routine_kinds and relation.function_signature is None: + raise ValueError( + f"{path}: routine relation {relation.name!r} requires an exact " + "function_signature" + ) + record( + schema=relation.schema, + name=relation.name, + kind=kind, + function_signature=relation.function_signature, + owner=relation.owner_declaration, + source_path=path, + ) + for database_object in manifest.database_objects: + if database_object.database_ref is None or database_object.schema is None: + continue + if database_object.owner_declaration != authority: + raise ValueError( + f"{path}: catalog object {database_object.name!r} owner " + f"{database_object.owner_declaration!r} conflicts with manifest " + f"authority {authority!r}" + ) + kind = EnumApplicationInventoryObjectKind(database_object.kind.value) + if kind in routine_kinds and database_object.function_signature is None: + raise ValueError( + f"{path}: routine object {database_object.name!r} requires an " + "exact function_signature" + ) + record( + schema=database_object.schema, + name=database_object.name, + kind=kind, + function_signature=database_object.function_signature, + owner=database_object.owner_declaration, + source_path=path, + ) + + duplicate_declarations = { + identity: owners + for identity, owners in declarations.items() + if len(owners) != 1 + } + if duplicate_declarations: + raise ValueError( + "application catalog identities require exactly one ownership " + f"declaration: {duplicate_declarations!r}" + ) + return tuple( + ModelApplicationDatabaseCatalogIdentity( + schema=schema, + name=name, + kind=kind, + function_signature=function_signature, + ) + for schema, name, kind, function_signature in sorted( + declarations, + key=lambda identity: tuple( + "" if item is None else str(item) for item in identity + ), + ) + ) + + +def _analyze_sql_fragment( + fragment: str, + *, + visible_ctes: set[str], + application_schemas: set[str], + violations: list[str], + target_locations: list[tuple[str, str]], +) -> None: + """Analyze one SQL scope, recursively checking each data-modifying CTE body.""" + lexical_violations = _postgresql_lexical_violations(fragment) + if lexical_violations: + violations.extend(lexical_violations) + return + if _requires_dynamic_sql_rejection(fragment): + violations.append( + "procedural block contains dynamic SQL whose relation targets cannot " + "be proven statically" + ) + return + if _routine_body_is_uninspectable(fragment): + violations.append( + "SQL or PL/pgSQL routine body cannot be proven statically; use a " + "standard single-quoted, dollar-quoted, RETURN, or BEGIN ATOMIC body" + ) + return + + for _language, body in _routine_sql_bodies(fragment): + _analyze_sql_fragment( + body, + visible_ctes=set(visible_ctes), + application_schemas=application_schemas, + violations=violations, + target_locations=target_locations, + ) + + masked_fragment = _mask_sql_string_bodies(fragment) + target_fragment = _without_leading_maintenance_options(fragment) + masked_target_fragment = _mask_sql_string_bodies(target_fragment) + if any( + pattern.fullmatch(masked_target_fragment) + for pattern in _NON_EXACT_MAINTENANCE_OPERATIONS + ): + violations.append( + "application maintenance operation must target an exact " + "schema-qualified table target" + ) + extension = re.match( + rf"^\s*create\s+extension\s+(?:if\s+not\s+exists\s+)?" + rf"(?P{_SQL_IDENTIFIER})", + masked_fragment, + re.IGNORECASE, + ) + if extension is not None: + extension_schema = re.search( + rf"\bwith\s+schema\s+(?P{_SQL_IDENTIFIER})", + masked_fragment[extension.end() :], + re.IGNORECASE, + ) + if extension_schema is None: + violations.append( + "CREATE EXTENSION must be schema-qualified with an explicit topology " + "WITH SCHEMA target" + ) + else: + schema = _unquote_identifier(extension_schema.group("schema")) + if schema == "public": + violations.append( + "application extension target schema 'public' is prohibited" + ) + elif schema not in application_schemas: + violations.append( + f"application extension target schema {schema!r} uses unknown " + "topology schema" + ) + if ( + re.match( + rf"^\s*create\s+type\s+" + rf"(?:(?:{_SQL_IDENTIFIER})\s*\.\s*)?{_SQL_IDENTIFIER}\s+" + r"as\s+range\b", + masked_fragment, + re.IGNORECASE, + ) + is not None + and re.search(r"\bmultirange_type_name\s*=", masked_fragment, re.IGNORECASE) + is None + ): + violations.append( + "CREATE TYPE AS RANGE requires an explicit MULTIRANGE_TYPE_NAME so " + "the generated catalog identity is authoritative" + ) + + leading_ctes = _leading_ctes(fragment) + if leading_ctes is not None: + recursive, definitions, tail = leading_ctes + definition_names = {name for name, _ in definitions} + prior_names = set(visible_ctes) + for name, body in definitions: + body_names = ( + prior_names.union(definition_names) if recursive else set(prior_names) + ) + _analyze_sql_fragment( + body, + visible_ctes=body_names, + application_schemas=application_schemas, + violations=violations, + target_locations=target_locations, + ) + prior_names.add(name) + _analyze_sql_fragment( + tail, + visible_ctes=visible_ctes.union(definition_names), + application_schemas=application_schemas, + violations=violations, + target_locations=target_locations, + ) + return + + view_offset = _view_query_offset(fragment) + if view_offset is not None: + view_body_ctes = _leading_ctes(fragment[view_offset:]) + if view_body_ctes is not None: + recursive, definitions, tail = view_body_ctes + definition_names = {name for name, _ in definitions} + prior_names = set(visible_ctes) + for name, body in definitions: + body_names = ( + prior_names.union(definition_names) + if recursive + else set(prior_names) + ) + _analyze_sql_fragment( + body, + visible_ctes=body_names, + application_schemas=application_schemas, + violations=violations, + target_locations=target_locations, + ) + prior_names.add(name) + # Re-join the view head to the post-WITH tail so the view's own name is + # still validated as a real relation target. Only the CTE definitions are + # elided here; they were each analyzed above under their own scope. + _analyze_sql_fragment( + f"{fragment[:view_offset]} {tail}", + visible_ctes=visible_ctes.union(definition_names), + application_schemas=application_schemas, + violations=violations, + target_locations=target_locations, + ) + return + + explained = _explained_statement(target_fragment) + main_dml_tail = _main_dml_tail(explained) + variants: tuple[str, ...] = (target_fragment,) + if main_dml_tail != target_fragment: + variants = (target_fragment, main_dml_tail) + is_privilege_statement = ( + re.match( + r"^\s*(?:grant|revoke)\b", + masked_target_fragment, + re.IGNORECASE, + ) + is not None + ) + for variant in variants: + masked_variant = _mask_sql_string_bodies(variant) + for pattern, permits_ephemeral, permits_system_read in _RELATION_TARGETS: + if permits_ephemeral and is_privilege_statement: + # GRANT/REVOKE FROM names principals, not relation read targets. + continue + for match in pattern.finditer(masked_variant): + _record_sql_target( + schema_token=match.group("schema"), + name_token=match.group("name"), + remaining=masked_variant[match.end() :], + permits_ephemeral=permits_ephemeral, + permits_system_read=permits_system_read, + cte_names=visible_ctes, + application_schemas=application_schemas, + violations=violations, + target_locations=target_locations, + ) + select_into = _select_into_target_match(variant) + if select_into is not None: + _record_sql_target( + schema_token=select_into.group("schema"), + name_token=select_into.group("name"), + remaining="", + permits_ephemeral=False, + permits_system_read=False, + cte_names=visible_ctes, + application_schemas=application_schemas, + violations=violations, + target_locations=target_locations, + ) + + for pattern, permits_ephemeral, permits_system_read in _RELATION_LISTS: + if permits_ephemeral and is_privilege_statement: + # REVOKE ... FROM names principals, not relation read targets. + continue + for list_match in pattern.finditer(masked_target_fragment): + targets = _split_top_level_commas(list_match.group("targets")) + for target in targets: + target_match = _LIST_TARGET.match(target) + if target_match is None: + continue + _record_sql_target( + schema_token=target_match.group("schema"), + name_token=target_match.group("name"), + remaining=target[target_match.end() :], + permits_ephemeral=permits_ephemeral, + permits_system_read=permits_system_read, + cte_names=visible_ctes, + application_schemas=application_schemas, + violations=violations, + target_locations=target_locations, + ) + + +def _analyze_application_database_sql( + sql: str, + topology: ModelDeploymentTopology, +) -> tuple[tuple[str, ...], tuple[tuple[str, str], ...]]: + """Return qualification violations and all qualified application targets.""" + violations: list[str] = [] + target_locations: list[tuple[str, str]] = [] + application_schemas = { + schema_name + for database in topology.databases.values() + for schema_name in database.schemas + } + lexical_violations = _postgresql_lexical_violations(sql) + if lexical_violations: + return lexical_violations, () + cleaned = sqlparse.format(sql, strip_comments=True) + for statement in sqlparse.split(cleaned): + _analyze_sql_fragment( + statement, + visible_ctes=set(), + application_schemas=application_schemas, + violations=violations, + target_locations=target_locations, + ) + return tuple(sorted(set(violations))), tuple(sorted(set(target_locations))) + + +def lint_application_database_sql( + sql: str, + topology: ModelDeploymentTopology, +) -> tuple[str, ...]: + """Reject public, unknown, or unqualified relation targets in changed SQL.""" + violations, _ = _analyze_application_database_sql(sql, topology) + return violations + + +def application_database_sql_target_locations( + sql: str, + topology: ModelDeploymentTopology, +) -> tuple[tuple[str, str], ...]: + """Return every qualified topology-application target referenced by SQL.""" + _, target_locations = _analyze_application_database_sql(sql, topology) + return target_locations + + +_EXPLICIT_TARGET_KIND_MAP: dict[str, frozenset[EnumApplicationInventoryObjectKind]] = { + "foreign table": frozenset({EnumApplicationInventoryObjectKind.FOREIGN_TABLE}), + "table": frozenset({EnumApplicationInventoryObjectKind.TABLE}), + "view": frozenset({EnumApplicationInventoryObjectKind.VIEW}), + "materialized view": frozenset( + {EnumApplicationInventoryObjectKind.MATERIALIZED_VIEW} + ), + "function": frozenset( + { + EnumApplicationInventoryObjectKind.FUNCTION, + EnumApplicationInventoryObjectKind.WINDOW_FUNCTION, + } + ), + "procedure": frozenset({EnumApplicationInventoryObjectKind.PROCEDURE}), + "aggregate": frozenset({EnumApplicationInventoryObjectKind.AGGREGATE}), + "sequence": frozenset({EnumApplicationInventoryObjectKind.SEQUENCE}), + "type": frozenset( + { + EnumApplicationInventoryObjectKind.TYPE, + EnumApplicationInventoryObjectKind.BASE_TYPE, + EnumApplicationInventoryObjectKind.RANGE_TYPE, + EnumApplicationInventoryObjectKind.MULTIRANGE_TYPE, + } + ), + "domain": frozenset({EnumApplicationInventoryObjectKind.TYPE}), + "column": _RELATION_OBJECT_KINDS, +} +_EXPLICIT_OBJECT_OPERATION = re.compile( + r"^\s*(?Palter|drop)\s+" + r"(?Pforeign\s+table|materialized\s+view|table|view|function|" + r"procedure|aggregate|sequence|type|domain)\s+" + r"(?:if\s+exists\s+)?(?:only\s+)?(?P[\s\S]*)$", + re.IGNORECASE, +) +_QUALIFIED_TARGET = re.compile( + rf"^\s*(?P{_SQL_IDENTIFIER})\s*\.\s*" + rf"(?P{_SQL_IDENTIFIER})", + re.IGNORECASE, +) +_COMMENT_OBJECT_OPERATION = re.compile( + r"^\s*comment\s+on\s+" + r"(?Pforeign\s+table|materialized\s+view|table|view|column|function|" + r"procedure|aggregate|sequence|type|domain)\s+" + r"(?P[\s\S]*?)\s+is\b", + re.IGNORECASE, +) + + +def _target_requirement( + target: str, + *, + allowed_kinds: frozenset[EnumApplicationInventoryObjectKind], + routine_identity: bool, + application_schemas: set[str], +) -> ApplicationDatabaseSqlTargetRequirement | None: + """Parse one already qualification-checked target into exact authority shape.""" + match = _QUALIFIED_TARGET.match(target) + if match is None: + return None + schema = _unquote_identifier(match.group("schema")) + if schema not in application_schemas: + return None + name = _unquote_identifier(match.group("name")) + signature: str | None = None + if routine_identity: + offset = _skip_whitespace(target, match.end()) + parenthesized = _balanced_parenthesized(target, offset) + if parenthesized is not None: + signature_body, _ = parenthesized + signature = f"({signature_body.strip()})" + return ApplicationDatabaseSqlTargetRequirement( + schema=schema, + name=name, + allowed_kinds=allowed_kinds, + function_signature=signature, + ) + + +def application_database_sql_target_requirements( + sql: str, + topology: ModelDeploymentTopology, +) -> tuple[ApplicationDatabaseSqlTargetRequirement, ...]: + """Return kind- and overload-aware ownership requirements for SQL targets.""" + if _postgresql_lexical_violations(sql): + return () + application_schemas = { + schema_name + for database in topology.databases.values() + for schema_name in database.schemas + } + all_locations = set(application_database_sql_target_locations(sql, topology)) + created_locations = { + (identity.schema, identity.name) + for identity in application_database_created_catalog_identities(sql) + } + requirements: set[ApplicationDatabaseSqlTargetRequirement] = set() + explicitly_typed_locations: set[tuple[str, str]] = set() + cleaned = sqlparse.format(sql, strip_comments=True) + for raw_statement in sqlparse.split(cleaned): + statement = _mask_sql_string_bodies(raw_statement).rstrip("; ") + explicit = _EXPLICIT_OBJECT_OPERATION.match(statement) + if explicit is not None: + kind_token = " ".join(explicit.group("kind").lower().split()) + allowed_kinds = _EXPLICIT_TARGET_KIND_MAP[kind_token] + target_text = re.split( + r"\s+(?:cascade|restrict)\b", + explicit.group("targets"), + maxsplit=1, + flags=re.IGNORECASE, + )[0] + targets = ( + _split_top_level_commas(target_text) + if explicit.group("operation").lower() == "drop" + else (target_text,) + ) + routine_identity = bool(allowed_kinds.intersection(_ROUTINE_OBJECT_KINDS)) + for target in targets: + requirement = _target_requirement( + target, + allowed_kinds=allowed_kinds, + routine_identity=routine_identity, + application_schemas=application_schemas, + ) + if requirement is not None: + requirements.add(requirement) + explicitly_typed_locations.add(requirement.location) + + comment = _COMMENT_OBJECT_OPERATION.match(statement) + if comment is not None: + kind_token = " ".join(comment.group("kind").lower().split()) + allowed_kinds = _EXPLICIT_TARGET_KIND_MAP[kind_token] + requirement = _target_requirement( + comment.group("target"), + allowed_kinds=allowed_kinds, + routine_identity=bool( + allowed_kinds.intersection(_ROUTINE_OBJECT_KINDS) + ), + application_schemas=application_schemas, + ) + if requirement is not None: + requirements.add(requirement) + explicitly_typed_locations.add(requirement.location) + + for call in re.finditer( + rf"\bcall\s+(?P{_SQL_IDENTIFIER}\s*\.\s*{_SQL_IDENTIFIER})", + statement, + re.IGNORECASE, + ): + requirement = _target_requirement( + statement[call.start("target") :], + allowed_kinds=frozenset({EnumApplicationInventoryObjectKind.PROCEDURE}), + routine_identity=True, + application_schemas=application_schemas, + ) + if requirement is not None: + requirements.add(requirement) + explicitly_typed_locations.add(requirement.location) + + for execution in re.finditer( + rf"\bexecute\s+(?Pfunction|procedure)\s+" + rf"(?P{_SQL_IDENTIFIER}\s*\.\s*{_SQL_IDENTIFIER})", + statement, + re.IGNORECASE, + ): + execution_kind = execution.group("kind").lower() + allowed_kinds = ( + frozenset( + { + EnumApplicationInventoryObjectKind.FUNCTION, + EnumApplicationInventoryObjectKind.WINDOW_FUNCTION, + } + ) + if execution_kind == "function" + else frozenset({EnumApplicationInventoryObjectKind.PROCEDURE}) + ) + requirement = _target_requirement( + statement[execution.start("target") :], + allowed_kinds=allowed_kinds, + routine_identity=False, + application_schemas=application_schemas, + ) + if requirement is not None: + # PostgreSQL trigger functions have catalog identity ``()`` even + # when CREATE TRIGGER supplies TG_ARGV values. + if execution_kind == "function": + requirement = ApplicationDatabaseSqlTargetRequirement( + schema=requirement.schema, + name=requirement.name, + allowed_kinds=requirement.allowed_kinds, + function_signature="()", + ) + requirements.add(requirement) + explicitly_typed_locations.add(requirement.location) + + privilege = re.match( + r"^\s*(?:grant|revoke)\s+[\s\S]*?\s+on\s+" + r"(?:(?Pforeign\s+table|materialized\s+view|table|view|function|" + r"procedure|sequence|type|domain)\s+)?" + r"(?P[\s\S]*?)\s+(?:to|from)\b", + statement, + re.IGNORECASE, + ) + if privilege is not None: + kind_token = privilege.group("kind") + allowed_kinds = ( + _RELATION_OBJECT_KINDS + if kind_token is None + else _EXPLICIT_TARGET_KIND_MAP[" ".join(kind_token.lower().split())] + ) + routine_identity = bool(allowed_kinds.intersection(_ROUTINE_OBJECT_KINDS)) + for target in _split_top_level_commas(privilege.group("targets")): + requirement = _target_requirement( + target, + allowed_kinds=allowed_kinds, + routine_identity=routine_identity, + application_schemas=application_schemas, + ) + if requirement is not None: + requirements.add(requirement) + explicitly_typed_locations.add(requirement.location) + + for schema, name in all_locations.difference( + created_locations.union(explicitly_typed_locations) + ): + requirements.add( + ApplicationDatabaseSqlTargetRequirement( + schema=schema, + name=name, + allowed_kinds=_RELATION_OBJECT_KINDS, + ) + ) + return tuple( + sorted( + requirements, + key=lambda requirement: ( + requirement.schema, + requirement.name, + tuple(sorted(kind.value for kind in requirement.allowed_kinds)), + requirement.function_signature or "", + ), + ) + ) + + +def validate_application_database_pool_identities( + identities: Sequence[ModelApplicationDatabasePoolIdentity], + topology: ModelDeploymentTopology, + *, + database_ref: str = "application", +) -> tuple[str, ...]: + """Prove one topology database with exact, distinct workload users.""" + violations: list[str] = [] + database = _database_for( + topology, + database_ref, + label="application pool evidence", + violations=violations, + ) + if database is None: + return tuple(sorted(set(violations))) + expected_pool_users = { + binding_name: binding.principal + for binding_name, binding in database.bindings.items() + } + by_pool = {identity.pool: identity for identity in identities} + if len(by_pool) != len(identities): + violations.append("application pool evidence contains duplicate pool names") + if set(by_pool) != set(expected_pool_users): + violations.append( + "application pool evidence must cover the exact typed topology binding set" + ) + databases = {identity.current_database for identity in identities} + if databases != {database.physical_name}: + violations.append( + "application pools must resolve to one physical database " + f"{database.physical_name!r}; observed {sorted(databases)!r}" + ) + for pool, expected_user in expected_pool_users.items(): + identity = by_pool.get(pool) + if identity is not None and identity.current_user != expected_user: + violations.append( + f"pool {pool!r} expected current_user {expected_user!r}, got " + f"{identity.current_user!r}" + ) + users = [identity.current_user for identity in identities] + if len(set(users)) != len(users): + violations.append("application pool current_user values must be distinct") + return tuple(sorted(set(violations))) + + +__all__ = [ + "ApplicationDatabaseSqlTargetRequirement", + "CANONICAL_TENANT_PREDICATE", + "application_database_created_catalog_identities", + "application_database_function_definition_sha256", + "application_database_sql_target_locations", + "application_database_sql_target_requirements", + "lint_application_database_sql", + "load_application_database_ownership_identities", + "validate_application_database_catalog_census", + "validate_application_database_pool_identities", + "validate_application_database_relation_states", +] diff --git a/src/omnibase_infra/validation/application_database_red_control_registry.py b/src/omnibase_infra/validation/application_database_red_control_registry.py new file mode 100644 index 0000000000..711999bf93 --- /dev/null +++ b/src/omnibase_infra/validation/application_database_red_control_registry.py @@ -0,0 +1,357 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Executable RED-control bindings for application database enforcement.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType + +from omnibase_infra.validation.enums.enum_application_database_enforcement_gate import ( + EnumApplicationDatabaseEnforcementGate, +) + +_CONTROL_ID = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + + +@dataclass(frozen=True, slots=True) +class ApplicationDatabaseRedControlBinding: + """One control's exact executable source proof and optional Docker result.""" + + control_id: str + pytest_node_id: str + docker_result: str | None = None + + def __post_init__(self) -> None: + """Reject broad or semantically unrelated pytest bindings.""" + if _CONTROL_ID.fullmatch(self.control_id) is None: + raise ValueError(f"invalid RED control id: {self.control_id!r}") + path, separator, test_case = self.pytest_node_id.partition("::") + if ( + not separator + or not path.startswith("tests/") + or not path.endswith(".py") + or not test_case.startswith("test_") + or "::" in test_case + ): + raise ValueError( + f"invalid exact pytest node id for {self.control_id!r}: " + f"{self.pytest_node_id!r}" + ) + normalized = self.control_id.replace("-", "_") + dedicated_test = f"test_red_control_{normalized}" + exact_case = f"[{self.control_id}]" + if test_case != dedicated_test and not test_case.endswith(exact_case): + raise ValueError( + f"RED control {self.control_id!r} must bind to an exact semantic " + f"pytest case named {dedicated_test!r} or ending in {exact_case!r}" + ) + if self.docker_result is not None: + expected = f"domain_control={self.control_id} status=PASS" + if self.docker_result != expected: + raise ValueError( + f"RED control {self.control_id!r} Docker result must be " + f"{expected!r}" + ) + + +ApplicationDatabaseRedControlRegistry = Mapping[ + EnumApplicationDatabaseEnforcementGate, + Mapping[str, ApplicationDatabaseRedControlBinding], +] + +_OWNERSHIP = "tests/unit/validation/test_application_relation_ownership.py" +_DOMAIN = "tests/unit/validation/test_application_database_domain_enforcement.py" +_ACL = "tests/unit/validation/test_application_database_acl.py" +_TOPOLOGY = "tests/unit/topology/test_application_database_topology.py" +_ADAPTER = "tests/unit/runtime/auto_wiring/test_projection_domain_adapters.py" +_SQL_GATE = "tests/ci/test_application_database_sql_gate.py" + + +def _case( + path: str, + test: str, + control_id: str, + *, + docker: bool = False, +) -> ApplicationDatabaseRedControlBinding: + return ApplicationDatabaseRedControlBinding( + control_id=control_id, + pytest_node_id=f"{path}::{test}[{control_id}]", + docker_result=(f"domain_control={control_id} status=PASS" if docker else None), + ) + + +def _control( + path: str, + control_id: str, + *, + docker: bool = False, +) -> ApplicationDatabaseRedControlBinding: + return ApplicationDatabaseRedControlBinding( + control_id=control_id, + pytest_node_id=(f"{path}::test_red_control_{control_id.replace('-', '_')}"), + docker_result=(f"domain_control={control_id} status=PASS" if docker else None), + ) + + +def _freeze_registry( + registry: dict[ + EnumApplicationDatabaseEnforcementGate, + dict[str, ApplicationDatabaseRedControlBinding], + ], +) -> ApplicationDatabaseRedControlRegistry: + """Freeze the reviewable control-to-result registry at both mapping levels.""" + bindings = [binding for proofs in registry.values() for binding in proofs.values()] + mismatched_keys = sorted( + control_id + for proofs in registry.values() + for control_id, binding in proofs.items() + if binding.control_id != control_id + ) + if mismatched_keys: + raise ValueError(f"RED control registry key drift: {mismatched_keys}") + node_ids = [binding.pytest_node_id for binding in bindings] + if len(set(node_ids)) != len(node_ids): + raise ValueError("RED controls must bind to unique exact pytest node ids") + return MappingProxyType( + {gate: MappingProxyType(dict(proofs)) for gate, proofs in registry.items()} + ) + + +APPLICATION_DATABASE_RED_CONTROL_REGISTRY = _freeze_registry( + { + EnumApplicationDatabaseEnforcementGate.CLASSIFICATION: { + "missing-owner": _case( + _OWNERSHIP, "test_seeded_ownership_defects_fail_closed", "missing-owner" + ), + "duplicate-owner": _case( + _OWNERSHIP, + "test_seeded_ownership_defects_fail_closed", + "duplicate-owner", + ), + "conflicting-location": _case( + _OWNERSHIP, + "test_seeded_ownership_defects_fail_closed", + "conflicting-location", + ), + "incomplete-retained-census": _control( + _OWNERSHIP, "incomplete-retained-census" + ), + "empty-authoritative-relation-set": _control( + _DOMAIN, "empty-authoritative-relation-set", docker=True + ), + "public-catalog-leak": _control( + _DOMAIN, "public-catalog-leak", docker=True + ), + }, + EnumApplicationDatabaseEnforcementGate.SCHEMA_QUALIFICATION: { + "public-application-table": _control(_DOMAIN, "public-application-table"), + "unqualified-application-table": _control( + _DOMAIN, "unqualified-application-table" + ), + "unqualified-application-mutation-target": _control( + _DOMAIN, "unqualified-application-mutation-target" + ), + "unknown-topology-schema": _control(_DOMAIN, "unknown-topology-schema"), + "wrong-object-kind": _control(_SQL_GATE, "wrong-object-kind"), + "wrong-routine-overload": _control(_SQL_GATE, "wrong-routine-overload"), + "dynamic-sql-target": _case( + _DOMAIN, + "test_valid_postgres_alternate_target_forms_fail_closed", + "dynamic-sql-target", + ), + "implicit-multirange-identity": _case( + _DOMAIN, + "test_valid_postgres_alternate_target_forms_fail_closed", + "implicit-multirange-identity", + ), + }, + EnumApplicationDatabaseEnforcementGate.TENANT_RLS: { + "tenant-text-key": _case( + _DOMAIN, + "test_seeded_tenant_shape_defects_fail_closed", + "tenant-text-key", + docker=True, + ), + "tenant-nullable": _case( + _DOMAIN, + "test_seeded_tenant_shape_defects_fail_closed", + "tenant-nullable", + docker=True, + ), + "tenant-default": _case( + _DOMAIN, + "test_seeded_tenant_shape_defects_fail_closed", + "tenant-default", + docker=True, + ), + "missing-enable-rls": _case( + _DOMAIN, + "test_seeded_tenant_shape_defects_fail_closed", + "missing-enable-rls", + docker=True, + ), + "missing-force-rls": _case( + _DOMAIN, + "test_seeded_tenant_shape_defects_fail_closed", + "missing-force-rls", + docker=True, + ), + "using-drift": _case( + _DOMAIN, + "test_seeded_policy_predicate_drift_fails_closed", + "using-drift", + docker=True, + ), + "with-check-drift": _case( + _DOMAIN, + "test_seeded_policy_predicate_drift_fails_closed", + "with-check-drift", + docker=True, + ), + "canonical-policy-unrelated-role": _control( + _DOMAIN, "canonical-policy-unrelated-role", docker=True + ), + "uncontracted-identity-root": _control( + _DOMAIN, "uncontracted-identity-root", docker=True + ), + "identity-root-runtime-login": _control( + _DOMAIN, "identity-root-runtime-login", docker=True + ), + "identity-root-unproven-enumeration": _control( + _DOMAIN, "identity-root-unproven-enumeration", docker=True + ), + "identity-root-runtime-membership": _control( + _DOMAIN, "identity-root-runtime-membership", docker=True + ), + "identity-root-runtime-set-role": _control( + _DOMAIN, "identity-root-runtime-set-role", docker=True + ), + "widening-permissive-policy": _control( + _DOMAIN, "widening-permissive-policy" + ), + "owner-security-view": _control( + _DOMAIN, "owner-security-view", docker=True + ), + "unsafe-security-definer": _control( + _DOMAIN, "unsafe-security-definer", docker=True + ), + "unproven-security-view": _control( + _DOMAIN, "unproven-security-view", docker=True + ), + "unproven-security-definer": _control( + _DOMAIN, "unproven-security-definer", docker=True + ), + "security-definer-volatility-drift": _control( + _DOMAIN, "security-definer-volatility-drift", docker=True + ), + }, + EnumApplicationDatabaseEnforcementGate.INTERNAL_CATALOG: { + "internal-tenant-id": _case( + _DOMAIN, + "test_non_tenant_domain_red_control_fails_closed", + "internal-tenant-id", + docker=True, + ), + "catalog-tenant-id": _case( + _DOMAIN, + "test_non_tenant_domain_red_control_fails_closed", + "catalog-tenant-id", + ), + "internal-tenant-policy": _case( + _DOMAIN, + "test_non_tenant_domain_red_control_fails_closed", + "internal-tenant-policy", + ), + "catalog-rls": _case( + _DOMAIN, + "test_non_tenant_domain_red_control_fails_closed", + "catalog-rls", + ), + "uncontracted-source-tenant": _case( + _DOMAIN, + "test_source_tenant_id_is_typed_non_authoritative_provenance", + "uncontracted-source-tenant", + docker=True, + ), + "source-tenant-generated-unique-alias": _control( + _DOMAIN, "source-tenant-generated-unique-alias", docker=True + ), + }, + EnumApplicationDatabaseEnforcementGate.ROLE_ACL: { + "public-connect": _case( + _ACL, + "test_seeded_acl_red_control_fails_closed", + "public-connect", + ), + "public-execute": _case( + _ACL, + "test_seeded_acl_red_control_fails_closed", + "public-execute", + ), + "runtime-owner": _case( + _ACL, "test_seeded_acl_red_control_fails_closed", "runtime-owner" + ), + "runtime-ddl": _case( + _ACL, + "test_seeded_acl_red_control_fails_closed", + "runtime-ddl", + ), + "runtime-bypassrls": _control(_ACL, "runtime-bypassrls"), + "cross-domain-grant": _control(_ACL, "cross-domain-grant"), + "unsafe-default-privilege": _case( + _ACL, + "test_seeded_acl_red_control_fails_closed", + "unsafe-default-privilege", + ), + }, + EnumApplicationDatabaseEnforcementGate.ONE_DATABASE: { + "old-application-database": _control( + _DOMAIN, "old-application-database", docker=True + ), + "duplicate-pool-user": _control( + _DOMAIN, "duplicate-pool-user", docker=True + ), + "wrong-pool-user": _control(_DOMAIN, "wrong-pool-user"), + "missing-pool-binding": _control(_DOMAIN, "missing-pool-binding"), + }, + EnumApplicationDatabaseEnforcementGate.ADAPTER: { + "untrusted-tenant-selection": _control( + _ADAPTER, "untrusted-tenant-selection" + ), + "mismatched-signer-binding": _control( + _ADAPTER, "mismatched-signer-binding" + ), + "nonlocal-tenant-guc": _control(_ADAPTER, "nonlocal-tenant-guc"), + "leaked-tenant-guc": _control(_ADAPTER, "leaked-tenant-guc"), + "internal-resolver-call": _control(_ADAPTER, "internal-resolver-call"), + "domain-blind-upsert": _control(_ADAPTER, "domain-blind-upsert"), + }, + EnumApplicationDatabaseEnforcementGate.TOPOLOGY_PARITY: { + "profile-instance-drift": _control(_TOPOLOGY, "profile-instance-drift"), + "database-user-drift": _case( + _TOPOLOGY, + "test_seeded_database_user_schema_and_dsn_drift_fail_closed", + "database-user-drift", + ), + "docker-profile-injection-drift": _control( + _TOPOLOGY, "docker-profile-injection-drift" + ), + "docker-dsn-consumer-drift": _case( + _TOPOLOGY, + "test_seeded_docker_database_or_dsn_drift_fails_closed", + "docker-dsn-consumer-drift", + ), + }, + } +) + + +__all__ = [ + "APPLICATION_DATABASE_RED_CONTROL_REGISTRY", + "ApplicationDatabaseRedControlBinding", + "ApplicationDatabaseRedControlRegistry", +] diff --git a/src/omnibase_infra/validation/application_database_source_tenant_authority.py b/src/omnibase_infra/validation/application_database_source_tenant_authority.py new file mode 100644 index 0000000000..02b5d44352 --- /dev/null +++ b/src/omnibase_infra/validation/application_database_source_tenant_authority.py @@ -0,0 +1,355 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Catalog-resolved authority closure for optional source-tenant provenance.""" + +from __future__ import annotations + +import re +from collections import defaultdict, deque +from collections.abc import Sequence + +from omnibase_infra.validation.models.model_application_database_routine_dependency_state import ( + ModelApplicationDatabaseRoutineDependencyState, +) + +_INSPECTABLE_ROUTINE_LANGUAGES = frozenset({"sql", "plpgsql"}) +_DYNAMIC_SQL_PATTERN = re.compile(r"\bexecute\b", re.IGNORECASE) +_QUALIFIED_CALL_PATTERN = re.compile( + r'(?"[^"]+"|[a-z_][a-z0-9_$]*)' + r'\s*\.\s*(?P"[^"]+"|[a-z_][a-z0-9_$]*)\s*\(', + re.IGNORECASE, +) + + +def _is_identifier_character(character: str) -> bool: + return ( + character == "_" + or character == "$" + or character.isalnum() + or ord(character) >= 128 + ) + + +def _has_escape_string_prefix(value: str, quote_index: int) -> bool: + return ( + quote_index > 0 + and value[quote_index - 1].lower() == "e" + and (quote_index == 1 or not _is_identifier_character(value[quote_index - 2])) + ) + + +def _dollar_quote_at(value: str, index: int) -> str | None: + match = re.match( + r"\$(?:(?:[a-z_]|[^\x00-\x7f])" + r"(?:[a-z0-9_]|[^\x00-\x7f])*)?\$", + value[index:], + re.IGNORECASE, + ) + return None if match is None else match.group(0) + + +def _without_sql_comments(value: str) -> str: + """Replace PostgreSQL comments with whitespace without touching quoted text.""" + normalized = list(value) + index = 0 + while index < len(value): + if value.startswith("--", index): + newline = value.find("\n", index + 2) + stop = len(value) if newline < 0 else newline + for offset in range(index, stop): + normalized[offset] = " " + index = stop + continue + if value.startswith("/*", index): + depth = 1 + start = index + index += 2 + while index < len(value) and depth: + if value.startswith("/*", index): + depth += 1 + index += 2 + elif value.startswith("*/", index): + depth -= 1 + index += 2 + else: + index += 1 + for offset in range(start, index): + if normalized[offset] not in {"\r", "\n"}: + normalized[offset] = " " + continue + character = value[index] + if character in {"'", '"'}: + quote = character + escape_backslashes = character == "'" and _has_escape_string_prefix( + value, + index, + ) + index += 1 + while index < len(value): + if escape_backslashes and value[index] == "\\": + index += 2 + continue + if value[index] != quote: + index += 1 + continue + if index + 1 < len(value) and value[index + 1] == quote: + index += 2 + continue + index += 1 + break + continue + if character == "$": + delimiter = _dollar_quote_at(value, index) + if delimiter is not None: + end = value.find(delimiter, index + len(delimiter)) + index = len(value) if end < 0 else end + len(delimiter) + continue + index += 1 + return "".join(normalized) + + +class ApplicationDatabaseAuthorityResolutionError(ValueError): + """Raised when catalog authority cannot be resolved without guessing.""" + + +def _normalize_identifier(value: str) -> str: + normalized = value.strip() + if normalized.startswith('"') and normalized.endswith('"'): + normalized = normalized[1:-1].replace('""', '"') + return normalized.lower() + + +def _identifier_is_referenced(text: str, identifier: str) -> bool: + return bool( + re.search( + rf'(? bool: + return bool( + re.search( + rf'(? bool: + """Detect a composite argument used as a value rather than one named field.""" + return bool( + re.search( + rf"\${position}(?!\d)(?:\s*\.\s*\*|(?!\s*\.))", + source_body, + re.IGNORECASE, + ) + ) + + +def _named_composite_argument_is_consumed( + source_body: str, + argument_name: str, +) -> bool: + """Detect a named composite argument consumed as a whole-row value.""" + escaped_quoted_name = re.escape(argument_name.replace('"', '""')) + escaped_unquoted_name = re.escape(argument_name) + return bool( + re.search( + rf'(? bool: + """Detect NEW/OLD whole-row use while excluding the structural RETURN row.""" + without_structural_return = re.sub( + r"\breturn\s+(?:new|old)\b", + "", + source_body, + flags=re.IGNORECASE, + ) + return bool( + re.search( + r"\b(?:new|old)\b(?:\s*\.\s*\*|(?!\s*\.))", + without_structural_return, + re.IGNORECASE, + ) + ) + + +def resolve_application_database_authority_columns( + *, + target_columns: Sequence[str], + target_composite_type_id: int, + direct_referenced_columns: Sequence[str], + direct_whole_row_reference: bool, + root_routine_ids: Sequence[int], + routines: Sequence[ModelApplicationDatabaseRoutineDependencyState], + governed_schemas: Sequence[str], +) -> tuple[str, ...]: + """Resolve columns used by an index, trigger, policy, or dependent view. + + PostgreSQL records the routine OID used by a parsed catalog expression, but + traditional SQL and PL/pgSQL string bodies do not record nested calls in + ``pg_depend``. The resolver therefore starts from exact catalog OIDs, walks + catalog-bound routine edges, and inspects only reachable routine definitions + for target-column and governed nested-call references. Unknown catalog OIDs, + opaque row-taking routines, dynamic SQL, and missing governed callees fail + closed. The visited-OID set bounds recursive routine cycles. + """ + ordered_target_columns = tuple(dict.fromkeys(target_columns)) + if len(ordered_target_columns) != len(tuple(target_columns)): + raise ApplicationDatabaseAuthorityResolutionError( + "target relation columns must be unique" + ) + target_column_set = frozenset(ordered_target_columns) + governed_schema_set = frozenset( + _normalize_identifier(schema) for schema in governed_schemas + ) + routines_by_id: dict[int, ModelApplicationDatabaseRoutineDependencyState] = {} + governed_by_qualified_name: dict[ + tuple[str, str], list[ModelApplicationDatabaseRoutineDependencyState] + ] = defaultdict(list) + governed_by_name: dict[ + str, list[ModelApplicationDatabaseRoutineDependencyState] + ] = defaultdict(list) + for routine in routines: + if routine.object_id in routines_by_id: + raise ApplicationDatabaseAuthorityResolutionError( + f"catalog routine oid {routine.object_id} is duplicated" + ) + routines_by_id[routine.object_id] = routine + if routine.namespace in governed_schema_set: + governed_by_qualified_name[(routine.namespace, routine.name)].append( + routine + ) + governed_by_name[routine.name].append(routine) + + referenced_columns = ( + set(ordered_target_columns) + if direct_whole_row_reference + else set(direct_referenced_columns) + ) + unknown_direct_columns = referenced_columns.difference(target_column_set) + if unknown_direct_columns: + raise ApplicationDatabaseAuthorityResolutionError( + "catalog authority references unknown target columns: " + f"{sorted(unknown_direct_columns)!r}" + ) + + pending: deque[int] = deque(dict.fromkeys(root_routine_ids)) + visited: set[int] = set() + while pending: + routine_id = pending.popleft() + if routine_id in visited: + continue + resolved_routine = routines_by_id.get(routine_id) + if resolved_routine is None: + raise ApplicationDatabaseAuthorityResolutionError( + f"catalog routine oid {routine_id} is unresolved" + ) + visited.add(routine_id) + + unknown_routine_columns = set( + resolved_routine.referenced_target_columns + ).difference(target_column_set) + if unknown_routine_columns: + raise ApplicationDatabaseAuthorityResolutionError( + f"catalog routine oid {routine_id} references unknown target " + f"columns: {sorted(unknown_routine_columns)!r}" + ) + referenced_columns.update(resolved_routine.referenced_target_columns) + if resolved_routine.references_target_whole_row: + referenced_columns.update(ordered_target_columns) + pending.extend(resolved_routine.referenced_routine_ids) + + receives_target_row = ( + target_composite_type_id in resolved_routine.argument_type_ids + or resolved_routine.returns_trigger + ) + governed = resolved_routine.namespace in governed_schema_set + if not governed and not receives_target_row: + continue + if resolved_routine.language not in _INSPECTABLE_ROUTINE_LANGUAGES: + raise ApplicationDatabaseAuthorityResolutionError( + f"catalog routine {resolved_routine.namespace}." + f"{resolved_routine.name} cannot inspect language " + f"{resolved_routine.language!r} safely" + ) + if not resolved_routine.source_body or not resolved_routine.source_body.strip(): + raise ApplicationDatabaseAuthorityResolutionError( + f"catalog routine {resolved_routine.namespace}." + f"{resolved_routine.name} has no inspectable definition" + ) + source_body = _without_sql_comments(resolved_routine.source_body) + if _DYNAMIC_SQL_PATTERN.search(source_body): + raise ApplicationDatabaseAuthorityResolutionError( + f"catalog routine {resolved_routine.namespace}." + f"{resolved_routine.name} uses dynamic SQL that cannot be " + "resolved fail-closed" + ) + composite_argument_positions = tuple( + position + for position, argument_type_id in enumerate( + resolved_routine.argument_type_ids, + start=1, + ) + if argument_type_id == target_composite_type_id + ) + argument_names = resolved_routine.argument_names + if any( + _composite_argument_is_consumed(source_body, position) + or ( + bool(argument_names) + and argument_names[position - 1] is not None + and _named_composite_argument_is_consumed( + source_body, + argument_names[position - 1] or "", + ) + ) + for position in composite_argument_positions + ) or ( + resolved_routine.returns_trigger and _trigger_row_is_consumed(source_body) + ): + referenced_columns.update(ordered_target_columns) + referenced_columns.update( + column + for column in ordered_target_columns + if _identifier_is_referenced(source_body, column) + ) + + for match in _QUALIFIED_CALL_PATTERN.finditer(source_body): + schema_name = _normalize_identifier(match.group("schema")) + if schema_name not in governed_schema_set: + continue + routine_name = _normalize_identifier(match.group("name")) + candidates = governed_by_qualified_name.get((schema_name, routine_name), ()) + if not candidates: + raise ApplicationDatabaseAuthorityResolutionError( + "qualified governed routine " + f"{schema_name}.{routine_name} is unresolved" + ) + pending.extend(candidate.object_id for candidate in candidates) + + for routine_name, candidates in governed_by_name.items(): + if _unqualified_routine_is_called(source_body, routine_name): + pending.extend(candidate.object_id for candidate in candidates) + + return tuple( + column for column in ordered_target_columns if column in referenced_columns + ) + + +__all__ = [ + "ApplicationDatabaseAuthorityResolutionError", + "resolve_application_database_authority_columns", +] diff --git a/src/omnibase_infra/validation/application_relation_ownership.py b/src/omnibase_infra/validation/application_relation_ownership.py new file mode 100644 index 0000000000..223eb3e3ee --- /dev/null +++ b/src/omnibase_infra/validation/application_relation_ownership.py @@ -0,0 +1,864 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed global application-relation ownership validation (OMN-15418). + +Node contracts and service migration manifests reuse core's +``ModelDbTableDeclaration`` as their table-location source of truth. This module +projects those distributed declarations into one validation report and proves +that a supplied live inventory has exactly one writer/owner for every relation. +Readers remain explicit and never count as owners. +""" + +from __future__ import annotations + +from collections import Counter, defaultdict +from collections.abc import Mapping, Sequence +from pathlib import Path + +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_core.models.contracts.subcontracts.model_db_table_declaration import ( + ModelDbTableDeclaration, +) +from omnibase_core.models.core.model_deployment_topology import ModelDeploymentTopology +from omnibase_core.models.errors import ModelOnexError +from omnibase_infra.validation.enums.enum_application_database_object_kind import ( + EnumApplicationDatabaseObjectKind, +) +from omnibase_infra.validation.enums.enum_application_inventory_object_kind import ( + EnumApplicationInventoryObjectKind, +) +from omnibase_infra.validation.enums.enum_application_relation_kind import ( + EnumApplicationRelationKind, +) +from omnibase_infra.validation.enums.enum_application_relation_purpose import ( + EnumApplicationRelationPurpose, +) +from omnibase_infra.validation.enums.enum_application_relation_violation import ( + EnumApplicationRelationViolation, +) +from omnibase_infra.validation.models.model_application_relation_declaration import ( + ModelApplicationRelationDeclaration, +) +from omnibase_infra.validation.models.model_application_relation_evidence_inventory import ( + ModelApplicationRelationEvidenceInventory, +) +from omnibase_infra.validation.models.model_application_relation_inventory import ( + ModelApplicationRelationInventory, +) +from omnibase_infra.validation.models.model_application_relation_ownership_report import ( + ModelApplicationRelationOwnershipReport, +) +from omnibase_infra.validation.models.model_application_relation_violation import ( + ModelApplicationRelationViolation, +) +from omnibase_infra.validation.models.model_live_application_relation import ( + ModelLiveApplicationRelation, + RelationIdentity, +) +from omnibase_infra.validation.models.model_migration_ownership_manifest import ( + ModelMigrationOwnershipManifest, +) +from omnibase_infra.validation.models.model_node_ownership_document import ( + ModelNodeOwnershipDocument, +) + + +def _load_yaml_mapping(path: Path) -> Mapping[str, object]: + # Why: PyYAML is a runtime dependency without complete inline typing. + import yaml + + raw: object = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(raw, Mapping): + raise ValueError(f"Expected YAML mapping in {path}, got {type(raw).__name__}") + return raw + + +def load_application_relation_inventory( + path: Path, +) -> ModelApplicationRelationInventory: + """Load a minimal inventory or project the exact rich OMN-15423 evidence.""" + raw = _load_yaml_mapping(path) + if "inventory_projection" not in raw: + return ModelApplicationRelationInventory.model_validate(raw) + + evidence = ModelApplicationRelationEvidenceInventory.model_validate(raw) + kind_map = { + EnumApplicationInventoryObjectKind.TABLE: EnumApplicationRelationKind.TABLE, + EnumApplicationInventoryObjectKind.VIEW: EnumApplicationRelationKind.VIEW, + EnumApplicationInventoryObjectKind.MATERIALIZED_VIEW: ( + EnumApplicationRelationKind.MATERIALIZED_VIEW + ), + EnumApplicationInventoryObjectKind.FOREIGN_TABLE: ( + EnumApplicationRelationKind.FOREIGN_TABLE + ), + EnumApplicationInventoryObjectKind.FUNCTION: ( + EnumApplicationRelationKind.FUNCTION + ), + } + live_relations = tuple( + ModelLiveApplicationRelation( + name=relation.name, + database_ref=evidence.database_ref, + schema=relation.target_schema, + kind=kind_map[relation.kind], + purpose=_inventory_relation_purpose(relation.name), + domain=relation.domain, + function_signature=relation.function_signature, + ) + for relation in evidence.relations + if relation.kind in kind_map + ) + excluded_database_objects = tuple( + relation.name + for relation in evidence.relations + if relation.kind not in kind_map + ) + return ModelApplicationRelationInventory( + schema_version=evidence.schema_version, + relations=live_relations, + completion_status=evidence.completion_status, + blocked_relations=evidence.blocked_relations, + retained_live_census=evidence.retained_live_census, + runtime_evidence={ + "full_day_datname_usename_activity": ( + evidence.runtime_evidence.full_day_datname_usename_activity + ), + "live_catalog_parity": evidence.runtime_evidence.live_catalog_parity, + }, + source_relation_count=len(evidence.relations), + excluded_database_objects=excluded_database_objects, + ) + + +def load_service_ownership_manifest(path: Path) -> ModelMigrationOwnershipManifest: + """Load an OMN-15423 service manifest with strict typed table locations.""" + return ModelMigrationOwnershipManifest.model_validate(_load_yaml_mapping(path)) + + +def _load_node_ownership_document(path: Path) -> ModelNodeOwnershipDocument: + return ModelNodeOwnershipDocument.model_validate(_load_yaml_mapping(path)) + + +def _table_purpose(table: ModelDbTableDeclaration) -> EnumApplicationRelationPurpose: + normalized_role = table.role.strip().lower() + if normalized_role == "migration_ledger" or normalized_role.endswith( + "_migration_ledger" + ): + return EnumApplicationRelationPurpose.MIGRATION_LEDGER + return EnumApplicationRelationPurpose.DATA + + +def _inventory_relation_purpose(name: str) -> EnumApplicationRelationPurpose: + """Classify conventional migration-ledger relation names from live evidence.""" + normalized_name = name.strip().lower() + if normalized_name in {"alembic_version", "migrations_log", "schema_migrations"}: + return EnumApplicationRelationPurpose.MIGRATION_LEDGER + if normalized_name.endswith("_schema_migrations"): + return EnumApplicationRelationPurpose.MIGRATION_LEDGER + return EnumApplicationRelationPurpose.DATA + + +def _lookup_violation_code(exc: ValueError) -> EnumApplicationRelationViolation: + if str(exc).startswith("Unknown database_ref"): + return EnumApplicationRelationViolation.UNKNOWN_DATABASE + return EnumApplicationRelationViolation.UNKNOWN_SCHEMA + + +def _topology_domain( + topology: ModelDeploymentTopology, + table: ModelDbTableDeclaration, + violations: list[ModelApplicationRelationViolation], + source_path: Path, +) -> EnumDatabaseSchemaDomain | None: + try: + return topology.table_domain(table) + except ValueError as exc: + violations.append( + ModelApplicationRelationViolation( + code=_lookup_violation_code(exc), + message=str(exc), + relation_name=table.name, + source_paths=(str(source_path),), + ) + ) + return None + + +def _declaration_from_table( + *, + table: ModelDbTableDeclaration, + authority: str, + topology: ModelDeploymentTopology, + source_path: Path, + violations: list[ModelApplicationRelationViolation], +) -> ModelApplicationRelationDeclaration: + is_reader = table.access == "read" + return ModelApplicationRelationDeclaration( + name=table.name, + database_ref=table.database_ref, + schema=table.schema, + kind=EnumApplicationRelationKind.TABLE, + purpose=_table_purpose(table), + domain=_topology_domain(topology, table, violations, source_path), + owner_declaration=None if is_reader else authority, + readers=(authority,) if is_reader else (), + access=table.access, + role=table.role, + source_path=str(source_path), + ) + + +def _status_is_blocking(status: str) -> bool: + return status.strip().lower() not in {"complete", "pass", "passed", "verified"} + + +def _append_inventory_evidence_violations( + inventory: ModelApplicationRelationInventory, + violations: list[ModelApplicationRelationViolation], +) -> None: + """Retain rich inventory blockers in the global fail-closed report.""" + if inventory.source_relation_count is not None: + projected_source_count = len(inventory.relations) + len( + inventory.excluded_database_objects + ) + if inventory.source_relation_count != projected_source_count: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.INCOMPLETE_CENSUS, + message=( + "Application inventory source_relation_count does not match " + "the typed relation projection plus explicitly excluded " + f"database objects: declared={inventory.source_relation_count}, " + f"projected={projected_source_count}" + ), + ) + ) + + if inventory.completion_status and _status_is_blocking(inventory.completion_status): + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.INCOMPLETE_CENSUS, + message=( + "Application inventory completion_status is " + f"{inventory.completion_status!r}" + ), + ) + ) + + for evidence_name, runtime_status in inventory.runtime_evidence.items(): + if _status_is_blocking(runtime_status.status): + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.INCOMPLETE_CENSUS, + message=( + f"Application inventory runtime evidence {evidence_name!r} " + f"is {runtime_status.status!r}: " + f"{runtime_status.reason or 'no reason supplied'}" + ), + ) + ) + + census = inventory.retained_live_census + if census is not None: + projected_kind_counts = Counter( + relation.kind for relation in inventory.relations + ) + census_comparisons = ( + ( + "base tables", + census.observed_base_tables, + projected_kind_counts[EnumApplicationRelationKind.TABLE], + ), + ( + "views and materialized views", + census.observed_views_and_materialized_views, + projected_kind_counts[EnumApplicationRelationKind.VIEW] + + projected_kind_counts[EnumApplicationRelationKind.MATERIALIZED_VIEW], + ), + ) + for relation_group, observed_count, projected_count in census_comparisons: + if observed_count is not None and observed_count != projected_count: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.INCOMPLETE_CENSUS, + message=( + f"Application inventory retained census {relation_group} " + "do not match typed relation rows: " + f"observed={observed_count}, projected={projected_count}" + ), + ) + ) + + if census.parity_status is not None and _status_is_blocking( + census.parity_status + ): + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.INCOMPLETE_CENSUS, + message=( + f"Application inventory retained census is " + f"{census.parity_status!r}: " + f"{census.reason or 'no reason supplied'}" + ), + ) + ) + + for blocked in inventory.blocked_relations: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.BLOCKED_RELATION, + message=f"Blocked {blocked.kind.value} {blocked.name!r}: {blocked.reason}", + relation_name=blocked.name, + ) + ) + + +def _append_service_declarations( + *, + path: Path, + manifest: ModelMigrationOwnershipManifest, + topology: ModelDeploymentTopology, + declarations: list[ModelApplicationRelationDeclaration], + violations: list[ModelApplicationRelationViolation], +) -> None: + authority = manifest.owner_declaration or f"service:{manifest.service}" + if manifest.completion_status and _status_is_blocking(manifest.completion_status): + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.INCOMPLETE_CENSUS, + message=( + f"Service manifest {manifest.service!r} completion_status is " + f"{manifest.completion_status!r}" + ), + source_paths=(str(path),), + ) + ) + + for evidence_name, runtime_status in manifest.runtime_evidence.items(): + if _status_is_blocking(runtime_status.status): + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.INCOMPLETE_CENSUS, + message=( + f"Service manifest {manifest.service!r} runtime evidence " + f"{evidence_name!r} is {runtime_status.status!r}: " + f"{runtime_status.reason or 'no reason supplied'}" + ), + source_paths=(str(path),), + ) + ) + + census = manifest.retained_live_census + if census is not None and census.parity_status is not None: + if _status_is_blocking(census.parity_status): + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.INCOMPLETE_CENSUS, + message=( + f"Service manifest {manifest.service!r} retained census is " + f"{census.parity_status!r}: {census.reason or 'no reason supplied'}" + ), + source_paths=(str(path),), + ) + ) + + for blocked in manifest.blocked_relations: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.BLOCKED_RELATION, + message=f"Blocked {blocked.kind.value} {blocked.name!r}: {blocked.reason}", + relation_name=blocked.name, + source_paths=(str(path),), + ) + ) + + tables_by_name: dict[str, list[ModelDbTableDeclaration]] = defaultdict(list) + for table in manifest.db_io.db_tables: + tables_by_name[table.name].append(table) + if table.database_ref != manifest.target_database_ref: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.CONFLICTING_LOCATION, + message=( + f"Service manifest target_database_ref " + f"{manifest.target_database_ref!r} conflicts with table " + f"{table.name!r} database_ref {table.database_ref!r}" + ), + relation_name=table.name, + source_paths=(str(path),), + ) + ) + declarations.append( + _declaration_from_table( + table=table, + authority=authority, + topology=topology, + source_path=path, + violations=violations, + ) + ) + + evidence_table_names: set[str] = set() + for evidence in manifest.relation_evidence: + if evidence.owner_declaration != authority: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.OWNER_MISMATCH, + message=( + f"Relation evidence owner {evidence.owner_declaration!r} does " + f"not match manifest owner {authority!r}" + ), + relation_name=evidence.name, + source_paths=(str(path),), + ) + ) + + if evidence.kind is EnumApplicationRelationKind.TABLE: + evidence_table_names.add(evidence.name) + candidates = tables_by_name.get(evidence.name, []) + if len(candidates) != 1: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.MISSING_TYPED_LOCATION, + message=( + f"Table evidence {evidence.name!r} resolves to " + f"{len(candidates)} typed db_io declarations" + ), + relation_name=evidence.name, + source_paths=(str(path),), + ) + ) + continue + table = candidates[0] + domain = _topology_domain(topology, table, violations, path) + if domain is not None and domain is not evidence.domain: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.DOMAIN_MISMATCH, + message=( + f"Table {table.name!r} evidence domain " + f"{evidence.domain.value} conflicts with topology domain " + f"{domain.value}" + ), + relation_name=table.name, + source_paths=(str(path),), + ) + ) + declarations.append( + ModelApplicationRelationDeclaration( + name=table.name, + database_ref=table.database_ref, + schema=table.schema, + kind=EnumApplicationRelationKind.TABLE, + purpose=_table_purpose(table), + domain=domain, + owner_declaration=None, + readers=tuple(sorted(set(evidence.readers))), + access="read", + role=f"{table.role}_evidence_readers", + source_path=str(path), + ) + ) + continue + + if evidence.database_ref is None or evidence.schema is None: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.MISSING_TYPED_LOCATION, + message=( + f"{evidence.kind.value} evidence {evidence.name!r} must " + "declare database_ref and schema" + ), + relation_name=evidence.name, + source_paths=(str(path),), + ) + ) + continue + try: + domain = topology.schema_domain(evidence.database_ref, evidence.schema) + except ValueError as exc: + violations.append( + ModelApplicationRelationViolation( + code=_lookup_violation_code(exc), + message=str(exc), + relation_name=evidence.name, + source_paths=(str(path),), + ) + ) + domain = None + if domain is not None and domain is not evidence.domain: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.DOMAIN_MISMATCH, + message=( + f"{evidence.kind.value} {evidence.name!r} evidence domain " + f"{evidence.domain.value} conflicts with topology domain " + f"{domain.value}" + ), + relation_name=evidence.name, + source_paths=(str(path),), + ) + ) + declarations.append( + ModelApplicationRelationDeclaration( + name=evidence.name, + database_ref=evidence.database_ref, + schema=evidence.schema, + kind=evidence.kind, + purpose=EnumApplicationRelationPurpose.DATA, + domain=domain, + owner_declaration=authority, + readers=tuple(sorted(set(evidence.readers))), + access="read_write", + role=evidence.kind.value, + source_path=str(path), + function_signature=evidence.function_signature, + ) + ) + + if census is not None and census.observed_base_tables is not None: + named_tables = len(evidence_table_names) + if census.observed_base_tables > named_tables: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.INCOMPLETE_CENSUS, + message=( + f"Service manifest {manifest.service!r} observed " + f"{census.observed_base_tables} base tables but names only " + f"{named_tables}; {census.observed_base_tables - named_tables} " + "live-only relations remain unresolved" + ), + source_paths=(str(path),), + ) + ) + + named_view_count = sum( + evidence.kind + in { + EnumApplicationRelationKind.VIEW, + EnumApplicationRelationKind.MATERIALIZED_VIEW, + } + for evidence in manifest.relation_evidence + ) + if ( + census is not None + and census.observed_views_and_materialized_views is not None + and census.observed_views_and_materialized_views > named_view_count + ): + unresolved = census.observed_views_and_materialized_views - named_view_count + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.INCOMPLETE_CENSUS, + message=( + f"Service manifest {manifest.service!r} observed " + f"{census.observed_views_and_materialized_views} views/materialized " + f"views but names only {named_view_count}; {unresolved} live-only " + "relations remain unresolved" + ), + source_paths=(str(path),), + ) + ) + + for database_object in manifest.database_objects: + if database_object.kind is not EnumApplicationDatabaseObjectKind.FUNCTION: + continue + if database_object.owner_declaration != authority: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.OWNER_MISMATCH, + message=( + f"Function owner {database_object.owner_declaration!r} does " + f"not match manifest owner {authority!r}" + ), + relation_name=database_object.name, + source_paths=(str(path),), + ) + ) + if database_object.database_ref is None or database_object.schema is None: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.MISSING_TYPED_LOCATION, + message=( + f"function {database_object.name!r} must declare database_ref " + "and schema" + ), + relation_name=database_object.name, + source_paths=(str(path),), + ) + ) + continue + try: + domain = topology.schema_domain( + database_object.database_ref, database_object.schema + ) + except ValueError as exc: + violations.append( + ModelApplicationRelationViolation( + code=_lookup_violation_code(exc), + message=str(exc), + relation_name=database_object.name, + source_paths=(str(path),), + ) + ) + domain = None + if domain is not None and domain is not database_object.domain: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.DOMAIN_MISMATCH, + message=( + f"function {database_object.name!r} evidence domain " + f"{database_object.domain.value} conflicts with topology " + f"domain {domain.value}" + ), + relation_name=database_object.name, + source_paths=(str(path),), + ) + ) + declarations.append( + ModelApplicationRelationDeclaration( + name=database_object.name, + database_ref=database_object.database_ref, + schema=database_object.schema, + kind=EnumApplicationRelationKind.FUNCTION, + purpose=EnumApplicationRelationPurpose.DATA, + domain=domain, + owner_declaration=authority, + readers=tuple(sorted(set(database_object.readers))), + access="read_write", + role="function", + source_path=str(path), + function_signature=database_object.function_signature, + ) + ) + + +def _append_location_conflicts( + declarations: Sequence[ModelApplicationRelationDeclaration], + violations: list[ModelApplicationRelationViolation], +) -> None: + locations_by_object: dict[ + tuple[str, EnumApplicationRelationKind], set[tuple[str, str]] + ] = defaultdict(set) + sources_by_object: dict[tuple[str, EnumApplicationRelationKind], set[str]] = ( + defaultdict(set) + ) + for declaration in declarations: + object_key = (declaration.name, declaration.kind) + locations_by_object[object_key].add( + (declaration.database_ref, declaration.schema) + ) + sources_by_object[object_key].add(declaration.source_path) + for (name, kind), locations in locations_by_object.items(): + if len(locations) > 1: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.CONFLICTING_LOCATION, + message=( + f"{kind.value} {name!r} has conflicting typed locations: " + f"{sorted(locations)!r}" + ), + relation_name=name, + source_paths=tuple(sorted(sources_by_object[(name, kind)])), + ) + ) + + +def validate_application_relation_ownership( + *, + topology: ModelDeploymentTopology, + node_contract_paths: Sequence[Path], + service_manifest_paths: Sequence[Path], + inventory: ModelApplicationRelationInventory, +) -> ModelApplicationRelationOwnershipReport: + """Project distributed declarations and validate exactly-one ownership.""" + declarations: list[ModelApplicationRelationDeclaration] = [] + violations: list[ModelApplicationRelationViolation] = [] + _append_inventory_evidence_violations(inventory, violations) + + for path in sorted(node_contract_paths): + document = _load_node_ownership_document(path) + authority = f"node:{document.name}" + for table in document.db_io.db_tables: + declarations.append( + _declaration_from_table( + table=table, + authority=authority, + topology=topology, + source_path=path, + violations=violations, + ) + ) + + for path in sorted(service_manifest_paths): + _append_service_declarations( + path=path, + manifest=load_service_ownership_manifest(path), + topology=topology, + declarations=declarations, + violations=violations, + ) + + _append_location_conflicts(declarations, violations) + + declarations_by_identity: dict[ + RelationIdentity, list[ModelApplicationRelationDeclaration] + ] = defaultdict(list) + for declaration in declarations: + declarations_by_identity[declaration.identity].append(declaration) + + duplicate_owner_identities: set[RelationIdentity] = set() + for identity, candidates in declarations_by_identity.items(): + owners = [candidate for candidate in candidates if candidate.owner_declaration] + if len(owners) <= 1: + continue + duplicate_owner_identities.add(identity) + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.DUPLICATE_OWNER, + message=( + f"Declared relation {identity!r} has {len(owners)} owners: " + f"{[owner.owner_declaration for owner in owners]!r}" + ), + relation_name=identity[2], + source_paths=tuple(owner.source_path for owner in owners), + ) + ) + + live_by_identity: dict[RelationIdentity, ModelLiveApplicationRelation] = {} + for live_relation in inventory.relations: + try: + topology_domain = topology.schema_domain( + live_relation.database_ref, live_relation.schema + ) + except ValueError as exc: + violations.append( + ModelApplicationRelationViolation( + code=_lookup_violation_code(exc), + message=str(exc), + relation_name=live_relation.name, + ) + ) + else: + if ( + live_relation.domain is not None + and live_relation.domain is not topology_domain + ): + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.DOMAIN_MISMATCH, + message=( + f"Live {live_relation.kind.value} {live_relation.name!r} " + f"inventory domain {live_relation.domain.value} conflicts " + f"with topology domain {topology_domain.value}" + ), + relation_name=live_relation.name, + ) + ) + if live_relation.identity in live_by_identity: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.DUPLICATE_OWNER, + message=f"Live inventory repeats relation {live_relation.identity!r}", + relation_name=live_relation.name, + ) + ) + live_by_identity[live_relation.identity] = live_relation + + for identity, live_relation in live_by_identity.items(): + candidates = declarations_by_identity.get(identity, []) + owners = [candidate for candidate in candidates if candidate.owner_declaration] + if not owners: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.MISSING_OWNER, + message=f"Live {live_relation.kind.value} {identity!r} has no owner", + relation_name=live_relation.name, + ) + ) + elif len(owners) > 1 and identity not in duplicate_owner_identities: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.DUPLICATE_OWNER, + message=( + f"Live {live_relation.kind.value} {identity!r} has " + f"{len(owners)} owners: " + f"{[owner.owner_declaration for owner in owners]!r}" + ), + relation_name=live_relation.name, + source_paths=tuple(owner.source_path for owner in owners), + ) + ) + for owner in owners: + if owner.purpose is not live_relation.purpose: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.PURPOSE_MISMATCH, + message=( + f"Relation {identity!r} inventory purpose " + f"{live_relation.purpose.value!r} conflicts with owner " + f"purpose {owner.purpose.value!r}" + ), + relation_name=live_relation.name, + source_paths=(owner.source_path,), + ) + ) + + for identity, candidates in declarations_by_identity.items(): + if identity not in live_by_identity: + violations.append( + ModelApplicationRelationViolation( + code=EnumApplicationRelationViolation.UNKNOWN_RELATION, + message=f"Declared relation {identity!r} is absent from live inventory", + relation_name=identity[2], + source_paths=tuple( + sorted({candidate.source_path for candidate in candidates}) + ), + ) + ) + + ordered_violations = tuple( + sorted( + violations, + key=lambda violation: ( + violation.code.value, + violation.relation_name or "", + violation.message, + ), + ) + ) + return ModelApplicationRelationOwnershipReport( + declarations=tuple(declarations), + violations=ordered_violations, + ) + + +def assert_application_relation_ownership( + report: ModelApplicationRelationOwnershipReport, +) -> None: + """Raise one sanitized startup/CI error when global ownership is invalid.""" + if report.is_valid: + return + summaries = "; ".join( + f"{violation.code.value}: {violation.message}" + for violation in report.violations + ) + raise ModelOnexError( + f"Application relation ownership validation failed with " + f"{len(report.violations)} violation(s): {summaries}" + ) + + +__all__ = [ + "EnumApplicationDatabaseObjectKind", + "EnumApplicationRelationKind", + "EnumApplicationRelationPurpose", + "EnumApplicationRelationViolation", + "ModelApplicationRelationInventory", + "ModelApplicationRelationOwnershipReport", + "ModelLiveApplicationRelation", + "ModelMigrationOwnershipManifest", + "assert_application_relation_ownership", + "load_application_relation_inventory", + "load_service_ownership_manifest", + "validate_application_relation_ownership", +] diff --git a/src/omnibase_infra/validation/enums/enum_application_database_acl_authorization_scope.py b/src/omnibase_infra/validation/enums/enum_application_database_acl_authorization_scope.py new file mode 100644 index 0000000000..8d0c4caf34 --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_database_acl_authorization_scope.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Authorization boundary for generated application-database ACL matrices.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationDatabaseAclAuthorizationScope(StrEnum): + """Separate deployment-authorized evidence from synthetic proof fixtures.""" + + DEPLOYMENT = "deployment" + SYNTHETIC_PROOF = "synthetic_proof" + + +__all__ = ["EnumApplicationDatabaseAclAuthorizationScope"] diff --git a/src/omnibase_infra/validation/enums/enum_application_database_acl_policy_source_kind.py b/src/omnibase_infra/validation/enums/enum_application_database_acl_policy_source_kind.py new file mode 100644 index 0000000000..a67765ff5c --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_database_acl_policy_source_kind.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Evidence source kinds accepted by application-database ACL policy.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationDatabaseAclPolicySourceKind(StrEnum): + """Distinguish authoritative topology policy from synthetic proof input.""" + + TOPOLOGY_CONTRACT = "topology_contract" + SYNTHETIC_FIXTURE = "synthetic_fixture" + + +__all__ = ["EnumApplicationDatabaseAclPolicySourceKind"] diff --git a/src/omnibase_infra/validation/enums/enum_application_database_acl_render_phase.py b/src/omnibase_infra/validation/enums/enum_application_database_acl_render_phase.py new file mode 100644 index 0000000000..3095361df2 --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_database_acl_render_phase.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed stages of the additive application-database ACL rollout.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationDatabaseAclRenderPhase(StrEnum): + """Separate the additive scaffold from materialized-object hardening.""" + + SCAFFOLD = "scaffold" + FULL = "full" + + +__all__ = ["EnumApplicationDatabaseAclRenderPhase"] diff --git a/src/omnibase_infra/validation/enums/enum_application_database_enforcement_gate.py b/src/omnibase_infra/validation/enums/enum_application_database_enforcement_gate.py new file mode 100644 index 0000000000..7b10294ffd --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_database_enforcement_gate.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Application database enforcement gate families.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationDatabaseEnforcementGate(StrEnum): + """Closed set of source and deployment assertions owned by OMN-15361.""" + + CLASSIFICATION = "classification" + SCHEMA_QUALIFICATION = "schema_qualification" + TENANT_RLS = "tenant_rls" + INTERNAL_CATALOG = "internal_catalog" + ROLE_ACL = "role_acl" + ONE_DATABASE = "one_database" + ADAPTER = "adapter" + TOPOLOGY_PARITY = "topology_parity" + + +__all__ = ["EnumApplicationDatabaseEnforcementGate"] diff --git a/src/omnibase_infra/validation/enums/enum_application_database_identity_root.py b/src/omnibase_infra/validation/enums/enum_application_database_identity_root.py new file mode 100644 index 0000000000..3bf2850604 --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_database_identity_root.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Narrow tenant identity-root exceptions declared by the approved plan.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationDatabaseIdentityRoot(StrEnum): + """Closed identity-root contracts; ordinary tenant tables are excluded.""" + + CANONICAL_TENANT = "canonical_tenant_identity_root" + PRE_TENANT_BOOTSTRAP = "pre_tenant_bootstrap_identity_root" + + +__all__ = ["EnumApplicationDatabaseIdentityRoot"] diff --git a/src/omnibase_infra/validation/enums/enum_application_database_identity_root_operation.py b/src/omnibase_infra/validation/enums/enum_application_database_identity_root_operation.py new file mode 100644 index 0000000000..a3625498ea --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_database_identity_root_operation.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Audited control-plane operations for tenant identity roots.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationDatabaseIdentityRootOperation(StrEnum): + """Operations that require an explicit non-runtime RLS bypass identity.""" + + TENANT_CREATION = "tenant_creation" + CROSS_TENANT_ENUMERATION = "cross_tenant_enumeration" + + +__all__ = ["EnumApplicationDatabaseIdentityRootOperation"] diff --git a/src/omnibase_infra/validation/enums/enum_application_database_object_kind.py b/src/omnibase_infra/validation/enums/enum_application_database_object_kind.py new file mode 100644 index 0000000000..9c2f8416bd --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_database_object_kind.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Non-table database object kinds represented by migration evidence.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationDatabaseObjectKind(StrEnum): + """Database object kinds that may be named by an ownership manifest.""" + + FUNCTION = "function" + AGGREGATE = "aggregate" + WINDOW_FUNCTION = "window_function" + PROCEDURE = "procedure" + SEQUENCE = "sequence" + EXTENSION = "extension" + TYPE = "type" + BASE_TYPE = "base_type" + RANGE_TYPE = "range_type" + MULTIRANGE_TYPE = "multirange_type" + + +__all__ = ["EnumApplicationDatabaseObjectKind"] diff --git a/src/omnibase_infra/validation/enums/enum_application_database_principal_inventory_source_kind.py b/src/omnibase_infra/validation/enums/enum_application_database_principal_inventory_source_kind.py new file mode 100644 index 0000000000..537987fc9e --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_database_principal_inventory_source_kind.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Evidence source kinds accepted by application-database principal census.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationDatabasePrincipalInventorySourceKind(StrEnum): + """Distinguish authorized catalog evidence from synthetic proof input.""" + + AUTHORIZED_CATALOG = "authorized_catalog" + SYNTHETIC_FIXTURE = "synthetic_fixture" + + +__all__ = ["EnumApplicationDatabasePrincipalInventorySourceKind"] diff --git a/src/omnibase_infra/validation/enums/enum_application_inventory_object_kind.py b/src/omnibase_infra/validation/enums/enum_application_inventory_object_kind.py new file mode 100644 index 0000000000..e63ebb9fb9 --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_inventory_object_kind.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Object kinds emitted by the OMN-15423 inventory projection.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationInventoryObjectKind(StrEnum): + """Typed superset of ownership-checked relations and supporting objects.""" + + TABLE = "table" + VIEW = "view" + MATERIALIZED_VIEW = "materialized_view" + FOREIGN_TABLE = "foreign_table" + FUNCTION = "function" + AGGREGATE = "aggregate" + WINDOW_FUNCTION = "window_function" + PROCEDURE = "procedure" + SEQUENCE = "sequence" + EXTENSION = "extension" + TYPE = "type" + BASE_TYPE = "base_type" + RANGE_TYPE = "range_type" + MULTIRANGE_TYPE = "multirange_type" + + +__all__ = ["EnumApplicationInventoryObjectKind"] diff --git a/src/omnibase_infra/validation/enums/enum_application_relation_kind.py b/src/omnibase_infra/validation/enums/enum_application_relation_kind.py new file mode 100644 index 0000000000..08db8476ca --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_relation_kind.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""PostgreSQL application relation kinds covered by ownership proof.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationRelationKind(StrEnum): + """Physical relation kinds represented in the application inventory.""" + + TABLE = "table" + VIEW = "view" + MATERIALIZED_VIEW = "materialized_view" + FOREIGN_TABLE = "foreign_table" + FUNCTION = "function" + + +__all__ = ["EnumApplicationRelationKind"] diff --git a/src/omnibase_infra/validation/enums/enum_application_relation_purpose.py b/src/omnibase_infra/validation/enums/enum_application_relation_purpose.py new file mode 100644 index 0000000000..6fe792ed89 --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_relation_purpose.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Semantic purposes for application relations.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationRelationPurpose(StrEnum): + """Semantic purpose kept separate from a PostgreSQL object kind.""" + + DATA = "data" + MIGRATION_LEDGER = "migration_ledger" + + +__all__ = ["EnumApplicationRelationPurpose"] diff --git a/src/omnibase_infra/validation/enums/enum_application_relation_violation.py b/src/omnibase_infra/validation/enums/enum_application_relation_violation.py new file mode 100644 index 0000000000..c419d57608 --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_application_relation_violation.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Fail-closed application-relation ownership violation codes.""" + +from enum import StrEnum, unique + + +@unique +class EnumApplicationRelationViolation(StrEnum): + """Deterministic global ownership violation codes.""" + + MISSING_OWNER = "missing_owner" + DUPLICATE_OWNER = "duplicate_owner" + CONFLICTING_LOCATION = "conflicting_location" + UNKNOWN_RELATION = "unknown_relation" + UNKNOWN_DATABASE = "unknown_database" + UNKNOWN_SCHEMA = "unknown_schema" + MISSING_TYPED_LOCATION = "missing_typed_location" + DOMAIN_MISMATCH = "domain_mismatch" + OWNER_MISMATCH = "owner_mismatch" + PURPOSE_MISMATCH = "purpose_mismatch" + BLOCKED_RELATION = "blocked_relation" + INCOMPLETE_CENSUS = "incomplete_census" + + +__all__ = ["EnumApplicationRelationViolation"] diff --git a/src/omnibase_infra/validation/enums/enum_internal_tenant_column_transform_status.py b/src/omnibase_infra/validation/enums/enum_internal_tenant_column_transform_status.py new file mode 100644 index 0000000000..62f53dbd56 --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_internal_tenant_column_transform_status.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Internal tenant-column transformation evidence states.""" + +from enum import StrEnum, unique + + +@unique +class EnumInternalTenantColumnTransformStatus(StrEnum): + """Finite states emitted by the OMN-15423 classification inventory.""" + + NOT_APPLICABLE_NO_SOURCE_TENANT_ID = "not_applicable_no_source_tenant_id" + SOURCE_DEPENDENCY_INVENTORY_COMPLETE_RUNTIME_COLLISION_SCAN_BLOCKED = ( + "source_dependency_inventory_complete_runtime_collision_scan_blocked" + ) + + +__all__ = ["EnumInternalTenantColumnTransformStatus"] diff --git a/src/omnibase_infra/validation/enums/enum_ownership_evidence_status.py b/src/omnibase_infra/validation/enums/enum_ownership_evidence_status.py new file mode 100644 index 0000000000..f60784eada --- /dev/null +++ b/src/omnibase_infra/validation/enums/enum_ownership_evidence_status.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed status values accepted from ownership evidence manifests.""" + +from enum import StrEnum, unique + + +@unique +class EnumOwnershipEvidenceStatus(StrEnum): + """Finite evidence states; only successful terminal states satisfy the gate.""" + + BLOCKED = "blocked" + COMPLETE = "complete" + FAIL = "fail" + FAILED = "failed" + PASS = "pass" + PASSED = "passed" + PENDING = "pending" + VERIFIED = "verified" + + +__all__ = ["EnumOwnershipEvidenceStatus"] diff --git a/src/omnibase_infra/validation/models/model_application_database_acl_matrix.py b/src/omnibase_infra/validation/models/model_application_database_acl_matrix.py new file mode 100644 index 0000000000..c8d3bce9f5 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_acl_matrix.py @@ -0,0 +1,362 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed, generated application-database ACL matrix.""" + +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_infra.validation.enums.enum_application_database_acl_authorization_scope import ( + EnumApplicationDatabaseAclAuthorizationScope, +) +from omnibase_infra.validation.models.model_application_database_acl_object import ( + ModelApplicationDatabaseAclObject, +) +from omnibase_infra.validation.models.model_application_database_acl_row import ( + ModelApplicationDatabaseAclRow, +) +from omnibase_infra.validation.models.model_application_database_acl_source import ( + ModelApplicationDatabaseAclSource, +) +from omnibase_infra.validation.models.model_application_database_catalog_object_evidence import ( + ModelApplicationDatabaseCatalogObjectEvidence, +) +from omnibase_infra.validation.models.model_application_database_default_acl_row import ( + ModelApplicationDatabaseDefaultAclRow, +) +from omnibase_infra.validation.models.model_application_database_observed_role_state import ( + ModelApplicationDatabaseObservedRoleState, +) +from omnibase_infra.validation.models.model_application_database_role_membership import ( + ModelApplicationDatabaseRoleMembership, +) +from omnibase_infra.validation.models.model_application_database_role_state import ( + ModelApplicationDatabaseRoleState, +) + + +class ModelApplicationDatabaseAclMatrix(BaseModel): + """Complete generated role/object/default-privilege projection.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: Literal["1.0"] = "1.0" + authorization_scope: EnumApplicationDatabaseAclAuthorizationScope + scaffold_status: Literal["READY", "BLOCKED"] + scaffold_blockers: tuple[str, ...] = () + status: Literal["READY", "BLOCKED"] + sources: tuple[ModelApplicationDatabaseAclSource, ...] = Field(..., min_length=1) + verified_evidence_source_keys: tuple[str, ...] = () + declared_principals: dict[str, tuple[str, ...]] + observed_principals: dict[str, tuple[str, ...]] + absent_principals: dict[str, tuple[str, ...]] + observed_owner_roles: tuple[str, ...] + absent_owner_roles: tuple[str, ...] + observed_role_states: tuple[ModelApplicationDatabaseObservedRoleState, ...] + governed_role_states: tuple[ModelApplicationDatabaseRoleState, ...] + retained_administrative_principals: tuple[str, ...] + database_owners: dict[str, str] + required_connect_databases: tuple[str, ...] + observed_connect_database_owners: dict[str, str] + allowed_connect_principals: dict[str, tuple[str, ...]] + observed_connect_principals: dict[str, tuple[str, ...]] + absent_connect_principals: dict[str, tuple[str, ...]] + schema_domains: dict[str, dict[str, EnumDatabaseSchemaDomain]] + observed_schema_owners: dict[str, dict[str, str]] + absent_schemas: dict[str, tuple[str, ...]] + principal_domains: dict[str, tuple[EnumDatabaseSchemaDomain, ...]] + allowed_memberships: tuple[ModelApplicationDatabaseRoleMembership, ...] + observed_objects: tuple[ModelApplicationDatabaseCatalogObjectEvidence, ...] + objects: tuple[ModelApplicationDatabaseAclObject, ...] + rows: tuple[ModelApplicationDatabaseAclRow, ...] + default_privileges: tuple[ModelApplicationDatabaseDefaultAclRow, ...] + blockers: tuple[str, ...] = () + excluded_objects: tuple[str, ...] = () + + @model_validator(mode="after") + def validate_matrix(self) -> ModelApplicationDatabaseAclMatrix: + """Keep the artifact deterministic, unique, and honest about blockers.""" + source_keys = [source.source_key for source in self.sources] + if len(set(source_keys)) != len(source_keys): + raise ValueError("ACL matrix source keys must be unique") + if len(set(self.verified_evidence_source_keys)) != len( + self.verified_evidence_source_keys + ): + raise ValueError("verified evidence source keys must be unique") + source_by_key = {source.source_key: source for source in self.sources} + unknown_verified = sorted( + set(self.verified_evidence_source_keys) - set(source_by_key) + ) + if unknown_verified: + raise ValueError( + f"verified evidence contains unknown source keys: {unknown_verified!r}" + ) + invalid_verified = sorted( + source_key + for source_key in self.verified_evidence_source_keys + if source_by_key[source_key].purpose + not in {"catalog_result_evidence", "activity_result_evidence"} + ) + if invalid_verified: + raise ValueError( + "only catalog/activity result blobs can be semantically verified: " + f"{invalid_verified!r}" + ) + if ( + self.authorization_scope + is EnumApplicationDatabaseAclAuthorizationScope.DEPLOYMENT + and (self.status == "READY" or self.scaffold_status == "READY") + ): + locked_result_keys = { + source.source_key + for source in self.sources + if source.purpose + in {"catalog_result_evidence", "activity_result_evidence"} + } + if not locked_result_keys or locked_result_keys != set( + self.verified_evidence_source_keys + ): + raise ValueError( + "READY deployment phase requires every locked catalog/activity " + "result source to be semantically verified" + ) + object_ids = [item.identity for item in self.objects] + if len(set(object_ids)) != len(object_ids): + raise ValueError("ACL matrix object identities must be unique") + row_ids = [row.identity for row in self.rows] + if len(set(row_ids)) != len(row_ids): + raise ValueError("ACL matrix row identities must be unique") + default_ids = [ + ( + row.owner, + row.database_ref, + row.schema_ref, + row.object_type, + row.grantee, + ) + for row in self.default_privileges + ] + if len(set(default_ids)) != len(default_ids): + raise ValueError("Default ACL row identities must be unique") + membership_ids = [item.identity for item in self.allowed_memberships] + if len(set(membership_ids)) != len(membership_ids): + raise ValueError("Role membership identities must be unique") + if set(self.declared_principals) != set(self.observed_principals) or set( + self.declared_principals + ) != set(self.absent_principals): + raise ValueError( + "declared/observed/absent principal database keys disagree" + ) + if set(self.schema_domains) != set(self.declared_principals): + raise ValueError("schema_domains and principal database keys disagree") + if set(self.observed_schema_owners) != set(self.schema_domains) or set( + self.absent_schemas + ) != set(self.schema_domains): + raise ValueError("schema presence/absence database keys are incomplete") + for database_ref, schemas in self.schema_domains.items(): + observed_schemas = set(self.observed_schema_owners[database_ref]) + absent_schemas = set(self.absent_schemas[database_ref]) + if observed_schemas & absent_schemas: + raise ValueError( + f"{database_ref}: schema presence/absence evidence overlaps" + ) + if self.scaffold_status == "READY" and set(schemas) != ( + observed_schemas.union(absent_schemas) + ): + raise ValueError( + f"{database_ref}: READY scaffold schema evidence is incomplete" + ) + observed_object_ids = [obj.identity for obj in self.observed_objects] + if len(set(observed_object_ids)) != len(observed_object_ids): + raise ValueError("observed catalog object identities must be unique") + expected_object_ids = { + ( + obj.catalog_kind, + obj.schema_ref, + obj.object_ref, + obj.function_signature or "", + ) + for obj in self.objects + } + if self.status == "READY" and set(observed_object_ids) != expected_object_ids: + raise ValueError( + "READY matrix requires exact live-catalog object identity coverage" + ) + if len(set(self.required_connect_databases)) != len( + self.required_connect_databases + ): + raise ValueError("required_connect_databases must be unique") + invalid_databases = sorted( + database + for database in self.required_connect_databases + if re.fullmatch(r"[a-z_][a-z0-9_]*", database) is None + ) + if invalid_databases: + raise ValueError( + f"required_connect_databases contain unsafe names: {invalid_databases!r}" + ) + if set(self.allowed_connect_principals) != set( + self.observed_connect_principals + ) or set(self.allowed_connect_principals) != set( + self.absent_connect_principals + ): + raise ValueError("allowed/observed/absent CONNECT database keys disagree") + if set(self.observed_connect_database_owners) != set( + self.allowed_connect_principals + ): + raise ValueError( + "observed CONNECT owner and principal database keys disagree" + ) + if not set(self.allowed_connect_principals) <= set( + self.required_connect_databases + ): + raise ValueError("CONNECT policies contain unrequired databases") + if self.status == "READY" and set(self.allowed_connect_principals) != set( + self.required_connect_databases + ): + raise ValueError("READY matrix requires CONNECT policy for every database") + for database, allowed in self.allowed_connect_principals.items(): + observed = self.observed_connect_principals[database] + absent = self.absent_connect_principals[database] + if set(observed) & set(absent): + raise ValueError( + f"{database}: CONNECT presence and absence evidence overlap" + ) + if not set(allowed) <= set(observed).union(absent): + raise ValueError( + f"{database}: allowed CONNECT principals lack presence/absence evidence" + ) + globally_observed = { + principal + for principals in ( + *self.observed_principals.values(), + *self.observed_connect_principals.values(), + self.observed_owner_roles, + tuple(self.observed_connect_database_owners.values()), + ) + for principal in principals + } + globally_absent = { + principal + for principals in ( + *self.absent_principals.values(), + *self.absent_connect_principals.values(), + self.absent_owner_roles, + ) + for principal in principals + } + global_evidence_conflicts = sorted(globally_observed & globally_absent) + if global_evidence_conflicts: + raise ValueError( + "cluster-global role presence/absence evidence conflicts: " + f"{global_evidence_conflicts!r}" + ) + managed_owner_evidence = set(self.observed_owner_roles).union( + self.absent_owner_roles + ) + workload_evidence = { + principal + for principals in ( + *self.declared_principals.values(), + *self.observed_principals.values(), + *self.absent_principals.values(), + *self.allowed_connect_principals.values(), + *self.observed_connect_principals.values(), + *self.absent_connect_principals.values(), + ) + for principal in principals + } + role_kind_overlap = sorted(managed_owner_evidence & workload_evidence) + if role_kind_overlap: + raise ValueError( + "managed owner and workload role evidence overlap: " + f"{role_kind_overlap!r}" + ) + invalid_owner_evidence = sorted( + role + for role in managed_owner_evidence.union( + self.observed_connect_database_owners.values() + ) + if re.fullmatch(r"[a-z_][a-z0-9_]*", role) is None + ) + if invalid_owner_evidence: + raise ValueError( + f"owner evidence contains unsafe role names: {invalid_owner_evidence!r}" + ) + desired_managed_owners = ( + set(self.database_owners.values()) + .union(obj.owner for obj in self.objects) + .union(row.owner for row in self.default_privileges) + ) + if self.scaffold_status == "READY" and not desired_managed_owners <= ( + managed_owner_evidence + ): + raise ValueError( + "READY scaffold managed owners lack presence/absence evidence: " + f"{sorted(desired_managed_owners - managed_owner_evidence)!r}" + ) + governed_roles = [state.role for state in self.governed_role_states] + if len(set(governed_roles)) != len(governed_roles): + raise ValueError("governed role states must have unique role names") + if self.scaffold_status == "READY" and not set(governed_roles) <= ( + globally_observed.union(globally_absent) + ): + raise ValueError( + "READY scaffold governed roles lack presence/absence evidence" + ) + observed_state_roles = [state.role for state in self.observed_role_states] + if len(set(observed_state_roles)) != len(observed_state_roles): + raise ValueError("observed role states must have unique role names") + observed_principal_roles = { + principal + for principals in ( + *self.observed_principals.values(), + *self.observed_connect_principals.values(), + self.observed_owner_roles, + ) + for principal in principals + } + if set(observed_state_roles) != observed_principal_roles: + raise ValueError( + "observed role states must cover the global observed principal and " + "managed-owner census" + ) + retained_admins = set(self.retained_administrative_principals) + if len(retained_admins) != len(self.retained_administrative_principals): + raise ValueError("retained administrative principals must be unique") + if not retained_admins <= observed_principal_roles: + raise ValueError( + "retained administrative principals must be in the observed census" + ) + if retained_admins & set(governed_roles): + raise ValueError( + "retained administrative principals cannot be governed roles" + ) + if not set(self.database_owners) <= set(self.declared_principals): + raise ValueError("database_owners contain unknown database keys") + if self.status == "READY" and set(self.database_owners) != set( + self.declared_principals + ): + raise ValueError("READY matrix requires an owner for every database") + if any( + len(set(principals)) != len(principals) + for principals in ( + *self.declared_principals.values(), + *self.observed_principals.values(), + *self.absent_principals.values(), + ) + ): + raise ValueError("principal collections must be unique") + if (self.status == "READY") != (not self.blockers): + raise ValueError("READY status and blockers disagree") + if (self.scaffold_status == "READY") != (not self.scaffold_blockers): + raise ValueError("READY scaffold_status and scaffold_blockers disagree") + return self + + +__all__ = ["ModelApplicationDatabaseAclMatrix"] diff --git a/src/omnibase_infra/validation/models/model_application_database_acl_object.py b/src/omnibase_infra/validation/models/model_application_database_acl_object.py new file mode 100644 index 0000000000..044eb68c22 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_acl_object.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Concrete owned object in a generated application-database ACL matrix.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_core.enums.enum_database_grant_object_type import ( + EnumDatabaseGrantObjectType, +) +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_infra.validation.types.type_application_database_function_signature import ( + ApplicationDatabaseFunctionSignature, +) + + +class ModelApplicationDatabaseAclObject(BaseModel): + """One concrete database object projected from typed ownership evidence.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + database_ref: str = Field(..., min_length=1) + physical_database: str = Field(..., min_length=1) + schema_ref: str = Field(..., min_length=1) + domain: EnumDatabaseSchemaDomain + object_type: EnumDatabaseGrantObjectType + object_ref: str = Field(..., min_length=1) + catalog_kind: Literal[ + "table", + "view", + "materialized_view", + "foreign_table", + "sequence", + "function", + "aggregate", + "window_function", + "procedure", + "type", + "base_type", + "range_type", + "multirange_type", + ] + owner: str = Field(..., min_length=1) + owner_declaration: str = Field(..., min_length=1) + target_materialized: bool + function_signature: ApplicationDatabaseFunctionSignature | None = None + source_keys: tuple[str, ...] = Field(..., min_length=1) + + @model_validator(mode="after") + def validate_catalog_identity(self) -> ModelApplicationDatabaseAclObject: + """Tie the grant target to one exact PostgreSQL catalog object shape.""" + expected_object_type = { + "table": EnumDatabaseGrantObjectType.TABLE, + "view": EnumDatabaseGrantObjectType.TABLE, + "materialized_view": EnumDatabaseGrantObjectType.TABLE, + "foreign_table": EnumDatabaseGrantObjectType.TABLE, + "sequence": EnumDatabaseGrantObjectType.SEQUENCE, + "function": EnumDatabaseGrantObjectType.FUNCTION, + "aggregate": EnumDatabaseGrantObjectType.FUNCTION, + "window_function": EnumDatabaseGrantObjectType.FUNCTION, + "procedure": EnumDatabaseGrantObjectType.FUNCTION, + "type": EnumDatabaseGrantObjectType.TYPE, + "base_type": EnumDatabaseGrantObjectType.TYPE, + "range_type": EnumDatabaseGrantObjectType.TYPE, + "multirange_type": EnumDatabaseGrantObjectType.TYPE, + }[self.catalog_kind] + if self.object_type is not expected_object_type: + raise ValueError( + f"catalog_kind={self.catalog_kind!r} requires " + f"object_type={expected_object_type.value!r}" + ) + is_routine = self.catalog_kind in { + "function", + "aggregate", + "window_function", + "procedure", + } + if not is_routine and self.function_signature is not None: + raise ValueError( + "function_signature is only valid for routine catalog kinds" + ) + return self + + @property + def identity( + self, + ) -> tuple[str, str, EnumDatabaseGrantObjectType, str, str | None]: + """Return the exact PostgreSQL object identity, including overloads.""" + return ( + self.database_ref, + self.schema_ref, + self.object_type, + self.object_ref, + self.function_signature, + ) + + +__all__ = ["ModelApplicationDatabaseAclObject"] diff --git a/src/omnibase_infra/validation/models/model_application_database_acl_policy.py b/src/omnibase_infra/validation/models/model_application_database_acl_policy.py new file mode 100644 index 0000000000..b71a4e3b62 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_acl_policy.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Independent typed domain policy for application database principals.""" + +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_infra.validation.enums.enum_application_database_acl_policy_source_kind import ( + EnumApplicationDatabaseAclPolicySourceKind, +) +from omnibase_infra.validation.models.model_application_database_connection_policy import ( + ModelApplicationDatabaseConnectionPolicy, +) +from omnibase_infra.validation.models.model_application_database_role_state import ( + ModelApplicationDatabaseRoleState, +) + +_SQL_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]*$") + + +class ModelApplicationDatabaseAclPolicy(BaseModel): + """Allowed schema domains declared independently from grants being checked.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: Literal["1.0"] = "1.0" + database_ref: str = Field(..., min_length=1) + physical_database: str = Field(..., min_length=1) + completion_status: str = Field(..., min_length=1) + source_kind: EnumApplicationDatabaseAclPolicySourceKind + principal_domains: dict[str, tuple[EnumDatabaseSchemaDomain, ...]] = Field( + ..., min_length=1 + ) + database_owner_role: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + migration_principal: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + migration_owner_roles: tuple[str, ...] = Field(..., min_length=1) + governed_role_states: tuple[ModelApplicationDatabaseRoleState, ...] = Field( + ..., + min_length=1, + ) + retained_administrative_principals: tuple[str, ...] = () + connection_policies: tuple[ModelApplicationDatabaseConnectionPolicy, ...] = Field( + ..., min_length=1 + ) + reason: str = Field(..., min_length=1) + + @field_validator("principal_domains") + @classmethod + def validate_principal_refs( + cls, + value: dict[str, tuple[EnumDatabaseSchemaDomain, ...]], + ) -> dict[str, tuple[EnumDatabaseSchemaDomain, ...]]: + """Require safe principals and unique, nonempty domain declarations.""" + invalid = sorted( + principal + for principal in value + if _SQL_IDENTIFIER.fullmatch(principal) is None + ) + if invalid: + raise ValueError( + f"principal_domains contain unsafe identifiers: {invalid!r}" + ) + return value + + @field_validator("migration_owner_roles") + @classmethod + def validate_migration_owner_roles(cls, values: tuple[str, ...]) -> tuple[str, ...]: + """Require unique canonical owner-role identifiers.""" + if len(set(values)) != len(values): + raise ValueError("migration_owner_roles must be unique") + invalid = sorted( + value for value in values if _SQL_IDENTIFIER.fullmatch(value) is None + ) + if invalid: + raise ValueError( + f"migration_owner_roles contain unsafe identifiers: {invalid!r}" + ) + return values + + @field_validator("retained_administrative_principals") + @classmethod + def validate_retained_administrative_principals( + cls, + values: tuple[str, ...], + ) -> tuple[str, ...]: + """Require an explicit, unique list of untouched administrative roles.""" + if len(set(values)) != len(values): + raise ValueError("retained administrative principals must be unique") + invalid = sorted( + value for value in values if _SQL_IDENTIFIER.fullmatch(value) is None + ) + if invalid: + raise ValueError( + "retained administrative principals contain unsafe identifiers: " + f"{invalid!r}" + ) + return values + + @model_validator(mode="after") + def validate_domains(self) -> ModelApplicationDatabaseAclPolicy: + """Reject empty or duplicate domain declarations.""" + invalid = sorted( + principal + for principal, domains in self.principal_domains.items() + if not domains or len(set(domains)) != len(domains) + ) + if invalid: + raise ValueError( + f"principal_domains must be nonempty and unique for {invalid!r}" + ) + policy_refs = [policy.database_ref for policy in self.connection_policies] + physical_databases = [ + policy.physical_database for policy in self.connection_policies + ] + if len(set(policy_refs)) != len(policy_refs): + raise ValueError("connection policy database_refs must be unique") + if len(set(physical_databases)) != len(physical_databases): + raise ValueError("connection policy physical databases must be unique") + governed_roles = [state.role for state in self.governed_role_states] + if len(set(governed_roles)) != len(governed_roles): + raise ValueError("governed role states must have unique role names") + return self + + +__all__ = ["ModelApplicationDatabaseAclPolicy"] diff --git a/src/omnibase_infra/validation/models/model_application_database_acl_row.py b/src/omnibase_infra/validation/models/model_application_database_acl_row.py new file mode 100644 index 0000000000..f9f6a1b603 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_acl_row.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Explicit principal-target cell in an application-database ACL matrix.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_core.enums.enum_database_grant_object_type import ( + EnumDatabaseGrantObjectType, +) +from omnibase_core.enums.enum_database_privilege import EnumDatabasePrivilege +from omnibase_infra.validation.types.type_application_database_function_signature import ( + ApplicationDatabaseFunctionSignature, +) + + +class ModelApplicationDatabaseAclRow(BaseModel): + """One explicit principal/target cell; empty privileges mean deny.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + principal: str = Field(..., min_length=1) + database_ref: str = Field(..., min_length=1) + physical_database: str = Field(..., min_length=1) + object_type: EnumDatabaseGrantObjectType + schema_ref: str | None = None + object_ref: str | None = None + function_signature: ApplicationDatabaseFunctionSignature | None = None + privileges: tuple[EnumDatabasePrivilege, ...] = () + + @model_validator(mode="after") + def validate_target(self) -> ModelApplicationDatabaseAclRow: + """Require an exact target shape and reject duplicate privileges.""" + if len(set(self.privileges)) != len(self.privileges): + raise ValueError("ACL row privileges must be unique") + if self.object_type is EnumDatabaseGrantObjectType.DATABASE: + if ( + self.schema_ref is not None + or self.object_ref is not None + or self.function_signature is not None + ): + raise ValueError("DATABASE ACL rows cannot name schema/object") + elif self.object_type is EnumDatabaseGrantObjectType.SCHEMA: + if ( + self.schema_ref is None + or self.object_ref is not None + or self.function_signature is not None + ): + raise ValueError("SCHEMA ACL rows require schema and no object") + elif self.schema_ref is None or self.object_ref is None: + raise ValueError("Object ACL rows require schema and object") + elif ( + self.object_type is not EnumDatabaseGrantObjectType.FUNCTION + and self.function_signature is not None + ): + raise ValueError("Only FUNCTION ACL rows can carry function_signature") + return self + + @property + def identity( + self, + ) -> tuple[ + str, + str, + EnumDatabaseGrantObjectType, + str | None, + str | None, + str | None, + ]: + return ( + self.principal, + self.database_ref, + self.object_type, + self.schema_ref, + self.object_ref, + self.function_signature, + ) + + +__all__ = ["ModelApplicationDatabaseAclRow"] diff --git a/src/omnibase_infra/validation/models/model_application_database_acl_source.py b/src/omnibase_infra/validation/models/model_application_database_acl_source.py new file mode 100644 index 0000000000..66824059cb --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_acl_source.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Immutable source record for a generated application-database ACL matrix.""" + +from __future__ import annotations + +from pathlib import PurePosixPath +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class ModelApplicationDatabaseAclSource(BaseModel): + """One immutable contract or evidence blob used to generate the matrix.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + source_key: str = Field(..., pattern=r"^[a-z][a-z0-9_]*$") + repository: str = Field( + ..., + pattern=r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", + ) + revision: str = Field(..., pattern=r"^[0-9a-f]{40}$") + path: str = Field(..., pattern=r"^[A-Za-z0-9_./-]+$") + sha256: str = Field(..., pattern=r"^[0-9a-f]{64}$") + purpose: Literal[ + "topology", + "relation_inventory", + "service_ownership", + "rendered_topology", + "typed_loader", + "legacy_fixture", + "principal_inventory", + "acl_policy", + "catalog_query_evidence", + "catalog_result_evidence", + "activity_query_evidence", + "activity_result_evidence", + ] + + @field_validator("path") + @classmethod + def validate_relative_path(cls, value: str) -> str: + """Reject absolute, parent-traversing, or noncanonical source paths.""" + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts or "." in path.parts: + raise ValueError("source path must be canonical and repository-relative") + return value + + +__all__ = ["ModelApplicationDatabaseAclSource"] diff --git a/src/omnibase_infra/validation/models/model_application_database_activity_evidence.py b/src/omnibase_infra/validation/models/model_application_database_activity_evidence.py new file mode 100644 index 0000000000..30339377ac --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_activity_evidence.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Durable full-day activity evidence for an application database.""" + +from __future__ import annotations + +from datetime import datetime, timedelta + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, model_validator + + +class ModelApplicationDatabaseActivityEvidence(BaseModel): + """Hash-addressed query/result evidence covering at least 24 hours.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + window_started_at: AwareDatetime + window_ended_at: AwareDatetime + query_sha256: str = Field(..., pattern=r"^[0-9a-f]{64}$") + result_sha256: str = Field(..., pattern=r"^[0-9a-f]{64}$") + query_source_key: str = Field(..., pattern=r"^[a-z][a-z0-9_]*$") + result_source_key: str = Field(..., pattern=r"^[a-z][a-z0-9_]*$") + observation_count: int = Field(..., ge=0) + + @model_validator(mode="after") + def validate_full_day_window(self) -> ModelApplicationDatabaseActivityEvidence: + """Require an ordered interval spanning a complete day.""" + started_at = datetime.fromisoformat(self.window_started_at.isoformat()) + ended_at = datetime.fromisoformat(self.window_ended_at.isoformat()) + if ended_at - started_at < timedelta(hours=24): + raise ValueError("activity evidence window must span at least 24 hours") + return self + + +__all__ = ["ModelApplicationDatabaseActivityEvidence"] diff --git a/src/omnibase_infra/validation/models/model_application_database_activity_principal_observation.py b/src/omnibase_infra/validation/models/model_application_database_activity_principal_observation.py new file mode 100644 index 0000000000..9ab2a078c0 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_activity_principal_observation.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""One activity observation in an application-database evidence result.""" + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelApplicationDatabaseActivityPrincipalObservation(BaseModel): + """One exact ``(datname, usename)`` pair with its sample count.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + principal: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + observation_count: int = Field(..., ge=1) + + +__all__ = ["ModelApplicationDatabaseActivityPrincipalObservation"] diff --git a/src/omnibase_infra/validation/models/model_application_database_activity_result_evidence.py b/src/omnibase_infra/validation/models/model_application_database_activity_result_evidence.py new file mode 100644 index 0000000000..71e9fa7baa --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_activity_result_evidence.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed full-day application-database activity-result evidence.""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import Literal + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, model_validator + +from omnibase_infra.validation.models.model_application_database_activity_principal_observation import ( + ModelApplicationDatabaseActivityPrincipalObservation, +) + + +class ModelApplicationDatabaseActivityResultEvidence(BaseModel): + """Canonical full-day activity-result content for one database.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: Literal["1.0"] = "1.0" + database_ref: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + physical_database: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + window_started_at: AwareDatetime + window_ended_at: AwareDatetime + activity_query_sha256: str = Field(..., pattern=r"^[0-9a-f]{64}$") + observation_count: int = Field(..., ge=0) + active_principals: tuple[ + ModelApplicationDatabaseActivityPrincipalObservation, + ..., + ] + + @model_validator(mode="after") + def validate_full_day_window( + self, + ) -> ModelApplicationDatabaseActivityResultEvidence: + """Reject unordered or partial-day result windows.""" + started_at = datetime.fromisoformat(self.window_started_at.isoformat()) + ended_at = datetime.fromisoformat(self.window_ended_at.isoformat()) + if ended_at - started_at < timedelta(hours=24): + raise ValueError("activity result window must span at least 24 hours") + principals = [row.principal for row in self.active_principals] + if len(set(principals)) != len(principals): + raise ValueError("activity result principal pairs must be unique") + observed_count = sum(row.observation_count for row in self.active_principals) + if observed_count != self.observation_count: + raise ValueError( + "activity result observation_count must equal exact pair-row counts" + ) + return self + + +__all__ = ["ModelApplicationDatabaseActivityResultEvidence"] diff --git a/src/omnibase_infra/validation/models/model_application_database_catalog_identity.py b/src/omnibase_infra/validation/models/model_application_database_catalog_identity.py new file mode 100644 index 0000000000..662afae43e --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_catalog_identity.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""PostgreSQL-catalog identity used by the exact application census gate.""" + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.validation.enums.enum_application_inventory_object_kind import ( + EnumApplicationInventoryObjectKind, +) +from omnibase_infra.validation.types.type_application_database_function_signature import ( + ApplicationDatabaseFunctionSignature, +) + + +class ModelApplicationDatabaseCatalogIdentity(BaseModel): + """One observed base relation, view, materialized view, or function.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + schema: str = Field( # type: ignore[assignment] + ..., pattern=r"^[a-z_][a-z0-9_]*$" + ) + name: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + kind: EnumApplicationInventoryObjectKind + function_signature: ApplicationDatabaseFunctionSignature | None = None + + @property + def identity( + self, + ) -> tuple[ + str, + str, + EnumApplicationInventoryObjectKind, + ApplicationDatabaseFunctionSignature | None, + ]: + """Return the exact application-catalog identity.""" + return (self.schema, self.name, self.kind, self.function_signature) + + +__all__ = ["ModelApplicationDatabaseCatalogIdentity"] diff --git a/src/omnibase_infra/validation/models/model_application_database_catalog_object_evidence.py b/src/omnibase_infra/validation/models/model_application_database_catalog_object_evidence.py new file mode 100644 index 0000000000..7993dd80e4 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_catalog_object_evidence.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Exact live-catalog identity and owner for one managed-schema object.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_infra.validation.types.type_application_database_function_signature import ( + ApplicationDatabaseFunctionSignature, +) + + +class ModelApplicationDatabaseCatalogObjectEvidence(BaseModel): + """Current object identity used by the transaction-start catalog guard.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_ref: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + catalog_kind: Literal[ + "table", + "view", + "materialized_view", + "foreign_table", + "sequence", + "function", + "aggregate", + "window_function", + "procedure", + "type", + "base_type", + "range_type", + "multirange_type", + ] + object_ref: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + function_signature: ApplicationDatabaseFunctionSignature | None = None + owner: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + + @model_validator(mode="after") + def validate_signature_shape( + self, + ) -> ModelApplicationDatabaseCatalogObjectEvidence: + """Require exact signatures only for routines.""" + routine = self.catalog_kind in { + "function", + "aggregate", + "window_function", + "procedure", + } + if routine != (self.function_signature is not None): + raise ValueError( + "catalog routine evidence requires a signature and non-routines forbid it" + ) + return self + + @property + def identity(self) -> tuple[str, str, str, str]: + """Return the stable live-catalog identity.""" + return ( + self.catalog_kind, + self.schema_ref, + self.object_ref, + self.function_signature or "", + ) + + +__all__ = ["ModelApplicationDatabaseCatalogObjectEvidence"] diff --git a/src/omnibase_infra/validation/models/model_application_database_catalog_result_evidence.py b/src/omnibase_infra/validation/models/model_application_database_catalog_result_evidence.py new file mode 100644 index 0000000000..20d55798a6 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_catalog_result_evidence.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed application-database catalog-result evidence.""" + +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from omnibase_infra.validation.models.model_application_database_catalog_object_evidence import ( + ModelApplicationDatabaseCatalogObjectEvidence, +) +from omnibase_infra.validation.models.model_application_database_observed_role_state import ( + ModelApplicationDatabaseObservedRoleState, +) + +_SQL_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]*$") + + +class ModelApplicationDatabaseCatalogResultEvidence(BaseModel): + """Canonical catalog-result content bound to an immutable source blob.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: Literal["1.0"] = "1.0" + database_ref: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + physical_database: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + completion_status: str = Field(..., min_length=1) + catalog_parity_status: str = Field(..., min_length=1) + catalog_query_sha256: str = Field(..., pattern=r"^[0-9a-f]{64}$") + database_owner_role: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + principal_refs: tuple[str, ...] = Field(..., min_length=1) + absent_principal_refs: tuple[str, ...] = () + owner_refs: tuple[str, ...] = () + absent_owner_refs: tuple[str, ...] = () + observed_role_states: tuple[ModelApplicationDatabaseObservedRoleState, ...] + observed_schema_owners: dict[str, str] = Field(default_factory=dict) + absent_schema_refs: tuple[str, ...] = () + observed_objects: tuple[ModelApplicationDatabaseCatalogObjectEvidence, ...] = () + + @field_validator( + "principal_refs", + "absent_principal_refs", + "owner_refs", + "absent_owner_refs", + "absent_schema_refs", + ) + @classmethod + def validate_role_refs(cls, values: tuple[str, ...]) -> tuple[str, ...]: + """Require unique, safely quotable role names in result content.""" + if len(set(values)) != len(values): + raise ValueError("catalog evidence role references must be unique") + invalid = sorted( + value for value in values if _SQL_IDENTIFIER.fullmatch(value) is None + ) + if invalid: + raise ValueError( + f"catalog evidence contains unsafe role references: {invalid!r}" + ) + return values + + @model_validator(mode="after") + def validate_role_census( + self, + ) -> ModelApplicationDatabaseCatalogResultEvidence: + """Keep positive, negative, workload, and managed-owner evidence disjoint.""" + principal_overlap = set(self.principal_refs) & set(self.absent_principal_refs) + if principal_overlap: + raise ValueError( + "catalog principal presence/absence evidence overlaps: " + f"{sorted(principal_overlap)!r}" + ) + owner_overlap = set(self.owner_refs) & set(self.absent_owner_refs) + if owner_overlap: + raise ValueError( + "catalog owner presence/absence evidence overlaps: " + f"{sorted(owner_overlap)!r}" + ) + role_kind_overlap = set(self.principal_refs).union( + self.absent_principal_refs + ) & set(self.owner_refs).union(self.absent_owner_refs) + if role_kind_overlap: + raise ValueError( + "catalog principal and managed-owner evidence overlaps: " + f"{sorted(role_kind_overlap)!r}" + ) + state_roles = [state.role for state in self.observed_role_states] + expected_state_roles = set(self.principal_refs).union(self.owner_refs) + if ( + len(set(state_roles)) != len(state_roles) + or set(state_roles) != expected_state_roles + ): + raise ValueError( + "catalog observed role states must cover the exact principal and " + "managed-owner census" + ) + invalid_schema_owners = sorted( + schema + for schema, owner in self.observed_schema_owners.items() + if _SQL_IDENTIFIER.fullmatch(schema) is None + or _SQL_IDENTIFIER.fullmatch(owner) is None + ) + if invalid_schema_owners: + raise ValueError( + "catalog observed schema owners contain unsafe identifiers: " + f"{invalid_schema_owners!r}" + ) + schema_overlap = set(self.observed_schema_owners) & set(self.absent_schema_refs) + if schema_overlap: + raise ValueError( + "catalog schema presence/absence evidence overlaps: " + f"{sorted(schema_overlap)!r}" + ) + object_identities = [obj.identity for obj in self.observed_objects] + if len(set(object_identities)) != len(object_identities): + raise ValueError("catalog observed object identities must be unique") + unknown_object_schemas = sorted( + {obj.schema_ref for obj in self.observed_objects} + - set(self.observed_schema_owners) + ) + if unknown_object_schemas: + raise ValueError( + "catalog objects require observed schema-owner evidence: " + f"{unknown_object_schemas!r}" + ) + classified_present_roles = ( + set(self.principal_refs) + .union(self.owner_refs) + .union({self.database_owner_role}) + ) + unclassified_catalog_owners = sorted( + set(self.observed_schema_owners.values()).union( + obj.owner for obj in self.observed_objects + ) + - classified_present_roles + ) + if unclassified_catalog_owners: + raise ValueError( + "catalog schema/object owners must be classified in the present " + f"role census: {unclassified_catalog_owners!r}" + ) + return self + + +__all__ = ["ModelApplicationDatabaseCatalogResultEvidence"] diff --git a/src/omnibase_infra/validation/models/model_application_database_column_state.py b/src/omnibase_infra/validation/models/model_application_database_column_state.py new file mode 100644 index 0000000000..286d3e0447 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_column_state.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Catalog-observed column state for application-domain enforcement.""" + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelApplicationDatabaseColumnState(BaseModel): + """One exact column shape after a migration is applied.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + data_type: str = Field(..., min_length=1) + nullable: bool + default_expression: str | None = None + generated_expression: str | None = None + + +__all__ = ["ModelApplicationDatabaseColumnState"] diff --git a/src/omnibase_infra/validation/models/model_application_database_connection_policy.py b/src/omnibase_infra/validation/models/model_application_database_connection_policy.py new file mode 100644 index 0000000000..7996ddad85 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_connection_policy.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Explicit CONNECT policy for one PostgreSQL database.""" + +from __future__ import annotations + +import re + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +_SQL_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]*$") + + +class ModelApplicationDatabaseConnectionPolicy(BaseModel): + """Principals allowed to connect to one protected physical database.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + database_ref: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + physical_database: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + observed_database_owner_role: str = Field( + ..., + pattern=r"^[a-z_][a-z0-9_]*$", + ) + allowed_principals: tuple[str, ...] = Field(..., min_length=1) + + @field_validator("allowed_principals") + @classmethod + def validate_allowed_principals(cls, values: tuple[str, ...]) -> tuple[str, ...]: + """Require a unique, safely quotable allowlist.""" + if len(set(values)) != len(values): + raise ValueError("allowed_principals must be unique") + invalid = sorted( + value for value in values if _SQL_IDENTIFIER.fullmatch(value) is None + ) + if invalid: + raise ValueError( + f"allowed_principals contain unsafe identifiers: {invalid!r}" + ) + return values + + +__all__ = ["ModelApplicationDatabaseConnectionPolicy"] diff --git a/src/omnibase_infra/validation/models/model_application_database_default_acl_row.py b/src/omnibase_infra/validation/models/model_application_database_default_acl_row.py new file mode 100644 index 0000000000..a1d5ac3a01 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_default_acl_row.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Future-object privilege cell in an application-database ACL matrix.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_core.enums.enum_database_grant_object_type import ( + EnumDatabaseGrantObjectType, +) +from omnibase_core.enums.enum_database_privilege import EnumDatabasePrivilege + + +class ModelApplicationDatabaseDefaultAclRow(BaseModel): + """One future-object privilege cell for an actual schema owner.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + owner: str = Field(..., min_length=1) + database_ref: str = Field(..., min_length=1) + physical_database: str = Field(..., min_length=1) + schema_ref: str = Field(..., min_length=1) + object_type: Literal[ + EnumDatabaseGrantObjectType.TABLE, + EnumDatabaseGrantObjectType.SEQUENCE, + EnumDatabaseGrantObjectType.FUNCTION, + EnumDatabaseGrantObjectType.TYPE, + ] + grantee: str = Field(..., min_length=1) + privileges: tuple[EnumDatabasePrivilege, ...] = () + + +__all__ = ["ModelApplicationDatabaseDefaultAclRow"] diff --git a/src/omnibase_infra/validation/models/model_application_database_enforcement_contract.py b/src/omnibase_infra/validation/models/model_application_database_enforcement_contract.py new file mode 100644 index 0000000000..73febb2f73 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_enforcement_contract.py @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed activation contract for application database domain gates.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from types import MappingProxyType +from typing import Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + field_serializer, + field_validator, + model_validator, +) + +from omnibase_infra.validation.application_database_red_control_registry import ( + APPLICATION_DATABASE_RED_CONTROL_REGISTRY, +) +from omnibase_infra.validation.enums.enum_application_database_enforcement_gate import ( + EnumApplicationDatabaseEnforcementGate, +) +from omnibase_infra.validation.models.model_application_database_enforcement_gate_state import ( + ModelApplicationDatabaseEnforcementGateState, +) + +_REVISION = re.compile(r"^[0-9a-f]{40}$") +_PIN_KEY = re.compile(r"^[a-z0-9_]+#[0-9]+$") + + +class ModelApplicationDatabaseEnforcementContract(BaseModel): + """Complete gate family, immutable source pins, and activation truth.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: Literal["1.0"] = "1.0" + ticket: Literal["OMN-15361"] + predecessor_pins: Mapping[str, str] = Field(..., min_length=1) + gates: Mapping[ + EnumApplicationDatabaseEnforcementGate, + ModelApplicationDatabaseEnforcementGateState, + ] = Field(..., min_length=1) + + @field_validator("predecessor_pins") + @classmethod + def validate_predecessor_pins(cls, pins: Mapping[str, str]) -> Mapping[str, str]: + """Require immutable full Git revisions with unambiguous PR identities.""" + invalid_keys = sorted(key for key in pins if _PIN_KEY.fullmatch(key) is None) + invalid_revisions = sorted( + revision + for revision in pins.values() + if _REVISION.fullmatch(revision) is None + ) + if invalid_keys: + raise ValueError(f"invalid predecessor pin keys: {invalid_keys}") + if invalid_revisions: + raise ValueError( + "predecessor pins must use full lowercase commit revisions" + ) + if len(set(pins.values())) != len(pins): + raise ValueError( + "predecessor revisions must identify distinct source heads" + ) + return MappingProxyType(dict(pins)) + + @field_validator("gates") + @classmethod + def freeze_gates( + cls, + gates: Mapping[ + EnumApplicationDatabaseEnforcementGate, + ModelApplicationDatabaseEnforcementGateState, + ], + ) -> Mapping[ + EnumApplicationDatabaseEnforcementGate, + ModelApplicationDatabaseEnforcementGateState, + ]: + """Freeze the complete validated gate family against in-place mutation.""" + return MappingProxyType(dict(gates)) + + @field_serializer("predecessor_pins") + def serialize_predecessor_pins(self, pins: Mapping[str, str]) -> dict[str, str]: + """Restore the YAML/JSON wire shape for the immutable mapping.""" + return dict(pins) + + @field_serializer("gates") + def serialize_gates( + self, + gates: Mapping[ + EnumApplicationDatabaseEnforcementGate, + ModelApplicationDatabaseEnforcementGateState, + ], + ) -> dict[ + EnumApplicationDatabaseEnforcementGate, + ModelApplicationDatabaseEnforcementGateState, + ]: + """Restore the YAML/JSON wire shape for the immutable mapping.""" + return dict(gates) + + @model_validator(mode="after") + def validate_complete_gate_set(self) -> ModelApplicationDatabaseEnforcementContract: + """Reject omitted gates and RED claims without executable test bindings.""" + expected = set(EnumApplicationDatabaseEnforcementGate) + actual = set(self.gates) + if actual != expected: + missing = sorted(gate.value for gate in expected - actual) + extra = sorted(str(gate) for gate in actual - expected) + raise ValueError( + f"application database enforcement gate set drift: " + f"missing={missing}, extra={extra}" + ) + for gate, state in self.gates.items(): + declared_controls = set(state.seeded_red_controls) + registered_controls = set(APPLICATION_DATABASE_RED_CONTROL_REGISTRY[gate]) + if declared_controls != registered_controls: + missing = sorted(registered_controls - declared_controls) + extra = sorted(declared_controls - registered_controls) + raise ValueError( + "application database RED control registry drift: " + f"gate={gate.value}, missing={missing}, extra={extra}" + ) + return self + + +__all__ = ["ModelApplicationDatabaseEnforcementContract"] diff --git a/src/omnibase_infra/validation/models/model_application_database_enforcement_gate_state.py b/src/omnibase_infra/validation/models/model_application_database_enforcement_gate_state.py new file mode 100644 index 0000000000..99fa33ad5e --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_enforcement_gate_state.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Phased state for one application database enforcement gate.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +class ModelApplicationDatabaseEnforcementGateState(BaseModel): + """Mandatory source proof and honest deployment activation state.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + source_enforcement: Literal["mandatory"] + deployment_enforcement: Literal["mandatory", "blocked"] + source_proofs: tuple[str, ...] = Field(..., min_length=1) + source_proof_paths: tuple[str, ...] = Field(..., min_length=1) + seeded_red_controls: tuple[str, ...] = Field(..., min_length=1) + deployment_blockers: tuple[str, ...] = () + + @field_validator( + "source_proofs", + "source_proof_paths", + "seeded_red_controls", + "deployment_blockers", + ) + @classmethod + def validate_unique_nonempty_values( + cls, values: tuple[str, ...] + ) -> tuple[str, ...]: + """Reject duplicates and whitespace-only evidence claims.""" + if len(set(values)) != len(values): + raise ValueError("gate evidence values must be unique") + if any(not value.strip() for value in values): + raise ValueError("gate evidence values cannot be blank") + return values + + @field_validator("source_proof_paths") + @classmethod + def validate_repository_paths(cls, paths: tuple[str, ...]) -> tuple[str, ...]: + """Keep proof paths repository-relative and traversal-free.""" + invalid = sorted( + path for path in paths if path.startswith("/") or ".." in path.split("/") + ) + if invalid: + raise ValueError( + f"source proof paths must be repository-relative: {invalid}" + ) + return paths + + @model_validator(mode="after") + def validate_deployment_state(self) -> ModelApplicationDatabaseEnforcementGateState: + """A blocked deployment needs reasons; a mandatory one cannot retain them.""" + if self.deployment_enforcement == "blocked" and not self.deployment_blockers: + raise ValueError("blocked deployment enforcement requires blockers") + if self.deployment_enforcement == "mandatory" and self.deployment_blockers: + raise ValueError("mandatory deployment enforcement cannot retain blockers") + return self + + +__all__ = ["ModelApplicationDatabaseEnforcementGateState"] diff --git a/src/omnibase_infra/validation/models/model_application_database_function_state.py b/src/omnibase_infra/validation/models/model_application_database_function_state.py new file mode 100644 index 0000000000..302b496311 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_function_state.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Catalog and audit state for one application-owned PostgreSQL function.""" + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from omnibase_infra.validation.models.model_application_database_tenant_isolation_evidence import ( + ModelApplicationDatabaseTenantIsolationEvidence, +) + + +class ModelApplicationDatabaseFunctionState(BaseModel): + """Security-relevant function state plus its independent audit proof.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + owner: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + security_definer: bool + search_path: tuple[str, ...] = () + public_execute: bool + audit_id: str | None = Field(default=None, min_length=1) + definition_sha256: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$") + audited_definition_sha256: str | None = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + tenant_isolation_evidence: ( + ModelApplicationDatabaseTenantIsolationEvidence | None + ) = None + + @field_validator("search_path") + @classmethod + def validate_unique_search_path(cls, values: tuple[str, ...]) -> tuple[str, ...]: + """Reject duplicate path entries without blessing unsafe entries.""" + if len(set(values)) != len(values): + raise ValueError("function search_path entries must be unique") + return values + + +__all__ = ["ModelApplicationDatabaseFunctionState"] diff --git a/src/omnibase_infra/validation/models/model_application_database_identity_root_control_state.py b/src/omnibase_infra/validation/models/model_application_database_identity_root_control_state.py new file mode 100644 index 0000000000..18aaffb36c --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_identity_root_control_state.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Live control-plane evidence for an exceptional tenant identity root.""" + +import re + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from omnibase_infra.validation.enums.enum_application_database_identity_root_operation import ( + EnumApplicationDatabaseIdentityRootOperation, +) + + +class ModelApplicationDatabaseIdentityRootControlState(BaseModel): + """Bind declared root operations to one audited, non-runtime role.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + role: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + role_can_login: bool + role_superuser: bool + role_bypass_rls: bool + runtime_membership_principals: tuple[str, ...] + runtime_set_role_denied_principals: tuple[str, ...] + declared_operations: tuple[EnumApplicationDatabaseIdentityRootOperation, ...] + observed_operations: tuple[EnumApplicationDatabaseIdentityRootOperation, ...] + behavioral_proof_ids: tuple[str, ...] + + @field_validator( + "runtime_membership_principals", + "runtime_set_role_denied_principals", + ) + @classmethod + def validate_runtime_principal_census( + cls, + principals: tuple[str, ...], + ) -> tuple[str, ...]: + """Keep role evidence deterministic and constrained to SQL identifiers.""" + invalid = [ + principal + for principal in principals + if re.fullmatch(r"[a-z_][a-z0-9_]*", principal) is None + ] + if invalid: + raise ValueError( + f"identity-root runtime principal names are invalid: {invalid!r}" + ) + if tuple(sorted(principals)) != principals: + raise ValueError( + "identity-root runtime principal evidence must be canonically sorted" + ) + return principals + + @model_validator(mode="after") + def validate_exact_proof_shape( + self, + ) -> "ModelApplicationDatabaseIdentityRootControlState": + """Every distinct observed operation carries one durable proof identity.""" + for label, values in { + "runtime membership": self.runtime_membership_principals, + "runtime SET ROLE denial": self.runtime_set_role_denied_principals, + "declared": self.declared_operations, + "observed": self.observed_operations, + "behavioral proof": self.behavioral_proof_ids, + }.items(): + if len(set(values)) != len(values): + raise ValueError(f"identity-root {label} entries must be unique") + if len(self.behavioral_proof_ids) != len(self.observed_operations): + raise ValueError( + "identity-root observed operations require one behavioral proof each" + ) + if any(not proof.strip() for proof in self.behavioral_proof_ids): + raise ValueError("identity-root behavioral proof IDs cannot be blank") + return self + + +__all__ = ["ModelApplicationDatabaseIdentityRootControlState"] diff --git a/src/omnibase_infra/validation/models/model_application_database_observed_role_state.py b/src/omnibase_infra/validation/models/model_application_database_observed_role_state.py new file mode 100644 index 0000000000..b14877b83d --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_observed_role_state.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Catalog-observed attributes for an application-database principal role.""" + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelApplicationDatabaseObservedRoleState(BaseModel): + """Catalog-observed attributes for a present principal role.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + role: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + login: bool + superuser: bool + bypass_rls: bool + create_database: bool + create_role: bool + replication: bool + inherit: bool + + +__all__ = ["ModelApplicationDatabaseObservedRoleState"] diff --git a/src/omnibase_infra/validation/models/model_application_database_policy_state.py b/src/omnibase_infra/validation/models/model_application_database_policy_state.py new file mode 100644 index 0000000000..8c12c358d5 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_policy_state.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Catalog-observed PostgreSQL policy state for domain enforcement.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class ModelApplicationDatabasePolicyState(BaseModel): + """One policy's exact composition and predicates.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + permissive: bool + command: Literal["ALL", "SELECT", "INSERT", "UPDATE", "DELETE"] + roles: tuple[str, ...] = Field(..., min_length=1) + using_expression: str | None = None + with_check_expression: str | None = None + + @field_validator("roles") + @classmethod + def canonicalize_roles(cls, roles: tuple[str, ...]) -> tuple[str, ...]: + """Keep exact PostgreSQL role names as unique immutable set evidence.""" + if any(not role for role in roles): + raise ValueError("policy roles cannot contain an empty role name") + if len(set(roles)) != len(roles): + raise ValueError("policy roles must be unique") + return tuple(sorted(roles)) + + +__all__ = ["ModelApplicationDatabasePolicyState"] diff --git a/src/omnibase_infra/validation/models/model_application_database_pool_identity.py b/src/omnibase_infra/validation/models/model_application_database_pool_identity.py new file mode 100644 index 0000000000..efacbba679 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_pool_identity.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Read-only identity evidence from one application PostgreSQL pool.""" + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelApplicationDatabasePoolIdentity(BaseModel): + """Exact current database and user observed through one real pool.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + pool: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + current_database: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + current_user: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + + +__all__ = ["ModelApplicationDatabasePoolIdentity"] diff --git a/src/omnibase_infra/validation/models/model_application_database_principal_inventory.py b/src/omnibase_infra/validation/models/model_application_database_principal_inventory.py new file mode 100644 index 0000000000..b4f7dca269 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_principal_inventory.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed principal census for an application database ACL matrix.""" + +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from omnibase_infra.validation.enums.enum_application_database_principal_inventory_source_kind import ( + EnumApplicationDatabasePrincipalInventorySourceKind, +) +from omnibase_infra.validation.models.model_application_database_activity_evidence import ( + ModelApplicationDatabaseActivityEvidence, +) +from omnibase_infra.validation.models.model_application_database_catalog_object_evidence import ( + ModelApplicationDatabaseCatalogObjectEvidence, +) +from omnibase_infra.validation.models.model_application_database_observed_role_state import ( + ModelApplicationDatabaseObservedRoleState, +) + +_SQL_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]*$") + + +class ModelApplicationDatabasePrincipalInventory(BaseModel): + """Exact observed non-owner principal universe for one database.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: Literal["1.0"] = "1.0" + database_ref: str = Field(..., min_length=1) + physical_database: str = Field(..., min_length=1) + completion_status: str = Field(..., min_length=1) + catalog_parity_status: str = Field(..., min_length=1) + source_kind: EnumApplicationDatabasePrincipalInventorySourceKind + database_owner_role: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + principal_refs: tuple[str, ...] = Field(..., min_length=1) + absent_principal_refs: tuple[str, ...] = () + owner_refs: tuple[str, ...] = () + absent_owner_refs: tuple[str, ...] = () + observed_role_states: tuple[ModelApplicationDatabaseObservedRoleState, ...] + activity_principal_refs: tuple[str, ...] = () + observed_schema_owners: dict[str, str] = Field(default_factory=dict) + absent_schema_refs: tuple[str, ...] = () + observed_objects: tuple[ModelApplicationDatabaseCatalogObjectEvidence, ...] = () + live_database_read: bool + activity_evidence: ModelApplicationDatabaseActivityEvidence | None = None + catalog_query_sha256: str | None = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + catalog_result_sha256: str | None = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + catalog_query_source_key: str | None = Field( + default=None, + pattern=r"^[a-z][a-z0-9_]*$", + ) + catalog_result_source_key: str | None = Field( + default=None, + pattern=r"^[a-z][a-z0-9_]*$", + ) + reason: str = Field(..., min_length=1) + + @field_validator( + "principal_refs", + "absent_principal_refs", + "owner_refs", + "absent_owner_refs", + "activity_principal_refs", + "absent_schema_refs", + ) + @classmethod + def validate_principal_refs(cls, values: tuple[str, ...]) -> tuple[str, ...]: + """Reject duplicates and identifiers that cannot be safely quoted.""" + if len(set(values)) != len(values): + raise ValueError("principal_refs must be unique") + invalid = sorted( + value for value in values if _SQL_IDENTIFIER.fullmatch(value) is None + ) + if invalid: + raise ValueError(f"principal_refs contain unsafe identifiers: {invalid!r}") + return values + + @model_validator(mode="after") + def validate_provenance(self) -> ModelApplicationDatabasePrincipalInventory: + """Keep synthetic and authorized-catalog evidence distinguishable.""" + overlap = set(self.principal_refs) & set(self.absent_principal_refs) + if overlap: + raise ValueError( + f"principal presence and absence evidence overlap: {sorted(overlap)!r}" + ) + owner_overlap = set(self.owner_refs) & set(self.absent_owner_refs) + if owner_overlap: + raise ValueError( + f"owner presence and absence evidence overlap: {sorted(owner_overlap)!r}" + ) + role_kind_overlap = set(self.principal_refs).union( + self.absent_principal_refs + ) & set(self.owner_refs).union(self.absent_owner_refs) + if role_kind_overlap: + raise ValueError( + f"principal and owner evidence overlap: {sorted(role_kind_overlap)!r}" + ) + state_roles = [state.role for state in self.observed_role_states] + if len(set(state_roles)) != len(state_roles): + raise ValueError("observed role states must have unique role names") + expected_state_roles = set(self.principal_refs).union(self.owner_refs) + if set(state_roles) != expected_state_roles: + raise ValueError( + "observed role states must cover the exact present principal and " + "managed-owner census" + ) + classified_activity_roles = ( + set(self.principal_refs) + .union(self.owner_refs) + .union({self.database_owner_role}) + ) + if not set(self.activity_principal_refs) <= classified_activity_roles: + raise ValueError( + "activity principals must be classified as present principals, " + "managed owners, or the observed database owner" + ) + invalid_schema_owners = sorted( + schema + for schema, owner in self.observed_schema_owners.items() + if _SQL_IDENTIFIER.fullmatch(schema) is None + or _SQL_IDENTIFIER.fullmatch(owner) is None + ) + if invalid_schema_owners: + raise ValueError( + "observed schema owners contain unsafe identifiers: " + f"{invalid_schema_owners!r}" + ) + schema_overlap = set(self.observed_schema_owners) & set(self.absent_schema_refs) + if schema_overlap: + raise ValueError( + "schema presence and absence evidence overlap: " + f"{sorted(schema_overlap)!r}" + ) + object_identities = [obj.identity for obj in self.observed_objects] + if len(set(object_identities)) != len(object_identities): + raise ValueError("observed catalog object identities must be unique") + unknown_object_schemas = sorted( + {obj.schema_ref for obj in self.observed_objects} + - set(self.observed_schema_owners) + ) + if unknown_object_schemas: + raise ValueError( + "observed objects require present schema-owner evidence: " + f"{unknown_object_schemas!r}" + ) + classified_present_roles = ( + set(self.principal_refs) + .union(self.owner_refs) + .union({self.database_owner_role}) + ) + unclassified_catalog_owners = sorted( + set(self.observed_schema_owners.values()).union( + obj.owner for obj in self.observed_objects + ) + - classified_present_roles + ) + if unclassified_catalog_owners: + raise ValueError( + "observed schema/object owners must be classified in the present " + f"role census: {unclassified_catalog_owners!r}" + ) + if ( + self.source_kind + is EnumApplicationDatabasePrincipalInventorySourceKind.AUTHORIZED_CATALOG + and not self.live_database_read + ): + raise ValueError("authorized_catalog requires live_database_read=true") + if self.source_kind is ( + EnumApplicationDatabasePrincipalInventorySourceKind.AUTHORIZED_CATALOG + ) and ( + self.activity_evidence is None + or self.catalog_query_sha256 is None + or self.catalog_result_sha256 is None + or self.catalog_query_source_key is None + or self.catalog_result_source_key is None + ): + raise ValueError( + "authorized_catalog requires durable activity and catalog query/result provenance" + ) + if ( + self.source_kind + is EnumApplicationDatabasePrincipalInventorySourceKind.SYNTHETIC_FIXTURE + and self.live_database_read + ): + raise ValueError("synthetic_fixture cannot claim a live database read") + if self.source_kind is ( + EnumApplicationDatabasePrincipalInventorySourceKind.SYNTHETIC_FIXTURE + ) and any( + item is not None + for item in ( + self.activity_evidence, + self.catalog_query_sha256, + self.catalog_result_sha256, + self.catalog_query_source_key, + self.catalog_result_source_key, + ) + ): + raise ValueError( + "synthetic_fixture cannot carry authorized-catalog provenance" + ) + return self + + +__all__ = ["ModelApplicationDatabasePrincipalInventory"] diff --git a/src/omnibase_infra/validation/models/model_application_database_relation_state.py b/src/omnibase_infra/validation/models/model_application_database_relation_state.py new file mode 100644 index 0000000000..238121de6d --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_relation_state.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed final-state evidence for one classified application relation.""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType +from typing import Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + field_serializer, + field_validator, + model_validator, +) + +from omnibase_infra.validation.enums.enum_application_database_identity_root import ( + EnumApplicationDatabaseIdentityRoot, +) +from omnibase_infra.validation.models.model_application_database_column_state import ( + ModelApplicationDatabaseColumnState, +) +from omnibase_infra.validation.models.model_application_database_function_state import ( + ModelApplicationDatabaseFunctionState, +) +from omnibase_infra.validation.models.model_application_database_identity_root_control_state import ( + ModelApplicationDatabaseIdentityRootControlState, +) +from omnibase_infra.validation.models.model_application_database_policy_state import ( + ModelApplicationDatabasePolicyState, +) +from omnibase_infra.validation.models.model_application_database_tenant_isolation_evidence import ( + ModelApplicationDatabaseTenantIsolationEvidence, +) +from omnibase_infra.validation.models.model_application_relation_declaration import ( + ModelApplicationRelationDeclaration, +) + + +class ModelApplicationDatabaseRelationState(BaseModel): + """Contract classification joined to catalog-observed security state.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + declaration: ModelApplicationRelationDeclaration + columns: tuple[ModelApplicationDatabaseColumnState, ...] = () + primary_key_columns: tuple[str, ...] = () + unique_index_column_sets: tuple[tuple[str, ...], ...] = () + foreign_key_column_sets: tuple[tuple[str, ...], ...] = () + partition_key_columns: tuple[str, ...] = () + deduplication_key_columns: tuple[str, ...] = () + authorization_dependency_columns: tuple[str, ...] = () + write_eligibility_dependency_columns: tuple[str, ...] = () + rls_enabled: bool = False + rls_forced: bool = False + policies: tuple[ModelApplicationDatabasePolicyState, ...] = () + tenant_identity_column: str | None = Field( + default=None, + pattern=r"^[a-z_][a-z0-9_]*$", + ) + identity_root_contract: EnumApplicationDatabaseIdentityRoot | None = None + identity_root_control_state: ( + ModelApplicationDatabaseIdentityRootControlState | None + ) = None + source_tenant_provenance_contract: ( + Literal["non_authoritative_provenance"] | None + ) = None + canonical_policy_name: str | None = Field( + default=None, + pattern=r"^[a-z_][a-z0-9_]*$", + ) + declared_restrictive_policy_names: tuple[str, ...] = () + restrictive_policy_proofs: Mapping[str, str] = Field(default_factory=dict) + security_invoker: bool | None = None + view_tenant_isolation_evidence: ( + ModelApplicationDatabaseTenantIsolationEvidence | None + ) = None + function_state: ModelApplicationDatabaseFunctionState | None = None + + @field_validator("restrictive_policy_proofs") + @classmethod + def freeze_restrictive_policy_proofs( + cls, + proofs: Mapping[str, str], + ) -> Mapping[str, str]: + """Freeze behavioral proof references against in-place mutation.""" + return MappingProxyType(dict(proofs)) + + @field_serializer("restrictive_policy_proofs") + def serialize_restrictive_policy_proofs( + self, + proofs: Mapping[str, str], + ) -> dict[str, str]: + """Restore the JSON/YAML mapping shape for immutable proof evidence.""" + return dict(proofs) + + @model_validator(mode="after") + def validate_unique_catalog_rows(self) -> ModelApplicationDatabaseRelationState: + """Require exact catalog rows while leaving policy verdicts to the gate.""" + column_names = [column.name for column in self.columns] + if len(set(column_names)) != len(column_names): + raise ValueError("relation columns must have unique names") + if len(set(self.primary_key_columns)) != len(self.primary_key_columns): + raise ValueError("relation primary-key columns must be unique") + unknown_primary_columns = set(self.primary_key_columns).difference(column_names) + if unknown_primary_columns: + raise ValueError( + "relation primary key names unknown columns: " + f"{sorted(unknown_primary_columns)!r}" + ) + column_set_fields = { + "unique index": self.unique_index_column_sets, + "foreign key": self.foreign_key_column_sets, + } + for label, column_sets in column_set_fields.items(): + if any(not column_set for column_set in column_sets): + raise ValueError(f"{label} column sets cannot be empty") + if len(set(column_sets)) != len(column_sets): + raise ValueError(f"{label} column sets must be unique") + unknown = { + column for column_set in column_sets for column in column_set + }.difference(column_names) + if unknown: + raise ValueError(f"{label} names unknown columns: {sorted(unknown)!r}") + semantic_column_fields = { + "partition key": self.partition_key_columns, + "deduplication key": self.deduplication_key_columns, + "authorization dependency": self.authorization_dependency_columns, + "write eligibility dependency": (self.write_eligibility_dependency_columns), + } + for label, columns in semantic_column_fields.items(): + if len(set(columns)) != len(columns): + raise ValueError(f"{label} columns must be unique") + unknown = set(columns).difference(column_names) + if unknown: + raise ValueError(f"{label} names unknown columns: {sorted(unknown)!r}") + policy_names = [policy.name for policy in self.policies] + if len(set(policy_names)) != len(policy_names): + raise ValueError("relation policies must have unique names") + if len(set(self.declared_restrictive_policy_names)) != len( + self.declared_restrictive_policy_names + ): + raise ValueError("declared restrictive policy names must be unique") + return self + + +__all__ = ["ModelApplicationDatabaseRelationState"] diff --git a/src/omnibase_infra/validation/models/model_application_database_role_membership.py b/src/omnibase_infra/validation/models/model_application_database_role_membership.py new file mode 100644 index 0000000000..ebd5be2b47 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_role_membership.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Explicit permitted role membership in an application database ACL matrix.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelApplicationDatabaseRoleMembership(BaseModel): + """One exact PostgreSQL role membership and its PG16 option state.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + database_ref: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + role: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + member: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + admin_option: bool = False + inherit_option: bool = False + set_option: bool = True + + @property + def identity(self) -> tuple[str, str, str]: + return (self.database_ref, self.role, self.member) + + +__all__ = ["ModelApplicationDatabaseRoleMembership"] diff --git a/src/omnibase_infra/validation/models/model_application_database_role_state.py b/src/omnibase_infra/validation/models/model_application_database_role_state.py new file mode 100644 index 0000000000..67f9123db5 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_role_state.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Explicit desired state for an ACL-governed PostgreSQL role.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelApplicationDatabaseRoleState(BaseModel): + """Role attributes that an ACL policy explicitly authorizes changing.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + role: str = Field(..., pattern=r"^[a-z_][a-z0-9_]*$") + role_kind: Literal["owner", "workload", "migration", "external_connect"] + manage_attributes: bool = True + manage_memberships: bool = True + login: bool + superuser: Literal[False] = False + bypass_rls: Literal[False] = False + create_database: Literal[False] = False + create_role: Literal[False] = False + replication: Literal[False] = False + inherit: Literal[False] = False + + +__all__ = ["ModelApplicationDatabaseRoleState"] diff --git a/src/omnibase_infra/validation/models/model_application_database_routine_dependency_state.py b/src/omnibase_infra/validation/models/model_application_database_routine_dependency_state.py new file mode 100644 index 0000000000..cab02f224d --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_routine_dependency_state.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed catalog state for one routine reachable from an authority surface.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +class ModelApplicationDatabaseRoutineDependencyState(BaseModel): + """Catalog-resolved routine metadata used for transitive authority analysis.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + object_id: int = Field(..., gt=0) + namespace: str = Field(..., min_length=1) + name: str = Field(..., min_length=1) + language: str = Field(..., min_length=1) + source_body: str | None + argument_type_ids: tuple[int, ...] = () + argument_names: tuple[str | None, ...] = () + returns_trigger: bool = False + referenced_routine_ids: tuple[int, ...] = () + referenced_target_columns: tuple[str, ...] = () + references_target_whole_row: bool = False + + @field_validator("namespace", "name", "language") + @classmethod + def normalize_catalog_name(cls, value: str) -> str: + """Normalize catalog identifiers without accepting empty values.""" + normalized = value.strip().lower() + if not normalized: + raise ValueError("catalog names must not be blank") + return normalized + + @model_validator(mode="after") + def validate_unique_dependencies( + self, + ) -> ModelApplicationDatabaseRoutineDependencyState: + """Reject ambiguous duplicate dependency evidence.""" + if self.argument_names and len(self.argument_names) != len( + self.argument_type_ids + ): + raise ValueError( + "argument names must be empty or align exactly with argument type IDs" + ) + if any(name is not None and not name.strip() for name in self.argument_names): + raise ValueError("argument names must not contain blank values") + if len(set(self.referenced_routine_ids)) != len(self.referenced_routine_ids): + raise ValueError("referenced routine IDs must be unique") + if len(set(self.referenced_target_columns)) != len( + self.referenced_target_columns + ): + raise ValueError("referenced target columns must be unique") + return self + + +__all__ = ["ModelApplicationDatabaseRoutineDependencyState"] diff --git a/src/omnibase_infra/validation/models/model_application_database_tenant_isolation_evidence.py b/src/omnibase_infra/validation/models/model_application_database_tenant_isolation_evidence.py new file mode 100644 index 0000000000..641880b3df --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_database_tenant_isolation_evidence.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Behavioral tenant-isolation evidence for a tenant-facing SQL surface.""" + +from collections.abc import Mapping +from types import MappingProxyType +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator + + +class ModelApplicationDatabaseTenantIsolationEvidence(BaseModel): + """Expected and observed results for two tenants plus denied bad contexts.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + expected_rows_by_tenant: Mapping[UUID, int] = Field(..., min_length=2) + observed_rows_by_tenant: Mapping[UUID, int] = Field(..., min_length=2) + unset_context_rows: int = Field(..., ge=0) + malformed_context_denied: bool + + @field_validator("expected_rows_by_tenant", "observed_rows_by_tenant") + @classmethod + def validate_nonnegative_counts( + cls, + values: Mapping[UUID, int], + ) -> Mapping[UUID, int]: + """Reject invalid counts and freeze evidence against in-place mutation.""" + if any(value < 0 for value in values.values()): + raise ValueError("tenant isolation row counts must be nonnegative") + return MappingProxyType(dict(values)) + + @field_serializer("expected_rows_by_tenant", "observed_rows_by_tenant") + def serialize_row_counts( + self, + values: Mapping[UUID, int], + ) -> dict[UUID, int]: + """Restore the JSON/YAML mapping shape for immutable row counts.""" + return dict(values) + + +__all__ = ["ModelApplicationDatabaseTenantIsolationEvidence"] diff --git a/src/omnibase_infra/validation/models/model_application_inventory_relation.py b/src/omnibase_infra/validation/models/model_application_inventory_relation.py new file mode 100644 index 0000000000..83b81973ce --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_inventory_relation.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Strict rich relation row emitted by the OMN-15423 inventory projection.""" + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_infra.validation.enums.enum_application_inventory_object_kind import ( + EnumApplicationInventoryObjectKind, +) +from omnibase_infra.validation.models.model_internal_tenant_column_transform import ( + ModelInternalTenantColumnTransform, +) +from omnibase_infra.validation.types.type_application_database_function_signature import ( + ApplicationDatabaseFunctionSignature, +) + + +class ModelApplicationInventoryRelation(BaseModel): + """One repository/live-census object with its complete typed evidence.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str = Field(..., min_length=1) + kind: EnumApplicationInventoryObjectKind + target_schema: str = Field(..., min_length=1) + domain: EnumDatabaseSchemaDomain | None = None + owner_declaration: str | None = None + producer: str | None = None + readers: tuple[str, ...] = () + writers: tuple[str, ...] = () + accessor_nodes: tuple[str, ...] = () + authoritative_sources: tuple[str, ...] = () + blocked_reasons: tuple[str, ...] = () + classification_evidence: str = Field(..., min_length=1) + classification_status: str = Field(..., min_length=1) + constraints: tuple[str, ...] = () + contract_sources: tuple[str, ...] = () + current_schema: tuple[str, ...] = () + dependencies: tuple[str, ...] = () + dependent_objects: tuple[str, ...] = () + dsn_consumers: tuple[str, ...] = () + foreign_keys: tuple[str, ...] = () + function_signature: ApplicationDatabaseFunctionSignature | None = None + grants: tuple[str, ...] = () + indexes: tuple[str, ...] = () + keys: tuple[str, ...] = () + migration_root: str = Field(..., min_length=1) + migration_stream: str | None = None + partitioning: tuple[str, ...] = () + internal_tenant_column_transform: ModelInternalTenantColumnTransform | None = None + + +__all__ = ["ModelApplicationInventoryRelation"] diff --git a/src/omnibase_infra/validation/models/model_application_inventory_relation_counts.py b/src/omnibase_infra/validation/models/model_application_inventory_relation_counts.py new file mode 100644 index 0000000000..60d2e2d283 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_inventory_relation_counts.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed aggregate counts from the OMN-15423 inventory projection.""" + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelApplicationInventoryRelationCounts(BaseModel): + """Counts retained for completeness checks across every projected kind.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + table: int = Field(..., ge=0) + view: int = Field(..., ge=0) + materialized_view: int = Field(default=0, ge=0) + function: int = Field(..., ge=0) + procedure: int | None = Field(default=None, ge=0) + sequence: int = Field(..., ge=0) + extension: int = Field(..., ge=0) + type: int | None = Field(default=None, ge=0) + + +__all__ = ["ModelApplicationInventoryRelationCounts"] diff --git a/src/omnibase_infra/validation/models/model_application_inventory_runtime_evidence.py b/src/omnibase_infra/validation/models/model_application_inventory_runtime_evidence.py new file mode 100644 index 0000000000..9074ed35f3 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_inventory_runtime_evidence.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed runtime evidence from the OMN-15423 inventory projection.""" + +from pydantic import BaseModel, ConfigDict + +from omnibase_infra.validation.models.model_runtime_evidence_status import ( + ModelRuntimeEvidenceStatus, +) + + +class ModelApplicationInventoryRuntimeEvidence(BaseModel): + """Runtime facts whose blocked status must survive inventory projection.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + dsn_key_provenance: dict[str, tuple[str, ...]] + full_day_datname_usename_activity: ModelRuntimeEvidenceStatus + live_catalog_parity: ModelRuntimeEvidenceStatus + + +__all__ = ["ModelApplicationInventoryRuntimeEvidence"] diff --git a/src/omnibase_infra/validation/models/model_application_relation_declaration.py b/src/omnibase_infra/validation/models/model_application_relation_declaration.py new file mode 100644 index 0000000000..8cc669ad42 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_relation_declaration.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Normalized application-relation owner or reader declaration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict + +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_infra.validation.enums.enum_application_relation_kind import ( + EnumApplicationRelationKind, +) +from omnibase_infra.validation.enums.enum_application_relation_purpose import ( + EnumApplicationRelationPurpose, +) +from omnibase_infra.validation.models.model_live_application_relation import ( + RelationIdentity, +) +from omnibase_infra.validation.types.type_application_database_function_signature import ( + ApplicationDatabaseFunctionSignature, +) + + +class ModelApplicationRelationDeclaration(BaseModel): + """Normalized owner or reader projected from one distributed source.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + database_ref: str + schema: str # type: ignore[assignment] + kind: EnumApplicationRelationKind + purpose: EnumApplicationRelationPurpose + domain: EnumDatabaseSchemaDomain | None + owner_declaration: str | None + readers: tuple[str, ...] = () + access: Literal["read", "write", "read_write"] + role: str + source_path: str + function_signature: ApplicationDatabaseFunctionSignature | None = None + + @property + def identity(self) -> RelationIdentity: + return ( + self.database_ref, + self.schema, + self.name, + self.kind, + self.function_signature, + ) + + +__all__ = ["ModelApplicationRelationDeclaration"] diff --git a/src/omnibase_infra/validation/models/model_application_relation_evidence_inventory.py b/src/omnibase_infra/validation/models/model_application_relation_evidence_inventory.py new file mode 100644 index 0000000000..075f4eb320 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_relation_evidence_inventory.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Strict typed model for the rich OMN-15423 inventory evidence artifact.""" + +from collections import Counter + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_infra.validation.models.model_application_inventory_relation import ( + ModelApplicationInventoryRelation, +) +from omnibase_infra.validation.models.model_application_inventory_relation_counts import ( + ModelApplicationInventoryRelationCounts, +) +from omnibase_infra.validation.models.model_application_inventory_runtime_evidence import ( + ModelApplicationInventoryRuntimeEvidence, +) +from omnibase_infra.validation.models.model_blocked_relation import ModelBlockedRelation +from omnibase_infra.validation.models.model_retained_live_census import ( + ModelRetainedLiveCensus, +) + + +class ModelApplicationRelationEvidenceInventory(BaseModel): + """Full evidence projection produced by OMN-15423 without lossy parsing.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: str = Field(..., min_length=1) + ticket: str = Field(..., min_length=1) + database_ref: str = Field(..., min_length=1) + physical_seed_database: str = Field(..., min_length=1) + ownership_authority: str = Field(..., min_length=1) + inventory_projection: str = Field(..., min_length=1) + completion_status: str = Field(..., min_length=1) + relation_counts: ModelApplicationInventoryRelationCounts + relations: tuple[ModelApplicationInventoryRelation, ...] + blocked_relations: tuple[ModelBlockedRelation, ...] + retained_live_census: ModelRetainedLiveCensus + runtime_evidence: ModelApplicationInventoryRuntimeEvidence + + @model_validator(mode="after") + def validate_relation_counts(self) -> "ModelApplicationRelationEvidenceInventory": + """Reject stale aggregate evidence instead of trusting declared counts.""" + observed = Counter(relation.kind.value for relation in self.relations) + declared = self.relation_counts.model_dump(mode="json") + mismatches = { + kind: (declared_count, observed.get(kind, 0)) + for kind, declared_count in declared.items() + if declared_count is not None and declared_count != observed.get(kind, 0) + } + if mismatches: + raise ValueError( + f"relation_counts do not match typed relation rows: {mismatches!r}" + ) + return self + + +__all__ = ["ModelApplicationRelationEvidenceInventory"] diff --git a/src/omnibase_infra/validation/models/model_application_relation_inventory.py b/src/omnibase_infra/validation/models/model_application_relation_inventory.py new file mode 100644 index 0000000000..425efe2fa2 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_relation_inventory.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed live application-relation inventory.""" + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.validation.models.model_blocked_relation import ModelBlockedRelation +from omnibase_infra.validation.models.model_live_application_relation import ( + ModelLiveApplicationRelation, +) +from omnibase_infra.validation.models.model_retained_live_census import ( + ModelRetainedLiveCensus, +) +from omnibase_infra.validation.models.model_runtime_evidence_status import ( + ModelRuntimeEvidenceStatus, +) + + +class ModelApplicationRelationInventory(BaseModel): + """Live inventory consumed by the global ownership validator.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + # string-version-ok: fixture and evidence schema version from YAML. + schema_version: str = Field(..., min_length=1) + relations: tuple[ModelLiveApplicationRelation, ...] + completion_status: str | None = None + blocked_relations: tuple[ModelBlockedRelation, ...] = () + retained_live_census: ModelRetainedLiveCensus | None = None + runtime_evidence: dict[str, ModelRuntimeEvidenceStatus] = Field( + default_factory=dict + ) + source_relation_count: int | None = Field(default=None, ge=0) + excluded_database_objects: tuple[str, ...] = () + + +__all__ = ["ModelApplicationRelationInventory"] diff --git a/src/omnibase_infra/validation/models/model_application_relation_ownership_report.py b/src/omnibase_infra/validation/models/model_application_relation_ownership_report.py new file mode 100644 index 0000000000..436a23e2f5 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_relation_ownership_report.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Global application-relation ownership validation report.""" + +from pydantic import BaseModel, ConfigDict + +from omnibase_infra.validation.models.model_application_relation_declaration import ( + ModelApplicationRelationDeclaration, +) +from omnibase_infra.validation.models.model_application_relation_violation import ( + ModelApplicationRelationViolation, +) +from omnibase_infra.validation.models.model_live_application_relation import ( + RelationIdentity, +) + + +class ModelApplicationRelationOwnershipReport(BaseModel): + """Complete normalized projection and its fail-closed violations.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + declarations: tuple[ModelApplicationRelationDeclaration, ...] + violations: tuple[ModelApplicationRelationViolation, ...] + + @property + def is_valid(self) -> bool: + return not self.violations + + def readers_for(self, identity: RelationIdentity) -> tuple[str, ...]: + readers = { + reader + for declaration in self.declarations + if declaration.identity == identity + for reader in declaration.readers + } + return tuple(sorted(readers)) + + +__all__ = ["ModelApplicationRelationOwnershipReport"] diff --git a/src/omnibase_infra/validation/models/model_application_relation_violation.py b/src/omnibase_infra/validation/models/model_application_relation_violation.py new file mode 100644 index 0000000000..fede867327 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_application_relation_violation.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""One deterministic application-relation ownership failure.""" + +from pydantic import BaseModel, ConfigDict + +from omnibase_infra.validation.enums.enum_application_relation_violation import ( + EnumApplicationRelationViolation, +) + + +class ModelApplicationRelationViolation(BaseModel): + """One deterministic global ownership validation failure.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + code: EnumApplicationRelationViolation + message: str + relation_name: str | None = None + source_paths: tuple[str, ...] = () + + +__all__ = ["ModelApplicationRelationViolation"] diff --git a/src/omnibase_infra/validation/models/model_blocked_relation.py b/src/omnibase_infra/validation/models/model_blocked_relation.py new file mode 100644 index 0000000000..51d845904e --- /dev/null +++ b/src/omnibase_infra/validation/models/model_blocked_relation.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Explicit unresolved relation from a migration ownership manifest.""" + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from omnibase_infra.validation.enums.enum_application_relation_kind import ( + EnumApplicationRelationKind, +) + + +class ModelBlockedRelation(BaseModel): + """Unresolved live-only relation retained as an explicit hard blocker.""" + + model_config = ConfigDict(frozen=True, extra="allow") + + name: str = Field(..., min_length=1) + kind: EnumApplicationRelationKind + reason: str = Field(..., min_length=1) + + @field_validator("kind", mode="before") + @classmethod + def normalize_kind(cls, value: object) -> object: + if not isinstance(value, str): + return value + return value.strip().lower().replace("-", "_").replace(" ", "_") + + +__all__ = ["ModelBlockedRelation"] diff --git a/src/omnibase_infra/validation/models/model_database_object_evidence.py b/src/omnibase_infra/validation/models/model_database_object_evidence.py new file mode 100644 index 0000000000..686509955b --- /dev/null +++ b/src/omnibase_infra/validation/models/model_database_object_evidence.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Non-table database object evidence from a migration ownership manifest.""" + +from typing import Self + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_infra.validation.enums.enum_application_database_object_kind import ( + EnumApplicationDatabaseObjectKind, +) +from omnibase_infra.validation.types.type_application_database_function_signature import ( + ApplicationDatabaseFunctionSignature, +) + + +class ModelDatabaseObjectEvidence(BaseModel): + """Ownership-relevant subset of a ``database_objects`` entry.""" + + model_config = ConfigDict(frozen=True, extra="allow") + + name: str = Field(..., min_length=1) + kind: EnumApplicationDatabaseObjectKind + database_ref: str | None = None + schema: str | None = None # type: ignore[assignment] + domain: EnumDatabaseSchemaDomain + owner_declaration: str = Field(..., min_length=1) + readers: tuple[str, ...] = () + writers: tuple[str, ...] = () + current_schemas: tuple[str, ...] = () + function_signature: ApplicationDatabaseFunctionSignature | None = None + audit_id: str | None = Field(default=None, min_length=1) + definition_sha256: str | None = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + + @field_validator("kind", mode="before") + @classmethod + def normalize_kind(cls, value: object) -> object: + if not isinstance(value, str): + return value + return value.strip().lower().replace("-", "_").replace(" ", "_") + + @model_validator(mode="after") + def validate_audit_pair(self) -> Self: + """An authoritative routine audit names both its record and exact body.""" + if (self.audit_id is None) != (self.definition_sha256 is None): + raise ValueError( + "database object audit_id and definition_sha256 must be declared together" + ) + if ( + self.audit_id is not None + and self.definition_sha256 is not None + and self.definition_sha256 not in self.audit_id + ): + raise ValueError("database object audit_id must bind definition_sha256") + return self + + +__all__ = ["ModelDatabaseObjectEvidence"] diff --git a/src/omnibase_infra/validation/models/model_internal_tenant_column_transform.py b/src/omnibase_infra/validation/models/model_internal_tenant_column_transform.py new file mode 100644 index 0000000000..c445aa19b7 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_internal_tenant_column_transform.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed internal-domain tenant-column transformation evidence.""" + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.validation.enums.enum_internal_tenant_column_transform_status import ( + EnumInternalTenantColumnTransformStatus, +) + + +class ModelInternalTenantColumnTransform(BaseModel): + """Evidence required before an internal relation drops tenant stamping.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + status: EnumInternalTenantColumnTransformStatus + source_occurrences: tuple[str, ...] + key_fk_index_partition_dependencies: tuple[str, ...] + runtime_collision_scan: str = Field(..., min_length=1) + + +__all__ = ["ModelInternalTenantColumnTransform"] diff --git a/src/omnibase_infra/validation/models/model_live_application_relation.py b/src/omnibase_infra/validation/models/model_live_application_relation.py new file mode 100644 index 0000000000..70a4bc6d5d --- /dev/null +++ b/src/omnibase_infra/validation/models/model_live_application_relation.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed live-catalog application relation.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_infra.validation.enums.enum_application_relation_kind import ( + EnumApplicationRelationKind, +) +from omnibase_infra.validation.enums.enum_application_relation_purpose import ( + EnumApplicationRelationPurpose, +) +from omnibase_infra.validation.types.type_application_database_function_signature import ( + ApplicationDatabaseFunctionSignature, +) + +RelationIdentity = tuple[ + str, + str, + str, + EnumApplicationRelationKind, + ApplicationDatabaseFunctionSignature | None, +] + + +class ModelLiveApplicationRelation(BaseModel): + """One relation observed by a live-catalog inventory projection.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str = Field(..., min_length=1) + database_ref: str = Field(..., min_length=1) + schema: str = Field(..., min_length=1) # type: ignore[assignment] + kind: EnumApplicationRelationKind + purpose: EnumApplicationRelationPurpose = EnumApplicationRelationPurpose.DATA + domain: EnumDatabaseSchemaDomain | None = None + function_signature: ApplicationDatabaseFunctionSignature | None = None + + @field_validator("kind", mode="before") + @classmethod + def normalize_kind(cls, value: object) -> object: + if not isinstance(value, str): + return value + return value.strip().lower().replace("-", "_").replace(" ", "_") + + @property + def identity(self) -> RelationIdentity: + return ( + self.database_ref, + self.schema, + self.name, + self.kind, + self.function_signature, + ) + + +__all__ = ["ModelLiveApplicationRelation", "RelationIdentity"] diff --git a/src/omnibase_infra/validation/models/model_migration_ownership_manifest.py b/src/omnibase_infra/validation/models/model_migration_ownership_manifest.py new file mode 100644 index 0000000000..f4a22a830f --- /dev/null +++ b/src/omnibase_infra/validation/models/model_migration_ownership_manifest.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed ownership manifest for a repository migration stream.""" + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_core.models.contracts.subcontracts.model_db_ownership_subcontract import ( + ModelDbOwnershipSubcontract, +) +from omnibase_infra.validation.models.model_blocked_relation import ModelBlockedRelation +from omnibase_infra.validation.models.model_database_object_evidence import ( + ModelDatabaseObjectEvidence, +) +from omnibase_infra.validation.models.model_relation_evidence import ( + ModelRelationEvidence, +) +from omnibase_infra.validation.models.model_retained_live_census import ( + ModelRetainedLiveCensus, +) +from omnibase_infra.validation.models.model_runtime_evidence_status import ( + ModelRuntimeEvidenceStatus, +) + + +class ModelMigrationOwnershipManifest(BaseModel): + """Typed ownership boundary for a repository migration stream.""" + + model_config = ConfigDict(frozen=True, extra="allow") + + # string-version-ok: external manifest schema version from YAML. + schema_version: str = Field(..., min_length=1) + service: str = Field(..., min_length=1) + owner_declaration: str | None = Field(default=None, min_length=1) + current_physical_database: str | None = Field(default=None, min_length=1) + materialized_physical_databases: tuple[str, ...] = () + target_database_ref: str = Field(..., min_length=1) + db_io: ModelDbOwnershipSubcontract + relation_evidence: tuple[ModelRelationEvidence, ...] = () + database_objects: tuple[ModelDatabaseObjectEvidence, ...] = () + blocked_relations: tuple[ModelBlockedRelation, ...] = () + completion_status: str | None = None + retained_live_census: ModelRetainedLiveCensus | None = None + runtime_evidence: dict[str, ModelRuntimeEvidenceStatus] = Field( + default_factory=dict + ) + + +__all__ = ["ModelMigrationOwnershipManifest"] diff --git a/src/omnibase_infra/validation/models/model_node_ownership_document.py b/src/omnibase_infra/validation/models/model_node_ownership_document.py new file mode 100644 index 0000000000..2d558266ce --- /dev/null +++ b/src/omnibase_infra/validation/models/model_node_ownership_document.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed ownership subset of a node contract.""" + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_core.models.contracts.subcontracts.model_db_ownership_subcontract import ( + ModelDbOwnershipSubcontract, +) + + +class ModelNodeOwnershipDocument(BaseModel): + """Ownership-relevant typed subset of a node contract.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + name: str = Field(..., min_length=1) + db_io: ModelDbOwnershipSubcontract + + +__all__ = ["ModelNodeOwnershipDocument"] diff --git a/src/omnibase_infra/validation/models/model_relation_evidence.py b/src/omnibase_infra/validation/models/model_relation_evidence.py new file mode 100644 index 0000000000..4b31b4c223 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_relation_evidence.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Relation evidence from a migration ownership manifest.""" + +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from omnibase_core.enums.enum_database_schema_domain import EnumDatabaseSchemaDomain +from omnibase_infra.validation.enums.enum_application_database_identity_root import ( + EnumApplicationDatabaseIdentityRoot, +) +from omnibase_infra.validation.enums.enum_application_database_identity_root_operation import ( + EnumApplicationDatabaseIdentityRootOperation, +) +from omnibase_infra.validation.enums.enum_application_relation_kind import ( + EnumApplicationRelationKind, +) +from omnibase_infra.validation.types.type_application_database_function_signature import ( + ApplicationDatabaseFunctionSignature, +) + + +class ModelRelationEvidence(BaseModel): + """Ownership-relevant subset of an OMN-15423 relation-evidence entry.""" + + model_config = ConfigDict(frozen=True, extra="allow") + + name: str = Field(..., min_length=1) + kind: EnumApplicationRelationKind + database_ref: str | None = None + schema: str | None = None # type: ignore[assignment] + current_schemas: tuple[str, ...] = () + domain: EnumDatabaseSchemaDomain + owner_declaration: str = Field(..., min_length=1) + readers: tuple[str, ...] = () + writers: tuple[str, ...] = () + dependent_objects: tuple[str, ...] = () + source_tenant_provenance_contract: ( + Literal["non_authoritative_provenance"] | None + ) = None + tenant_identity_column: str | None = Field( + default=None, + pattern=r"^[a-z_][a-z0-9_]*$", + ) + identity_root_contract: EnumApplicationDatabaseIdentityRoot | None = None + identity_root_control_role: str | None = Field( + default=None, + pattern=r"^[a-z_][a-z0-9_]*$", + ) + identity_root_control_operations: tuple[ + EnumApplicationDatabaseIdentityRootOperation, ... + ] = () + canonical_policy_name: str | None = Field( + default=None, + pattern=r"^[a-z_][a-z0-9_]*$", + ) + deduplication_key_columns: tuple[str, ...] | None = None + authorization_dependency_columns: tuple[str, ...] | None = None + write_eligibility_dependency_columns: tuple[str, ...] | None = None + function_signature: ApplicationDatabaseFunctionSignature | None = None + + @field_validator("kind", mode="before") + @classmethod + def normalize_kind(cls, value: object) -> object: + if not isinstance(value, str): + return value + return value.strip().lower().replace("-", "_").replace(" ", "_") + + @field_validator( + "deduplication_key_columns", + "authorization_dependency_columns", + "write_eligibility_dependency_columns", + ) + @classmethod + def validate_dependency_columns( + cls, values: tuple[str, ...] | None + ) -> tuple[str, ...] | None: + """Keep declared semantic dependencies exact and identifier-safe.""" + if values is None: + return None + if len(set(values)) != len(values): + raise ValueError("relation dependency columns must be unique") + invalid = [ + value + for value in values + if re.fullmatch(r"[a-z_][a-z0-9_]*", value) is None + ] + if invalid: + raise ValueError(f"invalid relation dependency columns: {invalid!r}") + return values + + @model_validator(mode="after") + def validate_identity_root_contract(self) -> ModelRelationEvidence: + """Identity-root exceptions always name their policy-bound column.""" + if ( + self.identity_root_contract is not None + and self.tenant_identity_column is None + ): + raise ValueError( + "identity_root_contract requires an explicit tenant_identity_column" + ) + if self.identity_root_contract is not None: + if self.identity_root_control_role is None: + raise ValueError( + "identity_root_contract requires an audited control role" + ) + if not self.identity_root_control_operations: + raise ValueError( + "identity_root_contract requires declared control operations" + ) + elif ( + self.identity_root_control_role is not None + or self.identity_root_control_operations + ): + raise ValueError( + "identity-root control authority requires identity_root_contract" + ) + if len(set(self.identity_root_control_operations)) != len( + self.identity_root_control_operations + ): + raise ValueError("identity-root control operations must be unique") + return self + + +__all__ = ["ModelRelationEvidence"] diff --git a/src/omnibase_infra/validation/models/model_retained_live_census.py b/src/omnibase_infra/validation/models/model_retained_live_census.py new file mode 100644 index 0000000000..e74790452f --- /dev/null +++ b/src/omnibase_infra/validation/models/model_retained_live_census.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Live-catalog completeness counters from an ownership manifest.""" + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelRetainedLiveCensus(BaseModel): + """Completeness counters projected by migration inventory manifests.""" + + model_config = ConfigDict(frozen=True, extra="allow") + + observed_base_tables: int | None = Field(default=None, ge=0) + observed_views_and_materialized_views: int | None = Field(default=None, ge=0) + observed_sequences: int | None = Field(default=None, ge=0) + observed_functions: int | None = Field(default=None, ge=0) + observed_procedures: int | None = Field(default=None, ge=0) + observed_types: int | None = Field(default=None, ge=0) + observed_extensions: int | None = Field(default=None, ge=0) + parity_status: str | None = None + reason: str | None = None + + +__all__ = ["ModelRetainedLiveCensus"] diff --git a/src/omnibase_infra/validation/models/model_runtime_evidence_status.py b/src/omnibase_infra/validation/models/model_runtime_evidence_status.py new file mode 100644 index 0000000000..366be09116 --- /dev/null +++ b/src/omnibase_infra/validation/models/model_runtime_evidence_status.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed runtime-evidence status from an ownership manifest.""" + +from pydantic import BaseModel, ConfigDict + +from omnibase_infra.validation.enums.enum_ownership_evidence_status import ( + EnumOwnershipEvidenceStatus, +) + + +class ModelRuntimeEvidenceStatus(BaseModel): + """Status-bearing runtime evidence entry from an ownership manifest.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + status: EnumOwnershipEvidenceStatus + reason: str | None = None + credentials_captured: bool | None = None + + +__all__ = ["ModelRuntimeEvidenceStatus"] diff --git a/src/omnibase_infra/validation/types/__init__.py b/src/omnibase_infra/validation/types/__init__.py new file mode 100644 index 0000000000..36de0c4eed --- /dev/null +++ b/src/omnibase_infra/validation/types/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Reusable strict validation types.""" diff --git a/src/omnibase_infra/validation/types/type_application_database_function_signature.py b/src/omnibase_infra/validation/types/type_application_database_function_signature.py new file mode 100644 index 0000000000..8862a6ad7e --- /dev/null +++ b/src/omnibase_infra/validation/types/type_application_database_function_signature.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Safe PostgreSQL routine identity-signature type.""" + +from __future__ import annotations + +from typing import Annotated + +from pydantic import AfterValidator + +_UNQUOTED_CHARACTERS = frozenset( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.$ []," +) + + +def _validate_function_signature(value: str) -> str: + """Reject target-list escapes while retaining catalog identity arguments.""" + if len(value) < 2 or value[0] != "(" or value[-1] != ")": + raise ValueError("function_signature must have one outer argument list") + + body = value[1:-1] + if not body: + return value + + arguments: list[str] = [] + current: list[str] = [] + quoted = False + index = 0 + while index < len(body): + character = body[index] + if quoted: + if character in {"\n", "\r", "\x00"}: + raise ValueError( + "function_signature quoted identifiers cannot contain controls" + ) + if character == '"': + if index + 1 < len(body) and body[index + 1] == '"': + current.extend(('"', '"')) + index += 2 + continue + quoted = False + current.append(character) + index += 1 + continue + + if character == '"': + quoted = True + current.append(character) + elif character in {"(", ")"}: + raise ValueError("function_signature cannot contain nested argument lists") + elif character == ",": + argument = "".join(current).strip() + if not argument: + raise ValueError("function_signature arguments cannot be empty") + arguments.append(argument) + current = [] + elif character not in _UNQUOTED_CHARACTERS: + raise ValueError( + f"function_signature contains unsafe character {character!r}" + ) + else: + current.append(character) + index += 1 + + if quoted: + raise ValueError("function_signature contains an unterminated identifier") + final_argument = "".join(current).strip() + if not final_argument: + raise ValueError("function_signature arguments cannot be empty") + arguments.append(final_argument) + if any( + "[" in argument.replace("[]", "") or "]" in argument.replace("[]", "") + for argument in arguments + ): + raise ValueError("function_signature array brackets must be complete [] pairs") + return value + + +type ApplicationDatabaseFunctionSignature = Annotated[ + str, + AfterValidator(_validate_function_signature), +] + +__all__ = ["ApplicationDatabaseFunctionSignature"] diff --git a/src/omnibase_infra/validation/validation_exemptions.yaml b/src/omnibase_infra/validation/validation_exemptions.yaml index b7e11d318d..30c0fc97a6 100644 --- a/src/omnibase_infra/validation/validation_exemptions.yaml +++ b/src/omnibase_infra/validation/validation_exemptions.yaml @@ -3789,6 +3789,99 @@ pattern_exemptions: target_id is polymorphic across recommendation types: a runner name string (RESTART_RUNNER) or a GitHub Actions run id rendered as str (CANCEL_RUN). Neither is a UUID. ticket: OMN-13942 + # ========================================================================== + # Hybrid gateway wire identity exemptions (OMN-12912) + # ========================================================================== + - file_pattern: 'model_gateway_tenant_identity\.py' + violation_pattern: "Field 'principal_id' should use UUID" + reason: > + principal_id is the canonical MSK tenant principal t-, whose t- prefix makes UUID an invalid representation. The underlying tenant_id remains strongly typed as UUID. + + ticket: OMN-12912 + - file_pattern: 'model_gateway_heartbeat\.py' + violation_pattern: "Field 'tenant_id' should use UUID" + reason: > + The heartbeat crosses the tenant-prefixed wire boundary and therefore carries the validated DNS-safe tenant slug used in Kafka topic names. + + ticket: OMN-12912 + - file_pattern: 'model_gateway_heartbeat\.py' + violation_pattern: "Field 'principal_id' should use UUID" + reason: > + principal_id is the canonical MSK tenant principal t-, whose t- prefix makes UUID an invalid representation. + + ticket: OMN-12912 + # ========================================================================== + # Gateway attach/session control-plane exemptions (OMN-15750) + # ========================================================================== + - file_pattern: 'model_gateway_session\.py' + violation_pattern: "Field 'principal_id' should use UUID type instead of str" + reason: > + Same t- canonical MSK principal as model_gateway_tenant_identity.py; the t- prefix makes UUID an invalid representation. The underlying tenant_id field remains strongly typed as UUID. + + ticket: OMN-15750 + - file_pattern: 'model_gateway_session\.py' + violation_pattern: "Field 'keycloak_client_id' should use UUID type instead of str" + reason: > + Keycloak clientId is an arbitrary realm-scoped string (this node sets it to the immutable principal_id, itself non-UUID -- see above), not a UUID. + + ticket: OMN-15750 + - file_pattern: 'model_gateway_session\.py' + violation_pattern: "Field 'edge_instance_id' should use UUID type instead of str" + reason: > + Caller-declared host label for observability/bookkeeping only, never used for authorization (that is derived entirely from the validated token claims). Free-form string, not a UUID. + + ticket: OMN-15750 + - file_pattern: 'model_gateway_attach_request\.py' + violation_pattern: "Field 'edge_instance_id' should use UUID type instead of str" + reason: > + Same caller-declared host label as model_gateway_session.py's edge_instance_id. + + ticket: OMN-15750 + - file_pattern: 'model_gateway_session_event\.py' + violation_pattern: "Field 'principal_id' should use UUID type instead of str" + reason: > + Same t- canonical MSK principal as model_gateway_tenant_identity.py. + + ticket: OMN-15750 + - file_pattern: 'model_gateway_session_event\.py' + violation_pattern: "Field 'edge_instance_id' should use UUID type instead of str" + reason: > + Same caller-declared host label as model_gateway_session.py's edge_instance_id. + + ticket: OMN-15750 + # ========================================================================== + # Gateway auth client exemptions (OMN-15922) + # ========================================================================== + # The client half of the OMN-15750 attach contract. These mirror the + # already-granted server-side exemptions above field-for-field -- the client + # holds the same identifiers the node does, so a divergent type here would be + # a contract mismatch, not a tightening. + - file_pattern: 'model_gateway_credential\.py' + violation_pattern: "Field 'client_id' should use UUID type instead of str" + reason: > + Keycloak clientId is an arbitrary realm-scoped string (e.g. "ga-acme") and IS the principal_id the gateway resolves authority from. Same field as model_gateway_session.py's keycloak_client_id, which carries the identical OMN-15750 exemption; typing it UUID here would make the credential unable to represent a real Keycloak client. + + ticket: OMN-15922 + - file_pattern: 'model_gateway_credential\.py' + violation_pattern: "Field 'edge_instance_id' should use UUID type instead of str" + reason: > + Same caller-declared host label as model_gateway_session.py's edge_instance_id -- observability/bookkeeping only, never used for authorization. Defaults to the machine hostname, which is not a UUID. + + ticket: OMN-15922 + - file_pattern: 'cli_auth\.py' + method_pattern: "Function 'auth_login'" + violation_pattern: 'has \d+ parameters' + reason: > + The parameters ARE the command-line surface -- click binds one function parameter per --option, so the count is fixed by the six flags 'onex auth login' accepts. Collapsing them into a model would not remove any input; it would add an assembly step between click and the store while leaving the same six flags on the CLI. + + ticket: OMN-15922 + - file_pattern: 'store_gateway_credential\.py' + class_pattern: "Class 'StoreGatewayCredential'" + violation_pattern: 'has \d+ methods' + reason: > + Three public operations (load/save/clear) over two on-disk files, plus the private readers/writers those files require and two path properties. Every method is bound to the same ~/.onex config.yaml + credentials.json pair; splitting the class would put the by-reference invariant (secret value in the 0600 file, never in config.yaml) across two objects that must then agree, which is the coupling the single owner exists to prevent. + + ticket: OMN-15922 # These handle one-model-per-file violations for domain-grouped protocols architecture_exemptions: # ========================================================================== diff --git a/src/omnibase_infra/validators/tenant_scoped_ingress_schema.py b/src/omnibase_infra/validators/tenant_scoped_ingress_schema.py index 96f9f49d01..9e08a74520 100644 --- a/src/omnibase_infra/validators/tenant_scoped_ingress_schema.py +++ b/src/omnibase_infra/validators/tenant_scoped_ingress_schema.py @@ -82,10 +82,24 @@ DEFAULT_SCAN_ROOT = Path("src/omnibase_infra") DEFAULT_ALLOWLIST_PATH = Path("config/validation/tenant_scoped_ingress_allowlist.yaml") -# SYNC with the canonical runtime stamp regex +# SYNC with the canonical shared-resolver wire-prefix shape. OMN-15792 unified +# the runtime path onto a single resolver (``resolve_physical_topic`` / +# ``resolve_tenant_from_wire_topic`` in +# ``omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform``, +# whose private ``_TENANT_WIRE_PREFIX_RE`` derives from that module's +# ``_TENANT_SLUG_PATTERN`` constant). The symbol formerly cited here, # ``omnibase_infra.runtime.auto_wiring.handler_wiring._TENANT_WIRE_PREFIX_RE`` -# (OMN-14349). The runtime form captures the slug; this gate only needs the -# match, so the capture group is dropped. Both accept ``tenant-.`` where +# (OMN-14349), was DELETED by OMN-15792 — that call site now calls the shared +# resolver directly instead of hand-rolling its own regex. +# +# This gate deliberately keeps its own copy rather than routing through the +# shared resolver: it statically checks a `contract.yaml` topic *string* +# shape (a boolean "is this prefixed" match), not a live wire topic, so it +# does not need the resolver's full ``RESERVED_TENANT_SLUGS``-aware +# validation — `resolve_tenant_from_wire_topic` *raises* on a malformed or +# reserved slug rather than reporting "not prefixed", which would change this +# gate's PASS/FAIL semantics, not just deduplicate the copy. See OMN-15792 +# ticket item (e) residual list. Both accept ``tenant-.`` where # ```` is a 3-63 char DNS-compatible lowercase label. _TENANT_WIRE_PREFIX_RE: re.Pattern[str] = re.compile( r"^tenant-[a-z][a-z0-9-]{1,61}[a-z0-9]\." diff --git a/tests/ci/fixtures/omn15496_merge_time_external_check_runs.json b/tests/ci/fixtures/omn15496_merge_time_external_check_runs.json new file mode 100644 index 0000000000..99192583cd --- /dev/null +++ b/tests/ci/fixtures/omn15496_merge_time_external_check_runs.json @@ -0,0 +1,3866 @@ +{ + "_provenance": { + "ticket": "OMN-15496", + "captured_utc": "2026-07-30", + "source": "GET /repos/OmniNode-ai/omnibase_infra/commits/{sha}/check-runs?per_page=100 (paginated)", + "filter": "rows whose name is in EXPECTED_EXTERNAL_CONTEXTS AND whose started_at <= the PR's mergedAt (merge-time state; post-merge runs excluded)", + "note": "unedited API rows, reduced to the fields the resolver reads" + }, + "pull_requests": { + "2567": { + "head_sha": "0fca3b5ef257dd7d15e7002cbcf88955d413fdb4", + "merged_at": "2026-07-30T14:54:07Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:10Z", + "id": 90891357905 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:17:40Z", + "id": 90899221389 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:23:29Z", + "id": 90900881915 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:03Z", + "id": 90891358861 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:04Z", + "id": 90891359065 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:17:33Z", + "id": 90899221155 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:23:29Z", + "id": 90900882206 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:59:56Z", + "id": 90894331977 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:10Z", + "id": 90891358860 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:04Z", + "id": 90891358554 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:17:40Z", + "id": 90899221475 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:23:29Z", + "id": 90900881309 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:29:56Z", + "id": 90902684273 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:18:27Z", + "id": 90899471625 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:24:15Z", + "id": 90901094188 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:30:54Z", + "id": 90902969585 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:05Z", + "id": 90891359690 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:05Z", + "id": 90891358749 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:04Z", + "id": 90891359681 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:10Z", + "id": 90891359905 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:08Z", + "id": 90891357715 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:17:42Z", + "id": 90899221908 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:23:29Z", + "id": 90900880990 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:04Z", + "id": 90891358278 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:17:32Z", + "id": 90899221048 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:23:29Z", + "id": 90900881637 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:04Z", + "id": 90891359528 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:17:46Z", + "id": 90899224277 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:23:35Z", + "id": 90900884508 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:05Z", + "id": 90891359878 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T13:49:25Z", + "id": 90891452329 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:23:29Z", + "id": 90900883381 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T14:29:59Z", + "id": 90902714145 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T14:30:07Z", + "id": 90902738151 + } + ] + }, + "2565": { + "head_sha": "712ccb070e93bb8de192d466b349c0e7afd6e4b8", + "merged_at": "2026-07-30T09:27:57Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:25Z", + "id": 90809348945 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:48:46Z", + "id": 90811193618 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T08:14:37Z", + "id": 90816644722 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:24Z", + "id": 90809349137 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:31Z", + "id": 90809349146 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:48:39Z", + "id": 90811192464 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T08:14:38Z", + "id": 90816644231 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:39:44Z", + "id": 90809571129 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:24Z", + "id": 90809348452 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:27Z", + "id": 90809348204 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:24Z", + "id": 90809348299 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:48:39Z", + "id": 90811192496 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T08:14:38Z", + "id": 90816644536 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T08:15:57Z", + "id": 90816940775 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T09:17:56Z", + "id": 90830216225 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T09:18:45Z", + "id": 90830378924 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T09:17:54Z", + "id": 90830202416 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:24Z", + "id": 90809348773 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:24Z", + "id": 90809349739 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:24Z", + "id": 90809349769 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:30Z", + "id": 90809348353 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:48:39Z", + "id": 90811193647 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T08:14:39Z", + "id": 90816644324 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:24Z", + "id": 90809348417 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:48:39Z", + "id": 90811194190 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T08:14:33Z", + "id": 90816644520 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:24Z", + "id": 90809349766 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:48:46Z", + "id": 90811194284 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T08:14:33Z", + "id": 90816645680 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:38:25Z", + "id": 90809349238 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T08:14:38Z", + "id": 90816646205 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T09:17:53Z", + "id": 90830199369 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T09:18:00Z", + "id": 90830219481 + } + ] + }, + "2564": { + "head_sha": "c79fd5c36a10fe5e3ca4d17b6f3e79d465d0ff64", + "merged_at": "2026-07-30T07:20:46Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:08Z", + "id": 90801000257 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:53:14Z", + "id": 90801197628 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:55:58Z", + "id": 90801547127 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:14Z", + "id": 90800999520 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:08Z", + "id": 90800999755 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:53:15Z", + "id": 90801198068 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:56:08Z", + "id": 90801547646 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:53:24Z", + "id": 90801224259 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:08Z", + "id": 90800999973 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:08Z", + "id": 90800999851 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:14Z", + "id": 90801000643 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:53:15Z", + "id": 90801198057 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:55:18Z", + "id": 90801546635 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:56:45Z", + "id": 90801821787 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:02:39Z", + "id": 90802918395 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:08:41Z", + "id": 90804058979 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:16Z", + "id": 90801001454 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:08Z", + "id": 90800999936 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:12Z", + "id": 90801001463 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:09Z", + "id": 90801001096 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:08Z", + "id": 90800999299 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:53:19Z", + "id": 90801197672 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:56:05Z", + "id": 90801547222 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:08Z", + "id": 90800999968 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:53:15Z", + "id": 90801198266 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:56:10Z", + "id": 90801547344 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:09Z", + "id": 90801001330 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:53:16Z", + "id": 90801199131 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:55:47Z", + "id": 90801548360 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:08Z", + "id": 90801000902 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:55:38Z", + "id": 90801548330 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:01:53Z", + "id": 90802751519 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:07:50Z", + "id": 90803890325 + } + ] + }, + "2563": { + "head_sha": "a8a5f89477d6b2c19208b06c0add039720beeb42", + "merged_at": "2026-07-30T07:05:42Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:02Z", + "id": 90792713615 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:09Z", + "id": 90800137327 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:03Z", + "id": 90792713582 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:02Z", + "id": 90792713738 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:07Z", + "id": 90800137248 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:01:30Z", + "id": 90792944590 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:03Z", + "id": 90792713750 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:03Z", + "id": 90792713630 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:05Z", + "id": 90792713772 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:15Z", + "id": 90800137171 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:48:45Z", + "id": 90800409341 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:02:29Z", + "id": 90802866374 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:03Z", + "id": 90792714478 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:03Z", + "id": 90792713600 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:03Z", + "id": 90792714581 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:08Z", + "id": 90792714324 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:02Z", + "id": 90792713666 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:14Z", + "id": 90800137299 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:03Z", + "id": 90792713847 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:14Z", + "id": 90800137427 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:14Z", + "id": 90792714641 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:14Z", + "id": 90800138442 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:00:10Z", + "id": 90792714434 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:50Z", + "id": 90800264250 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:01:42Z", + "id": 90802739834 + } + ] + }, + "2562": { + "head_sha": "89025826343ea40d66747ae4dbea119434cb9e2a", + "merged_at": "2026-07-30T06:25:32Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:29Z", + "id": 90794644405 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:22:12Z", + "id": 90796136375 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:29Z", + "id": 90794644147 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:30Z", + "id": 90794645006 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:22:18Z", + "id": 90796136194 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:14:17Z", + "id": 90794918310 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:30Z", + "id": 90794644399 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:29Z", + "id": 90794645326 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:30Z", + "id": 90794645066 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:22:17Z", + "id": 90796136275 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:13:10Z", + "id": 90794745790 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:22:53Z", + "id": 90796236254 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:24:38Z", + "id": 90796507447 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:30Z", + "id": 90794644327 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:30Z", + "id": 90794646379 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:30Z", + "id": 90794646274 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:30Z", + "id": 90794644726 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:22:19Z", + "id": 90796136402 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:29Z", + "id": 90794644250 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:22:12Z", + "id": 90796136004 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:31Z", + "id": 90794645699 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:22:14Z", + "id": 90796137514 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:30Z", + "id": 90794646165 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:12:31Z", + "id": 90794646034 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:22:20Z", + "id": 90796137171 + } + ] + }, + "2561": { + "head_sha": "4d33ed425c8d9134e171560933d4a5680406c525", + "merged_at": "2026-07-30T07:06:11Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:48Z", + "id": 90788724190 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:35:10Z", + "id": 90789063132 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:36:46Z", + "id": 90789308968 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:30Z", + "id": 90800206429 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:26Z", + "id": 90800883204 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:55:44Z", + "id": 90801563944 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:49Z", + "id": 90788724356 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:43Z", + "id": 90788725021 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:35:04Z", + "id": 90789063264 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:36:53Z", + "id": 90789309215 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:30Z", + "id": 90800206416 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:28Z", + "id": 90800883256 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:55:55Z", + "id": 90801564756 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:34:46Z", + "id": 90789019810 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:42Z", + "id": 90788724366 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:49Z", + "id": 90788724459 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "failure", + "started_at": "2026-07-30T05:32:43Z", + "id": 90788724284 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:35:04Z", + "id": 90789063146 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:36:46Z", + "id": 90789309070 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:30Z", + "id": 90800206268 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:27Z", + "id": 90800883159 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:55:36Z", + "id": 90801565527 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:48:21Z", + "id": 90800348658 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:49:25Z", + "id": 90800532658 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:21Z", + "id": 90801010304 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:56:59Z", + "id": 90801863254 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T07:01:49Z", + "id": 90802768342 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:02:32Z", + "id": 90802894327 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:01:43Z", + "id": 90802739042 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:42Z", + "id": 90788724113 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:43Z", + "id": 90788725596 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:49Z", + "id": 90788725321 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:43Z", + "id": 90788724674 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:35:04Z", + "id": 90789063274 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:36:47Z", + "id": 90789309038 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:30Z", + "id": 90800206209 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:27Z", + "id": 90800883605 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:56:35Z", + "id": 90801565223 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:42Z", + "id": 90788724758 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:35:04Z", + "id": 90789063236 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:36:53Z", + "id": 90789309179 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:30Z", + "id": 90800206256 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:33Z", + "id": 90800883321 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:55:53Z", + "id": 90801563956 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:49Z", + "id": 90788725866 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:35:06Z", + "id": 90789064068 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:36:47Z", + "id": 90789309976 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:39Z", + "id": 90800207488 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:27Z", + "id": 90800884198 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:56:13Z", + "id": 90801568872 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:32:42Z", + "id": 90788725285 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:31Z", + "id": 90800207761 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:48:34Z", + "id": 90800382837 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:31Z", + "id": 90800884055 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:55:43Z", + "id": 90801566616 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T07:01:45Z", + "id": 90802755082 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:01:49Z", + "id": 90802760706 + } + ] + }, + "2560": { + "head_sha": "182cef9dd02fc7cc46ad66e4420b876861ae6af5", + "merged_at": "2026-07-30T07:05:56Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:24Z", + "id": 90787770626 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:36:59Z", + "id": 90789336565 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:20Z", + "id": 90800172660 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:18Z", + "id": 90800859210 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:20Z", + "id": 90787770310 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:17Z", + "id": 90787770539 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:37:05Z", + "id": 90789336690 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:19Z", + "id": 90800172760 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:18Z", + "id": 90800859490 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:27:58Z", + "id": 90788030267 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:33Z", + "id": 90787770660 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:32Z", + "id": 90787770789 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:37:21Z", + "id": 90789390131 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:57Z", + "id": 90800282431 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:19Z", + "id": 90800859178 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:01:47Z", + "id": 90802752026 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:48:42Z", + "id": 90800411847 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:52:06Z", + "id": 90800992798 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T07:01:44Z", + "id": 90802752592 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:02:35Z", + "id": 90802904023 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:13Z", + "id": 90787771551 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:12Z", + "id": 90787770483 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:47Z", + "id": 90787771509 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:36Z", + "id": 90787771302 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:13Z", + "id": 90787770241 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:36:58Z", + "id": 90789336571 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:26Z", + "id": 90800172727 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:18Z", + "id": 90800859331 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:17Z", + "id": 90787770652 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:36:58Z", + "id": 90789336655 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:25Z", + "id": 90800173033 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:23Z", + "id": 90800859238 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:21Z", + "id": 90787771734 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:37:05Z", + "id": 90789337385 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:47:20Z", + "id": 90800174002 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:19Z", + "id": 90800861319 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:26:33Z", + "id": 90787771476 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T06:51:24Z", + "id": 90800861180 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T07:01:39Z", + "id": 90802738649 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T07:01:41Z", + "id": 90802742483 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T07:01:47Z", + "id": 90802751911 + } + ] + }, + "2559": { + "head_sha": "1e87c0e7cc516e0c8d0f061d9f5605ea4a4f6b05", + "merged_at": "2026-07-30T05:48:14Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T05:15:02Z", + "id": 90786135571 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:59Z", + "id": 90786250409 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:03Z", + "id": 90786135574 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:03Z", + "id": 90786135917 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:44Z", + "id": 90786239154 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:17:02Z", + "id": 90786421888 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:02Z", + "id": 90786135620 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:07Z", + "id": 90786135978 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T05:15:03Z", + "id": 90786136419 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:50Z", + "id": 90786259202 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "skipped", + "started_at": "2026-07-30T05:15:17Z", + "id": 90786184903 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:16:29Z", + "id": 90786357376 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:10Z", + "id": 90786136943 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:02Z", + "id": 90786135814 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:03Z", + "id": 90786136542 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:07Z", + "id": 90786136515 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:09Z", + "id": 90786135740 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:49Z", + "id": 90786238777 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:04Z", + "id": 90786136160 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:41Z", + "id": 90786238990 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:03Z", + "id": 90786136753 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:47Z", + "id": 90786239655 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:09Z", + "id": 90786136655 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "failure", + "started_at": "2026-07-30T05:15:03Z", + "id": 90786136994 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T05:15:46Z", + "id": 90786240076 + } + ] + }, + "2556": { + "head_sha": "c27137cca08d04b2d6147db7420b4eeb61518d83", + "merged_at": "2026-07-30T04:08:25Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:12Z", + "id": 90772850507 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:42:52Z", + "id": 90773583590 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:06Z", + "id": 90772851289 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:06Z", + "id": 90772850721 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:42:52Z", + "id": 90773583622 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:38:57Z", + "id": 90773087370 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:06Z", + "id": 90772851150 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:06Z", + "id": 90772851016 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:06Z", + "id": 90772851092 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:42:52Z", + "id": 90773583580 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:43:37Z", + "id": 90773670384 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:59:32Z", + "id": 90775741301 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:06Z", + "id": 90772852098 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:06Z", + "id": 90772851044 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:11Z", + "id": 90772851994 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:07Z", + "id": 90772851669 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:06Z", + "id": 90772851076 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:42:59Z", + "id": 90773583610 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:12Z", + "id": 90772850710 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:42:52Z", + "id": 90773583715 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:07Z", + "id": 90772851616 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:42:54Z", + "id": 90773584593 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:37:13Z", + "id": 90772851911 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:42:25Z", + "id": 90773528632 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:42:52Z", + "id": 90773584556 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T03:58:42Z", + "id": 90775632676 + } + ] + }, + "2555": { + "head_sha": "0dbedbca0ee0c3fa2c0258632021c86209028893", + "merged_at": "2026-07-30T04:25:09Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:03Z", + "id": 90775808019 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:14:36Z", + "id": 90777846871 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:03Z", + "id": 90775807753 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:03Z", + "id": 90775807836 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:14:36Z", + "id": 90777846967 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:01:38Z", + "id": 90776029477 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:04Z", + "id": 90775807706 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:04Z", + "id": 90775807854 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:11Z", + "id": 90775807733 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:14:37Z", + "id": 90777846720 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:01:08Z", + "id": 90775964158 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:15:22Z", + "id": 90777947334 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "failure", + "started_at": "2026-07-30T04:24:38Z", + "id": 90779162389 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:11Z", + "id": 90775807850 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:09Z", + "id": 90775808461 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:07Z", + "id": 90775808505 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:03Z", + "id": 90775807816 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:14:36Z", + "id": 90777846921 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:02Z", + "id": 90775807645 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:14:36Z", + "id": 90777846849 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:09Z", + "id": 90775808387 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:14:36Z", + "id": 90777847818 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:09Z", + "id": 90775808329 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:00:03Z", + "id": 90775808394 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T04:14:42Z", + "id": 90777847706 + } + ] + }, + "2554": { + "head_sha": "0bb3f099183a1dfb592b97f1de57a7ff3e97528d", + "merged_at": "2026-07-30T03:48:09Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T02:41:59Z", + "id": 90765288264 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:43:10Z", + "id": 90765406702 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:41:51Z", + "id": 90765288028 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:41:50Z", + "id": 90765288031 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:42:45Z", + "id": 90765378073 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:44:02Z", + "id": 90765581823 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:41:50Z", + "id": 90765288076 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:42:11Z", + "id": 90765288546 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T02:41:56Z", + "id": 90765288111 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:42:38Z", + "id": 90765389639 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "skipped", + "started_at": "2026-07-30T02:42:25Z", + "id": 90765371157 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:43:22Z", + "id": 90765475461 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:41:58Z", + "id": 90765288847 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:41:50Z", + "id": 90765288122 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:42:10Z", + "id": 90765288906 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:42:12Z", + "id": 90765288853 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:41:51Z", + "id": 90765288182 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:42:42Z", + "id": 90765377931 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:41:50Z", + "id": 90765288015 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:42:37Z", + "id": 90765377986 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:42:07Z", + "id": 90765288946 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:42:34Z", + "id": 90765378629 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:42:08Z", + "id": 90765288885 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "failure", + "started_at": "2026-07-30T02:42:07Z", + "id": 90765288945 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:42:38Z", + "id": 90765378634 + } + ] + }, + "2553": { + "head_sha": "dcb2cf6f0cd5a79c69cefa7aa42a0d447dab449e", + "merged_at": "2026-07-30T03:48:40Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:11Z", + "id": 90764807483 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:39:39Z", + "id": 90764952990 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:56:49Z", + "id": 90767259607 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:24Z", + "id": 90764820167 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:12Z", + "id": 90764808796 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:39:29Z", + "id": 90764953089 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:56:50Z", + "id": 90767259601 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:40:10Z", + "id": 90765022680 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:16Z", + "id": 90764801996 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:09Z", + "id": 90764801405 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:12Z", + "id": 90764808377 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:39:24Z", + "id": 90764953035 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:56:49Z", + "id": 90767259527 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:54Z", + "id": 90764889045 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:40:36Z", + "id": 90765125915 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:57:54Z", + "id": 90767402146 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:17Z", + "id": 90764802914 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:09Z", + "id": 90764801016 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:10Z", + "id": 90764802202 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:10Z", + "id": 90764802193 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:10Z", + "id": 90764801374 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:39:44Z", + "id": 90764953104 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:56:49Z", + "id": 90767259576 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:16Z", + "id": 90764801430 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:39:39Z", + "id": 90764952920 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:56:55Z", + "id": 90767259381 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:10Z", + "id": 90764803386 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:39:24Z", + "id": 90764954159 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:56:51Z", + "id": 90767260691 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:09Z", + "id": 90764801774 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:38:17Z", + "id": 90764808226 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:39:28Z", + "id": 90764954562 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:56:50Z", + "id": 90767260311 + } + ] + }, + "2552": { + "head_sha": "05e105cf7ede887dbf1617f9d7c1248c2a70f11c", + "merged_at": "2026-07-30T02:08:13Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756203571 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:37:54Z", + "id": 90756295266 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:39:10Z", + "id": 90756515139 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756203968 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756203736 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:37:29Z", + "id": 90756282801 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:39:13Z", + "id": 90756514950 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:38:50Z", + "id": 90756470437 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756203772 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756203725 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T01:37:04Z", + "id": 90756203350 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:37:49Z", + "id": 90756296355 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:39:25Z", + "id": 90756515111 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "skipped", + "started_at": "2026-07-30T01:37:17Z", + "id": 90756259443 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:38:38Z", + "id": 90756405196 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:39:55Z", + "id": 90756624595 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756204236 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756203943 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756204373 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:59Z", + "id": 90756204545 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756204172 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:37:30Z", + "id": 90756282748 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:39:11Z", + "id": 90756515086 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756203404 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:37:29Z", + "id": 90756282891 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:39:18Z", + "id": 90756515264 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756204564 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:37:30Z", + "id": 90756283348 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:39:21Z", + "id": 90756516286 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756204334 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "failure", + "started_at": "2026-07-30T01:36:58Z", + "id": 90756204347 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:37:30Z", + "id": 90756283736 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T01:39:11Z", + "id": 90756516507 + } + ] + }, + "2551": { + "head_sha": "bdcbeccd809f05f5fcb95e7359af9b0af1cfe23c", + "merged_at": "2026-07-30T01:04:49Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:47Z", + "id": 90748473086 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T00:47:01Z", + "id": 90748821931 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:47:42Z", + "id": 90748922123 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:48Z", + "id": 90748472596 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:54Z", + "id": 90748472365 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:47:01Z", + "id": 90748822136 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:47:27Z", + "id": 90748883849 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:46:37Z", + "id": 90748758539 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:47Z", + "id": 90748472481 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:53Z", + "id": 90748472642 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:54Z", + "id": 90748472581 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T00:47:07Z", + "id": 90748822039 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:47:31Z", + "id": 90748891544 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:45:35Z", + "id": 90748598762 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-30T00:47:28Z", + "id": 90748895765 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:48:19Z", + "id": 90749023803 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:53Z", + "id": 90748473606 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:48Z", + "id": 90748474941 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:48Z", + "id": 90748473551 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:54Z", + "id": 90748473437 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:53Z", + "id": 90748472432 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:47:01Z", + "id": 90748821926 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:47:33Z", + "id": 90748883971 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:48Z", + "id": 90748472592 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:47:01Z", + "id": 90748822019 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:47:27Z", + "id": 90748883790 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:47Z", + "id": 90748473330 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:47:05Z", + "id": 90748828482 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:47:29Z", + "id": 90748884897 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:54Z", + "id": 90748473284 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:44:49Z", + "id": 90748473548 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "failure", + "started_at": "2026-07-30T00:47:04Z", + "id": 90748828825 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:47:28Z", + "id": 90748884866 + } + ] + }, + "2550": { + "head_sha": "6efc6b6fb8c1f171b32ece934b79184c85f027dd", + "merged_at": "2026-07-30T00:48:11Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:31Z", + "id": 90740599072 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:23:50Z", + "id": 90745135172 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:37Z", + "id": 90740599339 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:31Z", + "id": 90740599460 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:23:50Z", + "id": 90745135516 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:57:14Z", + "id": 90740874553 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:37Z", + "id": 90740599419 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:31Z", + "id": 90740599479 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:38Z", + "id": 90740599444 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:23:50Z", + "id": 90745135104 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:24:39Z", + "id": 90745268277 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:36:27Z", + "id": 90747159049 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:32Z", + "id": 90740600547 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:31Z", + "id": 90740599308 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:32Z", + "id": 90740600369 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:32Z", + "id": 90740600301 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:31Z", + "id": 90740599510 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:23:57Z", + "id": 90745135100 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:31Z", + "id": 90740599402 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:23:56Z", + "id": 90745135261 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:32Z", + "id": 90740600339 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:23:57Z", + "id": 90745136427 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:55:31Z", + "id": 90740600555 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:23:59Z", + "id": 90745136298 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T00:35:37Z", + "id": 90747033832 + } + ] + }, + "2546": { + "head_sha": "746865732c45107f57ec876b717f035db195af20", + "merged_at": "2026-07-29T23:04:37Z", + "check_runs": [ + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-29T22:13:32Z", + "id": 90722749209 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:18:34Z", + "id": 90722882000 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:44:53Z", + "id": 90728693250 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:46:57Z", + "id": 90729039190 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:49:26Z", + "id": 90729441521 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:51:58Z", + "id": 90729937357 + }, + { + "name": "Canonical Inference Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:55:13Z", + "id": 90730451837 + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:16:51Z", + "id": 90722749164 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-29T22:13:32Z", + "id": 90722749133 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:18:56Z", + "id": 90722882269 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:44:53Z", + "id": 90728692769 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:46:58Z", + "id": 90729039076 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:49:26Z", + "id": 90729441596 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:51:55Z", + "id": 90729937166 + }, + { + "name": "Duplication Sweep", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:55:10Z", + "id": 90730452512 + }, + { + "name": "Omni Standards Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:22:09Z", + "id": 90723877967 + }, + { + "name": "Stale TODO Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:16:48Z", + "id": 90722749161 + }, + { + "name": "Type Safety Validation", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:17:29Z", + "id": 90722749032 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-29T22:13:32Z", + "id": 90722749241 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:18:25Z", + "id": 90722882090 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:44:54Z", + "id": 90728692758 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:47:16Z", + "id": 90729038915 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:49:18Z", + "id": 90729441612 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:51:53Z", + "id": 90729936928 + }, + { + "name": "URL Authority Gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:55:18Z", + "id": 90730452015 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-29T22:14:14Z", + "id": 90722884595 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:22:06Z", + "id": 90723884608 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:47:42Z", + "id": 90729190193 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:50:10Z", + "id": 90729616941 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-29T22:52:08Z", + "id": 90729985580 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:52:50Z", + "id": 90730096766 + }, + { + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:55:57Z", + "id": 90730628318 + }, + { + "name": "deploy-gate / deploy-gate", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:17:06Z", + "id": 90722750815 + }, + { + "name": "dispatcher-route-coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:17:22Z", + "id": 90722749102 + }, + { + "name": "gate / CodeRabbit Thread Check", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:17:23Z", + "id": 90722750836 + }, + { + "name": "imperative-contract-guard / Imperative Contract Guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:17:57Z", + "id": 90722750667 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-29T22:13:32Z", + "id": 90722748861 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:18:46Z", + "id": 90722882368 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:44:53Z", + "id": 90728693085 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:47:13Z", + "id": 90729039476 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:49:07Z", + "id": 90729441654 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:51:59Z", + "id": 90729936768 + }, + { + "name": "main-target-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:55:00Z", + "id": 90730452231 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-29T22:13:32Z", + "id": 90722749184 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:18:52Z", + "id": 90722882200 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:44:53Z", + "id": 90728693181 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:47:14Z", + "id": 90729039165 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:49:39Z", + "id": 90729441571 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:52:01Z", + "id": 90729936790 + }, + { + "name": "non-dev-base-guard", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:55:13Z", + "id": 90730451799 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-29T22:13:33Z", + "id": 90722750604 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:18:57Z", + "id": 90722885133 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:45:01Z", + "id": 90728694132 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:47:09Z", + "id": 90729039959 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:49:20Z", + "id": 90729442813 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:51:59Z", + "id": 90729938206 + }, + { + "name": "pr-title / check-title", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:54:54Z", + "id": 90730453213 + }, + { + "name": "required-check-skip-guard / check-skip-vectors", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:17:39Z", + "id": 90722750366 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-29T22:13:33Z", + "id": 90722750823 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:18:44Z", + "id": 90722883604 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:49:20Z", + "id": 90729442795 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-07-29T22:51:58Z", + "id": 90729938167 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T22:55:00Z", + "id": 90730453681 + }, + { + "name": "verify / verify", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-29T23:00:04Z", + "id": 90731344845 + } + ] + } + } +} diff --git a/tests/ci/fixtures/omn15532_dependabot_pr2522_check_runs.json b/tests/ci/fixtures/omn15532_dependabot_pr2522_check_runs.json new file mode 100644 index 0000000000..dc95861361 --- /dev/null +++ b/tests/ci/fixtures/omn15532_dependabot_pr2522_check_runs.json @@ -0,0 +1,694 @@ +{ + "_source": "live gh api repos/OmniNode-ai/omnibase_infra/commits/2cdf352d5577ac2e533da11ebdf38941697fb7ad/check-runs, captured 2026-07-30 for OMN-15532; PR #2522, author dependabot[bot]", + "head_sha": "2cdf352d5577ac2e533da11ebdf38941697fb7ad", + "pr": 2522, + "pr_author": "dependabot[bot]", + "check_runs": [ + { + "conclusion": "skipped", + "id": 90196373842, + "name": "auto-tag", + "started_at": "2026-07-28T06:20:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90196373499, + "name": "Publish PR Merged Event", + "started_at": "2026-07-28T06:20:41Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90196373180, + "name": "Trigger node_redeploy Start", + "started_at": "2026-07-28T06:20:41Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90196373143, + "name": "TODO Audit", + "started_at": "2026-07-28T06:20:40Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90196373123, + "name": "Publish PR Webhook Event", + "started_at": "2026-07-28T06:20:41Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "id": 90195249939, + "name": "Runtime Boot Smoke (compose)", + "started_at": "2026-07-28T06:13:45Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90195170999, + "name": "CI Tests Gate", + "started_at": "2026-07-28T06:13:42Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90195170992, + "name": "Test-Failure Ratchet Gate", + "started_at": "2026-07-28T06:13:30Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90194904062, + "name": "Tests (Split 1/1)", + "started_at": "2026-07-28T06:11:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90194655992, + "name": "Detect Changes", + "started_at": "2026-07-28T06:11:09Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193931522, + "name": "Hostile Review Gate", + "started_at": "2026-07-28T06:06:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193907946, + "name": "Integration Silent-Skip Guard (OMN-14172)", + "started_at": "2026-07-28T06:10:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193907771, + "name": "Migration Integration Test", + "started_at": "2026-07-28T06:10:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193617388, + "name": "Omni Standards Gate", + "started_at": "2026-07-28T06:05:31Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193587246, + "name": "reason-graph", + "started_at": "2026-07-28T06:05:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193560221, + "name": "Version Pin Compliance", + "started_at": "2026-07-28T06:10:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193560005, + "name": "Kafka Boundary Compat (OMN-3256)", + "started_at": "2026-07-28T06:10:08Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559971, + "name": "zone-filter / Zone Filter (docs-only check)", + "started_at": "2026-07-28T06:05:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559964, + "name": "Cross-Repo Migration Conflicts", + "started_at": "2026-07-28T06:10:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559953, + "name": "Topic Enum Drift Check", + "started_at": "2026-07-28T06:08:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559919, + "name": "Arch Invariants (OMN-3343)", + "started_at": "2026-07-28T06:05:30Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559891, + "name": "OCC Born-Path Trigger Coverage (OMN-14987)", + "started_at": "2026-07-28T06:06:25Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559873, + "name": "Contract Sync Gate (Wave C) [OMN-8915]", + "started_at": "2026-07-28T06:07:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559866, + "name": "Contract Compliance Check", + "started_at": "2026-07-28T06:08:58Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559863, + "name": "AI-Slop Pattern Check (strict, PR diff)", + "started_at": "2026-07-28T06:07:30Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559862, + "name": "Topic Drift Check", + "started_at": "2026-07-28T06:07:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559861, + "name": "Plugin ENV->Service Completeness (OMN-4313)", + "started_at": "2026-07-28T06:08:42Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559855, + "name": "Contract Path Pre-Flight", + "started_at": "2026-07-28T06:06:25Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559854, + "name": "Contract Compliance", + "started_at": "2026-07-28T06:07:37Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559849, + "name": "Topic Naming Lint", + "started_at": "2026-07-28T06:07:26Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559846, + "name": "Compose Required-Env Coverage (OMN-5439)", + "started_at": "2026-07-28T06:07:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559843, + "name": "Kafka Schema Handshake (OMN-3411)", + "started_at": "2026-07-28T06:08:37Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559841, + "name": "Demo Loop Gate", + "started_at": "2026-07-28T06:06:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559831, + "name": "Writer-Migration Coupling Check", + "started_at": "2026-07-28T06:06:55Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559815, + "name": "Migration Freeze Check", + "started_at": "2026-07-28T06:05:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559693, + "name": "Lint", + "started_at": "2026-07-28T06:06:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559681, + "name": "Effect-Assertion Gate (RT-5)", + "started_at": "2026-07-28T06:05:52Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559673, + "name": "Infra Node Handler Ownership", + "started_at": "2026-07-28T06:05:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559649, + "name": "ONEX Validators", + "started_at": "2026-07-28T06:05:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193559635, + "name": "Fingerprint Check", + "started_at": "2026-07-28T06:04:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193547415, + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-28T06:04:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90193452529, + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-28T06:05:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192737115, + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-28T06:02:34Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "id": 90192736937, + "name": "occ-autobind-manual-replay", + "started_at": "2026-07-28T05:58:28Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "id": 90192736644, + "name": "occ-autobind", + "started_at": "2026-07-28T05:58:28Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "id": 90192736600, + "name": "occ-companion-effect", + "started_at": "2026-07-28T05:58:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192736470, + "name": "call / validate-docs", + "started_at": "2026-07-28T06:02:54Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "id": 90192736416, + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-28T05:58:28Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "id": 90192736346, + "name": "gate", + "started_at": "2026-07-28T05:58:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192736344, + "name": "occ-preflight / eligibility", + "started_at": "2026-07-28T06:03:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192736335, + "name": "no-noncanonical-lifecycle-classes", + "started_at": "2026-07-28T06:03:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192736334, + "name": "CI Summary", + "started_at": "2026-07-28T06:03:26Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192736296, + "name": "Runtime Profiles / validate", + "started_at": "2026-07-28T06:03:18Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192736292, + "name": "Enforce validator-requirements.yaml (OMN-13291)", + "started_at": "2026-07-28T06:03:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192736273, + "name": "OCC Companion Merged Gate (OMN-15214)", + "started_at": "2026-07-28T06:02:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192736194, + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-28T06:03:37Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "id": 90192736166, + "name": "verify", + "started_at": "2026-07-28T05:58:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192736165, + "name": "pr-title / check-title", + "started_at": "2026-07-28T06:02:09Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192736082, + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-28T06:02:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192736072, + "name": "verify / verify", + "started_at": "2026-07-28T06:03:08Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735916, + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-28T06:03:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735849, + "name": "occ-preflight / eligibility", + "started_at": "2026-07-28T06:03:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735577, + "name": "pinned-wheel-skew", + "started_at": "2026-07-28T06:02:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735470, + "name": "dispatcher-route-coverage", + "started_at": "2026-07-28T06:03:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735431, + "name": "URL Authority Gate", + "started_at": "2026-07-28T06:03:13Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735417, + "name": "Integration Test Coverage", + "started_at": "2026-07-28T06:02:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735383, + "name": "non-dev-base-guard", + "started_at": "2026-07-28T06:02:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735355, + "name": "Stale TODO Gate", + "started_at": "2026-07-28T06:02:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735347, + "name": "main-target-guard", + "started_at": "2026-07-28T06:02:13Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735343, + "name": "Integration Test Removal Gate", + "started_at": "2026-07-28T06:02:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735311, + "name": "sibling-lock-pins", + "started_at": "2026-07-28T06:02:46Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735310, + "name": "Runner Disk Preflight", + "started_at": "2026-07-28T06:02:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735309, + "name": "No bare compose teardown", + "started_at": "2026-07-28T06:02:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735294, + "name": "CodeQL", + "started_at": "2026-07-28T06:02:11Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735271, + "name": "shell-hygiene", + "started_at": "2026-07-28T06:02:04Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735267, + "name": "context-field-presence", + "started_at": "2026-07-28T06:02:12Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735265, + "name": "skill-node-mapping-sync", + "started_at": "2026-07-28T06:02:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735259, + "name": "release-identity", + "started_at": "2026-07-28T06:02:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735254, + "name": "Duplication Sweep", + "started_at": "2026-07-28T06:02:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735249, + "name": "terminal-cost-completeness", + "started_at": "2026-07-28T06:02:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735214, + "name": "Handler Contract Compliance", + "started_at": "2026-07-28T06:02:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735201, + "name": "Type Safety Validation", + "started_at": "2026-07-28T06:01:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735187, + "name": "CI Naming Convention", + "started_at": "2026-07-28T06:02:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735144, + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-28T06:01:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735137, + "name": "Check architecture handshake", + "started_at": "2026-07-28T06:01:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735113, + "name": "lint (shadow)", + "started_at": "2026-07-28T06:02:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735110, + "name": "contract-validation", + "started_at": "2026-07-28T06:01:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735099, + "name": "receipt-honesty", + "started_at": "2026-07-28T06:02:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735093, + "name": "Runner Routing Audit", + "started_at": "2026-07-28T06:02:52Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735090, + "name": "tests+coverage (shadow)", + "started_at": "2026-07-28T06:01:58Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735058, + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-28T06:03:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735057, + "name": "Canonical Inference Gate", + "started_at": "2026-07-28T06:01:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735054, + "name": "node-migration-sync", + "started_at": "2026-07-28T06:03:55Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192735029, + "name": "typecheck (shadow)", + "started_at": "2026-07-28T06:01:58Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192734994, + "name": "check-env-reads-gate", + "started_at": "2026-07-28T06:02:25Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192734979, + "name": "Precommit Parity Gate", + "started_at": "2026-07-28T06:01:42Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192734958, + "name": "Dep Provenance Gate", + "started_at": "2026-07-28T06:03:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "id": 90192734893, + "name": "Publish PR Webhook Event", + "started_at": "2026-07-28T06:02:48Z", + "status": "completed" + } + ] +} diff --git a/tests/ci/fixtures/runner_diag_real_tail.log.gz b/tests/ci/fixtures/runner_diag_real_tail.log.gz new file mode 100644 index 0000000000..b26e965047 Binary files /dev/null and b/tests/ci/fixtures/runner_diag_real_tail.log.gz differ diff --git a/tests/ci/test_application_database_acl_contract.py b/tests/ci/test_application_database_acl_contract.py new file mode 100644 index 0000000000..f246dbc714 --- /dev/null +++ b/tests/ci/test_application_database_acl_contract.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Static contract for the pinned generated application ACL candidate.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + +from omnibase_core.enums.enum_database_grant_object_type import ( + EnumDatabaseGrantObjectType, +) +from omnibase_infra.validation.application_database_acl import ( + PUBLIC_PRINCIPAL, + validate_application_database_acl_matrix, +) +from omnibase_infra.validation.models.model_application_database_acl_matrix import ( + ModelApplicationDatabaseAclMatrix, + ModelApplicationDatabaseAclSource, +) + +_ROOT = Path(__file__).parents[2] +_PROOF = _ROOT / "docker" / "application-acl-proof" + + +def _candidate() -> ModelApplicationDatabaseAclMatrix: + return ModelApplicationDatabaseAclMatrix.model_validate( + yaml.safe_load( + (_PROOF / "generated" / "candidate-matrix.yaml").read_text(encoding="utf-8") + ) + ) + + +def test_candidate_is_a_complete_fail_closed_projection_of_locked_sources() -> None: + lock = yaml.safe_load((_PROOF / "source-lock.yaml").read_text(encoding="utf-8")) + locked_sources = tuple( + ModelApplicationDatabaseAclSource.model_validate(source) + for source in lock["sources"] + ) + matrix = _candidate() + + assert set(matrix.sources) == set(locked_sources) + assert matrix.status == "BLOCKED" + assert matrix.scaffold_status == "BLOCKED" + assert matrix.scaffold_blockers + assert len(matrix.objects) == 112 + assert len(matrix.rows) == 5 * (1 + 3 + len(matrix.objects)) + assert len(matrix.default_privileges) == 3 * 4 * 5 + assert matrix.declared_principals == { + "application": ( + "app_dashboard", + "omninode_runtime", + "onex_api", + "tenant_projection_writer", + ) + } + assert matrix.observed_principals == {"application": ()} + assert matrix.absent_principals == {"application": ()} + assert set(matrix.required_connect_databases) == { + "keycloak", + "omnibase_infra", + "omnidash_analytics", + "omninode_cloud", + "omniclaude", + "omniintelligence", + "omnimemory", + "umami", + } + assert not matrix.allowed_connect_principals + assert not matrix.observed_connect_principals + assert not matrix.absent_connect_principals + assert not matrix.observed_connect_database_owners + violations = validate_application_database_acl_matrix(matrix) + assert violations + assert any("cross-domain" in violation for violation in violations) + assert any("not materialized" in violation for violation in violations) + assert all( + f"ACL policy violation: {violation}" in matrix.blockers + for violation in violations + ) + assert not matrix.database_owners + assert not matrix.allowed_memberships + assert all(not row.privileges for row in matrix.default_privileges) + assert all( + not row.privileges for row in matrix.rows if row.principal == PUBLIC_PRINCIPAL + ) + + +def test_candidate_retains_real_blockers_and_never_emits_blocked_sql() -> None: + matrix = _candidate() + blocker_text = "\n".join(matrix.blockers) + + assert "full_day_datname_usename_activity='blocked'" in blocker_text + assert "live_catalog_parity='blocked'" in blocker_text + assert "schema 'unresolved'" in blocker_text + assert "no authoritative repository DDL" in blocker_text + assert "principal_inventory" in blocker_text + assert "acl_policy" in blocker_text + assert "relation_counts.type is not inventoried" in blocker_text + assert "relation_counts.procedure is not inventoried" in blocker_text + assert "requires exactly one CONNECT policy" in blocker_text + assert "full object ACL rendering is gated" in blocker_text + assert "explicit function_signature" in blocker_text + assert not (_PROOF / "generated" / "application-acl.sql").exists() + + +def test_matrix_spans_every_required_object_and_default_privilege_kind() -> None: + matrix = _candidate() + + assert {obj.object_type for obj in matrix.objects} == { + EnumDatabaseGrantObjectType.TABLE, + EnumDatabaseGrantObjectType.SEQUENCE, + EnumDatabaseGrantObjectType.FUNCTION, + } + # No source head declares a type object, but future TYPE privileges are still + # deny-by-default for every actual owner and workload principal. + assert {row.object_type for row in matrix.default_privileges} == { + EnumDatabaseGrantObjectType.TABLE, + EnumDatabaseGrantObjectType.SEQUENCE, + EnumDatabaseGrantObjectType.FUNCTION, + EnumDatabaseGrantObjectType.TYPE, + } diff --git a/tests/ci/test_application_database_domain_enforcement_contract.py b/tests/ci/test_application_database_domain_enforcement_contract.py new file mode 100644 index 0000000000..22431c13ff --- /dev/null +++ b/tests/ci/test_application_database_domain_enforcement_contract.py @@ -0,0 +1,527 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Mandatory-source and blocked-deployment ratchet for OMN-15361.""" + +from __future__ import annotations + +import json +import runpy +import subprocess +import sys +from dataclasses import FrozenInstanceError +from pathlib import Path + +import pytest +import yaml + +from omnibase_infra.validation.application_database_red_control_registry import ( + APPLICATION_DATABASE_RED_CONTROL_REGISTRY, + ApplicationDatabaseRedControlBinding, +) +from omnibase_infra.validation.application_relation_ownership import ( + load_service_ownership_manifest, +) +from omnibase_infra.validation.enums.enum_application_database_enforcement_gate import ( + EnumApplicationDatabaseEnforcementGate, +) +from omnibase_infra.validation.enums.enum_application_database_object_kind import ( + EnumApplicationDatabaseObjectKind, +) +from omnibase_infra.validation.enums.enum_application_relation_kind import ( + EnumApplicationRelationKind, +) +from omnibase_infra.validation.models.model_application_database_enforcement_contract import ( + ModelApplicationDatabaseEnforcementContract, +) +from omnibase_infra.validation.models.model_database_object_evidence import ( + ModelDatabaseObjectEvidence, +) +from omnibase_infra.validation.models.model_relation_evidence import ( + ModelRelationEvidence, +) + +_ROOT = Path(__file__).parents[2] +_CONTRACT = _ROOT / "config" / "application_database_domain_enforcement.yaml" +_OWNERSHIP = _ROOT / "config" / "application_database_domain_proof_ownership.yaml" +_CI_WORKFLOW = _ROOT / ".github" / "workflows" / "ci.yml" +_DOMAIN_PROOF = ( + _ROOT / "scripts" / "ci" / "prove_application_database_domain_enforcement.py" +) +_ACL_PROOF = _ROOT / "scripts" / "ci" / "prove_application_database_acl.py" +_ACL_POSTGRES16_INVENTORY = ( + _ROOT + / "tests" + / "fixtures" + / "application_database_acl" + / "principal-inventory-postgres16.yaml" +) +_ACL_FIXTURES = _ACL_POSTGRES16_INVENTORY.parent +_ACL_PRECHANGE = ( + _ROOT + / "docker" + / "application-acl-proof" + / "generated" + / "prechange-fixture-acl.json" +) + + +def _contract() -> ModelApplicationDatabaseEnforcementContract: + return ModelApplicationDatabaseEnforcementContract.model_validate( + yaml.safe_load(_CONTRACT.read_text(encoding="utf-8")) + ) + + +def test_acl_postgres16_evidence_preserves_range_type_identities() -> None: + inventory = yaml.safe_load(_ACL_POSTGRES16_INVENTORY.read_text(encoding="utf-8")) + inventory_kinds = { + item["catalog_kind"] + for item in inventory["observed_objects"] + if item["object_ref"] == "account_id_span" + and item.get("function_signature") is None + } + assert inventory_kinds == {"range_type"} + multirange_inventory_kinds = { + item["catalog_kind"] + for item in inventory["observed_objects"] + if item["object_ref"] == "account_id_span_set" + and item.get("function_signature") is None + } + assert multirange_inventory_kinds == {"multirange_type"} + + prechange = json.loads(_ACL_PRECHANGE.read_text(encoding="utf-8")) + for section in ("object_acl", "object_owners"): + object_kinds = { + item["object_name"]: item["catalog_kind"] + for item in prechange[section] + if item["object_type"] == "TYPE" + } + assert object_kinds["account_id_span"] == "range_type", section + assert object_kinds["account_id_span_set"] == "multirange_type", section + + +def test_acl_prechange_artifact_rows_are_canonically_sorted() -> None: + prechange = json.loads(_ACL_PRECHANGE.read_text(encoding="utf-8")) + for section, rows in prechange.items(): + if not isinstance(rows, list) or not rows: + continue + assert rows == sorted( + rows, + key=lambda row: json.dumps( + row, + sort_keys=True, + separators=(",", ":"), + ), + ), section + + +def test_acl_live_snapshot_classifies_postgres_type_subkinds_exactly() -> None: + source = _ACL_PROOF.read_text(encoding="utf-8") + exact_catalog_kind_case = "\n".join( + ( + "CASE type.typtype", + " WHEN 'b' THEN 'base_type'", + " WHEN 'r' THEN 'range_type'", + " WHEN 'm' THEN 'multirange_type'", + " ELSE 'type'", + " END AS catalog_kind", + ) + ) + + assert source.count(exact_catalog_kind_case) == 2 + assert "'type' AS catalog_kind, 'TYPE' AS owner_keyword" not in source + assert "'TYPE' AS object_type, 'type' AS catalog_kind" not in source + + +def test_acl_postgres16_fixture_matrix_is_ready_and_rollback_keywords_are_typed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("ADMIN_DSN", "dbname=unused") + monkeypatch.setenv("ACL_FIXTURES", str(_ACL_FIXTURES)) + proof = runpy.run_path(str(_ACL_PROOF), run_name="acl_proof_contract") + + matrix = proof["_fixture_matrix"]() + assert matrix.status == "READY", matrix.blockers + typed_objects = { + obj.catalog_kind: obj + for obj in matrix.objects + if obj.object_ref in {"account_id_span", "account_id_span_set"} + and obj.function_signature is None + } + assert set(typed_objects) == {"range_type", "multirange_type"} + assert proof["_acl_object_keyword"](typed_objects["range_type"]) == "TYPE" + assert proof["_acl_object_keyword"](typed_objects["multirange_type"]) == "TYPE" + + +def test_every_domain_gate_is_mandatory_in_source_and_has_red_green_proof() -> None: + contract = _contract() + + assert set(contract.gates) == set(EnumApplicationDatabaseEnforcementGate) + for gate, state in contract.gates.items(): + assert state.source_enforcement == "mandatory", gate + assert state.source_proofs, gate + assert state.seeded_red_controls, gate + for proof_path in state.source_proof_paths: + assert (_ROOT / proof_path).is_file(), (gate, proof_path) + + +def test_seeded_red_control_registry_rejects_a_phantom_control() -> None: + source = yaml.safe_load(_CONTRACT.read_text(encoding="utf-8")) + source["gates"]["classification"]["seeded_red_controls"].append("phantom-control") + + with pytest.raises(ValueError, match="RED control registry"): + ModelApplicationDatabaseEnforcementContract.model_validate(source) + + +def test_seeded_red_control_registry_rejects_a_renamed_control() -> None: + source = yaml.safe_load(_CONTRACT.read_text(encoding="utf-8")) + source["gates"]["classification"]["seeded_red_controls"][0] = "renamed-control" + + with pytest.raises(ValueError, match="RED control registry"): + ModelApplicationDatabaseEnforcementContract.model_validate(source) + + +@pytest.mark.parametrize( + ("control_id", "mismatched_node_id"), + [ + ( + "unknown-topology-schema", + "tests/unit/validation/" + "test_application_database_domain_enforcement.py::" + "test_topology_schema_domain_drift_fails_closed", + ), + ( + "identity-root-unproven-enumeration", + "tests/unit/validation/" + "test_application_database_domain_enforcement.py::" + "test_tenant_identity_root_requires_closed_contract_relation_and_primary_key", + ), + ], +) +def test_red_control_binding_rejects_a_semantically_mismatched_node( + control_id: str, + mismatched_node_id: str, +) -> None: + with pytest.raises(ValueError, match="exact semantic pytest case"): + ApplicationDatabaseRedControlBinding( + control_id=control_id, + pytest_node_id=mismatched_node_id, + ) + + +def test_red_control_registry_is_deeply_immutable_and_one_to_one() -> None: + gate = EnumApplicationDatabaseEnforcementGate.CLASSIFICATION + controls = APPLICATION_DATABASE_RED_CONTROL_REGISTRY[gate] + binding = controls["missing-owner"] + all_bindings = tuple( + item + for gate_bindings in APPLICATION_DATABASE_RED_CONTROL_REGISTRY.values() + for item in gate_bindings.values() + ) + + assert len({item.pytest_node_id for item in all_bindings}) == len(all_bindings) + with pytest.raises(TypeError, match="does not support item assignment"): + controls["missing-owner"] = binding # type: ignore[index] + with pytest.raises(FrozenInstanceError): + binding.pytest_node_id = "tests/phantom.py::test_phantom" # type: ignore[misc] + + +def test_seeded_red_controls_execute_exactly_and_pass_in_the_required_ci_path() -> None: + contract = _contract() + bindings = tuple( + binding + for proofs in APPLICATION_DATABASE_RED_CONTROL_REGISTRY.values() + for binding in proofs.values() + ) + node_ids = tuple(binding.pytest_node_id for binding in bindings) + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", *node_ids], + cwd=_ROOT, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert f"{len(node_ids)} passed" in result.stdout, result.stdout + result.stderr + workflow = _CI_WORKFLOW.read_text(encoding="utf-8") + required_step = workflow.split( + "- name: Execute mandatory source assertions and seeded RED controls", + maxsplit=1, + )[1].split("- name:", maxsplit=1)[0] + + for gate, proofs in APPLICATION_DATABASE_RED_CONTROL_REGISTRY.items(): + state = contract.gates[gate] + assert set(proofs) == set(state.seeded_red_controls), gate + for control, binding in proofs.items(): + assert binding.control_id == control + node_id = binding.pytest_node_id + proof_path = node_id.partition("::")[0] + assert proof_path in state.source_proof_paths, (gate, control, proof_path) + assert proof_path in required_step, (gate, control, proof_path) + + +def test_docker_backed_red_controls_bind_to_exact_emitted_pass_results() -> None: + proof_source = _DOMAIN_PROOF.read_text(encoding="utf-8") + + for gate, proofs in APPLICATION_DATABASE_RED_CONTROL_REGISTRY.items(): + for control, binding in proofs.items(): + if binding.docker_result is None: + continue + assert binding.docker_result == f"domain_control={control} status=PASS" + assert f'"{control}"' in proof_source, (gate, control) + + +def test_unmet_deployment_preconditions_are_explicit_blockers_not_green_claims() -> ( + None +): + contract = _contract() + + assert all( + state.deployment_enforcement == "blocked" for state in contract.gates.values() + ) + blocker_text = "\n".join( + blocker + for state in contract.gates.values() + for blocker in state.deployment_blockers + ) + assert "OMN-15423" in blocker_text + assert "OMN-15358" in blocker_text + assert "OMN-15416" in blocker_text + assert "OMN-15424" in blocker_text + assert "OMN-15425" in blocker_text + assert "OMN-15426" in blocker_text + assert "full-day" in blocker_text + assert "secret" in blocker_text + assert "deploy" in blocker_text + + +def test_kubernetes_parity_remains_a_typed_blocker_not_a_source_green_claim() -> None: + contract = _contract() + topology_gate = contract.gates[ + EnumApplicationDatabaseEnforcementGate.TOPOLOGY_PARITY + ] + + source_text = "\n".join(topology_gate.source_proofs).lower() + assert "kubernetes" not in source_text or "blocked" in source_text + assert any("Kubernetes" in blocker for blocker in topology_gate.deployment_blockers) + + +def test_exact_predecessor_pins_are_immutable_and_complete() -> None: + contract = _contract() + + assert contract.predecessor_pins == { + "omnibase_core#1529": "1f4549d71d4d39560ac5a162ac1d39e54d86e688", + "omnibase_infra#2547": "95351f5d8e806fcf7fa2c276d9065df93ccf92b9", + "omnibase_infra#2548": "7228ce0c0934ae096dd6effd0f84ff1913fec6c0", + "omnibase_infra#2558": "2a2cfb275810b34197d4d5baf55bdcddc443e6dc", + "omnimarket#1956": "4637e625c99ef17c190aa471a5e51b7f646c6dfd", + "omninode_infra#771": "39033d55147ef22a061b665345b506246d3aa543", + } + + +def test_exact_domain_adapter_predecessor_is_checked_out_and_executed() -> None: + workflow = _CI_WORKFLOW.read_text(encoding="utf-8") + + assert "ref: 2a2cfb275810b34197d4d5baf55bdcddc443e6dc" in workflow + assert "path: .proof-dependencies/domain-adapter" in workflow + assert "working-directory: .proof-dependencies/domain-adapter" in workflow + + +def test_private_ownership_pin_is_pat_authenticated_and_fork_fail_closed() -> None: + workflow = _CI_WORKFLOW.read_text(encoding="utf-8") + + assert "token: ${{ secrets.CROSS_REPO_PAT }}" in workflow + assert "Enforce schema qualification in changed SQL (trusted)" in workflow + assert ( + "Enforce schema qualification in changed SQL (public fork, fail closed)" + in workflow + ) + fork_step = workflow.split( + "- name: Enforce schema qualification in changed SQL (public fork, fail closed)", + maxsplit=1, + )[1].split("- name:", maxsplit=1)[0] + assert ".proof-dependencies/omninode-infra" not in fork_step + assert "config/application_database_domain_proof_ownership.yaml" not in fork_step + + +def test_live_omnimarket_head_resolution_survives_fork_prs_without_org_secrets() -> ( + None +): + """OMN-15703 forkfix: the live-resolve step must not hard-depend on + CROSS_REPO_PAT. Fork-triggered pull_request runs receive no org secrets, + so a bare `secrets.CROSS_REPO_PAT` GH_TOKEN resolves empty, `gh api` + fails unauthenticated, and `set -euo pipefail` aborts the job before its + dedicated fork-lane proof step + ("Enforce schema qualification in changed SQL (public fork, fail closed)") + ever runs. omnimarket is a public repo, so github.token (always present, + including on fork PRs) is sufficient to read its commits API -- the step + must fall back to it rather than failing closed on token absence alone. + """ + workflow = _CI_WORKFLOW.read_text(encoding="utf-8") + + live_resolve_step = workflow.split( + "- name: Live-resolve omnimarket dev HEAD", + maxsplit=1, + )[1].split("- name:", maxsplit=1)[0] + + # Must not hard-depend on CROSS_REPO_PAT alone -- a fork PR run has no + # org secrets, so a bare `${{ secrets.CROSS_REPO_PAT }}` here means an + # empty GH_TOKEN and an unauthenticated `gh api` failure under + # `set -euo pipefail`. + assert "GH_TOKEN: ${{ secrets.CROSS_REPO_PAT }}" not in live_resolve_step + # Must fall back to the always-present github.token so the fork lane + # keeps resolving (omnimarket is public; no elevated scope is needed). + assert ( + "GH_TOKEN: ${{ secrets.CROSS_REPO_PAT || github.token }}" in live_resolve_step + ) + # Fail-closed behavior is preserved: no mutable-tag/latest/dev-ref + # fallback, hard exit on invalid resolution, still gated to the + # not-yet-pinned ("dev") case only. + assert "set -euo pipefail" in live_resolve_step + assert "exit 1" in live_resolve_step + assert "if: steps.resolve-omnimarket-ref.outputs.ref == 'dev'" in live_resolve_step + + +def test_predecessor_pins_reject_in_place_mutation() -> None: + contract = _contract() + + with pytest.raises(TypeError, match="does not support item assignment"): + # Intentional runtime mutation probe against a statically read-only Mapping. + contract.predecessor_pins["omnibase_core#1529"] = "0" * 40 # type: ignore[index] + + +def test_gates_reject_in_place_mutation() -> None: + contract = _contract() + gate = EnumApplicationDatabaseEnforcementGate.CLASSIFICATION + + with pytest.raises(TypeError, match="does not support item assignment"): + # Intentional runtime mutation probe against a statically read-only Mapping. + contract.gates[gate] = contract.gates[gate] # type: ignore[index] + + +def test_yaml_contract_remains_json_serializable_without_shape_drift() -> None: + source = yaml.safe_load(_CONTRACT.read_text(encoding="utf-8")) + contract = ModelApplicationDatabaseEnforcementContract.model_validate(source) + + assert contract.model_dump(mode="json") == source + assert json.loads(contract.model_dump_json()) == source + + +def test_source_tenant_provenance_is_declared_by_typed_ownership_evidence() -> None: + manifest = load_service_ownership_manifest(_OWNERSHIP) + matches = tuple( + evidence + for evidence in manifest.relation_evidence + if evidence.database_ref == "application" + and evidence.schema == "omninode_internal" + and evidence.name == "runtime_state" + ) + + assert len(matches) == 1 + assert ( + matches[0].source_tenant_provenance_contract == "non_authoritative_provenance" + ) + assert matches[0].deduplication_key_columns == ("state_id",) + assert matches[0].authorization_dependency_columns == () + assert matches[0].write_eligibility_dependency_columns == () + + +def test_tenant_identity_and_function_audit_are_checked_manifest_authority() -> None: + manifest = load_service_ownership_manifest(_OWNERSHIP) + identity_root = tuple( + evidence + for evidence in manifest.relation_evidence + if evidence.database_ref == "application" + and evidence.schema == "tenant" + and evidence.name == "tenants" + ) + tenant = tuple( + evidence + for evidence in manifest.relation_evidence + if evidence.database_ref == "application" + and evidence.schema == "tenant" + and evidence.name == "events" + ) + function = tuple( + database_object + for database_object in manifest.database_objects + if database_object.database_ref == "application" + and database_object.schema == "tenant" + and database_object.name == "safe_report" + ) + + assert len(identity_root) == 1 + assert identity_root[0].identity_root_control_role == "tenant_control_admin" + assert tuple( + operation.value + for operation in identity_root[0].identity_root_control_operations + ) == ("tenant_creation", "cross_tenant_enumeration") + assert len(tenant) == 1 + assert tenant[0].tenant_identity_column == "tenant_id" + assert tenant[0].identity_root_contract is None + assert tenant[0].canonical_policy_name == "tenant_isolation" + assert len(function) == 1 + expected_hash = "58b47971e3234c0117f153a4d3d7c7d0efdfb611804ba729153dfac19e503cfe" + assert function[0].function_signature == "()" + assert function[0].definition_sha256 == expected_hash + assert function[0].audit_id == f"OMN-15361:tenant.safe_report:{expected_hash}" + + +def test_owner_manifest_can_express_every_observed_application_object_kind() -> None: + assert EnumApplicationRelationKind.FOREIGN_TABLE.value == "foreign_table" + assert {kind.value for kind in EnumApplicationDatabaseObjectKind}.issuperset( + { + "function", + "aggregate", + "window_function", + "procedure", + "sequence", + "extension", + "type", + "base_type", + "range_type", + "multirange_type", + } + ) + assert { + "tenant_identity_column", + "identity_root_contract", + "identity_root_control_role", + "identity_root_control_operations", + "canonical_policy_name", + "deduplication_key_columns", + "authorization_dependency_columns", + "write_eligibility_dependency_columns", + }.issubset(ModelRelationEvidence.model_fields) + assert {"audit_id", "definition_sha256"}.issubset( + ModelDatabaseObjectEvidence.model_fields + ) + + +def test_contract_derived_table_grant_gate_is_wired_and_strict() -> None: + """OMN-15656: the TABLE-grant derivation gate must run in the STRICT job. + + ``deploy-gate`` never saw the 43/43 strict-wiring failure because nothing + resolved real node contracts against the real shipped topology. This asserts + the gate exists, consumes the pinned cross-repo checkout, and runs both + directions of the drift check. + """ + workflow = _CI_WORKFLOW.read_text(encoding="utf-8") + + assert ( + "- name: Enforce contract-derived application-database TABLE grants" in workflow + ) + step = workflow.split( + "- name: Enforce contract-derived application-database TABLE grants", + maxsplit=1, + )[1].split("- name:", maxsplit=1)[0] + assert "scripts/generate_application_database_table_grants.py" in step + assert ".proof-dependencies/omnimarket/src/omnimarket/nodes" in step + # --check catches grant/contract drift; --prove catches a grant that exists + # but does not actually satisfy the wiring validator. + assert "--check --prove" in step + + gate_module = (_ROOT / "scripts" / "ci" / "ci_summary_gate.py").read_text( + encoding="utf-8" + ) + assert '"Application Database Domain Enforcement (OMN-15361)"' in gate_module diff --git a/tests/ci/test_application_database_sql_authority_isolation.py b/tests/ci/test_application_database_sql_authority_isolation.py new file mode 100644 index 0000000000..4bba1be93b --- /dev/null +++ b/tests/ci/test_application_database_sql_authority_isolation.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Authority-universe isolation controls for the OMN-15361 SQL gate.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from omnibase_infra.topology.application_database import load_topology_profile +from omnibase_infra.validation.application_database_domain_enforcement import ( + application_database_created_catalog_identities, + application_database_sql_target_requirements, + lint_application_database_sql, + load_application_database_ownership_identities, +) +from scripts.ci.check_application_database_sql import changed_sql_paths + +pytestmark = pytest.mark.unit + +_ROOT = Path(__file__).parents[2] +_CI_WORKFLOW = _ROOT / ".github" / "workflows" / "ci.yml" +_PROOF_MANIFEST = _ROOT / "config" / "application_database_domain_proof_ownership.yaml" +_PROOF_SEED = _ROOT / "docker" / "application-domain-enforcement" / "seed.sql" + + +def _git(repository: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", *arguments], + cwd=repository, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def _commit(repository: Path, message: str) -> str: + _git(repository, "add", ".") + _git( + repository, + "-c", + "user.name=OMN-15361 proof", + "-c", + "user.email=omn-15361@example.invalid", + "commit", + "-m", + message, + ) + return _git(repository, "rev-parse", "HEAD") + + +def test_ephemeral_proof_seed_is_not_a_deployable_changed_sql_path( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + (repository / "baseline.txt").write_text("baseline\n", encoding="utf-8") + base_revision = _commit(repository, "baseline") + + proof_directory = repository / "docker" / "application-domain-enforcement" + proof_directory.mkdir(parents=True) + (proof_directory / "seed.sql").write_text( + "CREATE TABLE tenant.proof_only (id uuid);\n", + encoding="utf-8", + ) + migration_directory = repository / "migrations" + migration_directory.mkdir() + deployed = migration_directory / "deployed.sql" + deployed.write_text( + "CREATE TABLE tenant.deployed (id uuid);\n", + encoding="utf-8", + ) + head_revision = _commit(repository, "changed SQL") + + assert changed_sql_paths(repository, base_revision, head_revision) == (deployed,) + + +def test_fixture_and_control_bootstrap_sql_are_not_deployable_changed_sql( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + (repository / "baseline.txt").write_text("baseline\n", encoding="utf-8") + base_revision = _commit(repository, "baseline") + + excluded_paths = ( + repository / "docker/legacy-rds-fixture/legacy-seed.sql", + repository + / "docker/legacy-rds-fixture/ledger-control/forward/000_db_metadata.sql", + repository / "docker/migrations/forward/_ledger/bootstrap.sql", + repository / "src/omnibase_infra/migration/cutover/sql/bootstrap.sql", + ) + for path in excluded_paths: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "CREATE TABLE public.fixture_only (id uuid);\n", encoding="utf-8" + ) + + deployed = repository / "docker/migrations/forward/nodes/node_real/0001.sql" + deployed.parent.mkdir(parents=True) + deployed.write_text("CREATE TABLE tenant.deployed (id uuid);\n", encoding="utf-8") + head_revision = _commit(repository, "changed SQL") + + assert changed_sql_paths(repository, base_revision, head_revision) == (deployed,) + + +def test_production_sql_workflow_never_composes_ephemeral_proof_authority() -> None: + workflow = _CI_WORKFLOW.read_text(encoding="utf-8") + trusted_step = workflow.split( + "- name: Enforce schema qualification in changed SQL (trusted)", + maxsplit=1, + )[1].split("- name:", maxsplit=1)[0] + fork_step = workflow.split( + "- name: Enforce schema qualification in changed SQL (public fork, fail closed)", + maxsplit=1, + )[1].split("- name:", maxsplit=1)[0] + + assert "config/application_database_domain_proof_ownership.yaml" not in trusted_step + assert "config/application_database_domain_proof_ownership.yaml" not in fork_step + assert ".proof-dependencies/omninode-infra" in trusted_step + assert ".proof-dependencies/omninode-infra" not in fork_step + + +def test_mandatory_source_lane_executes_sql_regression_and_isolation_controls() -> None: + workflow = _CI_WORKFLOW.read_text(encoding="utf-8") + source_step = workflow.split( + "- name: Execute mandatory source assertions and seeded RED controls", + maxsplit=1, + )[1].split("- name:", maxsplit=1)[0] + + assert ( + "tests/unit/validation/" + "test_application_database_sql_enforcement_regressions.py" in source_step + ) + assert ( + "tests/ci/test_application_database_sql_authority_isolation.py" in source_step + ) + + +def test_ephemeral_proof_seed_has_an_isolated_exact_authority_universe() -> None: + topology = load_topology_profile("local") + sql = _PROOF_SEED.read_text(encoding="utf-8") + authoritative = load_application_database_ownership_identities((_PROOF_MANIFEST,)) + authoritative_identities = {identity.identity for identity in authoritative} + + assert not lint_application_database_sql(sql, topology) + assert all( + any( + identity.schema == requirement.schema + and identity.name == requirement.name + and identity.kind in requirement.allowed_kinds + and ( + requirement.function_signature is None + or identity.function_signature == requirement.function_signature + ) + for identity in authoritative + ) + for requirement in application_database_sql_target_requirements(sql, topology) + ) + assert { + identity.identity + for identity in application_database_created_catalog_identities(sql) + } == authoritative_identities diff --git a/tests/ci/test_application_database_sql_gate.py b/tests/ci/test_application_database_sql_gate.py new file mode 100644 index 0000000000..4e97eb8b40 --- /dev/null +++ b/tests/ci/test_application_database_sql_gate.py @@ -0,0 +1,565 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""The migration gate must inspect actual changed SQL, not fixture strings.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from scripts.ci.check_application_database_sql import ( + validate_changed_sql, + violation_key, +) + +pytestmark = pytest.mark.unit + +_ROOT = Path(__file__).parents[2] +_OMN15547_FIXTURE = ( + _ROOT + / "tests" + / "fixtures" + / "omn15547" + / "node-service-registry-tenant-rls-unqualified.sql.captured" +) + + +def _git(repository: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", *arguments], + cwd=repository, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def _commit(repository: Path, message: str) -> str: + _git(repository, "add", ".") + _git( + repository, + "-c", + "user.name=OMN-15361 proof", + "-c", + "user.email=omn-15361@example.invalid", + "commit", + "-m", + message, + ) + return _git(repository, "rev-parse", "HEAD") + + +def _ownership_manifest( + repository: Path, + table_name: str, + *, + filename: str = "ownership.yaml", + service: str = "sql_gate_test", +) -> Path: + path = repository / filename + path.write_text( + f"""schema_version: \"1.0\" +service: {service} +target_database_ref: application +db_io: + db_tables: + - name: {table_name} + database_ref: application + schema: tenant + migration: qualified.sql + access: read_write + role: test_relation +""", + encoding="utf-8", + ) + return path + + +def _function_ownership_manifest(repository: Path) -> Path: + path = repository / "function-ownership.yaml" + path.write_text( + """schema_version: "1.0" +service: sql_gate_function_test +target_database_ref: application +db_io: + db_tables: [] +database_objects: + - name: safe_report + kind: function + database_ref: application + schema: tenant + domain: TENANT + owner_declaration: service:sql_gate_function_test + function_signature: "()" +""", + encoding="utf-8", + ) + return path + + +def test_gate_checks_only_real_changed_sql_files(tmp_path: Path) -> None: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + + (repository / "unchanged_legacy.sql").write_text( + "CREATE TABLE legacy_unqualified (id uuid);\n", + encoding="utf-8", + ) + base_revision = _commit(repository, "baseline") + + (repository / "qualified.sql").write_text( + "CREATE TABLE tenant.changed_table (id uuid);\n", + encoding="utf-8", + ) + manifest = _ownership_manifest(repository, "changed_table") + green_head = _commit(repository, "qualified") + assert not validate_changed_sql( + repository, + base_revision, + green_head, + ownership_manifest_paths=(manifest,), + ).violations + + (repository / "unqualified.sql").write_text( + "SELECT * FROM changed_table;\n", + encoding="utf-8", + ) + red_head = _commit(repository, "unqualified") + violations = validate_changed_sql( + repository, + base_revision, + red_head, + ownership_manifest_paths=(manifest,), + ).violations + assert any("unqualified.sql" in violation for violation in violations) + assert any("schema-qualified" in violation for violation in violations) + assert all("unchanged_legacy.sql" not in violation for violation in violations) + + +def test_gate_exempts_only_the_omn15503_legacy_node_migration_path( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + (repository / "baseline.txt").write_text("baseline\n", encoding="utf-8") + base_revision = _commit(repository, "baseline") + + exempt = ( + repository + / "docker" + / "migrations" + / "forward" + / "nodes" + / "node_projection_delegation" + / "0029_delegation_terminal_failure_cause.sql" + ) + exempt.parent.mkdir(parents=True) + exempt.write_text( + "ALTER TABLE delegation_events ADD COLUMN terminal_ok boolean;\n", + encoding="utf-8", + ) + exempt_head = _commit(repository, "exempt legacy migration") + assert not validate_changed_sql( + repository, + base_revision, + exempt_head, + ownership_manifest_paths=(), + ).violations + + adjacent = exempt.with_name("0030_adjacent_unqualified.sql") + adjacent.write_text( + "ALTER TABLE delegation_events ADD COLUMN still_blocked text;\n", + encoding="utf-8", + ) + adjacent_head = _commit(repository, "adjacent legacy migration") + violations = validate_changed_sql( + repository, + exempt_head, + adjacent_head, + ownership_manifest_paths=(), + ).violations + assert any("0030_adjacent_unqualified.sql" in item for item in violations) + assert any("schema-qualified" in item for item in violations) + + +def test_gate_exempts_omn15655_legacy_root_shape_repair_paths( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + (repository / "baseline.txt").write_text("baseline\n", encoding="utf-8") + base_revision = _commit(repository, "baseline") + + migration_dir = repository / "docker" / "migrations" / "forward" + migration_dir.mkdir(parents=True) + for filename in ( + "031_create_llm_call_metrics_and_cost_aggregates.sql", + "050_create_baselines_tables.sql", + ): + (migration_dir / filename).write_text( + "ALTER TABLE public.legacy_shape ADD COLUMN tenant_id uuid;\n", + encoding="utf-8", + ) + + exempt_head = _commit(repository, "exempt legacy root shape repairs") + assert not validate_changed_sql( + repository, + base_revision, + exempt_head, + ownership_manifest_paths=(), + ).violations + + adjacent = migration_dir / "051_adjacent_unqualified.sql" + adjacent.write_text( + "ALTER TABLE legacy_shape ADD COLUMN still_blocked text;\n", + encoding="utf-8", + ) + adjacent_head = _commit(repository, "adjacent root migration") + violations = validate_changed_sql( + repository, + exempt_head, + adjacent_head, + ownership_manifest_paths=(), + ).violations + assert any("051_adjacent_unqualified.sql" in item for item in violations) + assert any("schema-qualified" in item for item in violations) + + +def test_qualified_create_requires_an_authoritative_ownership_declaration( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + (repository / "baseline.txt").write_text("baseline\n", encoding="utf-8") + base_revision = _commit(repository, "baseline") + (repository / "unclassified.sql").write_text( + "CREATE TABLE tenant.unclassified (id uuid);\n", + encoding="utf-8", + ) + head_revision = _commit(repository, "unclassified") + + violations = validate_changed_sql( + repository, + base_revision, + head_revision, + ownership_manifest_paths=(), + ).violations + assert any("ownership declaration" in violation for violation in violations) + + +def test_qualified_non_create_target_requires_authoritative_ownership( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + (repository / "baseline.txt").write_text("baseline\n", encoding="utf-8") + base_revision = _commit(repository, "baseline") + (repository / "undeclared_alter.sql").write_text( + "ALTER TABLE tenant.undeclared ADD COLUMN payload text;\n", + encoding="utf-8", + ) + head_revision = _commit(repository, "qualified alter") + + violations = validate_changed_sql( + repository, + base_revision, + head_revision, + ownership_manifest_paths=(), + ).violations + assert any("exactly one ownership declaration" in item for item in violations) + + +def test_duplicate_or_conflicting_owner_declarations_fail_closed( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + (repository / "baseline.txt").write_text("baseline\n", encoding="utf-8") + base_revision = _commit(repository, "baseline") + (repository / "qualified.sql").write_text( + "CREATE TABLE tenant.changed_table (id uuid);\n", + encoding="utf-8", + ) + first = _ownership_manifest( + repository, + "changed_table", + filename="owner-a.yaml", + service="owner_a", + ) + second = _ownership_manifest( + repository, + "changed_table", + filename="owner-b.yaml", + service="owner_b", + ) + head_revision = _commit(repository, "duplicate owners") + + violations = validate_changed_sql( + repository, + base_revision, + head_revision, + ownership_manifest_paths=(first, second), + ).violations + assert any("exactly one ownership declaration" in item for item in violations) + + +def test_red_control_wrong_routine_overload(tmp_path: Path) -> None: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + (repository / "baseline.txt").write_text("baseline\n", encoding="utf-8") + base_revision = _commit(repository, "baseline") + (repository / "wrong-overload.sql").write_text( + "ALTER FUNCTION tenant.safe_report(uuid) OWNER TO owner_onex_tenant;\n", + encoding="utf-8", + ) + manifest = _function_ownership_manifest(repository) + head_revision = _commit(repository, "wrong overload") + + violations = validate_changed_sql( + repository, + base_revision, + head_revision, + ownership_manifest_paths=(manifest,), + ).violations + assert any("exact routine ownership declaration" in item for item in violations) + + +def test_red_control_wrong_object_kind( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + (repository / "baseline.txt").write_text("baseline\n", encoding="utf-8") + base_revision = _commit(repository, "baseline") + (repository / "wrong-kind.sql").write_text( + "ALTER TABLE tenant.safe_report ADD COLUMN payload text;\n", + encoding="utf-8", + ) + manifest = _function_ownership_manifest(repository) + head_revision = _commit(repository, "wrong kind") + + violations = validate_changed_sql( + repository, + base_revision, + head_revision, + ownership_manifest_paths=(manifest,), + ).violations + assert any("exact object-kind ownership declaration" in item for item in violations) + + +def test_omn15361_replay_rejects_real_unqualified_application_migration( + tmp_path: Path, +) -> None: + """Replay the real node_service_registry migration shape that motivated the gate.""" + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + (repository / "baseline.txt").write_text("baseline\n", encoding="utf-8") + base_revision = _commit(repository, "baseline") + migration = ( + repository + / "docker" + / "migrations" + / "forward" + / "nodes" + / "node_projection_registration" + / "0002_node_service_registry_tenant_rls.sql" + ) + migration.parent.mkdir(parents=True) + migration.write_bytes(_OMN15547_FIXTURE.read_bytes()) + head_revision = _commit(repository, "captured unqualified migration") + + violations = validate_changed_sql( + repository, + base_revision, + head_revision, + ownership_manifest_paths=(), + ).violations + + rendered = "\n".join(violations) + assert "0002_node_service_registry_tenant_rls.sql" in rendered + assert "schema-qualified" in rendered + + +# --------------------------------------------------------------------------- +# OMN-15361 frozen-baseline ratchet. Mirrors the OMN-14443 deploy-gate +# grandfather pattern: pre-existing violations soft-pass, everything else is +# held to the full bar, and the list may only shrink. +# --------------------------------------------------------------------------- + + +def _baseline_repository(tmp_path: Path) -> tuple[Path, str, str, Path]: + """A repo whose changed SQL carries exactly one unqualified relation.""" + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "--initial-branch=main") + (repository / "seed.sql").write_text( + "CREATE TABLE tenant.seeded (id uuid);\n", encoding="utf-8" + ) + base_revision = _commit(repository, "baseline") + + (repository / "offender.sql").write_text( + "SELECT * FROM unqualified_relation;\n", encoding="utf-8" + ) + manifest = _ownership_manifest(repository, "seeded") + head = _commit(repository, "offender") + return repository, base_revision, head, manifest + + +def _write_baseline(path: Path, entries: list[tuple[str, str]]) -> Path: + lines = ['generated_at: "2026-08-15"', f"count: {len(entries)}", "violations:"] + for key, recorded_path in entries: + lines.append(f' - key: "{key}"') + lines.append(f' path: "{recorded_path}"') + lines.append(' violation: "recorded"') + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return path + + +def test_unbaselined_violation_is_held_to_the_full_bar(tmp_path: Path) -> None: + repository, base, head, manifest = _baseline_repository(tmp_path) + empty = _write_baseline(tmp_path / "baseline.yaml", []) + + outcome = validate_changed_sql( + repository, + base, + head, + ownership_manifest_paths=(manifest,), + baseline_path=empty, + ) + + assert any("unqualified_relation" in v for v in outcome.violations) + assert outcome.grandfathered == 0 + + +def test_baselined_violation_is_grandfathered_and_counted(tmp_path: Path) -> None: + repository, base, head, manifest = _baseline_repository(tmp_path) + raw = validate_changed_sql( + repository, base, head, ownership_manifest_paths=(manifest,) + ) + offender = next(v for v in raw.violations if "unqualified_relation" in v) + baseline = _write_baseline( + tmp_path / "baseline.yaml", [(violation_key(offender), "offender.sql")] + ) + + outcome = validate_changed_sql( + repository, + base, + head, + ownership_manifest_paths=(manifest,), + baseline_path=baseline, + ) + + assert not any("unqualified_relation" in v for v in outcome.violations) + assert outcome.grandfathered == 1 + + +def test_a_corrupt_baseline_grandfathers_nothing(tmp_path: Path) -> None: + repository, base, head, manifest = _baseline_repository(tmp_path) + corrupt = tmp_path / "baseline.yaml" + corrupt.write_text("{ this is: not: valid yaml ][", encoding="utf-8") + + outcome = validate_changed_sql( + repository, + base, + head, + ownership_manifest_paths=(manifest,), + baseline_path=corrupt, + ) + + assert any("unqualified_relation" in v for v in outcome.violations) + assert outcome.grandfathered == 0 + + +def test_a_missing_baseline_grandfathers_nothing(tmp_path: Path) -> None: + repository, base, head, manifest = _baseline_repository(tmp_path) + + outcome = validate_changed_sql( + repository, + base, + head, + ownership_manifest_paths=(manifest,), + baseline_path=tmp_path / "does-not-exist.yaml", + ) + + assert any("unqualified_relation" in v for v in outcome.violations) + assert outcome.grandfathered == 0 + + +def test_a_baseline_entry_for_a_deleted_file_fails_stale(tmp_path: Path) -> None: + repository, base, head, manifest = _baseline_repository(tmp_path) + baseline = _write_baseline( + tmp_path / "baseline.yaml", [("deadbeefdeadbeef", "removed_migration.sql")] + ) + + outcome = validate_changed_sql( + repository, + base, + head, + ownership_manifest_paths=(manifest,), + baseline_path=baseline, + ) + + assert any( + "stale baseline entry" in v and "no longer exists" in v + for v in outcome.violations + ) + + +def test_a_no_longer_firing_baseline_entry_fails_stale(tmp_path: Path) -> None: + repository, base, head, manifest = _baseline_repository(tmp_path) + # offender.sql IS linted this run, but this key never fires against it. + baseline = _write_baseline( + tmp_path / "baseline.yaml", [("cafebabecafebabe", "offender.sql")] + ) + + outcome = validate_changed_sql( + repository, + base, + head, + ownership_manifest_paths=(manifest,), + baseline_path=baseline, + ) + + assert any( + "stale baseline entry" in v and "no longer fires" in v + for v in outcome.violations + ) + + +def test_an_entry_for_an_unscanned_file_is_not_reported_stale(tmp_path: Path) -> None: + """A dev PR touching two files must not mass-fail on the other 160 entries.""" + repository, base, head, manifest = _baseline_repository(tmp_path) + # seed.sql exists but is NOT in this run's changed set, so its entry is + # unobservable — not stale. + baseline = _write_baseline( + tmp_path / "baseline.yaml", [("0123456789abcdef", "seed.sql")] + ) + + outcome = validate_changed_sql( + repository, + base, + head, + ownership_manifest_paths=(manifest,), + baseline_path=baseline, + ) + + assert not any("stale baseline entry" in v for v in outcome.violations) + + +def test_violation_key_is_content_addressed_not_positional() -> None: + same = "a.sql: application relation target 'x' must be schema-qualified" + assert violation_key(same) == violation_key(same) + assert violation_key(same) != violation_key(same.replace("a.sql", "b.sql")) + assert violation_key(same) != violation_key(same.replace("'x'", "'y'")) diff --git a/tests/ci/test_assert_image_config_paths.py b/tests/ci/test_assert_image_config_paths.py new file mode 100644 index 0000000000..856fdd31b6 --- /dev/null +++ b/tests/ci/test_assert_image_config_paths.py @@ -0,0 +1,261 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Coverage for the in-image required-config-path assertion (OMN-15676). + +``runner_fleet.yaml`` was tracked in the repo, valid, and referenced by config +code -- and absent from every built runtime image, because no Dockerfile +``COPY`` shipped it. The deployed runtime raised +``FileNotFoundError: /app/config/runner_fleet.yaml`` during auto-wiring. Every +repo-level check passed the whole time; the third occurrence of that class +(after the grants fixture and ``routing_tiers.yaml``) is what this mechanism +exists to be the last of. + +Two properties are held here: + +* the assertion is **fail-closed** -- a probe that cannot run, a path with no + verdict, or an emptied registry must all be failures, never green; +* the Dockerfile actually ships every path the typed registry declares, so the + registry cannot drift into decoration. + +The Dockerfile check is deliberately *supplementary*: it inspects the working +tree, which is exactly the surface that was green during all three incidents. +The load-bearing gate is ``assert_image_config_paths.py`` running inside the +built image, wired ahead of the push step in both runtime image builds. +""" + +from __future__ import annotations + +import importlib.util +import re +import stat +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +pytestmark = pytest.mark.unit + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT_PATH = _REPO_ROOT / "scripts" / "ci" / "assert_image_config_paths.py" +_DOCKERFILE = _REPO_ROOT / "docker" / "Dockerfile.runtime" + + +def _load_script() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "assert_image_config_paths", _SCRIPT_PATH + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def script() -> ModuleType: + return _load_script() + + +def _write_fake_docker(tmp_path: Path, body: str) -> str: + """Write a stub `docker` executable and return its path.""" + fake = tmp_path / "fake-docker" + fake.write_text(f"#!/bin/sh\n{body}\n", encoding="utf-8") + fake.chmod(fake.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + return str(fake) + + +def test_all_paths_present_passes( + script: ModuleType, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Every declared path reported PRESENT -> exit 0.""" + paths = script.REQUIRED_IMAGE_CONFIG_PATHS + echoes = "\n".join(f"echo 'PRESENT {entry.image_path}'" for entry in paths) + docker = _write_fake_docker(tmp_path, f"{echoes}\nexit 0") + + rc = script.main(["--image", "img:test", "--docker-bin", docker]) + + assert rc == 0 + assert "PASS" in capsys.readouterr().out + + +def test_missing_path_fails_and_names_it( + script: ModuleType, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A single missing path fails the assertion and is named in the output. + + This is the runner_fleet.yaml scenario reproduced against the mechanism. + """ + entries = script.REQUIRED_IMAGE_CONFIG_PATHS + target = entries[0].image_path + lines = [ + f"echo 'MISSING {entry.image_path}'" + if entry.image_path == target + else f"echo 'PRESENT {entry.image_path}'" + for entry in entries + ] + docker = _write_fake_docker(tmp_path, "\n".join(lines) + "\nexit 0") + + rc = script.main(["--image", "img:test", "--docker-bin", docker]) + + assert rc == 1 + captured = capsys.readouterr() + assert target in captured.err + assert "COPY" in captured.err + + +def test_probe_container_failure_fails_closed( + script: ModuleType, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A probe that cannot run proves nothing and must NOT report green.""" + docker = _write_fake_docker(tmp_path, "echo 'no such image' >&2\nexit 125") + + rc = script.main(["--image", "img:test", "--docker-bin", docker]) + + assert rc == 1 + assert "failing closed" in capsys.readouterr().err + + +def test_missing_docker_binary_fails_closed(script: ModuleType, tmp_path: Path) -> None: + """An absent container CLI is a failure, not a skip.""" + rc = script.main( + ["--image", "img:test", "--docker-bin", str(tmp_path / "not-a-real-binary")] + ) + assert rc == 1 + + +def test_silent_probe_yields_no_verdict_failure( + script: ModuleType, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Exit 0 with no verdict lines must not be read as 'all present'.""" + docker = _write_fake_docker(tmp_path, "exit 0") + + rc = script.main(["--image", "img:test", "--docker-bin", docker]) + + assert rc == 1 + assert "no verdict" in capsys.readouterr().err + + +def test_empty_registry_fails_closed( + script: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A registry emptied by a bad refactor must fail, not vacuously pass.""" + monkeypatch.setattr(script, "REQUIRED_IMAGE_CONFIG_PATHS", ()) + docker = _write_fake_docker(tmp_path, "exit 0") + + assert script.main(["--image", "img:test", "--docker-bin", docker]) == 1 + + +def test_registry_rejects_unsafe_paths(script: ModuleType, tmp_path: Path) -> None: + """Relative or shell-metacharacter paths are rejected before any probe runs.""" + template = script.REQUIRED_IMAGE_CONFIG_PATHS[0] + bad = ( + template.model_copy(update={"image_path": "config/relative.yaml"}), + template.model_copy(update={"image_path": "/app/config/$(whoami).yaml"}), + ) + problems = script._validate_paths(bad) + assert any("absolute" in p for p in problems) + assert any("quoting" in p for p in problems) + + +def test_registry_is_not_empty(script: ModuleType) -> None: + """The live registry must actually declare something.""" + assert len(script.REQUIRED_IMAGE_CONFIG_PATHS) >= 2 + + +def test_runner_fleet_config_is_registered(script: ModuleType) -> None: + """The OMN-15676 defect path is covered by the registry.""" + paths = {entry.image_path for entry in script.REQUIRED_IMAGE_CONFIG_PATHS} + assert "/app/config/runner_fleet.yaml" in paths + + +def test_dockerfile_ships_every_registered_path(script: ModuleType) -> None: + """Supplementary static check: every registry entry has a COPY destination. + + Not a substitute for the in-image assertion -- the working tree was green + during all three incidents. This only catches the drift where an entry is + added to the registry with no matching COPY, before the (slower) image + build reports it. + """ + dockerfile = _DOCKERFILE.read_text(encoding="utf-8") + # Join Dockerfile line continuations so multi-line COPYs are one token run. + flattened = re.sub(r"\\\s*\n\s*", " ", dockerfile) + copy_destinations: list[str] = [] + for line in flattened.splitlines(): + stripped = line.strip() + if not stripped.upper().startswith("COPY "): + continue + tokens = [t for t in stripped.split()[1:] if not t.startswith("--")] + if tokens: + copy_destinations.append(tokens[-1]) + + missing = [ + entry.image_path + for entry in script.REQUIRED_IMAGE_CONFIG_PATHS + if not any( + dest == entry.image_path + or (dest.endswith("/") and entry.image_path.startswith(dest)) + for dest in copy_destinations + ) + ] + assert not missing, ( + "registry entries with no COPY destination in docker/Dockerfile.runtime: " + f"{missing}" + ) + + +def test_registered_source_files_exist_in_build_context() -> None: + """The repo-side source of the runner-fleet COPY is present and tracked.""" + assert (_REPO_ROOT / "config" / "runner_fleet.yaml").is_file() + + +def test_replay_real_prefix_image_probe_is_rejected( + script: ModuleType, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """OMN-15676 replay: the real pre-fix image's probe bytes must be REJECTED. + + The fixture is the verbatim stdout of this guard's own probe, run inside + ghcr.io/omninode-ai/omnibase-infra-runtime@sha256:403b6092 (the image family + that was live on onex-dev), captured on omni-201-ts. Not a reconstruction: + the same two lines the probe emitted, MISSING for runner_fleet.yaml and + PRESENT for routing_tiers.yaml. + + The PRESENT line is load-bearing as a control. A broken implementation that + reported every path missing would satisfy an all-missing capture and look + correct; it cannot satisfy this one, because the guard must simultaneously + accept the path that really is in the image and reject the one that is not. + """ + fixture = ( + _REPO_ROOT + / "tests" + / "fixtures" + / "omn15676" + / "runtime-image-config-probe-403b6092.txt.captured" + ) + captured_bytes = fixture.read_text(encoding="utf-8") + + # Sanity-bind the replay to the registry: the capture predates any registry + # entry added later, so a case that grew a third path would silently stop + # being replayed by these bytes. + registry_paths = {entry.image_path for entry in script.REQUIRED_IMAGE_CONFIG_PATHS} + captured_paths = { + line.split(" ", 1)[1] for line in captured_bytes.splitlines() if " " in line + } + assert registry_paths == captured_paths, ( + "the captured probe no longer covers the registry; re-capture against a " + "current image rather than editing the fixture" + ) + + replay = tmp_path / "captured-stdout" + replay.write_text(captured_bytes, encoding="utf-8") + docker = _write_fake_docker(tmp_path, f"cat {replay}") + + rc = script.main(["--image", "replay:403b6092", "--docker-bin", docker]) + + assert rc == 1, "the guard accepted the image that boot-crashed onex-dev" + out = capsys.readouterr() + assert "/app/config/runner_fleet.yaml" in out.err + # The control: the path that IS in the image must not be reported missing. + assert "/app/config/delegation/routing_tiers.yaml" not in out.err + assert "OK /app/config/delegation/routing_tiers.yaml" in out.out diff --git a/tests/ci/test_auto_tag_release_dispatch.py b/tests/ci/test_auto_tag_release_dispatch.py index bde9787d95..0737158c95 100644 --- a/tests/ci/test_auto_tag_release_dispatch.py +++ b/tests/ci/test_auto_tag_release_dispatch.py @@ -175,3 +175,42 @@ def test_dependency_cascade_polls_until_pypi_visible_with_real_ceiling() -> None assert ":-900" in run_script # default 15-minute ceiling # The old too-short 12x10s ceiling is gone. assert "seq 1 12" not in run_script + + +def test_dependency_cascade_checks_movability_before_locking() -> None: + """OMN-15604 AC4: a git-pinned package cannot be moved by `uv lock + --upgrade-package` (uv always prefers an explicit [tool.uv.sources] + override over registry resolution), so the previous no-op re-lock + silently reported "no lockfile changes -- already on latest" even when + the repo was still stuck on the git pin. The lock step must now run the + movability check FIRST and fail loud+explicit, not attempt the + guaranteed-no-op lock.""" + workflow = _load_yaml(CASCADE_WORKFLOW) + steps = workflow["jobs"]["open-bump-pr"]["steps"] + + checkout_step = next( + step + for step in steps + if step.get("name") == "Checkout omnibase_infra dep-provenance script" + ) + assert checkout_step["with"]["repository"] == "OmniNode-ai/omnibase_infra" + assert "check_dep_provenance.py" in checkout_step["with"]["sparse-checkout"] + + lock_step = next( + step + for step in steps + if step.get("name") == "Create branch and upgrade lockfile" + ) + run_script = lock_step["run"] + movability_check_index = run_script.index("--check-movable") + lock_command_index = run_script.index("uv lock \\") + # The movability pre-check must run BEFORE the (potentially no-op) lock + # command, not after -- checking after would already have silently + # produced the misleading "no lockfile changes" state. + assert movability_check_index < lock_command_index + assert "check_dep_provenance.py" in run_script + + pre_lock_block = run_script[:lock_command_index] + assert "--check-movable" in pre_lock_block + assert "::error::" in pre_lock_block + assert "exit 1" in pre_lock_block diff --git a/tests/ci/test_canonical_clone_guard_chains_hook_chain.py b/tests/ci/test_canonical_clone_guard_chains_hook_chain.py new file mode 100644 index 0000000000..8ef22246c5 --- /dev/null +++ b/tests/ci/test_canonical_clone_guard_chains_hook_chain.py @@ -0,0 +1,317 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Regression guard for the canonical-clone hook guard's CHAINING behaviour +(OMN-15071, on top of the OMN-7018 worktree-discipline guard). + +Root cause this prevents regressing: the guard is installed by pointing +`core.hooksPath` at `scripts/git-hooks/canonical-clone/`. `core.hooksPath` +REPLACES git's hook lookup -- git never falls back to `$GIT_COMMON_DIR/hooks/` +-- so any path through the guard that returns success without invoking the real +hook silently disables the ENTIRE hook chain for that repository. + +The pre-fix revision did exactly that: `exit 0` for every worktree path. On +`.200`, which root CLAUDE.md rule 11a makes the DEFAULT host for pushes and gate +runs, every `git commit` in a worktree ran zero hooks and reported success, with +no output to distinguish it from a commit that had passed every gate. Live A/B +on `.200` 2026-07-30: an identical staged violation committed clean (exit 0, +zero hook output) under the pre-fix guard and was refused (exit 1, 135 lines of +hook output, HEAD unchanged) under the fixed one; an identical `git push` +created a remote ref with zero hook output under the pre-fix guard and was +refused with zero refs created under the fixed one. + +Assertion classes: + +1. Behavioural, chaining -- a commit the guard PERMITS must reach the real hook. + Proven with a sentinel hook that records its own invocation, so the test + fails if the guard returns success on its own. This is the assertion the + pre-fix script fails. +2. Behavioural, canonical-clone protection -- a commit in the main worktree of a + registry clone is still refused, and the sentinel must NOT have run. Chaining + must not be bought by trading away the OMN-7018 guarantee. +3. Behavioural, both directions -- when the real hook passes, the commit lands. + A guard that refuses everything would satisfy (2) alone. +4. Static -- the no-runnable-hook branch fails closed rather than reporting a + vacuous pass. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +GUARD_SCRIPT = REPO_ROOT / "scripts" / "git-hooks" / "canonical_clone_guard.sh" +HOOKS_DIR = REPO_ROOT / "scripts" / "git-hooks" / "canonical-clone" +HOOK_TYPES = ("pre-commit", "pre-push", "commit-msg", "pre-merge-commit") + +_SENTINEL_NAME = "sentinel-ran" + + +def _git( + *args: str, cwd: Path, env: dict[str, str] +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=cwd, + env=env, + capture_output=True, + text=True, + check=False, + ) + + +def _base_env(registry: Path) -> dict[str, str]: + env = dict(os.environ) + # Isolate from the developer's / runner's own git configuration so the test + # asserts the guard's behaviour, not the ambient environment's. + env["GIT_CONFIG_GLOBAL"] = os.devnull + env["GIT_CONFIG_SYSTEM"] = os.devnull + env["GIT_AUTHOR_NAME"] = "OMN-15071 Test" + env["GIT_AUTHOR_EMAIL"] = "omn15071@example.invalid" + env["GIT_COMMITTER_NAME"] = env["GIT_AUTHOR_NAME"] + env["GIT_COMMITTER_EMAIL"] = env["GIT_AUTHOR_EMAIL"] + env["OMNI_HOME"] = str(registry) + env.pop("ALLOW_CANONICAL_CLONE_COMMIT", None) + # git EXPORTS repo-scoping variables into hook processes, and they OVERRIDE + # both `-C` and the cwd for every descendant git call (memory + # `reference_git_env_vars_override_c_and_cwd`). When this suite is run from + # the pre-push hook these leak in and every `git` below would silently + # operate on the omnibase_infra worktree instead of the throwaway registry. + for leaked in ( + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_COMMON_DIR", + "GIT_PREFIX", + ): + env.pop(leaked, None) + return env + + +def _write_sentinel_hook(clone: Path, sentinel: Path, exit_code: int) -> None: + """Install a real hook file at the location the guard must chain to.""" + hooks_dir = clone / ".git" / "hooks" + hooks_dir.mkdir(parents=True, exist_ok=True) + hook = hooks_dir / "pre-commit" + hook.write_text( + f'#!/usr/bin/env bash\nprintf "chained\\n" > "{sentinel}"\nexit {exit_code}\n', + encoding="utf-8", + ) + hook.chmod(0o755) + + +@pytest.fixture +def registry(tmp_path: Path) -> Path: + """A miniature omni_home: `/` clone + `omni_worktrees/`.""" + reg = tmp_path / "omni_home" + (reg / "omni_worktrees").mkdir(parents=True) + return reg + + +@pytest.fixture +def clone(registry: Path) -> Path: + """A canonical clone with `core.hooksPath` pointed at the guard, plus one + commit so a linked worktree can be created from it.""" + repo = registry / "some_repo" + repo.mkdir(parents=True) + env = _base_env(registry) + _git("init", "-q", "-b", "dev", cwd=repo, env=env) + _git("config", "core.hooksPath", str(HOOKS_DIR), cwd=repo, env=env) + (repo / "seed.txt").write_text("seed\n", encoding="utf-8") + _git("add", "seed.txt", cwd=repo, env=env) + # The guard is already active, so seed via the documented emergency override + # rather than by disabling it -- this also exercises that the override still + # works end to end. + seed_env = dict(env) + seed_env["ALLOW_CANONICAL_CLONE_COMMIT"] = "1" + seeded = _git("commit", "-m", "seed", cwd=repo, env=seed_env) + assert seeded.returncode == 0, seeded.stderr + return repo + + +def _linked_worktree(clone: Path, registry: Path, env: dict[str, str]) -> Path: + worktree = registry / "omni_worktrees" / "OMN-15071" / "some_repo" + result = _git( + "worktree", + "add", + "-q", + "-b", + "jonah/omn-15071-test", + str(worktree), + "HEAD", + cwd=clone, + env=env, + ) + assert result.returncode == 0, result.stderr + return worktree + + +def test_guard_script_and_hook_symlinks_exist() -> None: + assert GUARD_SCRIPT.is_file(), f"expected the guard at {GUARD_SCRIPT}" + assert os.access(GUARD_SCRIPT, os.X_OK), f"{GUARD_SCRIPT} must be executable" + for hook_type in HOOK_TYPES: + link = HOOKS_DIR / hook_type + assert link.is_symlink(), f"{link} must be a symlink to the guard" + assert link.resolve() == GUARD_SCRIPT.resolve(), ( + f"{link} must resolve to {GUARD_SCRIPT}" + ) + + +def test_permitted_commit_chains_to_the_real_hook( + registry: Path, clone: Path, tmp_path: Path +) -> None: + """Assertion class 1 -- the class the pre-fix guard fails. + + A worktree commit the guard permits must REACH `$GIT_COMMON_DIR/hooks/`. + The sentinel exits 1, so a guard that chains produces a refused commit AND + a sentinel file; a guard that returns success on its own produces a landed + commit and NO sentinel file. + """ + env = _base_env(registry) + worktree = _linked_worktree(clone, registry, env) + sentinel = tmp_path / _SENTINEL_NAME + _write_sentinel_hook(clone, sentinel, exit_code=1) + + (worktree / "change.txt").write_text("change\n", encoding="utf-8") + _git("add", "change.txt", cwd=worktree, env=env) + head_before = _git("rev-parse", "HEAD", cwd=worktree, env=env).stdout.strip() + result = _git( + "commit", "-m", "should be refused by the chained hook", cwd=worktree, env=env + ) + head_after = _git("rev-parse", "HEAD", cwd=worktree, env=env).stdout.strip() + + assert sentinel.is_file(), ( + "the guard returned without invoking $GIT_COMMON_DIR/hooks/pre-commit -- " + "core.hooksPath replaces git's hook lookup, so this silently disables " + "the entire hook chain (OMN-15071)" + ) + assert result.returncode != 0, ( + "the chained hook exited 1; the commit must be refused" + ) + assert head_after == head_before, "a refused commit must not move HEAD" + + +def test_permitted_commit_succeeds_when_the_real_hook_passes( + registry: Path, clone: Path, tmp_path: Path +) -> None: + """Assertion class 3 -- chaining must not refuse legitimate work.""" + env = _base_env(registry) + worktree = _linked_worktree(clone, registry, env) + sentinel = tmp_path / _SENTINEL_NAME + _write_sentinel_hook(clone, sentinel, exit_code=0) + + (worktree / "change.txt").write_text("change\n", encoding="utf-8") + _git("add", "change.txt", cwd=worktree, env=env) + head_before = _git("rev-parse", "HEAD", cwd=worktree, env=env).stdout.strip() + result = _git("commit", "-m", "legitimate worktree commit", cwd=worktree, env=env) + head_after = _git("rev-parse", "HEAD", cwd=worktree, env=env).stdout.strip() + + assert sentinel.is_file(), "the real hook must still be reached on the passing path" + assert result.returncode == 0, f"legitimate commit was refused: {result.stderr}" + assert head_after != head_before, "a permitted, clean commit must land" + + +def test_canonical_clone_commit_is_still_blocked_before_any_chaining( + registry: Path, clone: Path, tmp_path: Path +) -> None: + """Assertion class 2 -- OMN-7018 is not traded away for OMN-15071.""" + env = _base_env(registry) + sentinel = tmp_path / _SENTINEL_NAME + _write_sentinel_hook(clone, sentinel, exit_code=0) + + (clone / "change.txt").write_text("change\n", encoding="utf-8") + _git("add", "change.txt", cwd=clone, env=env) + head_before = _git("rev-parse", "HEAD", cwd=clone, env=env).stdout.strip() + result = _git( + "commit", "-m", "must be blocked in the canonical clone", cwd=clone, env=env + ) + head_after = _git("rev-parse", "HEAD", cwd=clone, env=env).stdout.strip() + + assert result.returncode != 0, "a canonical-clone commit must be refused" + assert "blocked pre-commit in canonical clone" in result.stderr, result.stderr + assert head_after == head_before, "a blocked commit must not move HEAD" + assert not sentinel.exists(), ( + "the canonical-clone refusal must short-circuit BEFORE the chain runs" + ) + + +def test_canonical_clone_override_still_chains( + registry: Path, clone: Path, tmp_path: Path +) -> None: + """The documented emergency override suppresses the refusal only -- it is + not a hook-chain bypass (root CLAUDE.md rule #10).""" + env = _base_env(registry) + env["ALLOW_CANONICAL_CLONE_COMMIT"] = "1" + sentinel = tmp_path / _SENTINEL_NAME + _write_sentinel_hook(clone, sentinel, exit_code=1) + + (clone / "change.txt").write_text("change\n", encoding="utf-8") + _git("add", "change.txt", cwd=clone, env=env) + result = _git( + "commit", "-m", "override must still run the hooks", cwd=clone, env=env + ) + + assert sentinel.is_file(), ( + "ALLOW_CANONICAL_CLONE_COMMIT must not skip the hook chain" + ) + assert result.returncode != 0, ( + "the chained hook exited 1; the commit must be refused" + ) + + +def test_missing_hook_with_a_precommit_config_fails_closed() -> None: + """Assertion class 4 -- static. + + `pre-commit install` refuses to write hook files while `core.hooksPath` is + set, so "no installed hook" says nothing about "no configured hooks". The + guard must never answer that situation with a silent success. + """ + text = GUARD_SCRIPT.read_text(encoding="utf-8") + assert "hook-impl" in text, ( + "expected a `pre-commit hook-impl` fallback when no hook file is installed" + ) + assert "refuses to report a vacuous pass" in text, ( + "expected an explicit fail-closed branch when the chain cannot be run" + ) + + +def test_prepush_hook_unsets_leaked_git_scoping_vars_before_running_pytest() -> None: + """Chaining turns the pre-push hook ON for the first time on `.200`, and git + hands hooks a `GIT_DIR` that overrides `-C`/cwd for every descendant git + call. Without the unset, every test that builds a throwaway repository under + `tmp_path` operates on the real worktree instead and errors at setup -- + proven live 2026-07-30: `tests/scripts/test_check_deployed_migration_tree_sync.py` + is 9 errors with `GIT_DIR` exported and green without it, identically under + the pre-fix and post-fix guard (so the breakage is the leak, not the guard). + """ + hook = REPO_ROOT / "scripts" / "hooks" / "prepush_smart_tests.sh" + text = hook.read_text(encoding="utf-8") + unset_index = text.find("unset GIT_DIR") + assert unset_index != -1, ( + f"{hook} must unset the repo-scoping GIT_* variables git exports into " + "hooks before handing control to pytest" + ) + for var in ("GIT_WORK_TREE", "GIT_INDEX_FILE"): + assert var in text[unset_index : unset_index + 200], ( + f"{var} must be unset alongside GIT_DIR" + ) + first_pytest = text.find("uv run pytest") + assert first_pytest != -1, "expected a pytest invocation in the pre-push hook" + assert unset_index < first_pytest, ( + "the unset must precede every pytest invocation, or the leak still applies" + ) + + +def test_guard_contains_no_hardcoded_absolute_user_paths() -> None: + """Root CLAUDE.md rule #6. The pre-fix guard defaulted OMNI_HOME to a + literal `/Users/...` path and matched two more literal worktree roots.""" + text = GUARD_SCRIPT.read_text(encoding="utf-8") + for forbidden in ("/Users/", "/Volumes/"): + assert forbidden not in text, ( + f"hardcoded absolute path {forbidden!r} in {GUARD_SCRIPT}" + ) diff --git a/tests/ci/test_check_occ_companion_merged.py b/tests/ci/test_check_occ_companion_merged.py new file mode 100644 index 0000000000..a5e928acc8 --- /dev/null +++ b/tests/ci/test_check_occ_companion_merged.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Verdict tests for the occ-companion-merged STRICT gate (OMN-15214). + +The gate makes the 2026-07-26 hygiene-sweep trigger state — an OPEN +onex_change_control companion whose product PR has already MERGED — +unreachable via the merge path: the product PR's required ``CI Summary`` +context cannot go green until the cited companion is MERGED (or the cited +SHA is already an ancestor of an OCC durable branch). + +These tests pin the fail-closed verdict table: + +* companion MERGED → PASS +* companion OPEN → PENDING (poll; deadline converts to FAIL) +* companion CLOSED unmerged → FAIL immediately (the incident state) +* SHA ancestor of dev/main → PASS +* SHA not an ancestor → FAIL (OMN-15216 strandable pre-merge pin) +* missing Evidence-Source → PENDING (autobind mint may be in flight) +* malformed Evidence-Source → FAIL +* dependency-bot author → PASS (mirrors occ-preflight OMN-13762) +* non-PR event → PASS (gate not applicable) +* unresolvable PR number → FAIL (fail closed) +* API errors → PENDING (retryable), never PASS +""" + +from __future__ import annotations + +import pytest + +from scripts.ci.check_occ_companion_merged import ( + EXIT_FAIL, + EXIT_PASS, + EXIT_PENDING, + evaluate_once, + main, + parse_evidence_source, + resolve_pr_number, +) + +pytestmark = pytest.mark.unit + +PRODUCT_REPO = "OmniNode-ai/omnibase_infra" +OCC_REPO = "OmniNode-ai/onex_change_control" + + +class FakeFetcher: + """Deterministic stand-in for GhFetcher.""" + + def __init__( + self, + *, + prs: dict[tuple[str, str], dict[str, object] | None] | None = None, + compare: dict[tuple[str, str], str | None] | None = None, + ) -> None: + self._prs = prs or {} + self._compare = compare or {} + + def pr_view(self, repo: str, number: str, fields: str) -> dict[str, object] | None: + return self._prs.get((repo, str(number))) + + def compare_status(self, repo: str, base: str, head_sha: str) -> str | None: + return self._compare.get((base, head_sha)) + + +def _product_pr(body: str, author: str = "jonahgabriel") -> dict[str, object]: + return {"body": body, "author": {"login": author}} + + +def _evaluate(fetcher: FakeFetcher, **kwargs: object): + defaults: dict[str, object] = { + "event_name": "pull_request", + "repo": PRODUCT_REPO, + "pr_number": "2500", + "occ_repo": OCC_REPO, + } + defaults.update(kwargs) + return evaluate_once(fetcher, **defaults) # type: ignore[arg-type] + + +class TestEvidenceSourceParsing: + def test_first_line_wins_and_is_case_insensitive(self) -> None: + body = "intro\nevidence-source: OCC#5032 \nEvidence-Source: OCC#9999\n" + assert parse_evidence_source(body) == "OCC#5032" + + def test_absent_returns_none(self) -> None: + assert parse_evidence_source("no evidence here") is None + assert parse_evidence_source("") is None + + def test_indented_line_is_not_matched(self) -> None: + # occ-preflight anchors at line start; mirror it. + assert parse_evidence_source(" Evidence-Source: OCC#1") is None + + +class TestPrNumberResolution: + def test_pull_request_number_passthrough(self) -> None: + assert resolve_pr_number("pull_request", "123", "") == "123" + + def test_merge_group_head_ref_parse(self) -> None: + ref = "refs/heads/gh-readonly-queue/dev/pr-456-0123abc" + assert resolve_pr_number("merge_group", "", ref) == "456" + + def test_unresolvable_returns_empty(self) -> None: + assert resolve_pr_number("merge_group", "", "refs/heads/whatever") == "" + + +class TestCompanionPrVerdicts: + def _fetcher_with_companion(self, occ_state: dict[str, object]) -> FakeFetcher: + return FakeFetcher( + prs={ + (PRODUCT_REPO, "2500"): _product_pr("Evidence-Source: OCC#5032"), + (OCC_REPO, "5032"): occ_state, + } + ) + + def test_merged_companion_is_pass(self) -> None: + fetcher = self._fetcher_with_companion( + {"state": "MERGED", "mergeCommit": {"oid": "abc123"}} + ) + verdict = _evaluate(fetcher) + assert verdict.code == EXIT_PASS + assert "abc123" in verdict.reason + + def test_open_companion_is_pending_not_fail(self) -> None: + # OPEN may still auto-merge; the poll loop absorbs the latency and the + # deadline converts PENDING to FAIL. + fetcher = self._fetcher_with_companion({"state": "OPEN", "mergeCommit": None}) + verdict = _evaluate(fetcher) + assert verdict.code == EXIT_PENDING + assert "OPEN" in verdict.reason + + def test_closed_unmerged_companion_is_immediate_fail(self) -> None: + # The 2026-07-26 incident state: hygiene sweep closed the companion + # without merging. Evidence destroyed — terminal, never poll. + fetcher = self._fetcher_with_companion({"state": "CLOSED", "mergeCommit": None}) + verdict = _evaluate(fetcher) + assert verdict.code == EXIT_FAIL + assert "CLOSED" in verdict.reason + + def test_companion_fetch_error_is_pending_never_pass(self) -> None: + fetcher = FakeFetcher( + prs={ + (PRODUCT_REPO, "2500"): _product_pr("Evidence-Source: OCC#5032"), + (OCC_REPO, "5032"): None, + } + ) + assert _evaluate(fetcher).code == EXIT_PENDING + + +class TestShaVerdicts: + SHA = "a" * 40 + + def test_sha_ancestor_of_dev_is_pass(self) -> None: + fetcher = FakeFetcher( + prs={(PRODUCT_REPO, "2500"): _product_pr(f"Evidence-Source: {self.SHA}")}, + compare={("dev", self.SHA): "behind"}, + ) + assert _evaluate(fetcher).code == EXIT_PASS + + def test_sha_identical_to_main_is_pass(self) -> None: + fetcher = FakeFetcher( + prs={(PRODUCT_REPO, "2500"): _product_pr(f"Evidence-Source: {self.SHA}")}, + compare={("dev", self.SHA): "diverged", ("main", self.SHA): "identical"}, + ) + assert _evaluate(fetcher).code == EXIT_PASS + + def test_floating_sha_is_fail(self) -> None: + # A feature-branch head SHA on squash-only OCC can never become an + # ancestor of dev/main — terminal (OMN-15216). + fetcher = FakeFetcher( + prs={(PRODUCT_REPO, "2500"): _product_pr(f"Evidence-Source: {self.SHA}")}, + compare={("dev", self.SHA): "diverged", ("main", self.SHA): "ahead"}, + ) + verdict = _evaluate(fetcher) + assert verdict.code == EXIT_FAIL + assert "ancestor" in verdict.reason + + def test_compare_api_error_is_pending_never_fail(self) -> None: + fetcher = FakeFetcher( + prs={(PRODUCT_REPO, "2500"): _product_pr(f"Evidence-Source: {self.SHA}")}, + compare={("dev", self.SHA): None, ("main", self.SHA): None}, + ) + assert _evaluate(fetcher).code == EXIT_PENDING + + +class TestBodyAndScopeVerdicts: + def test_missing_evidence_source_is_pending(self) -> None: + fetcher = FakeFetcher(prs={(PRODUCT_REPO, "2500"): _product_pr("no line yet")}) + verdict = _evaluate(fetcher) + assert verdict.code == EXIT_PENDING + assert "Evidence-Source" in verdict.reason + + def test_malformed_evidence_source_is_fail(self) -> None: + fetcher = FakeFetcher( + prs={(PRODUCT_REPO, "2500"): _product_pr("Evidence-Source: not-a-ref!")} + ) + assert _evaluate(fetcher).code == EXIT_FAIL + + def test_dependency_bot_author_is_exempt(self) -> None: + fetcher = FakeFetcher( + prs={(PRODUCT_REPO, "2500"): _product_pr("", author="dependabot[bot]")} + ) + verdict = _evaluate(fetcher) + assert verdict.code == EXIT_PASS + assert "dependency-bot" in verdict.reason + + def test_non_pr_event_is_not_applicable_pass(self) -> None: + verdict = _evaluate(FakeFetcher(), event_name="push") + assert verdict.code == EXIT_PASS + assert "not applicable" in verdict.reason + + def test_unresolvable_pr_number_fails_closed(self) -> None: + verdict = _evaluate(FakeFetcher(), pr_number="") + assert verdict.code == EXIT_FAIL + + def test_product_pr_fetch_error_is_pending(self) -> None: + fetcher = FakeFetcher(prs={(PRODUCT_REPO, "2500"): None}) + assert _evaluate(fetcher).code == EXIT_PENDING + + def test_evidence_source_override_skips_body_fetch(self) -> None: + fetcher = FakeFetcher( + prs={(OCC_REPO, "5032"): {"state": "MERGED", "mergeCommit": {"oid": "x"}}} + ) + verdict = _evaluate(fetcher, evidence_source_override="OCC#5032") + assert verdict.code == EXIT_PASS + + +class TestMainEntrypoint: + def test_once_mode_returns_pending_exit_code(self, monkeypatch) -> None: + # --once with an OPEN companion must surface PENDING (2), not PASS. + import scripts.ci.check_occ_companion_merged as mod + + fetcher = FakeFetcher( + prs={ + (PRODUCT_REPO, "77"): _product_pr("Evidence-Source: OCC#5032"), + (OCC_REPO, "5032"): {"state": "OPEN", "mergeCommit": None}, + } + ) + monkeypatch.setattr(mod, "GhFetcher", lambda: fetcher) + rc = main( + [ + "--once", + "--repo", + PRODUCT_REPO, + "--pr-number", + "77", + "--occ-repo", + OCC_REPO, + ] + ) + assert rc == EXIT_PENDING + + def test_deadline_converts_pending_to_fail(self, monkeypatch) -> None: + import scripts.ci.check_occ_companion_merged as mod + + fetcher = FakeFetcher( + prs={ + (PRODUCT_REPO, "77"): _product_pr("Evidence-Source: OCC#5032"), + (OCC_REPO, "5032"): {"state": "OPEN", "mergeCommit": None}, + } + ) + monkeypatch.setattr(mod, "GhFetcher", lambda: fetcher) + rc = main( + [ + "--repo", + PRODUCT_REPO, + "--pr-number", + "77", + "--occ-repo", + OCC_REPO, + "--deadline-seconds", + "0", + "--poll-interval-seconds", + "0", + ] + ) + assert rc == EXIT_FAIL diff --git a/tests/ci/test_ci_summary_gate.py b/tests/ci/test_ci_summary_gate.py index ba2f5904b4..138eaf15fa 100644 --- a/tests/ci/test_ci_summary_gate.py +++ b/tests/ci/test_ci_summary_gate.py @@ -11,19 +11,113 @@ from __future__ import annotations +import json +from pathlib import Path +from typing import Any + import pytest +import yaml from scripts.ci.ci_summary_gate import ( + ACTOR_CONDITIONAL_CONTEXTS, EXIT_FAILURE, EXIT_PENDING, EXIT_SUCCESS, + EXPECTED_EXTERNAL_CONTEXTS, + MEASURED_NOT_ENFORCED_CONTEXTS, SKIPPABLE_GATE_JOBS, STRICT_GATE_JOBS, + applicable_external_contexts, evaluate, + evaluate_external_contexts, + latest_check_run_by_name, ) pytestmark = pytest.mark.unit +REPO_ROOT = Path(__file__).resolve().parents[2] +CI_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci.yml" +DEPLOY_AGENT_GATE = "Deploy Agent Tests (OMN-15378) / deploy-agent-tests" +APPLICATION_DB_GATE = "Application Database Domain Enforcement (OMN-15361)" + +# Real, unedited `commits/{sha}/check-runs` rows captured from the 16 dev PRs +# merged 2026-07-29T23:04Z → 2026-07-30T14:54Z, filtered to merge-time state. +# See the file's `_provenance` block for the exact capture command. +EXTERNAL_FIXTURE = ( + REPO_ROOT + / "tests" + / "ci" + / "fixtures" + / "omn15496_merge_time_external_check_runs.json" +) + + +def _load_external_fixture() -> dict[str, Any]: + loaded = json.loads(EXTERNAL_FIXTURE.read_text(encoding="utf-8")) + assert isinstance(loaded, dict) + return loaded + + +def _external_fixture(pr: str) -> list[dict[str, object]]: + """Merge-time external check-runs for one real merged dev PR.""" + entry = _load_external_fixture()["pull_requests"][pr] + runs = entry["check_runs"] + assert isinstance(runs, list) and runs + return [dict(row) for row in runs] + + +def _load_workflow(path: Path) -> dict[str, Any]: + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) + assert isinstance(loaded, dict) + return loaded + + +def _inner_job_names(workflow: dict[str, Any]) -> set[str]: + """Job ids and display names declared by a (called) workflow.""" + jobs: dict[str, Any] = workflow["jobs"] + names = {str(job_id) for job_id in jobs} + names |= { + str((body or {}).get("name")) + for body in jobs.values() + if (body or {}).get("name") + } + return names + + +def _observable_job_names(workflow: dict[str, Any]) -> tuple[set[str], set[str]]: + """Names the Actions jobs API can report for ``workflow``'s own run. + + Returns ``(exact_names, remote_caller_prefixes)``. Verified against live run + 30506617326: a plain job reports its display name (falling back to its job + id); a reusable-workflow caller that EXECUTES reports only + ``" / "`` rows and never a row under + its own job id (``occ-preflight / eligibility``, not ``occ-preflight``) — a + bare caller row appears only when the caller itself skipped (``zone-filter``, + ``Runtime Boot Smoke (compose)``). Inner jobs of a REMOTE reusable cannot be + resolved from this repo, so those callers are returned as prefixes. + """ + exact: set[str] = set() + remote_prefixes: set[str] = set() + for job_id, raw in workflow["jobs"].items(): + body: dict[str, Any] = raw or {} + display = str(body.get("name") or job_id) + uses = str(body.get("uses") or "") + if not uses: + exact.add(display) + continue + if uses.startswith("./"): + called_path = REPO_ROOT / uses[2:] + assert called_path.is_file(), ( + f"ci.yml job {job_id!r} calls {uses!r}, which does not exist" + ) + exact |= { + f"{display} / {inner}" + for inner in _inner_job_names(_load_workflow(called_path)) + } + else: + remote_prefixes.add(display) + return exact, remote_prefixes + def _job( name: str, conclusion: str | None, *, status: str = "completed", attempt: int = 1 @@ -205,3 +299,503 @@ def test_effect_assertion_gate_is_strict_and_fails_closed(self) -> None: code, report = evaluate(jobs) assert code == EXIT_FAILURE assert "Effect-Assertion Gate (RT-5)" in report + + def test_occ_companion_merged_gate_is_strict_and_fails_closed(self) -> None: + # OMN-15214: the companion-merged gate makes the 2026-07-26 hygiene-sweep + # trigger state (OPEN companion + MERGED product PR) unreachable via the + # merge path. It must be STRICT so a red/absent result fails the required + # "CI Summary" context — folding into the umbrella instead of adding a + # new top-level required context avoids the never-reports wedge. + gate = "OCC Companion Merged Gate (OMN-15214)" + assert gate in STRICT_GATE_JOBS + jobs = [j for j in _all_gates("success") if j["name"] != gate] + jobs.append(_job(gate, "failure")) + code, report = evaluate(jobs) + assert code == EXIT_FAILURE + assert gate in report + # A skip must also fail closed — the job is unconditional in ci.yml. + jobs = [j for j in _all_gates("success") if j["name"] != gate] + jobs.append(_job(gate, "skipped")) + code, _ = evaluate(jobs) + assert code == EXIT_FAILURE + + def test_deploy_agent_tests_gate_is_strict_and_fails_closed(self) -> None: + # OMN-15378 AC3: scripts/deploy-agent/tests/ (201 tests) was wired to RUN + # on PRs but into no aggregator — absent from dev's required set (only + # "CI Summary") and absent from both gate lists here, so a RED run left + # the required context green: the guard was advisory, code not mechanism. + # It must be STRICT so absent/red/skipped all fail the required context. + assert DEPLOY_AGENT_GATE in STRICT_GATE_JOBS + + # (a) red → FAILURE + jobs = [j for j in _all_gates("success") if j["name"] != DEPLOY_AGENT_GATE] + jobs.append(_job(DEPLOY_AGENT_GATE, "failure")) + code, report = evaluate(jobs) + assert code == EXIT_FAILURE + assert DEPLOY_AGENT_GATE in report + + # (b) skipped → FAILURE (the caller job is unconditional in ci.yml, so a + # skip means someone re-added an `if:`/path filter, not a legitimate + # no-op). + jobs = [j for j in _all_gates("success") if j["name"] != DEPLOY_AGENT_GATE] + jobs.append(_job(DEPLOY_AGENT_GATE, "skipped")) + code, _ = evaluate(jobs) + assert code == EXIT_FAILURE + + # (c) absent from the run entirely → PENDING, never a vacuous SUCCESS. + # This is the exact pre-fix state (the tests ran in their own workflow, + # so they never appeared in ci.yml's job list); the poller converts + # PENDING to FAILURE at its deadline. + jobs = [j for j in _all_gates("success") if j["name"] != DEPLOY_AGENT_GATE] + code, report = evaluate(jobs) + assert code == EXIT_PENDING + assert DEPLOY_AGENT_GATE in report + + def test_application_database_gate_is_strict_and_unconditional(self) -> None: + """OMN-15361 source and rebuilt-Docker controls must gate CI Summary.""" + assert APPLICATION_DB_GATE in STRICT_GATE_JOBS + jobs = [ + job for job in _all_gates("success") if job["name"] != APPLICATION_DB_GATE + ] + jobs.append(_job(APPLICATION_DB_GATE, "failure")) + code, report = evaluate(jobs) + assert code == EXIT_FAILURE + assert APPLICATION_DB_GATE in report + + workflow_job = _load_workflow(CI_WORKFLOW)["jobs"][ + "application-database-domain-enforcement" + ] + assert workflow_job["name"] == APPLICATION_DB_GATE + assert "if" not in workflow_job + assert workflow_job["needs"] == "occ-preflight" + + +class TestGateNamesResolveToRealJobs: + """Every gate name must be a job the poller can actually observe. + + The poller reads ``actions/runs/${RUN_ID}/jobs`` for ci.yml's OWN run, so a + gate naming a job that ci.yml never produces (e.g. a job that lives in a + separately-triggered workflow) is never present → PENDING forever → the + required context fails closed at the deadline on EVERY PR. That is the + failure mode of "just add the standalone workflow's job name to + STRICT_GATE_JOBS", and it is what this test makes unshippable. + """ + + def test_every_gate_name_resolves_to_a_ci_yml_job(self) -> None: + observable, remote_caller_prefixes = _observable_job_names( + _load_workflow(CI_WORKFLOW) + ) + + for gate in (*STRICT_GATE_JOBS, *SKIPPABLE_GATE_JOBS): + if gate in observable: + continue + caller = gate.split(" / ", 1)[0] + assert caller in remote_caller_prefixes, ( + f"gate {gate!r} is not a name the jobs API can report for " + "ci.yml's own run. A plain job reports its display name; a " + "reusable caller reports ' / ' " + "and NEVER its own job id. A gate the poller cannot observe is " + "absent forever → PENDING → the required 'CI Summary' context " + "fails closed at its deadline on every PR. Observable names: " + f"{sorted(observable)}" + ) + + def test_deploy_agent_gate_caller_is_unconditional(self) -> None: + # A STRICT gate may never legitimately skip, so the caller job must + # carry no `if:` and no `needs:` (OMN-15378 AC3). + job = _load_workflow(CI_WORKFLOW)["jobs"]["deploy-agent-tests"] + assert job["name"] == "Deploy Agent Tests (OMN-15378)" + assert job["uses"] == "./.github/workflows/deploy-agent-tests.yml" + assert "if" not in job + assert "needs" not in job + + def test_called_deploy_agent_workflow_does_not_self_trigger(self) -> None: + # Self-triggering would double-run the suite on every PR (duplicate + # producer) and re-open the path-filter blind spot the caller closes. + called = _load_workflow( + REPO_ROOT / ".github" / "workflows" / "deploy-agent-tests.yml" + ) + # PyYAML parses the `on:` key as the boolean True (YAML 1.1). + triggers = called.get(True, called.get("on")) + assert isinstance(triggers, dict) + assert "workflow_call" in triggers + assert "pull_request" not in triggers + assert "push" not in triggers + assert "merge_group" not in triggers + + +class TestExternalContextAssertion: + """OMN-15496 — cross-workflow contexts must gate the required rollup. + + Checks 1-3 above read ``actions/runs/${RUN_ID}/jobs``: ci.yml's OWN run. A + check produced by any other workflow file is invisible to them, and + ``omnibase_infra``'s ``dev`` requires exactly one context (``CI Summary``, + ``strict=false``) — so those checks were enforced by neither layer. + + Every fixture here is an UNEDITED ``commits/{sha}/check-runs`` payload from a + real merged ``dev`` PR, filtered to merge-time state + (``started_at <= mergedAt``). No hand-built dict stands in for the API shape. + """ + + def test_red_before_the_real_incident_greened(self) -> None: + """The exact false green this gate exists to prevent. + + PR #2555 merged 2026-07-30T04:25:09Z with ``CI Summary`` = success while + ``deploy-gate / deploy-gate`` = failure on the same head SHA. All 53 + in-run jobs really were green, so the run-scoped checks CANNOT catch it — + this is "exists but wrong", not a missing import. + """ + jobs = _all_gates("success") + # Pre-condition — the run-scoped verdict alone (i.e. this module's + # behaviour before OMN-15496) is SUCCESS on that very head SHA. + assert evaluate(jobs)[0] == EXIT_SUCCESS + + # ...and the payload really does carry the red, so the RED below is not + # passing for some unrelated reason. + deploy_gate = [ + row + for row in _external_fixture("2555") + if row["name"] == "deploy-gate / deploy-gate" + ] + assert [row["conclusion"] for row in deploy_gate] == ["failure"] + + code, report = evaluate( + jobs, + check_runs=_external_fixture("2555"), + external_contexts=EXPECTED_EXTERNAL_CONTEXTS, + ) + assert code == EXIT_FAILURE + assert "deploy-gate / deploy-gate" in report + + def test_falsification_control_tuple_entry_is_load_bearing(self) -> None: + """Drop the context from the tuple and the SAME payload greens. + + Without this the RED above could be passing for an unrelated reason. + """ + remaining = tuple( + c for c in EXPECTED_EXTERNAL_CONTEXTS if c != "deploy-gate / deploy-gate" + ) + assert len(remaining) == len(EXPECTED_EXTERNAL_CONTEXTS) - 1 + code, _ = evaluate( + _all_gates("success"), + check_runs=_external_fixture("2555"), + external_contexts=remaining, + ) + assert code == EXIT_SUCCESS + + def test_absent_context_is_pending_never_success(self) -> None: + """A context that never reports must not read as passing. + + Absence and success are indistinguishable to branch protection; that is + the OMN-14456 AC4 hole. PENDING is converted to FAILURE by the caller's + deadline. + """ + payload = [ + row + for row in _external_fixture("2567") + if row["name"] != "URL Authority Gate" + ] + code, report = evaluate( + _all_gates("success"), + check_runs=payload, + external_contexts=EXPECTED_EXTERNAL_CONTEXTS, + ) + assert code == EXIT_PENDING + assert "URL Authority Gate" in report + + def test_missing_payload_is_pending_never_success(self) -> None: + """A failed check-runs fetch must not green the gate.""" + code, _ = evaluate( + _all_gates("success"), + check_runs=None, + external_contexts=EXPECTED_EXTERNAL_CONTEXTS, + ) + assert code == EXIT_PENDING + + def test_still_running_context_is_pending(self) -> None: + payload = [dict(row) for row in _external_fixture("2567")] + for row in payload: + if row["name"] == "CodeQL": + row["status"] = "in_progress" + row["conclusion"] = None + code, _ = evaluate( + _all_gates("success"), + check_runs=payload, + external_contexts=EXPECTED_EXTERNAL_CONTEXTS, + ) + assert code == EXIT_PENDING + + def test_skipped_external_context_fails_closed(self) -> None: + """`skipped` is not a pass for an external context (OMN-15057 vector).""" + payload = [dict(row) for row in _external_fixture("2567")] + for row in payload: + if row["name"] == "verify / verify": + row["conclusion"] = "skipped" + code, report = evaluate( + _all_gates("success"), + check_runs=payload, + external_contexts=EXPECTED_EXTERNAL_CONTEXTS, + ) + assert code == EXIT_FAILURE + assert "verify / verify" in report + + def test_no_external_contexts_means_no_assertion(self) -> None: + """merge_group / workflow_dispatch have no PR-scoped context set.""" + code, _ = evaluate(_all_gates("success"), check_runs=None, external_contexts=()) + assert code == EXIT_SUCCESS + + def test_latest_wins_resolution_matches_github(self) -> None: + """A rerun's green supersedes the earlier red for the same name. + + Check-runs accumulate on a SHA forever, so "any red fails" would make + every transient red permanent and delete rerun as a recovery path. + Measured: that rule blocks 6 of the 16 sampled merged PRs, 5 of them on + already-reruns-green contexts. + """ + red_then_green = [ + { + "name": "CodeQL", + "status": "completed", + "conclusion": "failure", + "started_at": "2026-07-30T01:00:00Z", + "id": 1, + }, + { + "name": "CodeQL", + "status": "completed", + "conclusion": "success", + "started_at": "2026-07-30T02:00:00Z", + "id": 2, + }, + ] + resolved = latest_check_run_by_name(red_then_green) + assert resolved["CodeQL"].conclusion == "success" + + # ...and the reverse order still resolves to the LATEST, not the best. + green_then_red = [dict(row) for row in red_then_green] + green_then_red[0]["conclusion"] = "success" + green_then_red[1]["conclusion"] = "failure" + assert ( + latest_check_run_by_name(green_then_red)["CodeQL"].conclusion == "failure" + ) + + def test_external_contexts_disjoint_from_in_run_gates(self) -> None: + """No context may be asserted on both surfaces. + + ``occ-preflight / eligibility`` is the one name observed both inside and + outside ci.yml's check suite; asserting it twice would double-count an + ambiguous name (OMN-15112). + """ + overlap = set(EXPECTED_EXTERNAL_CONTEXTS) & { + *STRICT_GATE_JOBS, + *SKIPPABLE_GATE_JOBS, + } + assert overlap == set(), f"asserted on both surfaces: {sorted(overlap)}" + + def test_excluded_contexts_are_recorded_with_a_reason(self) -> None: + """Exclusions are data, not silence — each carries its measurement.""" + assert MEASURED_NOT_ENFORCED_CONTEXTS + for context, reason in MEASURED_NOT_ENFORCED_CONTEXTS.items(): + assert context not in EXPECTED_EXTERNAL_CONTEXTS + assert len(reason) > 40, f"{context} needs a substantive reason" + + def test_no_wedge_replay_over_sixteen_merged_dev_prs(self) -> None: + """The admitted tuple must not block PRs that legitimately merged. + + A required gate that reds on healthy PRs is worse than the hole it + closes. Replaying the merge-time payload of every dev PR merged + 2026-07-29T23:04Z → 2026-07-30T14:54Z must yield exactly ONE block, and + it must be the real defect (#2555, deploy-gate red at merge). + """ + fixture = _load_external_fixture() + blocked: dict[str, list[str]] = {} + for pr, entry in fixture["pull_requests"].items(): + failures, unresolved = evaluate_external_contexts( + entry["check_runs"], EXPECTED_EXTERNAL_CONTEXTS + ) + if failures or unresolved: + blocked[pr] = failures + unresolved + + assert len(fixture["pull_requests"]) == 16 + assert blocked == {"2555": ["deploy-gate / deploy-gate"]}, ( + "seed membership changed the no-wedge profile. Re-measure per-context " + "merge-time report rate over recent merged dev PRs before admitting a " + f"context; got {blocked}" + ) + + def test_every_admitted_context_reported_on_every_sampled_pr(self) -> None: + """Admission rule, enforced: 100% merge-time presence or it can wedge dev.""" + fixture = _load_external_fixture() + missing: dict[str, list[str]] = {} + for pr, entry in fixture["pull_requests"].items(): + observed = latest_check_run_by_name(entry["check_runs"]) + absent = [c for c in EXPECTED_EXTERNAL_CONTEXTS if c not in observed] + if absent: + missing[pr] = absent + assert missing == {}, ( + "a context absent from any sampled PR does not report on every PR " + f"shape; requiring it burns the poll deadline and wedges dev: {missing}" + ) + + +class TestExternalAssertionIsWiredIntoCiYml: + """A rule is not a mechanism — the gate must be load-bearing, not merely defined.""" + + def test_poll_step_passes_check_runs_and_event_name(self) -> None: + job = _load_workflow(CI_WORKFLOW)["jobs"]["ci-summary"] + poll = next( + s for s in job["steps"] if "Poll run jobs" in str(s.get("name", "")) + ) + run = str(poll["run"]) + assert "--check-runs-file check_runs.json" in run + assert '--event-name "${EVENT_NAME}"' in run + assert "commits/${HEAD_SHA}/check-runs" in run + # The verdict step owns pass/fail: it must NOT be continue-on-error, or + # the assertion is decorative exactly like the report-only cascade step. + assert poll.get("continue-on-error") is not True + assert job["permissions"]["checks"] == "read" + assert "HEAD_SHA" in poll["env"] and "EVENT_NAME" in poll["env"] + + def test_failed_check_runs_fetch_removes_the_file(self) -> None: + """A stale check_runs.json from a previous iteration must never be reused.""" + job = _load_workflow(CI_WORKFLOW)["jobs"]["ci-summary"] + poll = next( + s for s in job["steps"] if "Poll run jobs" in str(s.get("name", "")) + ) + run = str(poll["run"]) + assert run.count("rm -f check_runs.json") >= 2 + + +# -------------------------------------------------------------------------- +# OMN-15532 — actor-conditional external contexts. +# +# OMN-15496 admitted `gate / CodeRabbit Thread Check` as a fail-closed external +# context after measuring it 16/16 present over #2546…#2567. That window held no +# Dependabot PR. cr-thread-gate-caller.yml skips the caller job when +# `github.actor == 'dependabot[bot]'`, and because the context is the +# `caller-job / reusable-job` form the inner job never materialises — the +# check-run is ABSENT, not `skipped`. Absent burns the 90 min deadline and then +# fails closed against the SOLE required context on infra dev. +# -------------------------------------------------------------------------- + +DEPENDABOT_FIXTURE = ( + REPO_ROOT + / "tests" + / "ci" + / "fixtures" + / "omn15532_dependabot_pr2522_check_runs.json" +) +CR_THREAD_CONTEXT = "gate / CodeRabbit Thread Check" + + +def _dependabot_check_runs() -> list[dict[str, Any]]: + """Real, unedited check-runs from Dependabot PR #2522 (head 2cdf352d).""" + return list(json.loads(DEPENDABOT_FIXTURE.read_text())["check_runs"]) + + +class TestActorConditionalExternalContexts: + def test_fixture_is_the_real_absent_case(self) -> None: + """Guard the premise: the fixture must genuinely lack ONLY this context.""" + rows = _dependabot_check_runs() + names = {str(r["name"]) for r in rows} + absent = [c for c in EXPECTED_EXTERNAL_CONTEXTS if c not in names] + # If this ever changes, the exemption below is no longer justified. + assert absent == [CR_THREAD_CONTEXT], ( + "fixture no longer isolates the CR-thread-gate absence; re-measure " + "before trusting the exemption" + ) + + def test_dependabot_pr_is_not_wedged(self) -> None: + """AC1 — the real absent-context payload resolves for dependabot[bot].""" + code, report = evaluate( + _all_gates("success"), + check_runs=_dependabot_check_runs(), + external_contexts=EXPECTED_EXTERNAL_CONTEXTS, + pr_author="dependabot[bot]", + ) + assert code == EXIT_SUCCESS, report + + def test_same_payload_still_blocks_a_human_author(self) -> None: + """AC2 — the control that matters: the exemption must not leak.""" + code, report = evaluate( + _all_gates("success"), + check_runs=_dependabot_check_runs(), + external_contexts=EXPECTED_EXTERNAL_CONTEXTS, + pr_author="jonahgabriel", + ) + assert code == EXIT_PENDING, report + assert CR_THREAD_CONTEXT in report + + def test_absent_pr_author_enforces_everything(self) -> None: + """A forgotten --pr-author must enforce, never exempt.""" + code, _ = evaluate( + _all_gates("success"), + check_runs=_dependabot_check_runs(), + external_contexts=EXPECTED_EXTERNAL_CONTEXTS, + pr_author=None, + ) + assert code == EXIT_PENDING + + def test_exemption_entry_is_load_bearing( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """AC3 — falsification control. + + Genuinely REMOVE the registry entry and confirm the very same fixture + and author go back to blocking. Without this, the AC1 green could come + from the fixture rather than from the exemption. + """ + import scripts.ci.ci_summary_gate as gate_mod + + monkeypatch.setattr(gate_mod, "ACTOR_CONDITIONAL_CONTEXTS", {}) + code, report = evaluate( + _all_gates("success"), + check_runs=_dependabot_check_runs(), + external_contexts=EXPECTED_EXTERNAL_CONTEXTS, + pr_author="dependabot[bot]", + ) + assert code == EXIT_PENDING, report + assert CR_THREAD_CONTEXT in report + + def test_exemption_drops_only_that_one_context(self) -> None: + """The dependabot exemption must not quietly widen.""" + pruned = applicable_external_contexts( + EXPECTED_EXTERNAL_CONTEXTS, "dependabot[bot]" + ) + assert set(EXPECTED_EXTERNAL_CONTEXTS) - set(pruned) == {CR_THREAD_CONTEXT} + + def test_registry_keys_are_asserted_contexts(self) -> None: + """AC4 — cannot exempt a context that was never asserted.""" + for context in ACTOR_CONDITIONAL_CONTEXTS: + assert context in EXPECTED_EXTERNAL_CONTEXTS, context + + def test_registry_actors_are_concrete_logins(self) -> None: + """AC4 — no wildcard/empty actor may blanket-disable a context.""" + for context, actors in ACTOR_CONDITIONAL_CONTEXTS.items(): + assert actors, f"{context} declares an empty actor tuple" + for actor in actors: + assert actor.strip(), f"{context} declares a blank actor" + assert actor not in {"*", "all"}, f"{context} declares wildcard {actor}" + + def test_a_failing_context_still_fails_for_dependabot(self) -> None: + """The exemption is applicability, not a bypass: reds still block.""" + rows = _dependabot_check_runs() + for row in rows: + if row["name"] == "deploy-gate / deploy-gate": + row["conclusion"] = "failure" + code, report = evaluate( + _all_gates("success"), + check_runs=rows, + external_contexts=EXPECTED_EXTERNAL_CONTEXTS, + pr_author="dependabot[bot]", + ) + assert code == EXIT_FAILURE, report + + def test_ci_yml_passes_pr_author(self) -> None: + """AC5 — a rule is not a mechanism: the wiring must exist in ci.yml.""" + job = _load_workflow(CI_WORKFLOW)["jobs"]["ci-summary"] + poll = next( + s for s in job["steps"] if "Poll run jobs" in str(s.get("name", "")) + ) + assert '--pr-author "${PR_AUTHOR:-}"' in str(poll["run"]) + assert "PR_AUTHOR" in poll["env"] diff --git a/tests/ci/test_ci_workflow_resilience.py b/tests/ci/test_ci_workflow_resilience.py index 8e012e8f4f..10f4458325 100644 --- a/tests/ci/test_ci_workflow_resilience.py +++ b/tests/ci/test_ci_workflow_resilience.py @@ -43,9 +43,12 @@ SETUP_PYTHON_UV_ACTION = ( REPO_ROOT / ".github" / "actions" / "setup-python-uv" / "action.yml" ) +CR_THREAD_GATE_CALLER_WORKFLOW = ( + REPO_ROOT / ".github" / "workflows" / "cr-thread-gate-caller.yml" +) CHECKOUT_V7_SHA = "9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" CODEQL_V4_SHA = "dc73d59c2d7bd4f8194098a91219eeee6d8a1719" -OMNICLAUDE_REJECT_SKIP_NO_CHECKOUT_SHA = "ff230264ac3300d7ced43564dc921f44558110fe" +OMNICLAUDE_REJECT_SKIP_NO_CHECKOUT_SHA = "80de61fd1fee04abdeb6918e7f91cf820717e6a8" def _load_yaml(path: Path) -> dict[str, Any]: @@ -908,3 +911,103 @@ def test_codeql_uses_repo_config_that_ignores_github_metadata() -> None: assert config["paths"] == ["src", "scripts", "tests"] assert ".github/**" in config["paths-ignore"] + + +_CR_GATE_FALLBACK_CHAIN = ( + "github.event.pull_request.number || " + "github.event.issue.number || " + "github.event.merge_group.head_sha || " + "github.run_id" +) + + +def _normalized_group(node: dict[str, Any]) -> str: + group = node["concurrency"]["group"] + assert isinstance(group, str) + return " ".join(group.split()) + + +def _render_cr_gate_group( + normalized_template: str, *, event_name: str, number: int +) -> str: + """Minimal renderer for cr-thread-gate-caller.yml's two group templates. + + Only understands the exact `||`-chained fallback this workflow uses + (`github.event.pull_request.number || github.event.issue.number || ...`, + resolved GitHub-Actions-style: first non-empty/non-null operand wins) + plus the literal `github.workflow` / `github.event_name` tokens. Not a + general expression evaluator — sufficient to prove the OMN-15815 + group-key non-collision property for pull_request vs. issue_comment. + """ + fallback_expr = f"${{{{ {_CR_GATE_FALLBACK_CHAIN} }}}}" + assert fallback_expr in normalized_template, ( + "cr-thread-gate-caller.yml concurrency group's fallback chain " + "drifted from the shape this test renders — update the renderer " + "alongside any real template change" + ) + if event_name in ("pull_request", "issue_comment"): + resolved_number = str(number) + else: # pragma: no cover - only two event types exercised below + resolved_number = "RUN_ID" + rendered = normalized_template.replace(fallback_expr, resolved_number) + rendered = rendered.replace("${{ github.workflow }}", "CR Thread Gate (caller)") + rendered = rendered.replace("${{ github.event_name }}", event_name) + return rendered + + +def test_cr_thread_gate_caller_concurrency_group_is_event_scoped() -> None: + """OMN-15815 regression: a CodeRabbit issue_comment run must never share + a concurrency group with the pull_request run of the same PR. + + Both the workflow-level and job-level `concurrency.group` templates + fall back through `github.event.pull_request.number || + github.event.issue.number || ...` with no event discriminator — a + pull_request run and an issue_comment run on the SAME PR resolve to the + identical group key. With cancel-in-progress: true, CodeRabbit editing + its summary comment (an issue_comment event) cancels the in-flight real + gate run; the CodeRabbit-actored run then skips its own gate job (see + the job's actor-filtered `if:`), so no replacement check-run is ever + emitted and the required "gate / CodeRabbit Thread Check" context is + left stuck cancelled with no successor. + """ + workflow = _load_yaml(CR_THREAD_GATE_CALLER_WORKFLOW) + gate_job = workflow["jobs"]["gate"] + + assert workflow["concurrency"]["cancel-in-progress"] is True + assert gate_job["concurrency"]["cancel-in-progress"] is True + + for node, label in ((workflow, "workflow-level"), (gate_job, "job-level")): + template = _normalized_group(node) + assert "${{ github.event_name }}" in template, ( + f"{label} concurrency group must key on github.event_name " + "(OMN-15815) so issue_comment runs can't cancel pull_request runs" + ) + + pull_request_key = _render_cr_gate_group( + template, event_name="pull_request", number=2663 + ) + issue_comment_key = _render_cr_gate_group( + template, event_name="issue_comment", number=2663 + ) + assert pull_request_key != issue_comment_key, ( + f"{label} concurrency group collides across pull_request and " + "issue_comment for the identical PR number — this is exactly " + "the OMN-15815 self-cancellation shape" + ) + + # Same event type + same PR/issue number must still coalesce: two + # rapid pushes to one PR (pull_request) still supersede each other, + # and rapid consecutive CodeRabbit comment edits on one PR + # (issue_comment) still dedup among themselves. + assert pull_request_key == _render_cr_gate_group( + template, event_name="pull_request", number=2663 + ) + assert issue_comment_key == _render_cr_gate_group( + template, event_name="issue_comment", number=2663 + ) + + # Different PRs of the same event type must still resolve to + # distinct groups (no cross-PR collision regression). + assert pull_request_key != _render_cr_gate_group( + template, event_name="pull_request", number=2666 + ) diff --git a/tests/ci/test_compose_required_env_coverage.py b/tests/ci/test_compose_required_env_coverage.py index 736099b1b3..e43dd20c71 100644 --- a/tests/ci/test_compose_required_env_coverage.py +++ b/tests/ci/test_compose_required_env_coverage.py @@ -1,69 +1,295 @@ # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT -"""CI guard: every :?-required env var in docker-compose.infra.yml must appear -in the test_compose_config_valid fixture dict. +"""CI guard: every `:?`-required env var in docker-compose.infra.yml must be +supplied by EVERY compose-render test fixture that layers that file. -Catches: PRs that add a new service with a required :? env var to compose -without updating the integration test fixture, which previously caused -cascading failures in #886, #890, #895 (OMN-5240 root cause analysis). +Catches: PRs that add a new service with a required `:?` env var to compose +without updating the render fixtures, which previously caused cascading +failures in #886, #890, #895 (OMN-5240 root cause analysis). + +OMN-15263 — why this gate was generalized. It used to check ONE fixture +(`tests/integration/docker/test_docker_integration.py`). OMN-15173 (#2495) +added a `:?` fail-fast for `DEV_REDPANDA_ADVERTISE_HOST` and updated exactly +that fixture, so this gate stayed green while the other three render fixtures — +which build their env hermetically (`env=` replaced, not inherited) — started +failing `docker compose config` on every hosted CI runner: 12 failures in +`Tests (Split 2/15)`, red `CI Summary`, on every PR whose selector escalated to +the full suite. A gate that covers one of four call sites is why that landed +green. + +Enforced here: + +1. **Coverage** — for each registered render fixture, every `:?`-required var in + the base infra compose file is supplied by that fixture's env dict, by one of + the `--env-file` files it passes to compose, or is explicitly declared as + intentionally unset with a reason. +2. **Fail-closed discovery** — a `*compose_render*.py` module under + `tests/integration/` that is not registered below fails this gate, so the + next render fixture cannot silently escape coverage the way three just did. + +This module is deliberately static (AST + text parsing, no `docker` invocation) +so it fires on hosts without Docker — the exact hosts where the 12 render tests +`skip` and where this regression was invisible locally. """ from __future__ import annotations +import ast import re from pathlib import Path +from typing import NamedTuple import pytest -COMPOSE_FILE = ( - Path(__file__).parent.parent.parent / "docker" / "docker-compose.infra.yml" -) -FIXTURE_FILE = ( - Path(__file__).parent.parent - / "integration" - / "docker" - / "test_docker_integration.py" +REPO_ROOT = Path(__file__).resolve().parents[2] +COMPOSE_FILE = REPO_ROOT / "docker" / "docker-compose.infra.yml" + +# Any fixture whose source references this file layers the base infra compose +# file and therefore inherits its `:?` vars — compose interpolates every input +# file before merge, regardless of `--profile`. +BASE_COMPOSE_FILENAME = "docker-compose.infra.yml" + +# Fail-closed discovery: every module matching this glob must be registered in +# RENDER_FIXTURES below. +RENDER_FIXTURE_GLOB = "tests/integration/**/*compose_render*.py" + + +class RenderFixture(NamedTuple): + """A test module that renders compose and must supply its required env.""" + + path: str + # Module-level dict constants holding the render env. Extracted by AST, so + # a fixture must expose them at module scope, not inside a test body. + env_dicts: tuple[str, ...] + # (var, reason) pairs. Escape hatch for a fixture that must NOT set a var + # because proving the unset behaviour is the point of that fixture. + intentionally_unset: tuple[tuple[str, str], ...] = () + + +RENDER_FIXTURES: tuple[RenderFixture, ...] = ( + RenderFixture( + path="tests/integration/docker/test_docker_integration.py", + env_dicts=("COMPOSE_CONFIG_RENDER_ENV",), + ), + RenderFixture( + path="tests/integration/infra/test_dev_runtime_compose_render.py", + env_dicts=("BASE_REQUIRED_ENV",), + intentionally_unset=( + ( + "DEV_REDPANDA_ADVERTISE_HOST", + ( + "OMN-15173 counter-test: this module's " + "test_dev_redpanda_advertise_host_fails_fast_when_unset proves " + "the render FAILS when the var is unset. Adding it to " + "BASE_REQUIRED_ENV would make that test vacuous and resurrect " + "the silent localhost-advertise regression." + ), + ), + ), + ), + RenderFixture( + path="tests/integration/infra/test_prod_runtime_compose_render.py", + env_dicts=("COMPOSE_RENDER_ENV",), + ), + RenderFixture( + path="tests/integration/infra/test_judge_compose_render.py", + env_dicts=("LAYERED_RENDER_DUMMY_ENV",), + ), + RenderFixture( + path="tests/integration/infra/test_stability_test_runtime_compose_render.py", + env_dicts=("COMPOSE_RENDER_ENV",), + ), ) def extract_required_compose_vars(compose_path: Path) -> set[str]: - """Return all variable names that use :? fail-fast syntax in the compose file.""" - text = compose_path.read_text() + """Return all variable names that use `:?` fail-fast syntax in a compose file.""" + text = compose_path.read_text(encoding="utf-8") return set(re.findall(r"\$\{([A-Z_][A-Z0-9_]*):\?", text)) -def extract_fixture_vars(fixture_path: Path) -> set[str]: - """Return all string keys in the env.update({...}) dict in test_compose_config_valid.""" - text = fixture_path.read_text() - # Narrow to the env.update block inside test_compose_config_valid - # Strategy: find the function, then extract all "KEY": patterns within it - match = re.search( - r"def test_compose_config_valid.*?env\.update\s*\(\s*\{(.*?)\}\s*\)", - text, - re.DOTALL, +def extract_module_env_vars(source: str, dict_names: tuple[str, ...]) -> set[str]: + """Return the string keys of the named module-level dict constants. + + AST-based: only literal `str` keys are collected, and only from assignments + at module scope, so a dict nested inside a test body is never silently + counted as coverage. + """ + tree = ast.parse(source) + wanted = set(dict_names) + found: dict[str, set[str]] = {} + for node in tree.body: + if isinstance(node, ast.AnnAssign): + targets: list[ast.expr] = [node.target] + value = node.value + elif isinstance(node, ast.Assign): + targets = list(node.targets) + value = node.value + else: + continue + if not isinstance(value, ast.Dict): + continue + for target in targets: + if isinstance(target, ast.Name) and target.id in wanted: + found[target.id] = { + key.value + for key in value.keys + if isinstance(key, ast.Constant) and isinstance(key.value, str) + } + missing_dicts = wanted - found.keys() + assert not missing_dicts, ( + "Registered env dict(s) not found as module-level dict literals: " + + ", ".join(sorted(missing_dicts)) + + ". Either the constant was renamed or moved into a function body, or " + "RENDER_FIXTURES in this file is stale." + ) + return set().union(*found.values()) if found else set() + + +def extract_env_file_vars(source: str) -> set[str]: + """Return vars supplied by every `--env-file` the fixture passes to compose. + + A fixture may legitimately omit a var from its Python dict when it loads an + env file that defines it (judge loads docker/judge.env.example; every lane + loads docker/runtime-policy.env). + """ + provided: set[str] = set() + for rel_path in re.findall(r'"--env-file",\s*"([^"]+)"', source): + env_path = REPO_ROOT / rel_path + assert env_path.is_file(), ( + f"A render fixture passes --env-file {rel_path!r}, which does not " + "exist relative to the repo root." + ) + provided |= set( + re.findall( + r"^\s*(?:export\s+)?([A-Z_][A-Z0-9_]*)\s*=", + env_path.read_text(encoding="utf-8"), + re.MULTILINE, + ) + ) + return provided + + +def _fixture_source(fixture: RenderFixture) -> str: + path = REPO_ROOT / fixture.path + assert path.is_file(), ( + f"Registered render fixture {fixture.path} does not exist. If it was " + "renamed or deleted, update RENDER_FIXTURES in this file." + ) + return path.read_text(encoding="utf-8") + + +@pytest.mark.unit +def test_every_compose_render_fixture_is_registered() -> None: + """A new compose-render module must be registered, or this gate goes RED. + + Fail-closed: the OMN-15263 breakage landed green precisely because three + render fixtures were outside this gate's field of view. + """ + discovered = { + str(path.relative_to(REPO_ROOT)) + for path in REPO_ROOT.glob(RENDER_FIXTURE_GLOB) + if path.name.startswith("test_") + } + registered = {fixture.path for fixture in RENDER_FIXTURES} + unregistered = discovered - registered + assert not unregistered, ( + "These compose-render test modules are not registered in " + "RENDER_FIXTURES in tests/ci/test_compose_required_env_coverage.py:\n" + + "\n".join(f" - {path}" for path in sorted(unregistered)) + + "\n\nFix: add a RenderFixture entry naming the module-level env dict " + "constant(s) it renders with, so this coverage gate can see it." ) - if not match: - return set() - block = match.group(1) - return set(re.findall(r'"([A-Z_][A-Z0-9_]*)"\s*:', block)) @pytest.mark.unit -def test_all_required_compose_vars_in_fixture() -> None: - """Every :?-required var in compose must be present in the test fixture dict. +@pytest.mark.parametrize( + "fixture", RENDER_FIXTURES, ids=[fixture.path for fixture in RENDER_FIXTURES] +) +def test_all_required_compose_vars_in_fixture(fixture: RenderFixture) -> None: + """Every `:?`-required var in compose must be supplied by each render fixture. - This is the CI twin for the contract: 'if you add a :? var to compose, - you must also add it to the test_compose_config_valid fixture dict'. - Fails on the PR that introduces the gap, not three PRs later. + This is the CI twin for the contract: 'if you add a `:?` var to compose, you + must also add it to every compose-render fixture'. Fails on the PR that + introduces the gap, not on every unrelated PR afterwards. """ + source = _fixture_source(fixture) + if BASE_COMPOSE_FILENAME not in source: + pytest.skip( + f"{fixture.path} does not layer {BASE_COMPOSE_FILENAME}; its `:?` " + "vars come from another compose file." + ) + required = extract_required_compose_vars(COMPOSE_FILE) - provided = extract_fixture_vars(FIXTURE_FILE) + provided = ( + extract_module_env_vars(source, fixture.env_dicts) + | extract_env_file_vars(source) + | {var for var, _reason in fixture.intentionally_unset} + ) missing = required - provided assert not missing, ( - "These :?-required env vars are in docker-compose.infra.yml but " - "NOT in the test_compose_config_valid fixture dict:\n" - + "\n".join(f" - {v}" for v in sorted(missing)) - + "\n\nFix: add each missing var to the env.update({...}) dict in " - "tests/integration/docker/test_docker_integration.py " - "(around the 'test_compose_config_valid' method)." + "These `:?`-required env vars are in docker/docker-compose.infra.yml but " + f"NOT supplied by the render fixture {fixture.path}:\n" + + "\n".join(f" - {var}" for var in sorted(missing)) + + "\n\nFix: add each missing var to " + + " / ".join(fixture.env_dicts) + + f" in {fixture.path} (a render-only dummy value is fine). Every " + "compose-render fixture layering the base infra file interpolates every " + "`:?` var in it, regardless of --profile." + ) + + +@pytest.mark.unit +def test_intentionally_unset_vars_are_justified() -> None: + """An omission escape hatch must name a real var and carry a reason. + + Keeps the hatch from decaying into a silent allowlist: the var must still be + `:?`-required in compose (stale exemptions go RED) and must actually appear + in the fixture that omits it, so the deliberate omission is visible there. + """ + required = extract_required_compose_vars(COMPOSE_FILE) + for fixture in RENDER_FIXTURES: + source = _fixture_source(fixture) + for var, reason in fixture.intentionally_unset: + assert var in required, ( + f"{fixture.path} declares {var} intentionally unset, but {var} is " + "no longer `:?`-required in docker/docker-compose.infra.yml. " + "Remove the stale exemption." + ) + assert len(reason.strip()) >= 40, ( + f"{fixture.path}: exemption for {var} needs a real reason, not " + f"{reason!r}." + ) + assert var in source, ( + f"{fixture.path} is exempted from supplying {var} but never " + "mentions it. An exemption is only valid for a fixture that " + "deliberately exercises the unset case." + ) + + +@pytest.mark.unit +def test_dev_advertise_host_keeps_fail_fast_form() -> None: + """OMN-15173 counter-test: the dev advertise host must never regain a default. + + OMN-15263's fix supplies the var in the render fixtures. The wrong fix — + giving compose a `:-localhost` default again — would also turn every render + green, while silently restoring the off-host regression OMN-15173 removed. + This test makes that shortcut RED, on every host, with or without Docker. + """ + text = COMPOSE_FILE.read_text(encoding="utf-8") + + defaulted = re.findall(r"\$\{DEV_REDPANDA_ADVERTISE_HOST:-[^}]*\}", text) + assert not defaulted, ( + "DEV_REDPANDA_ADVERTISE_HOST regained a silent default in " + "docker/docker-compose.infra.yml: " + + ", ".join(defaulted) + + ". OMN-15173: an unset advertise host must fail the render loudly, not " + "render an address no off-host client can reach. Supply the var in the " + "render fixtures instead." + ) + assert "DEV_REDPANDA_ADVERTISE_HOST" in extract_required_compose_vars( + COMPOSE_FILE + ), ( + "DEV_REDPANDA_ADVERTISE_HOST is no longer `:?`-required in " + "docker/docker-compose.infra.yml (OMN-15173)." ) diff --git a/tests/ci/test_dev_lane_liveness.py b/tests/ci/test_dev_lane_liveness.py new file mode 100644 index 0000000000..a3a715a61f --- /dev/null +++ b/tests/ci/test_dev_lane_liveness.py @@ -0,0 +1,503 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Tests for the OMN-15190 lab/dev lane liveness detector. + +Incident class: the `.201` lab/dev lane (compose project ``omnibase-infra``) +was GC/idle-reclaimed to zero containers at least five times (2026-07-13, +07-14, 07-24, 07-26 x2). Every occurrence was found REACTIVELY, by whichever +PR happened to hit the resulting org-wide ``occ-autobind`` / +``occ-companion-effect`` connection-refused cascade. No monitor existed: +``scripts/system_health_check.sh`` held most of the logic and was wired into +nothing, and its severity contract actively *passed* a down lane +(``runtime_containers: yellow`` -> exit 0). + +Operator ruling 2026-07-29 (WS-4) reverses the posture: the lab lane is +KEEP-ALIVE, so lane-down is a defect. These tests drive the REAL bash +artifact (never a reimplementation of its logic) with a stubbed ``docker`` +and real localhost sockets, and pin both directions of every verdict: + +* the failure modes that actually strand CI are RED, +* the states that do NOT strand CI are NOT red (a permanently-red check is a + disabled check — the same reasoning ``healthcheck.sh`` layer 4 documents). + +Ticket: OMN-15190 +""" + +from __future__ import annotations + +import http.server +import os +import socket +import subprocess +import threading +from collections.abc import Iterator +from pathlib import Path + +import pytest +import yaml + +# tests/ci/ is the project-recognized home for CI/CD parity tests (OMN-4307). +# Local subprocesses + localhost sockets only; no external infrastructure. +pytestmark = pytest.mark.ci + +REPO_ROOT = Path(__file__).resolve().parents[2] +HEALTH_GATE = REPO_ROOT / "scripts" / "system_health_check.sh" +LANE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "dev-lane-liveness.yml" +CANARY_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "runner-fleet-canary.yml" + +# A container census in the exact `docker ps --format '{{.Names}}|{{.State}}| +# {{.Status}}'` shape, transcribed from the live `.201` dev lane on +# 2026-07-29T00:5xZ (compose project omnibase-infra). Used as the "lane is +# fine" baseline so the healthy-path assertions run against the real input +# distribution rather than a hand-shrunk approximation. +LIVE_LANE_ROWS = "\n".join( + [ + "omninode-runtime|running|Up 9 hours (healthy)", + "omninode-runtime-effects|running|Up 2 hours (healthy)", + "omnibase-intelligence-api|running|Up 9 hours (healthy)", + "omnibase-infra-postgres|running|Up 10 hours (healthy)", + "omnibase-infra-redpanda|running|Up 10 hours (healthy)", + "omnibase-infra-valkey|running|Up 10 hours (healthy)", + "omnibase-infra-intelligence-migration|exited|Exited (0) 2 hours ago", + ] +) + +RUNTIME_CONTAINER_NAMES = "\n".join( + [ + "omninode-runtime", + "omninode-runtime-effects", + "omnibase-intelligence-api", + "omnibase-infra-redpanda", + ] +) + + +class _HealthHandler(http.server.BaseHTTPRequestHandler): + """Minimal /health responder; status code is set per-test on the class.""" + + status_code = 200 + + def do_GET(self) -> None: + self.send_response(self.status_code) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"status":"ok"}') + + def log_message(self, *args: object) -> None: + return + + +@pytest.fixture +def health_server() -> Iterator[tuple[int, type[_HealthHandler]]]: + """A localhost HTTP server standing in for the lane's runtime /health.""" + handler = type("_ScopedHealthHandler", (_HealthHandler,), {"status_code": 200}) + server = http.server.HTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server.server_address[1], handler + finally: + server.shutdown() + server.server_close() + + +@pytest.fixture +def open_broker_port() -> Iterator[int]: + """A listening TCP socket standing in for the lane's published broker port.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 0)) + sock.listen(8) + try: + yield sock.getsockname()[1] + finally: + sock.close() + + +def _closed_port() -> int: + """A port number with nothing listening on it.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 0)) + port: int = sock.getsockname()[1] + sock.close() + return port + + +def _install_docker_stub( + tmp_path: Path, + lane_rows: str, + running_names: str = RUNTIME_CONTAINER_NAMES, + daemon_ok: bool = True, +) -> Path: + """Write a ``docker`` stub whose census output the test controls. + + The stub answers the three invocations the ``--lane`` subset makes: + the compose-project-label census, the plain running-name list, and the + ``rpk cluster health`` exec. Nothing else is emulated. + """ + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + rows_file = tmp_path / "lane_rows.txt" + rows_file.write_text(lane_rows) + names_file = tmp_path / "running_names.txt" + names_file.write_text(running_names) + + fail_block = ( + "" + if daemon_ok + else ( + 'echo "Cannot connect to the Docker daemon at unix:///var/run/' + 'docker.sock. Is the docker daemon running?" >&2\nexit 1\n' + ) + ) + + stub = f"""#!/usr/bin/env bash +{fail_block}case "$*" in + *"label=com.docker.compose.project"*) + cat {rows_file!s} + ;; + "ps --format {{{{.Names}}}}") + cat {names_file!s} + ;; + *"rpk cluster health"*) + echo "Healthy: true" + ;; + *) + exit 0 + ;; +esac +""" + docker = bin_dir / "docker" + docker.write_text(stub) + docker.chmod(0o755) + return bin_dir + + +def _run_lane_gate( + tmp_path: Path, + lane_rows: str, + broker_port: int, + main_port: int, + keepalive: str = "1", + running_names: str = RUNTIME_CONTAINER_NAMES, + daemon_ok: bool = True, +) -> subprocess.CompletedProcess[str]: + bin_dir = _install_docker_stub( + tmp_path, lane_rows, running_names=running_names, daemon_ok=daemon_ok + ) + env = dict(os.environ) + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + env["LANE_PROBE_HOST"] = "127.0.0.1" + env["DEV_LANE_BROKER_PORT"] = str(broker_port) + env["DEV_LANE_MAIN_PORT"] = str(main_port) + env["ONEX_LANE_KEEPALIVE"] = keepalive + return subprocess.run( + ["bash", str(HEALTH_GATE), "--lane", "--ci"], + capture_output=True, + text=True, + env=env, + timeout=120, + check=False, + ) + + +# --------------------------------------------------------------------------- +# The failure mode the ticket exists for: the lane is GONE. +# --------------------------------------------------------------------------- + + +def test_lane_absent_is_red_under_keepalive( + tmp_path: Path, + open_broker_port: int, + health_server: tuple[int, type[_HealthHandler]], +) -> None: + """Zero containers in the compose project is the OMN-15190 signature. + + Note the probe targets are deliberately REACHABLE here: the verdict must + come from lane membership, not from an incidentally-dead port. + """ + main_port, _ = health_server + result = _run_lane_gate(tmp_path, "", open_broker_port, main_port) + + assert result.returncode == 1, result.stdout + assert '"overall": "red"' in result.stdout + assert "ZERO containers" in result.stdout + assert "OMN-15190" in result.stdout + + +def test_lane_absent_is_advisory_when_keepalive_disabled( + tmp_path: Path, + open_broker_port: int, + health_server: tuple[int, type[_HealthHandler]], +) -> None: + """Negative control for the ruling itself. + + Identical input, one env flip. ``ONEX_LANE_KEEPALIVE=0`` reproduces the + pre-ruling posture — the lane is torn down, the gate says yellow, and + yellow exits 0. That exit-0 IS the defect the WS-4 ruling closes: the + canonical health gate reported success while the lane stranded every + repo's receipt path. Without this control the RED above could be an + unconditional failure rather than a ruling-driven verdict. + """ + main_port, _ = health_server + result = _run_lane_gate(tmp_path, "", open_broker_port, main_port, keepalive="0") + + assert result.returncode == 0, result.stdout + assert '"overall": "red"' not in result.stdout + assert "ZERO containers" in result.stdout + + +# --------------------------------------------------------------------------- +# Reachability on the exact path CI publishers use. +# --------------------------------------------------------------------------- + + +def test_healthy_lane_is_green( + tmp_path: Path, + open_broker_port: int, + health_server: tuple[int, type[_HealthHandler]], +) -> None: + """Non-vacuity: the live lane census must not manufacture a red.""" + main_port, _ = health_server + result = _run_lane_gate(tmp_path, LIVE_LANE_ROWS, open_broker_port, main_port) + + assert result.returncode == 0, result.stdout + assert '"overall": "green"' in result.stdout + assert "up and reachable" in result.stdout + + +def test_broker_port_refused_is_red( + tmp_path: Path, health_server: tuple[int, type[_HealthHandler]] +) -> None: + """Containers up but the publish port dead still strands the whole org.""" + main_port, _ = health_server + result = _run_lane_gate(tmp_path, LIVE_LANE_ROWS, _closed_port(), main_port) + + assert result.returncode == 1, result.stdout + assert '"overall": "red"' in result.stdout + assert "refused connection" in result.stdout + assert "occ-autobind" in result.stdout + + +def test_health_endpoint_non_200_is_red( + tmp_path: Path, + open_broker_port: int, + health_server: tuple[int, type[_HealthHandler]], +) -> None: + main_port, handler = health_server + handler.status_code = 503 + result = _run_lane_gate(tmp_path, LIVE_LANE_ROWS, open_broker_port, main_port) + + assert result.returncode == 1, result.stdout + assert "/health" in result.stdout + assert "503" in result.stdout + + +# --------------------------------------------------------------------------- +# Degradation that every "is it up?" probe misses. +# --------------------------------------------------------------------------- + + +def test_nonzero_exit_oneshot_is_red( + tmp_path: Path, + open_broker_port: int, + health_server: tuple[int, type[_HealthHandler]], +) -> None: + """The OMN-15312 class: lane serving, schema silently unapplied. + + This exact shape was live on `.201` while this test was written — + ``omnibase-infra-forward-migration`` Exited (3) with every runtime + container healthy and :8085/health returning 200. + """ + main_port, _ = health_server + rows = ( + LIVE_LANE_ROWS + + "\nomnibase-infra-forward-migration|exited|Exited (3) 15 minutes ago" + ) + result = _run_lane_gate(tmp_path, rows, open_broker_port, main_port) + + assert result.returncode == 1, result.stdout + assert "nonzero-exit" in result.stdout + assert "omnibase-infra-forward-migration(exit 3)" in result.stdout + + +def test_zero_exit_oneshot_is_not_red( + tmp_path: Path, + open_broker_port: int, + health_server: tuple[int, type[_HealthHandler]], +) -> None: + """Discriminator for the test above: exited != failed. + + Migration/provisioner one-shots are SUPPOSED to exit. A check that + flagged every exited container would be red on every healthy lane. + """ + main_port, _ = health_server + rows = ( + LIVE_LANE_ROWS + + "\nomnibase-infra-redpanda-partition-cap|exited|Exited (0) 15 minutes ago" + ) + result = _run_lane_gate(tmp_path, rows, open_broker_port, main_port) + + assert result.returncode == 0, result.stdout + assert '"overall": "green"' in result.stdout + + +def test_restarting_container_is_red( + tmp_path: Path, + open_broker_port: int, + health_server: tuple[int, type[_HealthHandler]], +) -> None: + """A crash-looping lane service is a defect, not a transient.""" + main_port, _ = health_server + rows = LIVE_LANE_ROWS + "\nomnibase-infra-migration-gate|restarting|Restarting (1)" + result = _run_lane_gate(tmp_path, rows, open_broker_port, main_port) + + assert result.returncode == 1, result.stdout + assert "not-running" in result.stdout + assert "omnibase-infra-migration-gate(restarting)" in result.stdout + + +def test_docker_unhealthy_is_yellow_not_red( + tmp_path: Path, + open_broker_port: int, + health_server: tuple[int, type[_HealthHandler]], +) -> None: + """Docker health is a secondary signal, never the verdict. + + A running-but-unhealthy sidecar does not strand the CI publish path, and + the fleet has twice proven Docker health alone inverts (OMN-13915: + 37/48 "Up (healthy)" with dead listeners; OMN-15233: 59/64 unhealthy + while the registry read 64/64 online). + """ + main_port, _ = health_server + rows = LIVE_LANE_ROWS.replace( + "omninode-runtime-effects|running|Up 2 hours (healthy)", + "omninode-runtime-effects|running|Up 2 hours (unhealthy)", + ) + result = _run_lane_gate(tmp_path, rows, open_broker_port, main_port) + + assert result.returncode == 0, result.stdout + assert '"overall": "yellow"' in result.stdout + assert "docker-unhealthy" in result.stdout + + +def test_docker_daemon_unreachable_is_red_fail_closed( + tmp_path: Path, + open_broker_port: int, + health_server: tuple[int, type[_HealthHandler]], +) -> None: + """Indeterminate is not health — the inversion OMN-13915 shipped with.""" + main_port, _ = health_server + result = _run_lane_gate( + tmp_path, LIVE_LANE_ROWS, open_broker_port, main_port, daemon_ok=False + ) + + assert result.returncode == 1, result.stdout + assert "fail-closed" in result.stdout + + +def test_missing_runtime_containers_are_red_under_keepalive( + tmp_path: Path, + open_broker_port: int, + health_server: tuple[int, type[_HealthHandler]], +) -> None: + """The severity inversion this ticket fixes, on the second surface. + + ``runtime_containers`` scored yellow ("runtime profile not active") when + the lane had no runtime services at all — and yellow exits 0. + """ + main_port, _ = health_server + result = _run_lane_gate( + tmp_path, + LIVE_LANE_ROWS, + open_broker_port, + main_port, + running_names="omnibase-infra-redpanda", + ) + + assert result.returncode == 1, result.stdout + assert "KEEP-ALIVE" in result.stdout + + +def test_missing_runtime_containers_are_advisory_without_keepalive( + tmp_path: Path, + open_broker_port: int, + health_server: tuple[int, type[_HealthHandler]], +) -> None: + """Negative control for the severity flip above.""" + main_port, _ = health_server + result = _run_lane_gate( + tmp_path, + LIVE_LANE_ROWS, + open_broker_port, + main_port, + keepalive="0", + running_names="omnibase-infra-redpanda", + ) + + assert result.returncode == 0, result.stdout + assert "runtime profile not active" in result.stdout + + +# --------------------------------------------------------------------------- +# Enforcement wiring: detection that does not fire is not a control. +# --------------------------------------------------------------------------- + + +def test_lane_probe_is_wired_to_a_firing_surface() -> None: + workflow = yaml.safe_load(LANE_WORKFLOW.read_text()) + jobs = workflow["jobs"] + + assert "dev-lane-liveness" in jobs, ( + "the lane probe must be wired to a firing surface — an unwired script " + "is exactly the OMN-15190 pre-state" + ) + lane_job = jobs["dev-lane-liveness"] + probe_step = next( + step + for step in lane_job["steps"] + if "system_health_check.sh" in step.get("run", "") + ) + + assert "--lane" in probe_step["run"] + assert probe_step["env"]["LANE_PROBE_HOST"] == "host.docker.internal", ( + "inside the deploy-runner container localhost is the container itself; " + "an unset probe host manufactures a false RED (OMN-14958)" + ) + assert str(probe_step["env"]["ONEX_LANE_KEEPALIVE"]) == "1" + + # `on` parses as the boolean True in YAML 1.1 unless quoted. + triggers = workflow.get("on", workflow.get(True)) + assert "schedule" in triggers, "the probe must run on a schedule, not on demand" + + +def test_lane_probe_runs_where_the_lane_is_reachable() -> None: + """The lane job is self-hosted BY DESIGN, and only on the deploy runner. + + `omnibase-deploy` is the one runner carrying both docker.sock and the + host-gateway alias, so it is the only place either half of the probe + (compose-project membership, lane host-port reachability) is observable. + The lane's broker host-port is on the tailnet — GitHub-hosted compute + could only ever assert "I cannot see it." + """ + workflow = yaml.safe_load(LANE_WORKFLOW.read_text()) + assert workflow["jobs"]["dev-lane-liveness"]["runs-on"] == [ + "self-hosted", + "omnibase-deploy", + ] + + +def test_fleet_canary_is_not_borrowed_for_the_lane_probe() -> None: + """The OMN-13915 fate boundary stays intact and unshared. + + The lane probe deliberately does NOT live in runner-fleet-canary.yml: that + workflow asserts every one of its jobs is GitHub-hosted, because a canary + sharing fate with the fleet it watches proves nothing. Folding a + self-hosted job in would have required narrowing that guard. This pins the + separation so a later consolidation cannot silently weaken it. + """ + canary = yaml.safe_load(CANARY_WORKFLOW.read_text()) + assert list(canary["jobs"]) == ["fleet-status"], ( + "runner-fleet-canary.yml must stay single-job and GitHub-hosted; put " + "lane/host-scoped probes in dev-lane-liveness.yml instead" + ) + assert canary["jobs"]["fleet-status"]["runs-on"] == "ubuntu-latest" diff --git a/tests/ci/test_env_parity.py b/tests/ci/test_env_parity.py index 0175e861f2..2eb219faba 100644 --- a/tests/ci/test_env_parity.py +++ b/tests/ci/test_env_parity.py @@ -1,20 +1,56 @@ # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT -"""Env parity test: every x-runtime-env key has a k8s ConfigMap or Secret entry. +"""Env parity test: docker-compose and the onex-dev k8s manifests agree, BOTH ways. -Ensures that every variable in the docker-compose x-runtime-env anchor is -accounted for in the k8s ConfigMap, a known Secret, or the LOCAL_ONLY_KEYS -allowlist. This prevents environment divergence between local docker-compose -and the k8s cluster. +Forward direction (OMN-4307): every variable in the docker-compose +``x-runtime-env`` anchor is accounted for in the k8s ConfigMap, a known Secret, +or the LOCAL_ONLY_KEYS allowlist. + +Reverse direction (OMN-15628): every configuration key the k8s manifests bind — +ConfigMap ``data`` keys plus literal ``value:`` entries in the runtime +Deployments — is bound somewhere in ``docker-compose.infra.yml``, or is +explicitly classified as cluster-only / tracked parity debt. + +Scope of each direction (read this before extending either — they deliberately +use DIFFERENT k8s surfaces, and the reverse walk is deliberately single-file): + +* FORWARD is scoped to the compose ``x-runtime-env`` anchor, which compose + merges into all three runtime-family services at once. Its k8s counterpart is + therefore the RUNTIME-FAMILY surface: ConfigMap keys (every runtime Deployment + ``envFrom``-s ``onex-runtime-config``) plus keys bound inline on ALL THREE + runtime Deployments. A key bound inline on only one workload does NOT satisfy + it — see ``test_k8s_family_surface_requires_all_runtime_deployments``. +* REVERSE is scoped to ``docker-compose.infra.yml`` plus the typed service + manifests under ``docker/catalog/services/`` (rendered by the catalog CLI into + ``docker/docker-compose.generated.yml``, which is not committed — see + ``extract_catalog_bound_keys``), and asks the weaker question "does this key + reach ANY compose container at all?", so it uses the + UNION of every k8s workload's bindings. ``infra.yml`` is the base file that + ``resolve_compose_file_args`` layers first for every deployed lane, so a key + bound there reaches all of them. The standalone lanes (``judge``, ``e2e``) + are NOT reverse-walked — they run deliberately narrower service sets, so a + full reverse walk against them would report design, not drift. They are + covered for the OMN-15628 seam specifically, by value, in + ``test_delegation_routing_tiers_path_matches_k8s_pin``. + +The reverse direction exists because the forward-only gate was structurally +blind to the failure that shipped in OMN-15628: ``DELEGATION_ROUTING_TIERS_PATH`` +was bound on all three onex-dev runtime Deployments and on ZERO compose files, +so every local/lab/stability/prod container booted healthy and then fail-closed +on the first delegation-routing request (the omnimarket routing reducer resolves +it through ``resolve_required_path_config``, which raises rather than defaulting). +A compose→k8s-only walk can never see a key that is missing on the compose side. Run with: uv run pytest tests/ci/test_env_parity.py -Ticket: OMN-4307 +Tickets: OMN-4307 (forward), OMN-15628 (reverse) """ from __future__ import annotations +import ast +import fnmatch import os import re from pathlib import Path @@ -30,6 +66,11 @@ COMPOSE_PATH = _REPO_ROOT / "docker" / "docker-compose.infra.yml" +# Typed service manifests the catalog CLI renders into +# docker/docker-compose.generated.yml. Second compose-binding surface for the +# reverse walk — see extract_catalog_bound_keys. +CATALOG_SERVICES_DIR = _REPO_ROOT / "docker" / "catalog" / "services" + # CONFIGMAP_PATH: omninode_infra may live as a sibling in several layouts: # 1. Local worktrees: /Volumes/.../omni_worktrees//omnibase_infra/ → # sibling at ../omninode_infra/ @@ -37,33 +78,77 @@ # sibling at ../omninode_infra/ # 3. CI with dual checkout: both repos checked out side-by-side # 4. OMNINODE_INFRA_DIR env var override -_CONFIGMAP_SUBPATH = "k8s/onex-dev/runtime/configmap.yaml" +_K8S_RUNTIME_SUBPATH = "k8s/onex-dev/runtime" +_CONFIGMAP_SUBPATH = f"{_K8S_RUNTIME_SUBPATH}/configmap.yaml" + +def _resolve_k8s_runtime_dir() -> Path | None: + """Resolve the onex-dev k8s runtime manifest directory in omninode_infra. -def _resolve_configmap_path() -> Path | None: - """Resolve the ConfigMap path, trying multiple candidate locations.""" + Same candidate chain the ConfigMap resolution has always used; hoisted to + the directory so the reverse-direction check (OMN-15628) can read the + Deployment manifests alongside the ConfigMap from one resolved root. + """ # Env var override takes precedence override = os.environ.get("OMNINODE_INFRA_DIR", "").strip() if override: - candidate = Path(override) / _CONFIGMAP_SUBPATH - if candidate.exists(): + candidate = Path(override) / _K8S_RUNTIME_SUBPATH + if (candidate / "configmap.yaml").exists(): return candidate # Try sibling directories relative to the repo root candidates: list[Path] = [ - # Direct sibling (local worktree or CI dual-checkout) - _REPO_ROOT.parent / "omninode_infra" / _CONFIGMAP_SUBPATH, - # Two levels up (omni_home monorepo layout: omni_home/omnibase_infra) - _REPO_ROOT.parent.parent / "omninode_infra" / _CONFIGMAP_SUBPATH, + # Direct sibling (CI dual-checkout, or a ticket dir holding both repos) + _REPO_ROOT.parent / "omninode_infra" / _K8S_RUNTIME_SUBPATH, + # Two levels up (omni_home registry layout: omni_home/omnibase_infra) + _REPO_ROOT.parent.parent / "omninode_infra" / _K8S_RUNTIME_SUBPATH, + # Three levels up: the standard worktree layout + # omni_home/omni_worktrees//omnibase_infra, whose canonical + # omninode_infra clone sits at the omni_home root. Without this the + # gate SKIPS on every local worktree run (including the pre-push hook), + # which is a vacuous green — the pre-push run for OMN-15628 skipped all + # three parity assertions for exactly this reason. + _REPO_ROOT.parent.parent.parent / "omninode_infra" / _K8S_RUNTIME_SUBPATH, ] for candidate in candidates: - if candidate.exists(): + if (candidate / "configmap.yaml").exists(): return candidate return None -CONFIGMAP_PATH = _resolve_configmap_path() +K8S_RUNTIME_DIR = _resolve_k8s_runtime_dir() +CONFIGMAP_PATH = ( + (K8S_RUNTIME_DIR / "configmap.yaml") if K8S_RUNTIME_DIR is not None else None +) + +# Runtime-family Deployments: the k8s workloads whose compose counterparts merge +# the shared ``x-runtime-env`` anchor. These are the manifests that bind +# DELEGATION_ROUTING_TIERS_PATH. +K8S_RUNTIME_FAMILY_DEPLOYMENTS: tuple[str, ...] = ( + "deployment-omninode-runtime.yaml", + "deployment-omninode-runtime-effects.yaml", + "deployment-omninode-runtime-worker.yaml", +) + +# Compose services that merge the ``x-runtime-env`` anchor and correspond +# one-to-one with K8S_RUNTIME_FAMILY_DEPLOYMENTS. +COMPOSE_RUNTIME_FAMILY_SERVICES: tuple[str, ...] = ( + "omninode-runtime", + "runtime-effects", + "runtime-worker", +) + +# Every compose file that stands up a delegation-runtime container, mapped to +# the services in it that must carry the k8s-pinned routing-tiers VALUE (not +# merely the key). ``infra`` is the dev/lab base every deployed lane layers; +# ``judge`` and ``e2e`` are standalone and inherit nothing, so a typo in either +# would otherwise be invisible to a presence-only check. +COMPOSE_RUNTIME_SERVICES_BY_FILE: dict[str, tuple[str, ...]] = { + "docker-compose.infra.yml": COMPOSE_RUNTIME_FAMILY_SERVICES, + "docker-compose.judge.yml": ("omninode-runtime", "runtime-effects"), + "docker-compose.e2e.yml": ("runtime",), +} # --------------------------------------------------------------------------- # Key classification @@ -97,6 +182,8 @@ def _resolve_configmap_path() -> Path | None: # GitHub API/CLI auth — injected via Infisical at runtime "GITHUB_TOKEN", "GH_TOKEN", + # Deploy control-plane command authentication — injected via Infisical at runtime + "DEPLOY_AGENT_HMAC_SECRET", # Qdrant vector store API key — credential, sourced from Infisical "QDRANT_API_KEY", # Cloud-tier LLM route secret ref — credential, sourced from Infisical/k8s secret. @@ -141,6 +228,18 @@ def _resolve_configmap_path() -> Path | None: "LOCAL_LLM_SHARED_SECRET", # Topic provisioner partition cap — local-only tuning knob; k8s does not set it "ONEX_TOPIC_PROVISIONER_MAX_PARTITIONS", + # OMN-15529 / OMN-15362: OnexBot-OCC-Writer App identity for the OCC + # companion producer. Deliberately NOT propagated to k8s today — the + # operator mints the App private key onto the .201 bus runtime only + # (ruling 2026-07-30), and the onex-dev cluster does not run the OCC + # companion producer. Classified local-only rather than SECRET_KEYS on + # purpose: SECRET_KEYS asserts "k8s injects this via Secret/Infisical", + # which would be a false claim here. If the cluster ever runs this + # producer, ONEXBOT_OCC_* move to SECRET_KEYS and + # OMNI_OCC_GITHUB_AUTH_MODE moves to the ConfigMap. + "ONEXBOT_OCC_APP_ID", + "ONEXBOT_OCC_PRIVATE_KEY", + "OMNI_OCC_GITHUB_AUTH_MODE", } ) @@ -188,6 +287,255 @@ def _resolve_configmap_path() -> Path | None: ) +# --------------------------------------------------------------------------- +# Reverse-direction classification (k8s -> compose), OMN-15628 +# --------------------------------------------------------------------------- + +# Keys the k8s manifests bind that describe CLUSTER topology or a managed data +# plane, and therefore have no docker-compose counterpart by construction. +# Every entry below is justified by its live ConfigMap value. +K8S_ONLY_KEYS: frozenset[str] = frozenset( + { + # Managed bus (MSK + IAM auth). Local/lab lanes run a PLAINTEXT Redpanda + # broker, so none of these have a compose analogue. + # KAFKA_RUNTIME_TARGET=msk, KAFKA_SECURITY_PROTOCOL=SASL_SSL, + # KAFKA_SASL_MECHANISM=AWS_MSK_IAM, KAFKA_MSK_REGION=us-east-1 + "KAFKA_RUNTIME_TARGET", + "KAFKA_SECURITY_PROTOCOL", + "KAFKA_SASL_MECHANISM", + "KAFKA_JAVA_SASL_MECHANISM", + "KAFKA_NON_JAVA_SASL_MECHANISM", + "KAFKA_MSK_REGION", + # Split DSN form. k8s composes the infra DSN from host + port + a + # Secret-sourced password (see the OMNIBASE_INFRA_DB_URL note in + # SECRET_KEYS); compose binds the whole DSN in one variable. + "OMNIBASE_INFRA_DB_HOST", + "OMNIBASE_INFRA_DB_PORT", + "POSTGRES_SSLMODE", # 'require' — managed RDS; local postgres is in-network + # Cluster-DNS service addresses (*.svc.cluster.local). Compose reaches + # the same services by compose-network service name / HOST+PORT pairs. + "CONTRACT_RESOLVER_URL", + "INTELLIGENCE_API_URL", + "KREUZBERG_URL", + "QDRANT_URL", + # Cluster Infisical bootstrap toggle; the compose lanes gate Infisical on + # INFISICAL_ADDR being set instead (see config_discovery docs). + "INFISICAL_REQUIRED", + # OMN-15750 gateway-attach ingress (omninode_infra#886). Same cluster-DNS + # category as the block above — both values are *.svc.cluster.local: + # GATEWAY_ATTACH_KEYCLOAK_INTROSPECTION_URL= + # http://keycloak.auth.svc.cluster.local/realms/omninode/protocol/openid-connect/token/introspect + # GATEWAY_ATTACH_KEYCLOAK_JWKS_URL= + # http://keycloak.auth.svc.cluster.local/realms/omninode/protocol/openid-connect/certs + # They address the Keycloak in the cluster's OWN `auth` namespace, which + # has no compose analogue: docker-compose.infra.yml's `keycloak` service + # is a dev bootstrap on the compose network (KEYCLOAK_ADMIN_URL= + # http://keycloak:8080), seeded from docker/keycloak/omninode-realm.json. + # + # Binding these in compose would ALSO be wrong on two independent counts, + # so this is not debt deferred for convenience: + # 1. No compose lane resolves these refs. node_gateway_attach_effect + # reads them by LOGICAL ref (contract.yaml keycloak_introspection_ref + # = "gateway.attach.keycloak.introspection", keycloak_jwks_ref = + # "gateway.attach.keycloak.jwks") through the secret resolver. The + # k8s ConfigMap's ONEX_SECRET_RESOLVER_CONFIG_JSON maps those two + # logical names onto these env vars; the compose dev/lab/stability/ + # prod resolver config (docker/runtime-policy.env + # DEV_RUNTIME_*_SECRET_RESOLVER_CONFIG_JSON) declares only llm.* and + # slack.bot_token — no gateway.attach.* mapping exists, so no compose + # container ever asks for either key. + # 2. The gateway ingress is not in a deployed compose lane at all. + # resolve_compose_file_args (scripts/deploy-runtime.sh) layers + # docker-compose.infra.yml + one lane overlay; the gateway services + # live in docker-compose.gateway.yml / .gateway-attach-test-lane.yml, + # which no lane layers. And OMN-15750's own acceptance criteria + # forbid the binding outright: "No broker/Keycloak URL literal in + # source, docker-compose, or env — resolved from contract ref at the + # effect boundary." + "GATEWAY_ATTACH_KEYCLOAK_INTROSPECTION_URL", + "GATEWAY_ATTACH_KEYCLOAK_JWKS_URL", + # k8s readiness/liveness probe plumbing for the three standalone + # omnimarket projection-writer Deployments (OMN-15905). Values are 8093 + # (live-events), 8094 (registration), 8095 (delegation) — each is the + # port that Deployment's OWN readinessProbe (httpGet /ready) and + # livenessProbe (tcpSocket) target, so the binding exists only to serve + # the kubelet. The compose counterparts of those three workloads are + # catalog services (docker/catalog/services/omnimarket-projection-*.yaml) + # which declare `healthcheck: null` and `ports: null` — there is no probe + # to answer, so BaseProjectionRunner's opt-in health server stays off and + # the key has no compose counterpart by construction. + "PROJECTION_RUNNER_HEALTH_PORT", + } +) + +# Keys bound in k8s but NOT bound in docker-compose. Each entry here is TECH +# DEBT: the compose lanes run without a setting the cluster considers part of +# its runtime configuration, so the two surfaces are provably not equivalent. +# Tracked in: OMN-4307 (parity backlog). +# NOTE: Do NOT add new keys here — bind the key in docker-compose instead, or +# classify it in K8S_ONLY_KEYS with a value-backed justification. +COMPOSE_PARITY_DEBT_KEYS: frozenset[str] = frozenset( + { + # Runtime feature flags set cluster-side only. + "ENABLE_REAL_TIME_EVENTS", + "KAFKA_ENABLE_INTELLIGENCE", + "ONEX_BOOT_UNIVERSE_PROVISION", + # runtime-worker push-validation scratch root (a cluster mount path); + # the compose worker has no equivalent binding today. + "ONEX_PUSH_VALIDATION_WORKROOT", + # skill-lifecycle consumer knobs bound inline on the k8s Deployment but + # left to in-code defaults in compose. + "OMNIBASE_INFRA_SKILL_LIFECYCLE_DLQ_TOPIC", + "OMNIBASE_INFRA_SKILL_LIFECYCLE_SCHEMA_VERSION", + "OMNIBASE_INFRA_SKILL_LIFECYCLE_HEALTH_CHECK_STALENESS_SECONDS", + } +) + + +# --------------------------------------------------------------------------- +# Flags this repo has deleted (OMN-15659) +# --------------------------------------------------------------------------- +# OMN-8779 / OMN-8780 deleted a set of feature flags from omnibase_infra because +# they defaulted to false and were therefore silent non-enforcement gates. +# ``tests/audit/test_no_dead_delegation_flags.py`` keeps them deleted by +# rejecting their names anywhere in this tree. +# +# That puts the reverse walk in a genuine deadlock for any such flag the cluster +# still binds: EVERY remedy the reverse walk prescribes -- bind it in compose, +# list it in K8S_ONLY_KEYS, list it in COMPOSE_PARITY_DEBT_KEYS -- requires +# writing the rejected name. Both gates cannot be satisfied at once. That is the +# defect OMN-15659 fixes: OMN-15628 classified one of these flags as compose +# parity debt, the audit rejected the classification, and dev went red. +# +# Resolution: a flag THIS repo deleted is out of scope for compose parity. Its +# absence from the compose lanes is the intended end state, not a gap. The +# shared onex-dev ConfigMap may still bind it for another owner -- the canonical +# feature-flag registry in ``omnibase_core.feature_flags.registry`` assigns each +# flag an ``owning_repo`` -- and a binding owned by another repo is not +# omnibase_infra parity debt. +# +# The set is READ FROM THE AUDIT rather than restated, so the two gates cannot +# drift apart and so this module never names a rejected flag. Excluding these +# keys does not create a blind spot: the audit itself scans every ``*.yml`` / +# ``*.yaml`` in this tree, so a compose lane that re-bound one of them would +# fail that audit directly. +# +# Fail-closed: a missing file, a renamed symbol, or an empty list raises at +# import rather than silently yielding an empty exclusion set. +_DEAD_FLAG_AUDIT_PATH = ( + _REPO_ROOT / "tests" / "audit" / "test_no_dead_delegation_flags.py" +) +_DEAD_FLAG_AUDIT_SYMBOL = "_DEAD_FLAGS" + + +def _load_repo_deleted_flags() -> frozenset[str]: + """Return the flag names the dead-flag audit rejects from this tree.""" + try: + source = _DEAD_FLAG_AUDIT_PATH.read_text() + except OSError as exc: # pragma: no cover - fail closed + raise RuntimeError( + f"cannot read the dead-flag audit at {_DEAD_FLAG_AUDIT_PATH}: {exc}. " + "The reverse parity walk cannot resolve which flags this repo deleted." + ) from exc + + for node in ast.parse(source).body: + if not isinstance(node, ast.Assign): + continue + if not any( + isinstance(target, ast.Name) and target.id == _DEAD_FLAG_AUDIT_SYMBOL + for target in node.targets + ): + continue + names = frozenset(str(name) for name in ast.literal_eval(node.value)) + if not names: + raise RuntimeError( + f"{_DEAD_FLAG_AUDIT_PATH}: {_DEAD_FLAG_AUDIT_SYMBOL} is empty" + ) + return names + + raise RuntimeError( + f"{_DEAD_FLAG_AUDIT_SYMBOL} not found in {_DEAD_FLAG_AUDIT_PATH}. " + "The reverse parity walk cannot resolve which flags this repo deleted." + ) + + +REPO_DELETED_FLAG_KEYS: frozenset[str] = _load_repo_deleted_flags() + + +# --------------------------------------------------------------------------- +# Compose YAML loading +# --------------------------------------------------------------------------- +# The lane overlays (prod / stability-test / judge) use the Compose merge +# directives ``!override`` and ``!reset``, which plain ``yaml.safe_load`` refuses +# with ConstructorError. Preserve the tag instead of dropping it: whether a +# service's ``environment`` mapping carries one of those tags is exactly the +# fact the lane-coverage test needs to assert, so it must survive parsing. + + +class ComposeTagged: + """A YAML node that carried a Compose merge directive (``!override`` / ``!reset``).""" + + __slots__ = ("tag", "value") + + def __init__(self, tag: str, value: object) -> None: + self.tag = tag + self.value = value + + def __repr__(self) -> str: # pragma: no cover - debug aid only + return f"ComposeTagged({self.tag!r}, {self.value!r})" + + +class _ComposeLoader(yaml.SafeLoader): + """SafeLoader that keeps unknown tags as :class:`ComposeTagged` wrappers. + + Deliberately NOT shared with the same-named loader in + ``tests/unit/infra/test_judge_compose_profile.py``: that one UNWRAPS compose + merge tags to their bare value, which is the opposite of what this module + needs. Whether ``environment`` carries ``!override`` / ``!reset`` is the + fact ``test_every_runtime_compose_lane_binds_delegation_routing_tiers_path`` + asserts on, so the tag has to survive parsing here. + """ + + +# The complete set of Compose merge directives. Anything else stays a hard +# ConstructorError rather than being silently wrapped — a tag this module does +# not understand should fail loudly, not read as "no directive present". +COMPOSE_MERGE_DIRECTIVES: tuple[str, ...] = ("!override", "!reset") + + +def _construct_tagged(loader: yaml.SafeLoader, node: yaml.Node) -> ComposeTagged: + if isinstance(node, yaml.MappingNode): + value: object = loader.construct_mapping(node, deep=True) + elif isinstance(node, yaml.SequenceNode): + value = loader.construct_sequence(node, deep=True) + elif isinstance(node, yaml.ScalarNode): + value = loader.construct_scalar(node) + else: # pragma: no cover - PyYAML emits no other node kinds + raise TypeError(f"unsupported node type for {node.tag}: {type(node).__name__}") + return ComposeTagged(node.tag, value) + + +for _directive in COMPOSE_MERGE_DIRECTIVES: + _ComposeLoader.add_constructor(_directive, _construct_tagged) + + +def load_compose(compose_path: Path) -> dict[str, object]: + """Parse a compose file, tolerating ``!override`` / ``!reset`` directives.""" + # _ComposeLoader extends SafeLoader; the multi-constructor only wraps + # unknown tags in a ComposeTagged record and never instantiates arbitrary + # objects, so S506's arbitrary-deserialization concern does not apply. Same + # justification and same suppression the four sibling compose-parsing tests + # under tests/unit/infra/ already carry. + document = yaml.load(compose_path.read_text(), Loader=_ComposeLoader) # noqa: S506 + return document if isinstance(document, dict) else {} + + +def compose_services(compose_path: Path) -> dict[str, object]: + """Return the ``services`` mapping of a compose file (empty if absent).""" + services = load_compose(compose_path).get("services") + return services if isinstance(services, dict) else {} + + # --------------------------------------------------------------------------- # Extraction helpers # --------------------------------------------------------------------------- @@ -215,6 +563,242 @@ def extract_configmap_keys(configmap_path: Path) -> set[str]: return set(data.get("data", {}).keys()) +def service_environment(service: object) -> object: + """Return a compose service's raw ``environment`` node (tag preserved).""" + if not isinstance(service, dict): + return None + return service.get("environment") + + +def _service_env_keys(service: object) -> set[str]: + """Return the env keys a single compose service declares.""" + env = service_environment(service) + if isinstance(env, ComposeTagged): + env = env.value + if isinstance(env, dict): + return {str(k) for k in env} + if isinstance(env, list): + # ``- KEY=value`` / ``- KEY`` list form + return {str(item).split("=", 1)[0] for item in env} + return set() + + +def extract_compose_bound_keys(compose_path: Path) -> set[str]: + """Extract every env key bound by ANY service in a compose file. + + The forward check walks only the ``x-runtime-env`` anchor because that is + the shared surface it governs. The reverse check must ask a broader + question — "does this key reach a container at all?" — so it takes the union + over every service's resolved ``environment`` mapping (YAML merge keys and + anchors are resolved at parse time, so anchor-merged services contribute + the anchor's keys). + """ + keys: set[str] = set() + for service in compose_services(compose_path).values(): + keys |= _service_env_keys(service) + return keys + + +def extract_catalog_bound_keys(catalog_services_dir: Path) -> set[str]: + """Extract every env key the service catalog binds on a generated container. + + The reverse walk's question is "does this key reach ANY compose container at + all?", and ``docker-compose.infra.yml`` is not the whole answer. Services in + ``docker/catalog/services/*.yaml`` are compose services too — the catalog CLI + (``src/omnibase_infra/docker/catalog/cli.py``) renders them into + ``docker/docker-compose.generated.yml``, which is how ``onex up `` + starts them. That generated file is NOT committed, so the manifests are the + tracked surface and the only one CI can read. + + Without this term the walk reports a false positive for any key bound on a + workload that exists in the catalog but not in ``infra.yml``. The three + standalone omnimarket projection writers are exactly that shape: k8s runs + them as their own Deployments, compose runs them from the + ``omnimarket-projections`` bundle, and ``infra.yml`` has no projection-writer + service at all (only ``projection-api``). ``KAFKA_CONSUMER_GROUP`` was + reported as drift on that basis while + ``docker/catalog/services/omnimarket-projection-delegation.yaml`` had bound it + the whole time — and NEITHER classification bucket could honestly absorb it: + K8S_ONLY_KEYS asserts "no docker-compose counterpart by construction" and + COMPOSE_PARITY_DEBT_KEYS asserts "the compose lanes run without this + setting", both false claims here. An incomplete surface has to be fixed at + the surface, not papered over with a classification. + + The key set mirrors ``generator.py``'s ``environment`` assembly exactly — + ``hardcoded_env`` | ``operational_defaults`` | ``catalog_env`` | + ``required_env`` — so a manifest field that reaches a container is visible + here, and one that does not is not. + """ + keys: set[str] = set() + for manifest_path in sorted(catalog_services_dir.glob("*.yaml")): + manifest = yaml.safe_load(manifest_path.read_text()) + if not isinstance(manifest, dict): + continue + for field in ("hardcoded_env", "operational_defaults", "catalog_env"): + mapping = manifest.get(field) + if isinstance(mapping, dict): + keys |= {str(k) for k in mapping} + required = manifest.get("required_env") + if isinstance(required, list): + keys |= {str(item) for item in required} + return keys + + +def extract_k8s_bound_keys(runtime_dir: Path) -> dict[str, set[str]]: + """Map every k8s-bound configuration key to the manifests that bind it. + + Sources: + * ``configmap.yaml`` ``data:`` keys + * ``deployment-*.yaml`` container ``env:`` entries that carry a literal + ``value:`` + + ``valueFrom`` entries (``secretKeyRef``/``fieldRef``) are deliberately out of + scope: those are credentials/downward-API values whose compose-side + counterpart is the host-env + Infisical surface already governed by + SECRET_KEYS in the forward direction, not a compose literal. + """ + bound: dict[str, set[str]] = {} + + configmap = runtime_dir / "configmap.yaml" + for key in extract_configmap_keys(configmap): + bound.setdefault(key, set()).add(configmap.name) + + for manifest in sorted(runtime_dir.glob("deployment-*.yaml")): + document = yaml.safe_load(manifest.read_text()) + containers = ( + (document or {}) + .get("spec", {}) + .get("template", {}) + .get("spec", {}) + .get("containers", []) + ) + for container in containers: + for entry in container.get("env", []) or []: + if "value" in entry: + bound.setdefault(str(entry["name"]), set()).add(manifest.name) + + return bound + + +def extract_k8s_runtime_family_bound_keys(runtime_dir: Path) -> set[str]: + """Keys bound for EVERY runtime-family workload, not merely somewhere in k8s. + + This is the correct counterpart for the FORWARD direction. The compose + ``x-runtime-env`` anchor is merged into all three runtime services at once, + so "this anchor key exists in k8s" is only true if every runtime workload + actually receives it. Two sources qualify: + + * ConfigMap ``data`` keys — every runtime Deployment ``envFrom``-s + ``onex-runtime-config``, so a ConfigMap key reaches all of them. + * Keys bound inline with a literal ``value:`` on ALL of + :data:`K8S_RUNTIME_FAMILY_DEPLOYMENTS`. + + A key bound inline on only SOME runtime Deployments (today: + ``OMNIINTELLIGENCE_PUBLISH_INTROSPECTION``, ``ONEX_PUSH_VALIDATION_WORKROOT``) + is deliberately excluded — treating it as satisfied would let a per-workload + binding stand in for an anchor-wide one, which is granularity the union + surface used by the reverse walk cannot express. + """ + bound = extract_k8s_bound_keys(runtime_dir) + per_manifest = [ + {k for k, sources in bound.items() if manifest in sources} + for manifest in K8S_RUNTIME_FAMILY_DEPLOYMENTS + ] + inline_on_every_runtime_workload: set[str] = ( + set.intersection(*per_manifest) if per_manifest else set() + ) + return ( + extract_configmap_keys(runtime_dir / "configmap.yaml") + | inline_on_every_runtime_workload + ) + + +def extract_dockerfile_baked_aliases(dockerfile_path: Path) -> dict[str, str]: + """Map each in-image path that ``Dockerfile.runtime`` COPYs FROM -> the path it copies TO. + + ``docker/Dockerfile.runtime`` bakes the packaged ``routing_tiers.yaml`` out + of the installed venv into a stable, interpreter-version-free location + (OMN-15645). The compose lanes therefore legitimately pin a DIFFERENT + literal than the onex-dev k8s Deployments, which pin the venv source path — + both name the same file content. + + Parsed from the Dockerfile rather than hardcoded as a second literal: a + hardcoded alias table would drift from the COPY the moment either path + moved, which is the exact failure the value lock exists to prevent. + """ + if not dockerfile_path.exists(): + return {} + + aliases: dict[str, str] = {} + # ``COPY --from=... `` possibly spread over backslash continuations. + text = re.sub(r"\\\s*\n\s*", " ", dockerfile_path.read_text()) + for line in text.splitlines(): + stripped = line.strip() + if not stripped.upper().startswith("COPY "): + continue + operands = [ + token + for token in stripped.split()[1:] + if not token.startswith("--") and token.upper() != "COPY" + ] + if len(operands) != 2: + continue + source, destination = operands + if source.startswith("/") and destination.startswith("/"): + aliases[source] = destination + return aliases + + +def resolve_baked_aliases(value: str, aliases: dict[str, str]) -> set[str]: + """Return ``value`` plus every in-image path Dockerfile.runtime bakes it to. + + Glob segments in the COPY source (``python*``) are matched with + :meth:`Path.match`-style semantics via :func:`fnmatch.fnmatch`, so the + interpreter-minor wildcard in the Dockerfile lines up with the concrete + ``python3.12`` literal the k8s manifests pin. + """ + equivalent = {value} + for source, destination in aliases.items(): + if source == value or fnmatch.fnmatch(value, source): + equivalent.add(destination) + return equivalent + + +def extract_k8s_env_value(manifest_path: Path, key: str) -> str | None: + """Return the literal ``value:`` a Deployment binds for ``key``, if any.""" + document = yaml.safe_load(manifest_path.read_text()) + containers = ( + (document or {}) + .get("spec", {}) + .get("template", {}) + .get("spec", {}) + .get("containers", []) + ) + for container in containers: + for entry in container.get("env", []) or []: + if entry.get("name") == key and "value" in entry: + return str(entry["value"]) + return None + + +def extract_compose_service_env_value( + compose_path: Path, service: str, key: str +) -> str | None: + """Return the resolved env value a compose service binds for ``key``.""" + env = service_environment(compose_services(compose_path).get(service)) + if isinstance(env, ComposeTagged): + env = env.value + if isinstance(env, dict): + value = env.get(key) + return None if value is None else str(value) + if isinstance(env, list): + for item in env: + name, _, value = str(item).partition("=") + if name == key: + return value + return None + + # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- @@ -222,22 +806,37 @@ def extract_configmap_keys(configmap_path: Path) -> set[str]: @pytest.mark.ci def test_runtime_env_keys_have_k8s_entries() -> None: - """Every x-runtime-env key is in ConfigMap, SECRET_KEYS, or LOCAL_ONLY_KEYS. + """Every x-runtime-env key is bound in k8s, SECRET_KEYS, or LOCAL_ONLY_KEYS. If this test fails, a key was added to docker-compose x-runtime-env without - a corresponding entry in the k8s ConfigMap (omninode_infra), and it is not + a corresponding binding in the k8s manifests (omninode_infra), and it is not registered in SECRET_KEYS (for Infisical/k8s Secret sources), LOCAL_ONLY_KEYS (for local-dev-only bootstrap variables), or CONFIGMAP_DEBT_KEYS (known gaps tracked as tech debt). + The k8s surface is the RUNTIME-FAMILY surface (OMN-15628): ConfigMap + ``data`` keys PLUS keys bound inline on ALL THREE runtime Deployments. It + was ConfigMap-only before, which made a key bound inline on a Deployment + (the placement used for DELEGATION_ROUTING_TIERS_PATH and + BIFROST_CONTRACT_PATH) read as absent even though the cluster sets it. + + Widening it to the plain union of every k8s workload's bindings would have + been the easy fix and is WRONG: ``x-runtime-env`` is merged into all three + runtime services at once, so a key bound inline on one workload only (e.g. + ``ONEX_PUSH_VALIDATION_WORKROOT`` on runtime-worker) must not satisfy an + anchor-wide claim. ``extract_k8s_runtime_family_bound_keys`` keeps that + granularity; ``test_k8s_family_surface_requires_all_runtime_deployments`` + locks it. + To fix a failure, choose one of: - 1. Add the key to the k8s ConfigMap (omninode_infra/k8s/onex-dev/runtime/configmap.yaml) + 1. Bind the key in k8s (omninode_infra/k8s/onex-dev/runtime/configmap.yaml, + or inline on the Deployments that need it) 2. Add the key to SECRET_KEYS in this file if it is sourced from a k8s Secret 3. Add the key to LOCAL_ONLY_KEYS if it is intentionally absent from k8s - 4. Add temporarily to CONFIGMAP_DEBT_KEYS if the ConfigMap update is blocked + 4. Add temporarily to CONFIGMAP_DEBT_KEYS if the k8s update is blocked (but you MUST file a ticket to resolve the debt) """ - if CONFIGMAP_PATH is None: + if K8S_RUNTIME_DIR is None: pytest.skip( "omninode_infra not found as a sibling — set OMNINODE_INFRA_DIR to run this test" ) @@ -248,13 +847,15 @@ def test_runtime_env_keys_have_k8s_entries() -> None: "Has the anchor been renamed or removed?" ) - configmap_keys = extract_configmap_keys(CONFIGMAP_PATH) - accounted_for = configmap_keys | SECRET_KEYS | LOCAL_ONLY_KEYS | CONFIGMAP_DEBT_KEYS + k8s_keys = extract_k8s_runtime_family_bound_keys(K8S_RUNTIME_DIR) + accounted_for = k8s_keys | SECRET_KEYS | LOCAL_ONLY_KEYS | CONFIGMAP_DEBT_KEYS missing = compose_keys - accounted_for assert not missing, ( - "Keys in x-runtime-env but missing from ConfigMap " - "(and not in SECRET_KEYS, LOCAL_ONLY_KEYS, or CONFIGMAP_DEBT_KEYS):\n" + "Keys in x-runtime-env but not bound for EVERY runtime workload in the " + "onex-dev k8s manifests (and not in SECRET_KEYS, LOCAL_ONLY_KEYS, or " + "CONFIGMAP_DEBT_KEYS). A key bound inline on only some runtime " + "Deployments does not count — x-runtime-env reaches all of them:\n" + "\n".join(f" {k}" for k in sorted(missing)) + "\n\nFix: add each missing key to one of:\n" " • omninode_infra/k8s/onex-dev/runtime/configmap.yaml (preferred)\n" @@ -264,6 +865,382 @@ def test_runtime_env_keys_have_k8s_entries() -> None: ) +@pytest.mark.ci +def test_k8s_bound_keys_are_bound_in_compose() -> None: + """Reverse parity (OMN-15628): every k8s-bound config key reaches compose. + + A key bound on the onex-dev Deployments/ConfigMap but bound in NO compose + service means the docker lanes (dev / lab / stability-test / prod on .201) + run without configuration the cluster treats as required. When the consumer + resolves that key fail-closed — as omnimarket's delegation routing reducer + does via ``resolve_required_path_config`` — the container still boots + healthy and only fails on the first request that touches the seam, so no + health check and no forward-only parity walk can detect it. + + To fix a failure, choose one of: + 1. Bind the key in docker/docker-compose.infra.yml (preferred — put it on + the ``x-runtime-env`` anchor if every runtime service needs it), or on + the owning docker/catalog/services/*.yaml manifest when the workload is + a catalog service rather than an infra.yml one + 2. Add it to K8S_ONLY_KEYS with a value-backed justification if it + describes cluster topology or a managed data plane + 3. Add it to COMPOSE_PARITY_DEBT_KEYS only if the compose binding is + genuinely blocked (and file/cite a ticket) + + Before reaching for 2 or 3, check whether the key is already bound on a + compose surface this walk does not read — a false positive is fixed at the + surface, never by a classification that states something untrue. + + Keys in REPO_DELETED_FLAG_KEYS are exempt and CANNOT be classified by any of + the three remedies above -- the dead-flag audit rejects their names anywhere + in this tree. See that constant for why (OMN-15659). + """ + if K8S_RUNTIME_DIR is None: + pytest.skip( + "omninode_infra not found as a sibling — set OMNINODE_INFRA_DIR to run this test" + ) + + k8s_bound = extract_k8s_bound_keys(K8S_RUNTIME_DIR) + assert k8s_bound, ( + f"No k8s-bound env keys extracted from {K8S_RUNTIME_DIR}. " + "Have the runtime manifests moved or been renamed?" + ) + + compose_keys = extract_compose_bound_keys(COMPOSE_PATH) + assert compose_keys, ( + f"No service env keys extracted from {COMPOSE_PATH}. " + "Has the services block been restructured?" + ) + + catalog_keys = extract_catalog_bound_keys(CATALOG_SERVICES_DIR) + assert catalog_keys, ( + f"No env keys extracted from the service catalog at {CATALOG_SERVICES_DIR}. " + "Have the manifests moved, or has the env field naming changed?" + ) + + accounted_for = ( + compose_keys + | catalog_keys + | K8S_ONLY_KEYS + | COMPOSE_PARITY_DEBT_KEYS + | REPO_DELETED_FLAG_KEYS + ) + missing = {k: v for k, v in k8s_bound.items() if k not in accounted_for} + + assert not missing, ( + "Keys bound in the onex-dev k8s manifests but bound in NO " + f"docker-compose service ({COMPOSE_PATH.name}) and no service-catalog " + f"manifest ({CATALOG_SERVICES_DIR.name}/), and not classified as " + "K8S_ONLY_KEYS or COMPOSE_PARITY_DEBT_KEYS:\n" + + "\n".join( + f" {k} (k8s source: {', '.join(sorted(missing[k]))})" + for k in sorted(missing) + ) + + "\n\nFix: bind each key in docker/docker-compose.infra.yml, or classify it in\n" + " • K8S_ONLY_KEYS in tests/ci/test_env_parity.py (cluster-only, justify with the value)\n" + " • COMPOSE_PARITY_DEBT_KEYS in tests/ci/test_env_parity.py (temp — must cite a ticket)" + ) + + +@pytest.mark.ci +def test_delegation_routing_tiers_path_matches_k8s_pin() -> None: + """OMN-15628 seam lock: the routing-tiers path agrees across both surfaces. + + ``DELEGATION_ROUTING_TIERS_PATH`` is resolved fail-closed by omnibase_infra's + delegation routing consumers (omnimarket + ``handler_delegation_routing._get_config`` → + ``resolve_required_path_config``), so compose and k8s pointing at different + in-container paths is a silent request-path break, not a boot failure. + + The value is locked on EVERY compose file that stands up a runtime container + (:data:`COMPOSE_RUNTIME_SERVICES_BY_FILE`), not just the ``infra`` base. The + lane-coverage test below is presence-only, so a typo'd literal in + ``docker-compose.judge.yml`` or ``docker-compose.e2e.yml`` — the two lanes + that inherit nothing — would otherwise satisfy both checks while pointing + the container at a path that does not exist. That is precisely the + "bound on both sides, different paths" failure this lock exists to catch. + + KNOWN CROSS-REPO DIVERGENCE (discovered here, tracked separately, NOT + asserted): ``configmap.yaml`` also carries this key, pinned to the stale + ``/app/contracts/delegation/routing_tiers.yaml``. All three runtime + Deployments override it inline with the correct site-packages path, and + inline ``env`` beats ``envFrom`` in Kubernetes, so the runtime family is + unaffected — but any workload that ``envFrom``-s ``onex-runtime-config`` + WITHOUT an inline override would receive the stale path. The k8s pin used + below is therefore the Deployment inline value, which is what those runtime + containers actually see. Fixing the ConfigMap is an ``omninode_infra`` + change that cannot land in this repo's PR; asserting it here would leave a + permanently-red gate on ``omnibase_infra`` for a defect it cannot fix. + """ + if K8S_RUNTIME_DIR is None: + pytest.skip( + "omninode_infra not found as a sibling — set OMNINODE_INFRA_DIR to run this test" + ) + + key = "DELEGATION_ROUTING_TIERS_PATH" + + k8s_values: dict[str, str | None] = { + manifest: extract_k8s_env_value(K8S_RUNTIME_DIR / manifest, key) + for manifest in K8S_RUNTIME_FAMILY_DEPLOYMENTS + } + unbound_k8s = [m for m, v in k8s_values.items() if v is None] + assert not unbound_k8s, ( + f"{key} is not bound on these onex-dev runtime Deployments: " + f"{', '.join(unbound_k8s)}. The compose side is pinned to it; unbinding " + "one side reopens the OMN-15628 seam." + ) + + bound_k8s: dict[str, str] = {m: v for m, v in k8s_values.items() if v is not None} + distinct_k8s = set(bound_k8s.values()) + assert len(distinct_k8s) == 1, ( + f"{key} is pinned to different values across the onex-dev runtime " + f"Deployments: {bound_k8s}" + ) + expected = next(iter(distinct_k8s)) + + docker_dir = COMPOSE_PATH.parent + compose_values: dict[str, str | None] = {} + for filename, services in COMPOSE_RUNTIME_SERVICES_BY_FILE.items(): + compose_path = docker_dir / filename + assert compose_path.exists(), ( + f"{filename} is listed in COMPOSE_RUNTIME_SERVICES_BY_FILE but does " + "not exist. Update the map when a compose lane is renamed or removed." + ) + for service in services: + compose_values[f"{filename}::{service}"] = ( + extract_compose_service_env_value(compose_path, service, key) + ) + + # Accept either the k8s literal itself or a path Dockerfile.runtime bakes it + # to. Both name the same file inside the image; see + # extract_dockerfile_baked_aliases. + aliases = extract_dockerfile_baked_aliases(docker_dir / "Dockerfile.runtime") + acceptable = resolve_baked_aliases(expected, aliases) + + mismatched = {s: v for s, v in compose_values.items() if v not in acceptable} + + assert not mismatched, ( + f"{key} disagrees between docker-compose and the onex-dev k8s pin.\n" + f" k8s pin ({', '.join(K8S_RUNTIME_FAMILY_DEPLOYMENTS)}): {expected}\n" + f" accepted in compose (k8s pin + Dockerfile.runtime-baked aliases of it): " + f"{sorted(acceptable)}\n" + + "\n".join(f" compose {s}: {v!r}" for s, v in sorted(mismatched.items())) + + f"\n\nFix: bind {key} in each file above to one of the accepted paths " + "(in infra.yml and judge.yml the runtime-env anchor covers every runtime " + "service at once; e2e.yml binds on the service directly). If you meant to " + "introduce a NEW in-image location, add the COPY to docker/Dockerfile.runtime " + "first — this check reads the aliases from there, it does not take a literal " + "on trust." + ) + + distinct_compose = set(compose_values.values()) + assert len(distinct_compose) == 1, ( + f"{key} is pinned to different (individually acceptable) paths across the " + f"compose lanes: {compose_values}. Every lane builds the same " + "docker/Dockerfile.runtime image, so they must agree on one literal — " + "divergence here is how a lane-specific edit silently stops matching the " + "others." + ) + + +@pytest.mark.ci +def test_every_runtime_compose_lane_binds_delegation_routing_tiers_path() -> None: + """OMN-15628 lane coverage: no docker lane can come up without the pin. + + Three shapes of compose file stand up a runtime container: + + * ANCHOR OWNERS — files with their own ``x-runtime-env`` anchor + (``infra`` = dev/lab base, ``judge``). Each must bind the key in its own + anchor. + * LANE OVERLAYS — layered on ``infra`` with ``-f infra -f `` + (prod, stability-test, dev-lane; see ``resolve_compose_file_args`` in + scripts/deploy-runtime.sh). Compose merges ``environment`` mappings + key-by-key across ``-f`` layers, so these inherit the base pin. That + inheritance holds ONLY while no overlay replaces the mapping wholesale + with a Compose merge directive (``environment: !override`` or + ``environment: !reset``) — assert that no service in the overlay does. + * STANDALONE — files that layer nothing (e2e). Must bind the key directly. + + The directive check is STRUCTURAL, over the parsed service graph, not a + regex over raw text: the previous regex matched the literal string + ``environment: !override`` only, so it was blind to ``!reset`` (which + severs inheritance identically) and to any reformatting of the same tag. + """ + docker_dir = COMPOSE_PATH.parent + + # (filename, top-level anchor key) — judge names its anchor differently. + anchor_owners = ( + ("docker-compose.infra.yml", "x-runtime-env"), + ("docker-compose.judge.yml", "x-judge-runtime-env"), + ) + lane_overlays = ( + "docker-compose.prod.yml", + "docker-compose.stability-test.yml", + "docker-compose.dev-lane.yml", + ) + standalone = ("docker-compose.e2e.yml",) + + key = "DELEGATION_ROUTING_TIERS_PATH" + + for filename, anchor_key in anchor_owners: + raw = (docker_dir / filename).read_text() + anchor = re.search(rf"^{anchor_key}:.*?(?=\n\S|\Z)", raw, re.DOTALL | re.M) + assert anchor is not None, f"{filename} has no {anchor_key} anchor" + assert re.search(rf"^\s{{2}}{key}:", anchor.group(0), re.MULTILINE), ( + f"{filename} owns its own {anchor_key} anchor but does not bind {key}. " + "Every runtime container it starts will fail closed on the first " + "delegation-routing request (OMN-15628)." + ) + + for filename in lane_overlays: + severed = { + name: env.tag + for name, service in compose_services(docker_dir / filename).items() + if isinstance((env := service_environment(service)), ComposeTagged) + } + assert not severed, ( + f"{filename} replaces a service's `environment` mapping wholesale with " + "a Compose merge directive: " + + ", ".join(f"{name} -> {tag}" for name, tag in sorted(severed.items())) + + ". That severs the compose merge that carries " + f"{key} (and every other x-runtime-env key) from " + "docker-compose.infra.yml into this lane. Bind the key explicitly in " + "this overlay, or drop the directive." + ) + + for filename in standalone: + raw = (docker_dir / filename).read_text() + assert re.search(rf"^\s+{key}:", raw, re.MULTILINE), ( + f"{filename} layers no base compose file, so it inherits nothing — " + f"it must bind {key} on its runtime service directly." + ) + + +def _duplicate_mapping_keys(node: yaml.Node, path: str = "") -> list[str]: + """Report ``a.b.KEY`` for every mapping key declared more than once.""" + duplicates: list[str] = [] + if isinstance(node, yaml.MappingNode): + seen: set[str] = set() + for key_node, value_node in node.value: + name = str(getattr(key_node, "value", key_node)) + where = f"{path}.{name}" if path else name + if name in seen: + duplicates.append(where) + seen.add(name) + duplicates.extend(_duplicate_mapping_keys(value_node, where)) + elif isinstance(node, yaml.SequenceNode): + for index, item in enumerate(node.value): + duplicates.extend(_duplicate_mapping_keys(item, f"{path}[{index}]")) + return duplicates + + +@pytest.mark.ci +def test_no_duplicate_keys_in_compose_files() -> None: + """No compose mapping declares the same key twice (OMN-15628 / OMN-15645). + + THIS IS THE CHECK THAT WOULD HAVE CAUGHT THE COLLISION. On 2026-08-02, + omnibase_infra#2620 (OMN-15645) and #2621 (OMN-15628) each added + ``DELEGATION_ROUTING_TIERS_PATH`` to the SAME ``x-runtime-env`` anchor with + a DIFFERENT value, 14 minutes apart. Both PRs were individually green. + Merged together they produced a duplicate key, and YAML last-wins silently + elected one value — putting ``dev`` red on the parity gate below with no + single PR having introduced the failure. + + A duplicate key is never intentional in these files and is invisible to + every loader-based check in this module, because ``yaml.safe_load`` + collapses it before any assertion runs. It has to be caught at the NODE + level, pre-construction, which is what this test does. + """ + docker_dir = COMPOSE_PATH.parent + compose_files = sorted(docker_dir.glob("docker-compose*.yml")) + assert compose_files, f"No docker-compose*.yml files found in {docker_dir}" + + offenders: dict[str, list[str]] = {} + for compose_file in compose_files: + # yaml.compose stops at the node graph and constructs nothing, so + # unlike yaml.load below it raises no S506 concern at all. + node = yaml.compose(compose_file.read_text(), Loader=_ComposeLoader) + if node is not None and (duplicates := _duplicate_mapping_keys(node)): + offenders[compose_file.name] = duplicates + + assert not offenders, ( + "Compose files declare duplicate mapping keys. YAML keeps the LAST " + "occurrence, so the earlier declaration is silently dead — an edit to it " + "is a no-op, and two PRs that each add the same key to the same block " + "merge into a wrong value with neither PR ever going red:\n" + + "\n".join( + f" {name}: {', '.join(where)}" for name, where in sorted(offenders.items()) + ) + + "\n\nFix: keep exactly one declaration per key and delete the other." + ) + + +@pytest.mark.ci +def test_k8s_family_surface_requires_all_runtime_deployments(tmp_path: Path) -> None: + """The forward k8s surface must not accept a partially-bound key. + + Regression lock for the granularity the OMN-15628 fix could have silently + traded away. Widening the forward direction from ConfigMap-only to "bound + anywhere in k8s" would let a key bound inline on ONE runtime Deployment + satisfy an ``x-runtime-env`` claim that reaches all three — a strictly + weaker assertion than the one it replaced. + + Driven against the REAL manifests, copied to a temp dir with the binding + removed from exactly one Deployment. Asserts the two surfaces diverge in + the expected direction: the union surface still reports the key (it is + still bound somewhere), the family surface no longer does. + + The probe key is DERIVED, not hardcoded: it must be inline on all three + runtime Deployments AND absent from the ConfigMap, otherwise the ConfigMap + term would keep it in the family surface and the mutation would prove + nothing. ``DELEGATION_ROUTING_TIERS_PATH`` itself does not qualify — see + the ConfigMap-divergence note on + ``test_delegation_routing_tiers_path_matches_k8s_pin``. + """ + if K8S_RUNTIME_DIR is None: + pytest.skip( + "omninode_infra not found as a sibling — set OMNINODE_INFRA_DIR to run this test" + ) + + mutated_manifest = "deployment-omninode-runtime-worker.yaml" + + for source in K8S_RUNTIME_DIR.iterdir(): + if source.is_file(): + (tmp_path / source.name).write_text(source.read_text()) + + configmap_keys = extract_configmap_keys(tmp_path / "configmap.yaml") + inline_only_family_keys = sorted( + extract_k8s_runtime_family_bound_keys(tmp_path) - configmap_keys + ) + assert inline_only_family_keys, ( + "No key is bound inline on all three runtime Deployments while absent " + "from the ConfigMap, so the family-vs-union distinction cannot be " + "probed. If the manifests genuinely moved every inline binding into the " + "ConfigMap, delete extract_k8s_runtime_family_bound_keys' inline term " + "rather than leaving this test unable to fail." + ) + key = inline_only_family_keys[0] + + target = tmp_path / mutated_manifest + document = yaml.safe_load(target.read_text()) + for container in document["spec"]["template"]["spec"]["containers"]: + container["env"] = [e for e in container.get("env") or [] if e["name"] != key] + target.write_text(yaml.safe_dump(document, sort_keys=False)) + + assert key in extract_k8s_bound_keys(tmp_path), ( + f"{key} should still appear in the UNION surface — it is still bound on " + "the other two runtime Deployments. If this fails, the fixture mutation " + "removed more than intended." + ) + assert key not in extract_k8s_runtime_family_bound_keys(tmp_path), ( + f"{key} was removed from {mutated_manifest} yet the runtime-FAMILY " + "surface still reports it as bound. extract_k8s_runtime_family_bound_keys " + "has been widened to a union and the forward parity check is now weaker " + "than ConfigMap-only was: a per-workload binding can stand in for an " + "anchor-wide one (OMN-15628)." + ) + + @pytest.mark.ci def test_compose_path_exists() -> None: """Sanity guard: docker-compose file is present at the expected path.""" diff --git a/tests/ci/test_flat_migration_no_foreign_connect_gate.py b/tests/ci/test_flat_migration_no_foreign_connect_gate.py new file mode 100644 index 0000000000..68bb4d30a4 --- /dev/null +++ b/tests/ci/test_flat_migration_no_foreign_connect_gate.py @@ -0,0 +1,456 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""OMN-15819 — static gate: no NEW flat migration with an un-ledgered foreign `\\connect`. + +Companion to the `omninode_infra` runner-honesty fix +(`k8s/migrations/omnibase-infra-migrate.yaml`): that Job's flat loop has no +execution path for a migration whose `\\connect` names a database other +than its own (``omnibase_infra``) -- 098/099 sat with a false "applied" +ledger row while their real target, ``omnidash_analytics``, never saw +either file. This gate is the pre-merge half, enforced in THIS repo (where +the flat SQL corpus actually lives): a cross-DB flat file must be listed, +with a citation, in +``docker/migrations/forward/cross-database-flat-migrations.yaml`` -- +otherwise the gate fails closed. + +``test_gate_rejects_a_synthetic_new_cross_db_flat_file`` is the RED-first +proof this deliverable calls for: it builds an isolated fixture tree with a +migration the manifest does NOT know about and asserts +``check_flat_migration_foreign_connect.check()`` returns a violation naming +it. Every other synthetic-fixture test in this file follows the same +isolated-tmp-tree shape so it never depends on, or mutates, the live +``docker/migrations/forward/`` corpus. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from scripts.ci import check_flat_migration_foreign_connect as gate + +pytestmark = pytest.mark.unit + + +def _write_manifest(path: Path, entries: list[dict[str, str]]) -> None: + path.write_text(yaml.safe_dump({"entries": entries}), encoding="utf-8") + + +# --------------------------------------------------------------------------- +# The live repo: this gate must be clean against real HEAD (AC2, both +# directions) -- if this fails, either a new cross-DB flat file landed +# un-ledgered, or the manifest drifted from live reality. +# --------------------------------------------------------------------------- + + +def test_gate_passes_against_the_live_repo() -> None: + # frozen_seed=MANIFEST_FROZEN_SEED matches what main() (the real CI + # entrypoint) actually passes -- a bare gate.check() would not exercise + # the closed-ledger enforcement against the live manifest at all. + violations = gate.check(frozen_seed=gate.MANIFEST_FROZEN_SEED) + assert not violations, "\n".join(v.describe() for v in violations) + + +def test_manifest_frozen_seed_matches_the_live_manifest_exactly() -> None: + """MANIFEST_FROZEN_SEED is a hand-maintained pin, not derived from the + manifest at runtime (by design -- it must not move just because someone + edits the manifest). Catch drift between the two explicitly instead of + letting it show up only as a confusing rejection/acceptance elsewhere.""" + manifest = gate.load_manifest() + assert set(manifest) == gate.MANIFEST_FROZEN_SEED + + +def test_live_manifest_has_the_five_undeliverable_entries() -> None: + manifest = gate.load_manifest() + undeliverable = { + name for name, entry in manifest.items() if entry.disposition == "undeliverable" + } + # OMN-15846 re-audited 083/096/097 (previously grandfathered) and + # confirmed all three are ALSO undeliverable via the k8s Job -- the + # manifest is now a fully-audited closed ledger with zero grandfathered + # entries remaining. + assert undeliverable == { + "098_create_omninode_internal_schema.sql", + "099_create_omninode_internal_live_events.sql", + "083_create_log_entries.sql", + "096_grant_role_omnidash_omnidash_analytics.sql", + "097_grant_app_dashboard_connect_omnidash_analytics.sql", + } + for name in ( + "098_create_omninode_internal_schema.sql", + "099_create_omninode_internal_live_events.sql", + ): + assert "OMN-15819" in manifest[name].citation, name + for name in ( + "083_create_log_entries.sql", + "096_grant_role_omnidash_omnidash_analytics.sql", + "097_grant_app_dashboard_connect_omnidash_analytics.sql", + ): + assert "OMN-15846" in manifest[name].citation, name + + +def test_no_grandfathered_entries_remain() -> None: + # OMN-15846 closed out every entry the manifest was seeded with as + # "grandfathered" (not re-audited) -- the frozen seed set is fully + # audited now, so a NEW grandfathered entry would only ever come from a + # future manifest edit that ducks the re-audit this test guards against. + manifest = gate.load_manifest() + grandfathered = { + name for name, entry in manifest.items() if entry.disposition == "grandfathered" + } + assert grandfathered == set() + + +def test_live_manifest_entries_all_target_omnidash_analytics() -> None: + # Not a structural requirement of the gate (a future entry could target a + # different foreign DB) -- but every entry today does, and a silent + # change here is worth a loud diff rather than passing quietly. + manifest = gate.load_manifest() + assert manifest, "expected at least one grandfathered/undeliverable entry" + for name, entry in manifest.items(): + assert entry.connect_target == "omnidash_analytics", name + + +# --------------------------------------------------------------------------- +# Pure detector: mirrors the k8s Job's own `awk '$1 == "\\connect"'` field +# semantics (first line whose first token IS `\connect`, not a `--` comment +# that merely mentions it). +# --------------------------------------------------------------------------- + + +def test_connect_target_ignores_a_prose_comment_mentioning_connect( + tmp_path: Path, +) -> None: + sql = tmp_path / "001_example.sql" + sql.write_text( + "-- this file switches with `\\connect other_db` partway through\n" + "CREATE TABLE t (id int);\n" + ) + assert gate.flat_migration_connect_target(sql) is None + + +def test_connect_target_finds_a_real_directive_after_comments(tmp_path: Path) -> None: + sql = tmp_path / "001_example.sql" + sql.write_text( + "-- some header prose\n" + "-- more prose\n" + "\\connect omnidash_analytics\n" + "CREATE TABLE t (id int);\n" + ) + assert gate.flat_migration_connect_target(sql) == "omnidash_analytics" + + +def test_connect_target_takes_the_first_directive_only(tmp_path: Path) -> None: + sql = tmp_path / "001_example.sql" + sql.write_text("\\connect first_db\nSELECT 1;\n\\connect second_db\n") + assert gate.flat_migration_connect_target(sql) == "first_db" + + +def test_connect_target_matches_a_directive_with_leading_whitespace( + tmp_path: Path, +) -> None: + """The k8s Job's own `awk '$1 == "\\connect"'` predicate strips leading + whitespace via default field splitting -- an indented `\\connect` is a + real, live directive to the runner. This gate must see it too, or a + migration with ` \\connect other_db` reads as "no \\connect" here while + remaining foreign to the runner (OMN-15819 CodeRabbit thread + r3749990754).""" + sql = tmp_path / "001_example.sql" + sql.write_text(" \\connect omnidash_analytics\nCREATE TABLE t (id int);\n") + assert gate.flat_migration_connect_target(sql) == "omnidash_analytics" + + +def test_connect_target_matches_a_directive_with_a_leading_tab(tmp_path: Path) -> None: + sql = tmp_path / "001_example.sql" + sql.write_text("\t\\connect omnidash_analytics\nCREATE TABLE t (id int);\n") + assert gate.flat_migration_connect_target(sql) == "omnidash_analytics" + + +def test_flat_migration_files_excludes_the_nodes_subdirectory(tmp_path: Path) -> None: + (tmp_path / "001_flat.sql").write_text("SELECT 1;\n") + node_dir = tmp_path / "nodes" / "some_node" + node_dir.mkdir(parents=True) + (node_dir / "0000_create.sql").write_text("\\connect omnidash_analytics\n") + + files = gate.flat_migration_files(tmp_path) + assert [p.name for p in files] == ["001_flat.sql"] + + +# --------------------------------------------------------------------------- +# RED-first: the gate must FAIL (return a violation) against a synthetic new +# cross-DB flat file the manifest has never heard of. +# --------------------------------------------------------------------------- + + +def test_gate_rejects_a_synthetic_new_cross_db_flat_file(tmp_path: Path) -> None: + forward_dir = tmp_path / "forward" + forward_dir.mkdir() + (forward_dir / "001_ordinary.sql").write_text("CREATE TABLE ordinary (id int);\n") + # The defect class: a brand-new flat migration nobody has ever ledgered, + # targeting a database the runner does not own. + (forward_dir / "200_new_cross_db_migration.sql").write_text( + "\\connect some_other_database\nCREATE TABLE t (id int);\n" + ) + manifest_path = tmp_path / "manifest.yaml" + _write_manifest(manifest_path, entries=[]) + + violations = gate.check(forward_dir=forward_dir, manifest_path=manifest_path) + + assert len(violations) == 1 + assert violations[0].file == "200_new_cross_db_migration.sql" + assert "OMN-15819" in violations[0].reason + assert "no execution path" in violations[0].reason + + +def test_gate_without_frozen_seed_passes_a_new_file_with_a_matching_entry( + tmp_path: Path, +) -> None: + """`frozen_seed=None` (the default) does not enforce the closed-ledger + property -- unrelated tests in this file rely on that. This is the + permissive baseline the next two tests contrast against.""" + forward_dir = tmp_path / "forward" + forward_dir.mkdir() + (forward_dir / "200_new_cross_db_migration.sql").write_text( + "\\connect some_other_database\nCREATE TABLE t (id int);\n" + ) + manifest_path = tmp_path / "manifest.yaml" + _write_manifest( + manifest_path, + entries=[ + { + "file": "200_new_cross_db_migration.sql", + "connect_target": "some_other_database", + "disposition": "undeliverable", + "citation": "OMN-99999 -- test fixture", + } + ], + ) + + assert gate.check(forward_dir=forward_dir, manifest_path=manifest_path) == [] + + +# --------------------------------------------------------------------------- +# The closed-ledger property (OMN-15819 CodeRabbit thread r3749990788): a +# manifest entry, on its own, must never be able to authorize a brand-new +# cross-DB flat migration -- only a filename already present at +# gate-authorship time (``MANIFEST_FROZEN_SEED``) may pass, regardless of +# disposition or citation. This is what ``main()`` actually enforces. +# --------------------------------------------------------------------------- + + +def test_gate_with_frozen_seed_rejects_a_new_file_even_with_a_matching_entry( + tmp_path: Path, +) -> None: + """RED-first: the exact scenario the permissive test above shows passing + must now fail once ``frozen_seed`` is supplied -- a new file + a new, + perfectly well-formed manifest entry in the same PR is still a reject.""" + forward_dir = tmp_path / "forward" + forward_dir.mkdir() + (forward_dir / "200_new_cross_db_migration.sql").write_text( + "\\connect some_other_database\nCREATE TABLE t (id int);\n" + ) + manifest_path = tmp_path / "manifest.yaml" + _write_manifest( + manifest_path, + entries=[ + { + "file": "200_new_cross_db_migration.sql", + "connect_target": "some_other_database", + "disposition": "undeliverable", + "citation": "OMN-99999 -- test fixture", + } + ], + ) + + violations = gate.check( + forward_dir=forward_dir, + manifest_path=manifest_path, + frozen_seed=gate.MANIFEST_FROZEN_SEED, + ) + + assert len(violations) == 1 + assert violations[0].file == "200_new_cross_db_migration.sql" + assert "not part of the frozen OMN-15819 seed set" in violations[0].reason + assert "hard reject" in violations[0].reason + + +def test_gate_with_frozen_seed_passes_a_file_that_is_in_the_seed( + tmp_path: Path, +) -> None: + """Positive control: a filename genuinely in ``MANIFEST_FROZEN_SEED`` + (one of the two OMN-15819 undeliverable entries) with a matching, + correct manifest entry still passes -- the closed-ledger check only + rejects filenames OUTSIDE the frozen set.""" + forward_dir = tmp_path / "forward" + forward_dir.mkdir() + (forward_dir / "098_create_omninode_internal_schema.sql").write_text( + "\\connect omnidash_analytics\nCREATE SCHEMA IF NOT EXISTS x;\n" + ) + manifest_path = tmp_path / "manifest.yaml" + _write_manifest( + manifest_path, + entries=[ + { + "file": "098_create_omninode_internal_schema.sql", + "connect_target": "omnidash_analytics", + "disposition": "undeliverable", + "citation": "OMN-15819 -- test fixture", + } + ], + ) + + violations = gate.check( + forward_dir=forward_dir, + manifest_path=manifest_path, + frozen_seed=gate.MANIFEST_FROZEN_SEED, + ) + + assert violations == [] + + +def test_gate_ignores_a_same_database_flat_file(tmp_path: Path) -> None: + """A \\connect back to the runner's own DB is not foreign -- not a violation.""" + forward_dir = tmp_path / "forward" + forward_dir.mkdir() + (forward_dir / "001_same_db.sql").write_text( + "\\connect omnibase_infra\nCREATE TABLE t (id int);\n" + ) + manifest_path = tmp_path / "manifest.yaml" + _write_manifest(manifest_path, entries=[]) + + assert gate.check(forward_dir=forward_dir, manifest_path=manifest_path) == [] + + +def test_gate_ignores_a_flat_file_with_no_connect_directive(tmp_path: Path) -> None: + forward_dir = tmp_path / "forward" + forward_dir.mkdir() + (forward_dir / "001_no_connect.sql").write_text("CREATE TABLE t (id int);\n") + manifest_path = tmp_path / "manifest.yaml" + _write_manifest(manifest_path, entries=[]) + + assert gate.check(forward_dir=forward_dir, manifest_path=manifest_path) == [] + + +def test_gate_rejects_a_stale_manifest_entry_for_a_removed_file(tmp_path: Path) -> None: + forward_dir = tmp_path / "forward" + forward_dir.mkdir() + (forward_dir / "001_unrelated.sql").write_text("SELECT 1;\n") + manifest_path = tmp_path / "manifest.yaml" + _write_manifest( + manifest_path, + entries=[ + { + "file": "999_long_gone.sql", + "connect_target": "some_other_database", + "disposition": "grandfathered", + "citation": "OMN-1 -- test fixture", + } + ], + ) + + violations = gate.check(forward_dir=forward_dir, manifest_path=manifest_path) + assert len(violations) == 1 + assert violations[0].file == "999_long_gone.sql" + assert "no live counterpart" in violations[0].reason + + +def test_gate_rejects_a_manifest_entry_whose_target_drifted(tmp_path: Path) -> None: + forward_dir = tmp_path / "forward" + forward_dir.mkdir() + (forward_dir / "001_drifted.sql").write_text("\\connect new_target\nSELECT 1;\n") + manifest_path = tmp_path / "manifest.yaml" + _write_manifest( + manifest_path, + entries=[ + { + "file": "001_drifted.sql", + "connect_target": "old_target", + "disposition": "grandfathered", + "citation": "OMN-1 -- test fixture", + } + ], + ) + + violations = gate.check(forward_dir=forward_dir, manifest_path=manifest_path) + assert len(violations) == 1 + assert "old_target" in violations[0].reason + assert "new_target" in violations[0].reason + + +@pytest.mark.parametrize( + ("bad_field", "bad_value"), + [("disposition", "vibes-based"), ("citation", "")], +) +def test_malformed_manifest_entry_fails_to_load( + tmp_path: Path, bad_field: str, bad_value: str +) -> None: + manifest_path = tmp_path / "manifest.yaml" + entry = { + "file": "001_x.sql", + "connect_target": "other_db", + "disposition": "grandfathered", + "citation": "OMN-1 -- test fixture", + } + entry[bad_field] = bad_value + _write_manifest(manifest_path, entries=[entry]) + + with pytest.raises(AssertionError): + gate.load_manifest(manifest_path) + + +def test_duplicate_manifest_entry_fails_to_load(tmp_path: Path) -> None: + manifest_path = tmp_path / "manifest.yaml" + entry = { + "file": "001_x.sql", + "connect_target": "other_db", + "disposition": "grandfathered", + "citation": "OMN-1 -- test fixture", + } + _write_manifest(manifest_path, entries=[entry, dict(entry)]) + + with pytest.raises(AssertionError, match="duplicate"): + gate.load_manifest(manifest_path) + + +# --------------------------------------------------------------------------- +# CLI entrypoint. +# --------------------------------------------------------------------------- + + +def test_main_returns_nonzero_and_prints_on_violation( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + forward_dir = tmp_path / "forward" + forward_dir.mkdir() + (forward_dir / "200_new.sql").write_text("\\connect other_db\nSELECT 1;\n") + manifest_path = tmp_path / "manifest.yaml" + _write_manifest(manifest_path, entries=[]) + + rc = gate.main( + ["--forward-dir", str(forward_dir), "--manifest", str(manifest_path)] + ) + out = capsys.readouterr() + + assert rc != 0 + assert "OMN-15819" in out.err + assert "200_new.sql" in out.err + + +def test_main_returns_zero_when_clean( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + forward_dir = tmp_path / "forward" + forward_dir.mkdir() + (forward_dir / "001_clean.sql").write_text("SELECT 1;\n") + manifest_path = tmp_path / "manifest.yaml" + _write_manifest(manifest_path, entries=[]) + + rc = gate.main( + ["--forward-dir", str(forward_dir), "--manifest", str(manifest_path)] + ) + out = capsys.readouterr() + + assert rc == 0 + assert "OK" in out.out diff --git a/tests/ci/test_flat_node_migration_shape_parity.py b/tests/ci/test_flat_node_migration_shape_parity.py new file mode 100644 index 0000000000..0a87c91633 --- /dev/null +++ b/tests/ci/test_flat_node_migration_shape_parity.py @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Static gate: a table declared by BOTH a flat and a node migration must not +silently drift between the two declarations. + +## The class this closes + +OMN-15376 shipped `tests/ci/test_node_migration_shape_reconciliation.py`, +which reconciles a table's fresh-create path against its own +drifted-pre-existing path -- but ONLY within +``docker/migrations/forward/nodes/``. It says nothing about a table declared +a SECOND time by a top-level flat migration +(``docker/migrations/forward/*.sql``, applied against the ``omnibase_infra`` +DB) that names the exact same table. Two known instances of exactly that +class each cost a full deploy cycle before anything caught them +(``llm_cost_aggregates`` / OMN-15376, ``baselines_comparisons`` / OMN-15302). +A full overlap computation over dev finds **14** tables declared by both +corpora; this gate is the binding between them that neither producer's own +reconciliation gate can see. + +## What this asserts + +For every table name this repo's migrations declare via a guarded +``CREATE TABLE IF NOT EXISTS`` in BOTH corpora: + +* the column set and normalized column type must match the ledger + (``docker/migrations/forward/flat-node-shape-parity.yaml``) entry for that + table -- ``status: identical`` requires the two sides to actually agree + (columns are re-derived live from the SQL, never trusted from the ledger), + ``status: accepted_divergence`` requires a non-empty ``reason`` and that + the two sides are STILL actually divergent (a stale acceptance whose + shapes have since converged is also a failure -- flip it to ``identical``). +* the ledger's table set matches the LIVE overlap exactly, in both + directions: a newly-introduced dual-producer table with no ledger entry + fails closed (AC2 -- a table cannot dodge review by omission), and a + ledger entry for a table that is no longer dual-produced fails just as + loudly (a stale record is not free to keep around). + +This is the STATIC half only, matching the OMN-15376 sibling gate's own +split: no database is touched, no execution proof is claimed. + +Ticket: OMN-15384 +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, cast + +import pytest +import yaml + +from tests.helpers.util_migration_shape import ( + diff_column_shapes, + flat_migration_files, + node_migration_files, + table_column_shapes, +) + +pytestmark = [pytest.mark.unit] + +LEDGER_PATH = ( + Path(__file__).resolve().parents[2] + / "docker" + / "migrations" + / "forward" + / "flat-node-shape-parity.yaml" +) + +# The 14-table overlap OMN-15384's own audit found. Pinned here (not just +# derived live) so a corpus edit that shrinks the overlap below what was +# actually audited is visible as a assertion diff, not a quietly-smaller +# passing set. +_AUDITED_OVERLAP = frozenset( + { + "agent_routing_decisions", + "baselines_breakdown", + "baselines_comparisons", + "baselines_trend", + "capability_scores", + "evidence_correlation_trace_projection", + "evidence_dashboard_projection", + "evidence_readiness_aggregate_projection", + "llm_call_metrics", + "llm_cost_aggregates", + "llm_routing_decisions", + "savings_estimates", + "session_outcomes", + "swarm_runs", + } +) + + +def _load_ledger() -> dict[str, dict[str, Any]]: + raw = yaml.safe_load(LEDGER_PATH.read_text(encoding="utf-8")) + assert isinstance(raw, dict) and isinstance(raw.get("tables"), dict), ( + f"{LEDGER_PATH} must be a mapping with a top-level `tables:` mapping" + ) + return cast("dict[str, dict[str, Any]]", raw["tables"]) + + +def _live_overlap() -> tuple[dict[str, dict[str, str]], dict[str, dict[str, str]]]: + flat_shapes = table_column_shapes(flat_migration_files()) + node_shapes = table_column_shapes(node_migration_files()) + return flat_shapes, node_shapes + + +def test_the_corpus_is_non_empty_and_the_ledger_matches_the_live_overlap() -> None: + """Anti-vacuity + AC2: the ledger cannot silently drift from the corpus. + + A table can't dodge this gate by never getting a ledger entry (missing == + fail), and a ledger entry can't survive after its table stops being + dual-produced (stale == fail) -- so deleting one side's declaration to + "resolve" a divergence is not a way to make this test quietly pass; it + surfaces as a stale-entry failure demanding the ledger be updated too. + """ + flat_shapes, node_shapes = _live_overlap() + live_overlap = set(flat_shapes) & set(node_shapes) + ledger = _load_ledger() + ledger_tables = set(ledger) + + assert len(live_overlap) >= 14, sorted(live_overlap) + assert live_overlap >= _AUDITED_OVERLAP, sorted(_AUDITED_OVERLAP - live_overlap) + + missing_from_ledger = live_overlap - ledger_tables + assert not missing_from_ledger, ( + f"{sorted(missing_from_ledger)} are declared by BOTH a flat and a node " + f"migration but have no entry in {LEDGER_PATH}. Add one with " + f"status: identical (if the shapes agree) or status: accepted_divergence " + f"plus a reason (if they legitimately don't) -- see the module docstring." + ) + stale_ledger_entries = ledger_tables - live_overlap + assert not stale_ledger_entries, ( + f"{sorted(stale_ledger_entries)} have a {LEDGER_PATH} entry but are no " + f"longer declared by both corpora (one side's CREATE TABLE was removed " + f"or renamed). Remove the stale entry." + ) + + +@pytest.mark.parametrize("table", sorted(_AUDITED_OVERLAP)) +def test_dual_producer_table_shape_matches_the_ledger_disposition(table: str) -> None: + """Per-table enforcement: identical stays identical, divergence stays named.""" + flat_shapes, node_shapes = _live_overlap() + ledger = _load_ledger() + + entry = ledger.get(table) + assert entry is not None, f"{table}: no ledger entry (see the overlap test)" + status = entry.get("status") + assert status in ("identical", "accepted_divergence"), ( + f"{table}: unknown ledger status {status!r}" + ) + + diff = diff_column_shapes(flat_shapes[table], node_shapes[table]) + + if status == "identical": + assert not diff, ( + f"{table} is recorded `status: identical` in {LEDGER_PATH} but its " + f"flat and node declarations have drifted apart (OMN-15384 class):\n" + f"{diff.describe(table=table)}\n" + f"Either converge the SQL, or change the ledger entry to " + f"accepted_divergence with a reason." + ) + else: + reason = entry.get("reason", "") + assert isinstance(reason, str) and reason.strip(), ( + f"{table}: status: accepted_divergence requires a non-empty `reason` " + f"in {LEDGER_PATH}" + ) + assert diff, ( + f"{table} is recorded `status: accepted_divergence` in {LEDGER_PATH} " + f"but its flat and node declarations now agree -- the entry is " + f"stale. Flip it to `status: identical` (and drop `reason`)." + ) + + +def test_diff_column_shapes_detects_every_drift_class() -> None: + """The pure comparator's branches are all exercised, not just imported. + + Without this, `diff_column_shapes` could be broken (e.g. always return an + empty diff) and every case above would still pass -- a checker whose + branches never fire is the same theater the OMN-15639 sentinel's own + synthetic-input test exists to rule out. + """ + identical = diff_column_shapes( + {"a": "UUID", "b": "TEXT"}, {"a": "UUID", "b": "TEXT"} + ) + assert not identical, identical + + only_flat = diff_column_shapes({"a": "UUID", "b": "TEXT"}, {"a": "UUID"}) + assert only_flat.only_flat == ("b",) + assert not only_flat.only_node + assert not only_flat.type_diff + + only_node = diff_column_shapes({"a": "UUID"}, {"a": "UUID", "b": "TEXT"}) + assert only_node.only_node == ("b",) + assert not only_node.only_flat + assert not only_node.type_diff + + type_diff = diff_column_shapes({"a": "REAL"}, {"a": "DOUBLEPRECISION"}) + assert type_diff.type_diff == (("a", "REAL", "DOUBLEPRECISION"),) + assert not type_diff.only_flat + assert not type_diff.only_node + + described = type_diff.describe(table="t") + assert "t:" in described + assert "type diff a" in described diff --git a/tests/ci/test_incident_replay_coverage_omn15547.py b/tests/ci/test_incident_replay_coverage_omn15547.py new file mode 100644 index 0000000000..18b171176d --- /dev/null +++ b/tests/ci/test_incident_replay_coverage_omn15547.py @@ -0,0 +1,518 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""The incident-replay coverage lint must actually say no (OMN-15547). + +WHAT IS UNDER TEST + ``scripts/ci/check_incident_replay_coverage.py`` -- the module CI and + pre-commit invoke, imported by path, never a re-implementation of its rules. + +WHY EVERY RED HERE IS EXECUTED, NOT ASSERTED + This lint exists because three guards shipped green while enforcing nothing. + A test suite for it that only checked "the happy path passes" would be the + same defect one level up: a lint that returns 0 unconditionally satisfies + every green assertion and nothing else. So the load-bearing tests here all + take the REAL repository tree, break exactly one thing about it, and require + a non-zero exit naming the rule -- including the default-deny path, which is + the property that makes the convention self-sustaining. + +THE FIXTURE IS THE REPO + Each RED case is built by copying the live tree's registry (and, for R1, the + real captured artifact) into a tmp root and mutating that copy. Nothing here + hand-writes a registry from scratch: a synthetic registry would only prove + the lint can parse a shape this file invented, which is the exact class of + vacuous proof the lint was written to detect. +""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import shutil +import sys +from pathlib import Path +from typing import Any + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +LINT_PATH = REPO_ROOT / "scripts" / "ci" / "check_incident_replay_coverage.py" +REGISTRY_REL = "tests/incident_replays/registry.yaml" + + +def _load_lint() -> Any: + spec = importlib.util.spec_from_file_location( + "check_incident_replay_coverage_omn15547", LINT_PATH + ) + assert spec and spec.loader, LINT_PATH + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +LINT = _load_lint() + + +def _clone_tree(tmp_path: Path) -> Path: + """Copy the parts of the real tree the lint reads, so a mutation is isolated. + + The lint reads: the registry, `.pre-commit-config.yaml`, `.github/workflows`, + and -- to resolve R1/R4/R5 -- the guard, fixture and test files the registry + names. Copying the whole repo would be slow; copying only what it reads keeps + the case honest because anything the lint touches is present verbatim. + """ + root = tmp_path / "repo" + root.mkdir() + + shutil.copy2( + REPO_ROOT / ".pre-commit-config.yaml", root / ".pre-commit-config.yaml" + ) + shutil.copytree(REPO_ROOT / ".github" / "workflows", root / ".github" / "workflows") + (root / "tests" / "incident_replays").mkdir(parents=True) + shutil.copy2(REPO_ROOT / REGISTRY_REL, root / REGISTRY_REL) + + registry = yaml.safe_load((REPO_ROOT / REGISTRY_REL).read_text(encoding="utf-8")) + referenced: set[str] = set() + for case in registry.get("cases") or []: + referenced.add(case["guard"]) + referenced.add(case["artifact"]["fixture"]) + referenced.add(case["test"].split("::", 1)[0]) + if case.get("discriminator"): + referenced.add(case["discriminator"].split("::", 1)[0]) + for entry in (registry.get("scope") or {}).get("debt_baseline") or []: + referenced.add(entry) + for entry in (registry.get("scope") or {}).get("required_guards") or []: + referenced.add(entry) + + for rel in sorted(referenced): + src = REPO_ROOT / rel + if not src.is_file(): + continue # PENDING entries legitimately do not exist yet + dst = root / rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + return root + + +def _registry(root: Path) -> dict[str, Any]: + return yaml.safe_load((root / REGISTRY_REL).read_text(encoding="utf-8")) + + +def _write_registry(root: Path, data: dict[str, Any]) -> None: + (root / REGISTRY_REL).write_text( + yaml.safe_dump(data, sort_keys=False), encoding="utf-8" + ) + + +def _rules(findings: list[Any]) -> set[str]: + return {f.rule for f in findings} + + +# -------------------------------------------------------------------------- +# GREEN control -- on the unmutated tree the lint must pass. +# -------------------------------------------------------------------------- +def test_the_live_tree_passes() -> None: + """Without this, every RED below could be produced by a lint that always fails.""" + result = LINT.evaluate(REPO_ROOT) + assert not result.findings, "\n".join(f.render() for f in result.findings) + assert result.covered, "the lint reports zero covered guards -- vacuous green" + + +# -------------------------------------------------------------------------- +# RED 1 -- a required guard loses its case (COVERAGE). +# -------------------------------------------------------------------------- +def test_red_required_guard_with_no_case(tmp_path: Path) -> None: + root = _clone_tree(tmp_path) + data = _registry(root) + required = data["scope"]["required_guards"] + dropped = [ + c + for c in data["cases"] + if c["guard"] in required and (root / c["guard"]).exists() + ] + assert dropped, "no required guard is covered in the live registry -- fix the setup" + data["cases"] = [c for c in data["cases"] if c not in dropped] + _write_registry(root, data) + + result = LINT.evaluate(root) + assert "COVERAGE" in _rules(result.findings), [f.render() for f in result.findings] + assert any(dropped[0]["guard"] in f.subject for f in result.findings) + + +# -------------------------------------------------------------------------- +# RED 2 -- R1: the captured artifact is edited after capture. +# -------------------------------------------------------------------------- +def test_red_fixture_bytes_mutated_by_one_byte(tmp_path: Path) -> None: + """A one-byte edit must break the build. + + This is the rule that separates "these are the bytes that failed" from + "these are bytes somebody kept adjusting until the test passed". + """ + root = _clone_tree(tmp_path) + case = _registry(root)["cases"][0] + fixture = root / case["artifact"]["fixture"] + raw = fixture.read_bytes() + fixture.write_bytes(raw + b" ") + + result = LINT.evaluate(root) + assert "R1" in _rules(result.findings), [f.render() for f in result.findings] + + +# -------------------------------------------------------------------------- +# RED 3 -- R2: provenance degraded to free text. +# -------------------------------------------------------------------------- +@pytest.mark.parametrize( + "prose", + [ + "captured from the live .201 health endpoint", + "same shape as the production body", + "copied off the host", + ], +) +def test_red_free_text_provenance_is_not_a_locator(tmp_path: Path, prose: str) -> None: + """Exactly the sentences a hand-typed fixture gets defended with. + + Each of these reads like provenance and none of them can be re-fetched. If + the lint accepted them it would accept a reconstruction, which is the defect. + """ + root = _clone_tree(tmp_path) + data = _registry(root) + data["cases"][0]["capture"]["source"] = prose + _write_registry(root, data) + + result = LINT.evaluate(root) + assert "R2" in _rules(result.findings), [f.render() for f in result.findings] + + +# -------------------------------------------------------------------------- +# RED 4 -- DEFAULT-DENY: a newly wired guard with no case and no baseline row. +# -------------------------------------------------------------------------- +def test_red_newly_wired_guard_defaults_to_denied(tmp_path: Path) -> None: + """The property that makes the convention self-sustaining. + + Without it the registry is a snapshot: today's guards are covered, and every + guard added tomorrow is exempt by default -- which is how the detection shelf + grew faster than the proof behind it in the first place. + """ + root = _clone_tree(tmp_path) + new_guard = root / "scripts" / "ci" / "check_brand_new_thing_omn15547.py" + new_guard.parent.mkdir(parents=True, exist_ok=True) + new_guard.write_text("#!/usr/bin/env python3\nraise SystemExit(0)\n") + config = root / ".pre-commit-config.yaml" + config.write_text( + config.read_text(encoding="utf-8") + + "\n# OMN-15547 default-deny probe\n" + + "# entry: python scripts/ci/check_brand_new_thing_omn15547.py\n", + encoding="utf-8", + ) + + result = LINT.evaluate(root) + assert "DEFAULT-DENY" in _rules(result.findings), [ + f.render() for f in result.findings + ] + assert any( + "check_brand_new_thing_omn15547.py" in f.subject for f in result.findings + ) + + +# -------------------------------------------------------------------------- +# RED 5 -- R4: the registry claims a replay the test does not perform. +# -------------------------------------------------------------------------- +def test_red_test_does_not_reference_the_fixture(tmp_path: Path) -> None: + root = _clone_tree(tmp_path) + data = _registry(root) + decoy = root / "tests" / "unit" / "scripts" / "test_decoy_omn15547.py" + decoy.parent.mkdir(parents=True, exist_ok=True) + decoy.write_text("def test_nothing() -> None:\n assert True\n") + data["cases"][0]["test"] = "tests/unit/scripts/test_decoy_omn15547.py::test_nothing" + _write_registry(root, data) + + result = LINT.evaluate(root) + assert "R4" in _rules(result.findings), [f.render() for f in result.findings] + + +# -------------------------------------------------------------------------- +# RED 6 -- R5: a false_red case with no discriminator. +# -------------------------------------------------------------------------- +def test_red_false_red_case_without_a_discriminator(tmp_path: Path) -> None: + """An accept-only proof cannot tell a working guard from a stuck-open one.""" + root = _clone_tree(tmp_path) + data = _registry(root) + target = next( + (c for c in data["cases"] if c.get("regression_class") == "false_red"), None + ) + if target is None: + pytest.skip("no false_red case in the registry to degrade") + target.pop("discriminator", None) + _write_registry(root, data) + + result = LINT.evaluate(root) + assert "R5" in _rules(result.findings), [f.render() for f in result.findings] + + +# -------------------------------------------------------------------------- +# RED 7 -- RATCHET: the debt baseline must stay truthful. +# -------------------------------------------------------------------------- +def test_red_covered_guard_left_in_the_debt_baseline(tmp_path: Path) -> None: + root = _clone_tree(tmp_path) + data = _registry(root) + covered = data["cases"][0]["guard"] + data["scope"]["debt_baseline"] = sorted({*data["scope"]["debt_baseline"], covered}) + _write_registry(root, data) + + result = LINT.evaluate(root) + assert "RATCHET" in _rules(result.findings), [f.render() for f in result.findings] + + +def test_red_stale_entry_in_the_debt_baseline(tmp_path: Path) -> None: + root = _clone_tree(tmp_path) + data = _registry(root) + data["scope"]["debt_baseline"] = sorted( + {*data["scope"]["debt_baseline"], "scripts/ci/deleted_long_ago.py"} + ) + _write_registry(root, data) + + result = LINT.evaluate(root) + assert "RATCHET" in _rules(result.findings), [f.render() for f in result.findings] + + +# -------------------------------------------------------------------------- +# The pre-registered requirement must ARM, not silently stay pending forever. +# -------------------------------------------------------------------------- +def test_pending_requirement_arms_when_the_guard_lands(tmp_path: Path) -> None: + """OMN-15538's guard is required before it exists; it must bind on arrival. + + A pre-registered requirement is only useful if it BINDS on arrival. + Otherwise listing a guard before it exists would be a way to look covered + forever: permanently PENDING, permanently green. + + This was observed live rather than imagined. ``check_pin_reachability.py`` + was pre-registered while omnibase_infra#2583 was open; when #2583 merged as + ``1da8d3c5`` the very next run of this lint went from + ``0 findings, 1 pending`` to ``COVERAGE: scripts/ci/check_pin_reachability + .py`` with no edit to the registry. The test reconstructs both states from + the real tree rather than depending on one of them still being reachable -- + a version of this test that skipped once the guard landed would prove + nothing exactly when it started to matter. + """ + root = _clone_tree(tmp_path) + required = _registry(root)["scope"]["required_guards"] + target = next((g for g in required if (root / g).exists()), None) + assert target, "expected at least one required guard present in the clone" + + # State 1 -- the guard has not landed yet: PENDING, and NOT a failure. + landed = root / target + body = landed.read_bytes() + landed.unlink() + data = _registry(root) + data["cases"] = [c for c in data["cases"] if c["guard"] != target] + _write_registry(root, data) + + before = LINT.evaluate(root) + assert target in before.pending, before.pending + assert not before.findings, [f.render() for f in before.findings] + + # State 2 -- the guard lands with no case: the requirement must bind NOW. + landed.parent.mkdir(parents=True, exist_ok=True) + landed.write_bytes(body) + + after = LINT.evaluate(root) + assert target not in after.pending + assert "COVERAGE" in _rules(after.findings), [f.render() for f in after.findings] + assert any(target in f.subject for f in after.findings) + + +# -------------------------------------------------------------------------- +# The lint's own incident replay case (registry: omn15547-handtyped-fixture- +# passed-as-proof). Not a hypothetical -- these are the bytes that shipped. +# -------------------------------------------------------------------------- +HANDTYPED_FIXTURE = ( + REPO_ROOT + / "tests" + / "fixtures" + / "omn15547" + / "test_omninode_system_slack_report.handtyped-fixture.py.captured" +) +HANDTYPED_FIXTURE_SHA256 = ( + "10cfdc48ef1a80dbc10f2c9cf1a84cfc20a68877111e15b5cbe1e56c428c5a0c" +) + + +def test_the_handtyped_fixture_capture_is_unmodified() -> None: + """Provenance guard: this is the dev blob at 0f050394, verbatim.""" + digest = hashlib.sha256(HANDTYPED_FIXTURE.read_bytes()).hexdigest() + assert digest == HANDTYPED_FIXTURE_SHA256, ( + "the captured OMN-15525 repair blob no longer matches " + "omnibase_infra@0f05039434996594000db85cd8d3947523bfebcf; re-fetch it " + "with `git show :` rather than editing it" + ) + + +def test_the_shipped_handtyped_fixture_fails_r1_and_r2() -> None: + """Replay: the artifact that WAS accepted as proof must now be refused. + + ``tests/fixtures/omn15547/test_omninode_system_slack_report.handtyped- + fixture.py.captured`` is the verbatim blob that landed on dev as the repair + for OMN-15525. It fixed the byte-count symptom and kept the disease: a + literal somebody typed, defended by prose. Both halves must fail. + """ + source = HANDTYPED_FIXTURE.read_text(encoding="utf-8") + + # R1: the health body is a literal built inside the test, not a committed + # capture the lint can hash. That is why nothing could detect it drifting. + assert "HEALTHY_BODY = json.dumps(" in source, ( + "the captured blob is supposed to be the LITERAL-fixture version; if " + "this fails the capture is of the wrong revision" + ) + assert "HEALTHY_BODY_SHA256" not in source, ( + "the captured blob must predate the sha256-pinned capture, or it is not " + "the artifact that shipped without provenance" + ) + + # R2: harvest the provenance the file actually offers -- the comment block + # above the literal -- and require that NONE of it resolves as a locator. + provenance_lines = [ + line.strip().lstrip("#").strip() + for line in source.splitlines() + if line.strip().startswith("#") + and any( + token in line.lower() + for token in ("live body", "mirrors", "realistic", ".201", "real ") + ) + ] + assert provenance_lines, "expected the blob's hand-written provenance comments" + assert any("mirrors the live body" in line.lower() for line in provenance_lines), ( + "the specific sentence this case exists to refuse is missing from the capture" + ) + for line in provenance_lines: + assert not any( + pattern.match(line) for pattern in LINT.LOCATOR_GRAMMARS.values() + ), ( + f"prose accepted as a locator: {line!r} -- R2 would admit a " + "hand-typed fixture, which is the whole defect" + ) + + +# -------------------------------------------------------------------------- +# Locator grammar is the R2 bar -- assert both polarities on real strings. +# -------------------------------------------------------------------------- +@pytest.mark.parametrize( + "locator", + [ + "gh-api:repos/OmniNode-ai/omnibase_infra/actions/runs/30574058377/jobs", + "gh-api:repos/OmniNode-ai/omnimarket/compare/dev...879d6fc6825f876458c6d45ed670c8715de8ac95", + "git-object:OmniNode-ai/omnimarket@879d6fc6ed6c4f6c86c2d3f0f4c1a0f8b7c6d5e4:.github/workflows/merge-hold-gate-reusable.yml", + "host-file:omni-201-ts:/data/maintenance/bin/omninode-system-slack-report.sh", + "live-http:omni-201-ts:8085/health", + "ci-artifact:30574058377/coverage-shard-3", + ], +) +def test_locator_grammars_accept_real_locators(locator: str) -> None: + assert any(p.match(locator) for p in LINT.LOCATOR_GRAMMARS.values()), locator + + +@pytest.mark.parametrize( + "locator", + [ + "the live 201 host", + "gh-api:repos/OmniNode-ai/omnibase_infra/actions/runs", # no numeric id + "git-object:OmniNode-ai/omnimarket@879d6fc6:.github/workflows/x.yml", # short sha + "host-file:omni-201-ts:data/maintenance/bin/x.sh", # not absolute + "https://example.invalid/whatever", + "", + ], +) +def test_locator_grammars_reject_unresolvable_provenance(locator: str) -> None: + assert not any(p.match(locator) for p in LINT.LOCATOR_GRAMMARS.values()), locator + + +# -------------------------------------------------------------------------- +# OMN-15536: the outage pin, replayed against the real predicate that now +# exists (registry: omn15536-unreachable-pin-outage). +# -------------------------------------------------------------------------- +COMPARE_DEAD = ( + REPO_ROOT + / "tests" + / "fixtures" + / "omn15547" + / "omnimarket-compare-dev-879d6fc6.gh-api.json.captured" +) +COMPARE_LIVE = ( + REPO_ROOT + / "tests" + / "fixtures" + / "omn15547" + / "omnimarket-compare-dev-454c429f.gh-api.json.captured" +) +COMPARE_DEAD_SHA256 = "4e957324574cf01581701cc662adba35cb092670ee67db48f31118f17239f3b5" +COMPARE_LIVE_SHA256 = "e669f0c1389440d41ba4e077fe94eea1596438d9cf866291ba298ceb2c30d5c7" + + +def _pin_reachability() -> Any: + """Import the real OMN-15538 guard, by path, exactly as CI invokes it.""" + spec = importlib.util.spec_from_file_location( + "check_pin_reachability_omn15547", + REPO_ROOT / "scripts" / "ci" / "check_pin_reachability.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_the_compare_captures_are_unmodified() -> None: + """Both captures are GitHub's own answers, byte for byte.""" + for path, expected in ( + (COMPARE_DEAD, COMPARE_DEAD_SHA256), + (COMPARE_LIVE, COMPARE_LIVE_SHA256), + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + assert digest == expected, ( + f"{path.name} no longer matches the captured compare response; " + "re-fetch it with gh api rather than editing it" + ) + + +def test_the_omn15536_outage_pin_is_unreachable_by_the_real_predicate() -> None: + """Replay the outage against the predicate, not against a hardcoded string. + + The shape-only validators that shipped before OMN-15538 passed ``879d6fc6`` + because it is 40 hex characters. What actually mattered was GitHub's answer + to a question they never asked, and this is that answer: ``status: + diverged`` -- the ref is reachable from no protected branch. + + Driving ``status_is_reachable`` over the captured payload keeps the test + hermetic while still exercising the predicate CI runs, and the ``behind`` + control below is what stops a blanket-reject implementation passing. + """ + module = _pin_reachability() + dead = json.loads(COMPARE_DEAD.read_text(encoding="utf-8")) + + assert dead["status"] == "diverged", ( + "the captured payload is supposed to be the UNREACHABLE case; if GitHub " + "now answers differently the capture is stale, not the guard wrong" + ) + assert module.status_is_reachable(dead["status"]) is False, ( + "the pin-reachability predicate accepts the exact ref that wedged every " + "open omnibase_infra PR for ~2.5h on 2026-07-30 (OMN-15536)" + ) + + +def test_the_squash_that_actually_landed_is_reachable() -> None: + """Discriminating control: reject-everything must not look correct. + + ``454c429f`` is the squash commit omnimarket#1976 really produced -- the ref + #2577 repointed to. A guard that refused it too would break every legitimate + pin, so the outage case above only means something alongside this one. + """ + module = _pin_reachability() + live = json.loads(COMPARE_LIVE.read_text(encoding="utf-8")) + + assert live["status"] == "behind" + assert module.status_is_reachable(live["status"]) is True, ( + "the predicate rejects a ref that IS an ancestor of dev -- a " + "blanket-reject guard is as broken as a blanket-accept one" + ) diff --git a/tests/ci/test_integration_guard_pull_fatality.py b/tests/ci/test_integration_guard_pull_fatality.py new file mode 100644 index 0000000000..ab82e9394f --- /dev/null +++ b/tests/ci/test_integration_guard_pull_fatality.py @@ -0,0 +1,366 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""OMN-15249 — the Integration Silent-Skip Guard must die AT the image pull. + +Live defect (omnibase_infra#2492, heads ``1729c7c3`` / ``dbfc0c98``, job +``89995662337``). The ``integration-guard`` job provisioned Postgres through a +GitHub-managed ``services:`` block. ``registry-1.docker.io`` timed out, and the +run looked like this:: + + ##[command]/usr/bin/docker pull postgres:16-alpine + Error response from daemon: Get "https://registry-1.docker.io/v2/": context deadline exceeded + ##[warning]Docker pull failed with exit code 1, back off 8.077 seconds before retry. + ... + ##[error]Docker pull failed with exit code 1 + ... (checkout, setup-python-uv, migrations, curated proofs: ALL skipped) ... + /home/runner/work/_temp/....sh: line 1: uv: command not found + ##[error]Process completed with exit code 127. + +Two defects, both fixed here: + +1. **The guard emitted a verdict for a run whose container never materialized.** + ``Enforce no missing-service silent-skips`` carried a bare ``if: always()``, + so it fired past a failed container init. Its toolchain had been skipped, so + it died ``exit 127``; had ``uv`` survived, ``check_integration_skips.py`` + would have exited 2 on the missing JUnit report — either way a verdict from a + run that never provisioned Postgres. +2. **The terminal signal was misattributed.** ``exit 127`` is the LAST error in + the log, several steps removed from the registry timeout that caused it, so + triage lands on a phantom missing-binary problem and reaches for "transient". + +The fix moves the pull out of the GitHub-managed ``services:`` block and into an +explicit, bounded-retry, fail-closed step this repo owns, so the pull failure is +fatal *at the pull*, names registry/image/timeout, and every downstream step +(including the verdict) is unreachable. + +Test posture: the pull script is executed for real as a bash subprocess against +a stubbed ``docker`` on ``PATH`` (a simulated registry timeout, not a source-text +grep), and the step graph is replayed through a GitHub-``if``-semantics +simulator that fails closed on any condition form it does not understand. +""" + +from __future__ import annotations + +import os +import re +import stat +import subprocess +from pathlib import Path +from typing import Any + +import pytest +import yaml + +pytestmark = pytest.mark.unit + +REPO_ROOT = Path(__file__).resolve().parents[2] +CI_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci.yml" + +GUARD_JOB_ID = "integration-guard" +PULL_STEP_ID = "pull_postgres" +START_STEP_ID = "start_postgres" +PROOFS_STEP_ID = "run_curated_proofs" +ENFORCE_STEP_ID = "enforce_no_silent_skips" +UPLOAD_STEP_ID = "upload_guard_results" +CLEANUP_STEP_ID = "stop_postgres" + +# Tokens whose presence in an executed step's shell body means that step depends +# on a toolchain the pull/setup steps install. Reaching one of these after a +# failed pull is exactly how the live run produced ``exit 127``. +_TOOLCHAIN_TOKENS = ("uv ", "uv\n", "pytest", "python ", "python\n") + + +def _load_ci_workflow() -> dict[str, Any]: + loaded = yaml.safe_load(CI_WORKFLOW.read_text(encoding="utf-8")) + assert isinstance(loaded, dict) + return loaded + + +def _guard_job() -> dict[str, Any]: + workflow = _load_ci_workflow() + jobs = workflow["jobs"] + assert GUARD_JOB_ID in jobs, ( + f"ci.yml no longer defines the `{GUARD_JOB_ID}` job — OMN-14172's guard " + "is the surface OMN-15249 hardens." + ) + job = jobs[GUARD_JOB_ID] + assert isinstance(job, dict) + return job + + +def _steps() -> list[dict[str, Any]]: + steps = _guard_job()["steps"] + assert isinstance(steps, list) + return [step for step in steps if isinstance(step, dict)] + + +def _step_by_id(step_id: str) -> dict[str, Any]: + for step in _steps(): + if step.get("id") == step_id: + return step + raise AssertionError( + f"`{GUARD_JOB_ID}` has no step with id `{step_id}`. OMN-15249 requires " + "the Postgres pull/start/proof/verdict steps to be individually " + "addressable so the verdict can be gated on the proof having run." + ) + + +# --------------------------------------------------------------------------- +# 1. The pull must be a step this repo owns. +# --------------------------------------------------------------------------- + + +def test_guard_does_not_delegate_the_pull_to_a_services_block() -> None: + """A ``services:`` image pull happens in GitHub's ``Initialize containers``. + + That step is not addressable from the workflow: its failure cannot be given + a named message, its retry budget cannot be bounded by this repo, and — as + #2492 proved — steps carrying ``if: always()`` still fire past it. Owning the + pull is what makes every other assertion in this module enforceable. + """ + job = _guard_job() + assert "services" not in job, ( + "`integration-guard` still provisions Postgres via a GitHub-managed " + "`services:` block. OMN-15249 requires an explicit, fail-closed pull " + "step so a registry timeout terminates the job at the pull with a named " + "error instead of surfacing as a downstream `exit 127`." + ) + + +def test_pull_is_the_first_step_and_cannot_be_softened() -> None: + steps = _steps() + assert steps[0].get("id") == PULL_STEP_ID, ( + "The Postgres pull must be the FIRST step: nothing after a failed pull " + f"can be trusted. Got first step id={steps[0].get('id')!r}." + ) + for step_id in (PULL_STEP_ID, START_STEP_ID): + step = _step_by_id(step_id) + assert "continue-on-error" not in step, ( + f"step `{step_id}` sets continue-on-error — the OMN-15249 defect " + "class is precisely 'container failure downgraded to a warning'." + ) + + +def test_pull_retry_budget_is_bounded_and_declared() -> None: + job_env = _guard_job()["env"] + attempts = int(str(job_env["GUARD_PG_PULL_ATTEMPTS"])) + timeout_seconds = int(str(job_env["GUARD_PG_PULL_TIMEOUT_SECONDS"])) + assert 1 <= attempts <= 5, ( + f"pull attempts must be bounded and small; got {attempts}. An unbounded " + "retry re-creates the warn-and-continue failure mode as a hang." + ) + assert 0 < timeout_seconds <= 600, ( + f"per-attempt pull timeout must be bounded; got {timeout_seconds}." + ) + assert str(job_env["GUARD_PG_IMAGE"]) == "postgres:16-alpine" + assert str(job_env["GUARD_PG_REGISTRY"]) == "registry-1.docker.io" + + +# --------------------------------------------------------------------------- +# 2. Behavioral: run the real pull script against a simulated registry timeout. +# --------------------------------------------------------------------------- + + +def _write_docker_stub(bin_dir: Path, *, exit_code: int, message: str) -> Path: + """Install a ``docker`` stub on PATH that records every invocation.""" + attempt_log = bin_dir / "attempts.log" + stub = bin_dir / "docker" + stub.write_text( + "#!/usr/bin/env bash\n" + f'echo "$@" >> "{attempt_log}"\n' + f'echo "{message}" >&2\n' + f"exit {exit_code}\n", + encoding="utf-8", + ) + stub.chmod(stub.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + return attempt_log + + +def _run_pull_script(tmp_path: Path, *, docker_exit_code: int) -> tuple[int, str, int]: + """Execute the workflow's real pull ``run:`` body; return (rc, output, attempts).""" + job_env = _guard_job()["env"] + script = tmp_path / "pull_step.sh" + script.write_text(_step_by_id(PULL_STEP_ID)["run"], encoding="utf-8") + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + attempt_log = _write_docker_stub( + bin_dir, + exit_code=docker_exit_code, + message=( + 'Error response from daemon: Get "https://registry-1.docker.io/v2/": ' + "context deadline exceeded (Client.Timeout exceeded while awaiting headers)" + ), + ) + + env = dict(os.environ) + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + # Take the real configured values from the workflow, except the inter-attempt + # backoff: boundedness is the property under test, wall-clock sleep is not. + for key in ( + "GUARD_PG_IMAGE", + "GUARD_PG_REGISTRY", + "GUARD_PG_PULL_ATTEMPTS", + "GUARD_PG_PULL_TIMEOUT_SECONDS", + ): + env[key] = str(job_env[key]) + env["GUARD_PG_PULL_BACKOFF_SECONDS"] = "0" + env["GITHUB_OUTPUT"] = str(tmp_path / "github_output") + + completed = subprocess.run( + ["bash", str(script)], + capture_output=True, + text=True, + env=env, + timeout=120, + check=False, + ) + attempts = ( + len( + [ + line + for line in attempt_log.read_text(encoding="utf-8").splitlines() + if line + ] + ) + if attempt_log.exists() + else 0 + ) + return completed.returncode, completed.stdout + completed.stderr, attempts + + +def test_simulated_registry_timeout_fails_the_pull_step_with_a_named_error( + tmp_path: Path, +) -> None: + """DoD-1: pull failure terminates AT the pull, naming registry/image/timeout.""" + job_env = _guard_job()["env"] + rc, output, attempts = _run_pull_script(tmp_path, docker_exit_code=1) + + assert rc != 0, ( + "A docker pull that never succeeds must fail the step. Warning-and-" + "continuing is the OMN-15249 defect." + ) + assert attempts == int(str(job_env["GUARD_PG_PULL_ATTEMPTS"])), ( + f"expected exactly {job_env['GUARD_PG_PULL_ATTEMPTS']} bounded pull " + f"attempts, observed {attempts} — retry must fail closed on exhaustion." + ) + assert "::error" in output, "the terminal failure must be a GitHub error annotation" + for token in ( + str(job_env["GUARD_PG_IMAGE"]), + str(job_env["GUARD_PG_REGISTRY"]), + str(job_env["GUARD_PG_PULL_TIMEOUT_SECONDS"]), + ): + assert token in output, ( + f"the failure message must name {token!r} so triage lands on the " + "registry pull instead of a phantom missing-binary problem" + ) + + +def test_pull_step_succeeds_and_stops_retrying_when_the_registry_answers( + tmp_path: Path, +) -> None: + """The fail-closed loop must not be a permanently-red check.""" + rc, output, attempts = _run_pull_script(tmp_path, docker_exit_code=0) + assert rc == 0, f"a successful pull must exit 0; output:\n{output}" + assert attempts == 1, f"a successful pull must not retry; observed {attempts}" + + +# --------------------------------------------------------------------------- +# 3. Step-graph replay: nothing toolchain-dependent is reachable after a bad pull. +# --------------------------------------------------------------------------- + +_ALWAYS_ONLY = re.compile(r"^always\(\)$") +_ALWAYS_AND_NOT_SKIPPED = re.compile( + r"^always\(\)\s*&&\s*steps\.(?P[A-Za-z0-9_\-]+)\.conclusion\s*!=\s*'skipped'$" +) + + +def _evaluate_step_if(condition: str, conclusions: dict[str, str]) -> bool: + """Evaluate the ``if:`` forms this job is allowed to use. Fail closed.""" + condition = condition.strip() + if _ALWAYS_ONLY.match(condition): + return True + match = _ALWAYS_AND_NOT_SKIPPED.match(condition) + if match: + return conclusions.get(match.group("step"), "skipped") != "skipped" + pytest.fail( + f"`{GUARD_JOB_ID}` uses an unrecognised step `if:` form: {condition!r}. " + "OMN-15249 gates the guard verdict on step conclusions; an unmodelled " + "condition could silently re-open the warn-and-continue path, so this " + "simulator fails closed rather than guessing." + ) + + +def _simulate(failing_step_id: str | None) -> tuple[list[str], dict[str, str]]: + """Replay GitHub's step-execution semantics; return (executed ids, conclusions).""" + executed: list[str] = [] + conclusions: dict[str, str] = {} + job_failed = False + for index, step in enumerate(_steps()): + step_id = step.get("id") or f"__unnamed_{index}" + condition = step.get("if") + runs = ( + (not job_failed) + if condition is None + else _evaluate_step_if(str(condition), conclusions) + ) + if not runs: + conclusions[step_id] = "skipped" + continue + executed.append(step_id) + if step_id == failing_step_id: + conclusions[step_id] = "failure" + job_failed = True + else: + conclusions[step_id] = "success" + return executed, conclusions + + +def test_no_toolchain_step_is_reachable_after_a_failed_pull() -> None: + """DoD-2: no ``exit 127`` downstream is reachable from a failed pull.""" + executed, conclusions = _simulate(failing_step_id=PULL_STEP_ID) + + assert executed[0] == PULL_STEP_ID + assert conclusions[ENFORCE_STEP_ID] == "skipped", ( + "the silent-skip verdict still runs after a failed pull — this is the " + "exact `if: always()` path that produced `uv: command not found` / " + "exit 127 on omnibase_infra#2492." + ) + assert conclusions[UPLOAD_STEP_ID] == "skipped", ( + "artifact upload still runs after a failed pull and warns 'No files were " + "found', adding noise to an already-misattributed failure." + ) + assert CLEANUP_STEP_ID in executed, ( + "container cleanup must still run after a failed pull so a partially " + "started container is never leaked." + ) + + by_id = {step.get("id"): step for step in _steps()} + for step_id in executed: + if step_id in (PULL_STEP_ID, CLEANUP_STEP_ID): + continue + body = str(by_id.get(step_id, {}).get("run", "")) + for token in _TOOLCHAIN_TOKENS: + assert token not in body, ( + f"step `{step_id}` runs after a failed pull and invokes " + f"{token.strip()!r}; the toolchain that provides it is installed " + "by a step that a failed pull skips, so this is a latent " + "exit-127 misattribution." + ) + + +def test_verdict_still_runs_when_the_curated_proofs_actually_fail() -> None: + """The OMN-14172 guard must keep firing on real integration FAILURES.""" + _, conclusions = _simulate(failing_step_id=PROOFS_STEP_ID) + assert conclusions[ENFORCE_STEP_ID] == "success", ( + "gating the verdict on container materialization must not disable it on " + "a genuine test failure — that would trade one false-green for another." + ) + assert conclusions[UPLOAD_STEP_ID] == "success" + + +def test_verdict_runs_on_the_fully_healthy_path() -> None: + executed, conclusions = _simulate(failing_step_id=None) + assert conclusions[ENFORCE_STEP_ID] == "success" + for step_id in (PULL_STEP_ID, START_STEP_ID, PROOFS_STEP_ID, CLEANUP_STEP_ID): + assert step_id in executed diff --git a/tests/ci/test_legacy_rds_fixture_contract.py b/tests/ci/test_legacy_rds_fixture_contract.py new file mode 100644 index 0000000000..278a089d23 --- /dev/null +++ b/tests/ci/test_legacy_rds_fixture_contract.py @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Static anti-vacuity contract for the OMN-15422 PostgreSQL fixture.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +pytestmark = pytest.mark.unit + +REPO_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_ROOT = REPO_ROOT / "docker" / "legacy-rds-fixture" +COMPOSE = FIXTURE_ROOT / "compose.yml" +DOCKERFILE = FIXTURE_ROOT / "Dockerfile" +MANIFEST = FIXTURE_ROOT / "fixture-manifest.json" +LEGACY_SEED = FIXTURE_ROOT / "legacy-seed.sql" +PROOF = FIXTURE_ROOT / "prove.sh" +PROOF_REPLAY_CAPTURE = ( + REPO_ROOT + / "tests" + / "fixtures" + / "omn15547" + / "legacy-rds-fixture-prove.sh.captured" +) +CUTOVER_PROOF = FIXTURE_ROOT / "cutover-proof" / "prove.sh" +CUTOVER_BOOTSTRAP = ( + REPO_ROOT + / "src" + / "omnibase_infra" + / "migration" + / "cutover" + / "sql" + / "bootstrap.sql" +) +CI_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "legacy-rds-fixture-proof.yml" +ROOT_COST_MIGRATION = ( + REPO_ROOT + / "docker" + / "migrations" + / "forward" + / "031_create_llm_call_metrics_and_cost_aggregates.sql" +) +ROOT_BASELINES_MIGRATION = ( + REPO_ROOT / "docker" / "migrations" / "forward" / "050_create_baselines_tables.sql" +) +REQUIRED_CASES = { + "mapping_ambiguity", + "checksum_conflict", + "owner_drift", + "unsafe_rls_policy", + "unsafe_view", + "unsafe_function", + "transformation_collision", + "flat_node_shape_collision", + "legacy_shape_collision", + "application_migration_ledger", + "cutover_receipts_and_rollback_boundary", +} + + +def test_fixture_manifest_is_synthetic_complete_and_discriminating() -> None: + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + + assert manifest["ticket"] == "OMN-15422" + assert manifest["postgres_major"] == 16 + assert manifest["provenance"]["live_database_read"] is False + assert manifest["sanitization"]["customer_data"] is False + assert manifest["sanitization"]["credentials"] is False + assert set(manifest["database_names"]) >= { + "omnibase_infra", + "omnidash_analytics", + "omninode_cloud", + } + assert set(manifest["ledger_shapes"]) == { + "migration_id_checksum_source_set", + "version_nullable_checksum", + "filename_applied_at", + } + + cases = {case["id"]: case for case in manifest["cases"]} + assert set(cases) == REQUIRED_CASES + for case_id, case in cases.items(): + assert case["positive_fixture"], case_id + assert case["red_fixture"], case_id + assert case["detector"], case_id + assert case["expected_red_signature"], case_id + + +def test_fixture_builds_all_paths_from_postgresql_16_without_credentials() -> None: + compose = yaml.safe_load(COMPOSE.read_text(encoding="utf-8")) + services = compose["services"] + assert set(services) == {"fresh-postgres", "legacy-postgres", "proof"} + + assert not any("ports" in service for service in services.values()) + for name in ("fresh-postgres", "legacy-postgres"): + service = services[name] + assert service["build"]["context"] == "../.." + assert service["build"]["dockerfile"] == ( + "docker/legacy-rds-fixture/Dockerfile" + ) + assert service["build"]["target"] == name.removesuffix("-postgres") + assert service["environment"] == {"POSTGRES_HOST_AUTH_METHOD": "trust"} + + proof = services["proof"] + assert proof["build"]["target"] == "proof" + assert proof["depends_on"]["fresh-postgres"]["condition"] == "service_healthy" + assert proof["depends_on"]["legacy-postgres"]["condition"] == "service_healthy" + + dockerfile = DOCKERFILE.read_text(encoding="utf-8") + assert dockerfile.count("FROM postgres:16-alpine") == 3 + assert "PASSWORD" not in dockerfile.upper() + + compose_text = COMPOSE.read_text(encoding="utf-8").upper() + assert "POSTGRES_PASSWORD" not in compose_text + assert "PASSWORD:" not in compose_text + + +def test_legacy_seed_reproduces_the_named_catalog_collision_classes() -> None: + sql = LEGACY_SEED.read_text(encoding="utf-8") + for database in ("omnibase_infra", "omnidash_analytics", "omninode_cloud"): + assert database in sql + for role in ("omninodeadmin", "role_omnidash", "app_dashboard", "onex_api"): + assert role in sql + for relation in ( + "schema_migrations", + "llm_cost_aggregates", + "baselines_comparisons", + "tenant_usage_legacy", + ): + assert relation in sql + for dependency in ( + "CREATE VIEW", + "CREATE FUNCTION", + "CREATE INDEX", + "REFERENCES", + "ENABLE ROW LEVEL SECURITY", + "ALTER DEFAULT PRIVILEGES", + ): + assert dependency in sql + for tenant_value in ( + "legacy-acme", + "omninode", + "00000000-0000-0000-0000-000000000000", + ): + assert tenant_value in sql + + +def test_proof_runs_real_migrations_twice_and_pins_the_blocked_upgrade() -> None: + proof = PROOF.read_text(encoding="utf-8") + replay = PROOF_REPLAY_CAPTURE.read_text(encoding="utf-8") + assert proof.count("run-forward-migrations.sh") >= 1 + assert "for pass in 1 2" in proof + assert "fresh-postgres" in proof + assert "legacy-postgres" in proof + assert "LEDGER_BLOCKER=" not in proof + assert "fixture_status=PASS blocker=none" in proof + assert "fixture_case=legacy_upgrade status=PASS" in proof + assert "Sentinel set. Migration gate will report HEALTHY." in proof + assert "second pass was not idempotent" in proof + assert "platform_catalog.schema_migrations" in proof + assert "fixture_case=application_ledger_fresh" in proof + assert "fixture_case=application_ledger_legacy" in proof + assert "selected_oid_preserved=true" in proof + assert "estimated_coverage_pct" in ROOT_COST_MIGRATION.read_text(encoding="utf-8") + assert ( + "legacy shape reconciliation: llm_cost_aggregates" + in ROOT_COST_MIGRATION.read_text(encoding="utf-8") + ) + assert ( + "ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS estimated_coverage_pct" + in ROOT_COST_MIGRATION.read_text(encoding="utf-8") + ) + root_cost_migration = ROOT_COST_MIGRATION.read_text(encoding="utf-8") + assert "ALTER TABLE llm_cost_aggregates ADD COLUMN IF NOT EXISTS id UUID;" in ( + root_cost_migration + ) + assert ( + "ALTER TABLE llm_cost_aggregates ALTER COLUMN id SET DEFAULT gen_random_uuid()" + in (root_cost_migration) + ) + assert "LIMIT 10000" in root_cost_migration + baselines_migration = ROOT_BASELINES_MIGRATION.read_text(encoding="utf-8") + assert "legacy shape reconciliation: baselines_comparisons" in baselines_migration + assert ( + "ALTER TABLE baselines_comparisons ADD COLUMN IF NOT EXISTS comparison_date" + in baselines_migration + ) + assert "DROP COLUMN IF EXISTS id CASCADE" in baselines_migration + assert "operator data mapping required" in baselines_migration + assert "fixture_case=legacy_upgrade status=BLOCKED" in replay + assert "fixture_status=PASS_WITH_EXPECTED_BLOCKER blocker=OMN-15423" in replay + dockerfile = DOCKERFILE.read_text(encoding="utf-8") + assert "ledger-control/" in dockerfile + assert "ledger-control/forward/fenced-node-migrations.yaml" in dockerfile + assert ( + "ledger-control/forward/grandfathered-force-rls-migrations.yaml" in dockerfile + ) + assert "ledger-control/forward/_ledger/bootstrap.sql" in dockerfile + + +def test_required_ci_executes_rebuilt_fixture_and_always_cleans_it() -> None: + workflow = yaml.safe_load(CI_WORKFLOW.read_text(encoding="utf-8")) + job = workflow["jobs"]["fixture-proof"] + assert "needs" not in job + assert "OMNI_RUNNER_SELECTOR_V1" in CI_WORKFLOW.read_text(encoding="utf-8") + assert "OMNI_DOCKER_CI_RUNS_ON_JSON" in job["runs-on"] + assert "OMNI_TRUSTED_CI_RUNS_ON_JSON" in job["runs-on"] + steps = job["steps"] + proof = next( + step + for step in steps + if step.get("name") == "Sanitized legacy-RDS Docker proof" + ) + assert "docker compose" in proof["run"] + assert "--build" in proof["run"] + assert "--exit-code-from proof" in proof["run"] + + cleanup = next( + step + for step in steps + if step.get("name") == "Clean sanitized legacy-RDS Docker proof" + ) + assert cleanup["if"] == "always()" + assert "down --volumes --remove-orphans" in cleanup["run"] + + +def test_cutover_extension_is_durable_and_red_proven_in_the_rebuilt_image() -> None: + proof = CUTOVER_PROOF.read_text(encoding="utf-8") + bootstrap = CUTOVER_BOOTSTRAP.read_text(encoding="utf-8") + outer_proof = PROOF.read_text(encoding="utf-8") + dockerfile = DOCKERFILE.read_text(encoding="utf-8") + + assert "cutover-proof/prove.sh" in outer_proof + assert "cutover-proof/prove.sh" in dockerfile + assert "migration/cutover/sql/bootstrap.sql" in dockerfile + for table in ( + "cutover_family_contracts", + "transformation_receipts", + "cutover_journal", + "reverse_delta_proofs", + "reverse_delta_entries", + ): + assert table in bootstrap + for signature in ( + "cutover_family_mismatch_isolation", + "cutover_pre_checkpoint_dsn_rollback", + "cutover_blind_dual_write", + "cutover_post_checkpoint_direct_rollback", + "cutover_reverse_delta_coverage", + "cutover_reverse_delta_complete", + "cutover_forward_fix_only", + "cutover_durable_journal", + ): + assert signature in proof diff --git a/tests/ci/test_managed_staging_proof_kit_seam.py b/tests/ci/test_managed_staging_proof_kit_seam.py new file mode 100644 index 0000000000..f5090a015b --- /dev/null +++ b/tests/ci/test_managed_staging_proof_kit_seam.py @@ -0,0 +1,297 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# Copyright (c) 2026 OmniNode Team +"""Seam test for the managed-staging proof kit. + +Tickets: OMN-15123 (frozen one-tenant contract), OMN-15124 (candidate-in-isolation +compatibility proof), OMN-15125 (Aug-5 readiness/rollback packet), OMN-10858 +(end-to-end cloud workflow proof harness). + +The seam has three sides and this test matches them field-by-field: + +1. **Ticket acceptance criteria** -- pinned here as ``REQUIRED_FIELDS`` / + ``REQUIRED_STAGES``, transcribed from the ticket bodies. This side is the + contract; it does not read the manifest, so a field silently dropped from the + manifest fails here rather than passing vacuously. +2. **The manifest** -- ``docs/runbooks/managed-staging-proof-kit/fields.yaml``: + every required field present, every field carrying a real evidence source + (a placeholder such as ``TBD`` is a failure, because "the evidence source is + named" is exactly what the tickets ask for). +3. **The rendered surfaces** -- each markdown packet template must carry one + table row per manifest field id with a non-placeholder evidence cell, and the + harness module must expose one ``stage_`` callable per manifest stage, + with ``--live`` defaulting OFF. + +Failure mode this guards: a packet that *looks* complete but leaves an evidence +source unnamed, so the go/no-go decision is made on prose instead of a readback. +""" + +from __future__ import annotations + +import importlib.util +import re +import sys +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +MANIFEST_PATH = ( + REPO_ROOT / "docs" / "runbooks" / "managed-staging-proof-kit" / "fields.yaml" +) + +# Placeholder tokens that mean "the evidence source is not actually named". +PLACEHOLDER_RE = re.compile(r"\b(TBD|TODO|FIXME|N/?A|XXX|\?{3,})\b", re.IGNORECASE) + +# --- side 1: ticket acceptance criteria, transcribed ------------------------ + +REQUIRED_FIELDS: dict[str, tuple[str, ...]] = { + # OMN-15123: "account/region/namespace, one gateway, one synthetic tenant, + # source/image/config digests, approved onex.mstg1. catalog, unique MSK + # epoch, signed start/reset policy, rollback authority, zero-prod-diff + # assertion, omnidash exclusion" + digest readback + plan-row binding. + "one_tenant_contract_freeze": ( + "aws_account", + "aws_region", + "k8s_namespace", + "gateway_endpoint", + "synthetic_tenant_id", + "source_digest", + "image_digest", + "config_digest", + "topic_catalog", + "zero_collision_readback", + "msk_epoch", + "group_start_reset_policy", + "rollback_authority", + "zero_prod_diff", + "omnidash_exclusion", + "plan_row_binding", + ), + # OMN-15124: IAM/TLS signer + token refresh, explicit bootstrap with + # auto-create off, broker/group perms, RDS verify-full, typed config + # authority, no raw endpoint fallback, dashboard zero authority, negative + # control, isolation lane, plan-row binding. + "candidate_isolation_compatibility": ( + "isolation_lane", + "msk_iam_signer", + "token_refresh_cycle", + "auto_create_off", + "explicit_topic_bootstrap", + "negative_control_out_of_catalog", + "broker_group_perms", + "rds_verify_full", + "typed_config_authority", + "no_raw_endpoint_fallback", + "dashboard_zero_authority", + "plan_row_binding", + ), + # OMN-15125: source + previous digests, linux/amd64 manifest, config/policy + # hashes, vulnerability result, A6 thresholds with live samples, staffed + # monitoring owner/actions, B12 psql readback, OMN-14772 teardown readback, + # executable rollback + the three items that convert Aug 5 from target to + # forecast (blocker graph, dated chain with slack, T20 handoff). + "aug5_readiness_rollback": ( + "source_digest", + "previous_digest", + "amd64_manifest", + "config_hash", + "policy_hash", + "vulnerability_result", + "a6_thresholds_with_live_samples", + "monitoring_owner_actions", + "b12_psql_readback", + "teardown_readback", + "executable_rollback", + "reconciled_blocker_graph", + "dated_chain_with_slack", + "t20_handoff", + "plan_row_binding", + ), +} + +# OMN-10858 reference chain: login -> tenant -> submit -> terminal readback -> +# cross-tenant denial. +REQUIRED_STAGES: tuple[str, ...] = ( + "login", + "tenant", + "submit", + "terminal_readback", + "cross_tenant_denial", +) + + +@pytest.fixture(scope="module") +def manifest() -> dict[str, Any]: + assert MANIFEST_PATH.is_file(), f"proof-kit manifest missing: {MANIFEST_PATH}" + loaded = yaml.safe_load(MANIFEST_PATH.read_text(encoding="utf-8")) + assert isinstance(loaded, dict), "manifest must be a mapping" + return loaded + + +def _packet(manifest: dict[str, Any], key: str) -> dict[str, Any]: + packets = manifest.get("packets") or {} + assert key in packets, f"manifest is missing packet '{key}'" + packet = packets[key] + assert isinstance(packet, dict), f"packet '{key}' must be a mapping" + return packet + + +def _fields(packet: dict[str, Any]) -> list[dict[str, Any]]: + fields = packet.get("fields") or [] + assert isinstance(fields, list), "packet.fields must be a list" + return [f for f in fields if isinstance(f, dict)] + + +@pytest.mark.parametrize("packet_key", sorted(REQUIRED_FIELDS)) +def test_manifest_covers_ticket_required_fields( + manifest: dict[str, Any], packet_key: str +) -> None: + """Side 1 -> side 2: every field the ticket names exists in the manifest.""" + packet = _packet(manifest, packet_key) + present = {f.get("id") for f in _fields(packet)} + missing = [fid for fid in REQUIRED_FIELDS[packet_key] if fid not in present] + assert not missing, ( + f"packet '{packet_key}' ({packet.get('ticket')}) is missing required " + f"fields: {missing}" + ) + + +@pytest.mark.parametrize("packet_key", sorted(REQUIRED_FIELDS)) +def test_every_field_names_a_real_evidence_source( + manifest: dict[str, Any], packet_key: str +) -> None: + """No field may ship with a placeholder evidence source.""" + packet = _packet(manifest, packet_key) + bad: list[str] = [] + for field in _fields(packet): + fid = str(field.get("id")) + source = str(field.get("evidence_source") or "").strip() + if not source or PLACEHOLDER_RE.search(source): + bad.append(f"{fid}={source!r}") + if not str(field.get("label") or "").strip(): + bad.append(f"{fid}=") + assert not bad, f"packet '{packet_key}' has unnamed evidence sources: {bad}" + + +@pytest.mark.parametrize("packet_key", sorted(REQUIRED_FIELDS)) +def test_template_has_a_row_per_manifest_field( + manifest: dict[str, Any], packet_key: str +) -> None: + """Side 2 -> side 3: the markdown template renders every manifest field.""" + packet = _packet(manifest, packet_key) + template_rel = str(packet.get("template") or "") + assert template_rel, f"packet '{packet_key}' declares no template" + template_path = REPO_ROOT / template_rel + assert template_path.is_file(), f"template missing: {template_path}" + text = template_path.read_text(encoding="utf-8") + + rows: dict[str, str] = {} + for line in text.splitlines(): + if not line.startswith("|"): + continue + cells = [c.strip() for c in line.strip().strip("|").split("|")] + if len(cells) < 3: + continue + rows[cells[0].strip("`")] = cells[-1] + + missing = [f["id"] for f in _fields(packet) if str(f.get("id")) not in rows] + assert not missing, ( + f"template {template_rel} is missing a table row for manifest fields: {missing}" + ) + + placeholder = [ + fid + for fid in (str(f.get("id")) for f in _fields(packet)) + if not rows[fid].strip("` ") or PLACEHOLDER_RE.search(rows[fid]) + ] + assert not placeholder, ( + f"template {template_rel} leaves the evidence-source cell unnamed for: {placeholder}" + ) + + +def _load_harness(manifest: dict[str, Any]) -> ModuleType: + script_rel = str((manifest.get("harness") or {}).get("script") or "") + assert script_rel, "manifest declares no harness script" + script_path = REPO_ROOT / script_rel + assert script_path.is_file(), f"harness script missing: {script_path}" + spec = importlib.util.spec_from_file_location( + "managed_staging_e2e_harness_under_test", script_path + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + # dataclasses resolves ``sys.modules[cls.__module__]`` while building a + # frozen dataclass, so the module must be registered before exec. + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(spec.name, None) + raise + return module + + +def test_harness_declares_the_required_stages(manifest: dict[str, Any]) -> None: + stages = [ + str(s.get("id")) + for s in ((manifest.get("harness") or {}).get("stages") or []) + if isinstance(s, dict) + ] + assert tuple(stages) == REQUIRED_STAGES, ( + "harness stage order must be the OMN-10858 chain " + f"{REQUIRED_STAGES}, got {tuple(stages)}" + ) + for stage in (manifest.get("harness") or {}).get("stages") or []: + surface = str(stage.get("surface") or "").strip() + assert surface and not PLACEHOLDER_RE.search(surface), ( + f"stage {stage.get('id')!r} does not name a real endpoint/topic surface: {surface!r}" + ) + + +def test_harness_implements_every_manifest_stage(manifest: dict[str, Any]) -> None: + module = _load_harness(manifest) + registry = getattr(module, "STAGES", None) + assert isinstance(registry, dict), "harness must expose a STAGES registry mapping" + declared = [ + str(s.get("id")) + for s in ((manifest.get("harness") or {}).get("stages") or []) + if isinstance(s, dict) + ] + assert list(registry) == declared, ( + f"harness STAGES {list(registry)} does not match manifest stages {declared}" + ) + for stage_id, fn in registry.items(): + assert callable(fn), f"stage {stage_id} is not callable" + + +def test_live_flag_defaults_off_and_dry_run_touches_nothing( + manifest: dict[str, Any], +) -> None: + """The harness must be safe to run with no flags: plan only, no I/O.""" + module = _load_harness(manifest) + parser = module.build_parser() + args = parser.parse_args([]) + assert args.live is False, "--live must default to OFF" + + plan = module.run(args) + assert plan.live is False + assert [r.stage_id for r in plan.results] == list(module.STAGES) + assert all(r.status == "PLANNED" for r in plan.results), ( + "a dry run must not execute any assertion: " + f"{[(r.stage_id, r.status) for r in plan.results]}" + ) + assert all(r.surface for r in plan.results), ( + "every planned stage must print its surface" + ) + + +def test_live_mode_fails_closed_without_config(manifest: dict[str, Any]) -> None: + """--live with no resolved config must refuse, not silently no-op.""" + module = _load_harness(manifest) + args = module.build_parser().parse_args(["--live"]) + with pytest.raises(module.HarnessConfigError): + module.run(args) diff --git a/tests/ci/test_merge_hold_gate_omn15484.py b/tests/ci/test_merge_hold_gate_omn15484.py new file mode 100644 index 0000000000..81fdcc9b0b --- /dev/null +++ b/tests/ci/test_merge_hold_gate_omn15484.py @@ -0,0 +1,274 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""OMN-15484: the Merge Hold Gate must be wired into a REQUIRED context here. + +OMN-15483 built a required check that fails while a PR carries a hold marker, so +the merge sweep cannot land it inside the adversarial-verification window — and +shipped it in ``omnimarket`` only. ``omnibase_infra`` carries incident §C (#2560) +in that ticket's table and had zero coverage. + +The fan-out adds a ``merge-hold-gate`` job calling the shared reusable workflow +in omnimarket, plus one entry in ``STRICT_GATE_JOBS``. Two things are proven +here, and they are different in kind: + +* **The strict registration behaves as claimed** — by driving this repo's REAL + ``ci_summary_gate.evaluate()`` over each possible job result. Registration, + not existence, is the mechanism: the poller's default-deny sweep already + catches a hold job that FAILS, so a test that only checked ``failure`` would + pass identically with the registration deleted. The load-bearing vectors are + ``skipped`` and ``absent``, and each is asserted against its unregistered + control so the assertion cannot be vacuous. +* **The ci.yml wiring produces the exact string that is registered** — the + check-run for a reusable call is `` / ``. If either half is renamed the required ``CI Summary`` context + goes permanently PENDING, which is a repo-wide outage, not a soft failure. + +Not proven here, deliberately: the hold vocabulary itself, and that a held title +produces exit 1. Those live in omnimarket — one definition, fleet-wide — and are +re-proven live in THIS repo's CI on every run by the shared workflow's own +self-proof step. Asserting them here would require a local copy of the +vocabulary, which is precisely what OMN-15484 AC1 forbids. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from scripts.ci.ci_summary_gate import ( + EXIT_FAILURE, + EXIT_PENDING, + EXIT_SUCCESS, + SKIPPABLE_GATE_JOBS, + STRICT_GATE_JOBS, + evaluate, +) + +pytestmark = pytest.mark.unit + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_CI_YAML = _REPO_ROOT / ".github" / "workflows" / "ci.yml" + +_JOB_ID = "merge-hold-gate" +_INNER_JOB_ID = "evaluate" +_CONTEXT = f"{_JOB_ID} / {_INNER_JOB_ID}" +_REUSABLE = "OmniNode-ai/omnimarket/.github/workflows/merge-hold-gate-reusable.yml" + + +def _job( + name: str, conclusion: str | None, *, status: str = "completed" +) -> dict[str, object]: + return { + "name": name, + "status": status, + "conclusion": conclusion, + "run_attempt": 1, + } + + +def _all_gates_green() -> list[dict[str, object]]: + """Every strict + skippable gate present and successful.""" + return [_job(g, "success") for g in (*STRICT_GATE_JOBS, *SKIPPABLE_GATE_JOBS)] + + +def _with_hold( + conclusion: str | None, *, status: str = "completed" +) -> list[dict[str, object]]: + """A fully green snapshot in which only the hold gate has the given state.""" + jobs = [j for j in _all_gates_green() if j["name"] != _CONTEXT] + jobs.append(_job(_CONTEXT, conclusion, status=status)) + return jobs + + +def _without_hold() -> list[dict[str, object]]: + """A fully green snapshot in which the hold gate never reported at all.""" + return [j for j in _all_gates_green() if j["name"] != _CONTEXT] + + +# --------------------------------------------------------------------------- +# AC2 — strict registration, driven through the real evaluator +# --------------------------------------------------------------------------- + + +class TestStrictRegistration: + def test_the_hold_gate_is_registered_strict(self) -> None: + assert _CONTEXT in STRICT_GATE_JOBS + assert _CONTEXT not in SKIPPABLE_GATE_JOBS, ( + "a skippable slot accepts `skipped` as good, which is the exact " + "bypass this registration exists to close" + ) + + def test_green_hold_gate_is_success(self) -> None: + code, report = evaluate(_with_hold("success")) + assert code == EXIT_SUCCESS, report + + def test_skipped_hold_gate_is_failure(self) -> None: + """The load-bearing vector. + + The job is unconditional, so a `skipped` conclusion means something went + wrong — not that the gate legitimately opted out. Unregistered, this is + SUCCESS (see the control below), and a held PR is required-green. + """ + code, report = evaluate(_with_hold("skipped")) + assert code == EXIT_FAILURE, report + assert _CONTEXT in report + + def test_absent_hold_gate_is_pending_not_success(self) -> None: + """Deleting the job must not silently re-open the gap. + + PENDING blocks the merge without asserting a false green. Unregistered, + an absent job is invisible to the poller and CI Summary reports SUCCESS. + """ + code, report = evaluate(_without_hold()) + assert code == EXIT_PENDING, report + assert _CONTEXT in report + + @pytest.mark.parametrize("conclusion", ["failure", "cancelled"]) + def test_non_success_conclusions_fail(self, conclusion: str) -> None: + code, report = evaluate(_with_hold(conclusion)) + assert code == EXIT_FAILURE, report + + def test_incomplete_hold_gate_is_pending(self) -> None: + """Still running is not yet clear — the completeness anchor holds.""" + code, _ = evaluate(_with_hold(None, status="in_progress")) + assert code == EXIT_PENDING + + # -- the controls that make the above non-vacuous ------------------------ + + def test_control_unregistered_skipped_would_pass(self) -> None: + """RED-before, expressed as a control rather than a claim. + + Same real evaluator, same snapshot, with the hold gate removed from the + strict tuple: a skipped hold gate yields SUCCESS. This is the state + omnibase_infra was in before this PR, and it is what makes the + `skipped -> FAILURE` assertion above evidence rather than decoration. + """ + unregistered = tuple(g for g in STRICT_GATE_JOBS if g != _CONTEXT) + code, _ = evaluate(_with_hold("skipped"), strict_gates=unregistered) + assert code == EXIT_SUCCESS + + def test_control_unregistered_absent_would_pass(self) -> None: + unregistered = tuple(g for g in STRICT_GATE_JOBS if g != _CONTEXT) + code, _ = evaluate(_without_hold(), strict_gates=unregistered) + assert code == EXIT_SUCCESS + + def test_control_unregistered_failure_still_fails(self) -> None: + """Why `failure` alone would have been a worthless test. + + The default-deny sweep catches any present+completed non-good job, so + `failure` fails with OR without the registration. A test suite that only + covered `failure` would pass unchanged if someone deleted the strict + entry. + """ + unregistered = tuple(g for g in STRICT_GATE_JOBS if g != _CONTEXT) + code, _ = evaluate(_with_hold("failure"), strict_gates=unregistered) + assert code == EXIT_FAILURE + + def test_the_hold_gate_is_not_soft_allowlisted(self) -> None: + """The allowlist is prefix-aware, so a caller-segment entry would cover it.""" + from scripts.ci.ci_summary_gate import SOFT_ALLOWLIST, _is_allowlisted + + assert not _is_allowlisted(_CONTEXT, SOFT_ALLOWLIST) + + +# --------------------------------------------------------------------------- +# The ci.yml wiring must produce exactly the registered string +# --------------------------------------------------------------------------- + + +class TestCiWiring: + @staticmethod + def _workflow() -> dict[str, Any]: + return dict(yaml.safe_load(_CI_YAML.read_text(encoding="utf-8"))) + + @staticmethod + def _hold_job() -> dict[str, Any]: + jobs = TestCiWiring._workflow()["jobs"] + assert _JOB_ID in jobs, f"{_JOB_ID} is not a job in ci.yml" + return dict(jobs[_JOB_ID]) + + def test_the_job_key_is_the_first_segment_of_the_registered_context(self) -> None: + """A `name:` here would override the job id and unwire the registration. + + The composed check-run is ` / `; with no `name:` the caller's display name IS its job id. This is + the trap already documented on `deploy-agent-tests`, made mechanical. + """ + assert _CONTEXT.split(" / ")[0] == _JOB_ID + assert "name" not in self._hold_job(), ( + "adding a `name:` changes the check-run context and makes the " + "required CI Summary gate permanently PENDING" + ) + + def test_the_job_is_unconditional(self) -> None: + """AC4: no upstream may cascade-skip the hold gate.""" + job = self._hold_job() + assert "needs" not in job + assert "if" not in job + + def test_the_job_calls_the_shared_gate(self) -> None: + """AC1: no local re-implementation and therefore no local vocabulary.""" + job = self._hold_job() + assert job["uses"].startswith(f"{_REUSABLE}@") + assert "steps" not in job + + def test_the_declared_context_name_matches_the_registration(self) -> None: + """AC5 seam: the string is validated in the OTHER repo. + + `context_name` is handed to omnimarket's workflow, which checks it + against the canonical vocabulary. If it is not the context GitHub really + mints, the remote guard validates a name that does not exist while the + real one goes unchecked — and the poller waits forever for a context + nobody produces. + """ + assert self._hold_job()["with"]["context_name"] == _CONTEXT + + def test_the_workflow_and_vocabulary_refs_are_the_same(self) -> None: + """Split-brain guard: gate logic and tokens must be one vintage. + + `uses:` selects the workflow FILE; `vocabulary_ref` selects the SOURCE + it reads. Drifting them runs vintage-X logic against vintage-Y tokens. + """ + job = self._hold_job() + assert job["with"]["vocabulary_ref"] == job["uses"].split("@", 1)[1] + + def test_the_pin_is_immutable_or_mainline(self) -> None: + """A feature-branch pin breaks this repo when that branch is deleted.""" + ref = self._hold_job()["uses"].split("@", 1)[1] + assert ref in {"dev", "main"} or re.fullmatch(r"[0-9a-f]{40}", ref), ( + f"the shared gate is pinned at {ref!r}; use a 40-hex SHA or a " + "mainline ref so a squash-merge branch deletion cannot wedge CI here" + ) + + def test_no_hold_vocabulary_is_declared_in_this_repository(self) -> None: + """The AC1 property this repo owns, as a fast local echo. + + The enforcing surface is the shared workflow's scan, which runs on every + CI run here. This test uses the identifier rule only — it deliberately + carries no token list, so it cannot itself become the second vocabulary + it exists to forbid. + """ + offenders: list[str] = [] + for root in (_REPO_ROOT / "src", _REPO_ROOT / "scripts"): + if not root.is_dir(): + continue + for path in root.rglob("*.py"): + for line in path.read_text( + encoding="utf-8", errors="ignore" + ).splitlines(): + normalized = line.upper().replace("-", "_").replace(" ", "") + if "RE.COMPILE" not in normalized: + continue + if any( + fragment in normalized + for fragment in ("DO_NOT_MERGE", "HOLD_MARKER") + ): + offenders.append(f"{path.relative_to(_REPO_ROOT)}: {line!r}") + assert offenders == [], ( + "a hold vocabulary is declared in this repository; it must live only " + f"in omnimarket and be read through the shared gate: {offenders}" + ) diff --git a/tests/ci/test_node_migration_shape_reconciliation.py b/tests/ci/test_node_migration_shape_reconciliation.py new file mode 100644 index 0000000000..61fd2718fc --- /dev/null +++ b/tests/ci/test_node_migration_shape_reconciliation.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Static gate: every guarded CREATE TABLE must reconcile its own shape. + +## The class this closes + +``CREATE TABLE IF NOT EXISTS t (...)`` SILENTLY NO-OPS when a table named ``t`` +already exists with a DIFFERENT shape. The statements that follow it in the same +file are not so forgiving: ``CREATE INDEX IF NOT EXISTS ... ON t (col)`` guards +the index NAME, not the COLUMN, so it raises ``column "col" does not exist`` and +``ON_ERROR_STOP=1`` kills the whole migration Job at that point. + +Because the runner halts at the FIRST failure, instances of this class surface +strictly one per deploy cycle. Two of them each cost a full cycle to discover: + +* OMN-15376 — ``llm_cost_aggregates.aggregation_key``, deploy-onex-dev run + 30418878385, ``0001_create_llm_cost_aggregates.sql:64``. +* OMN-15302 — ``baselines_comparisons.snapshot_id``, + ``0001_create_baselines_tables.sql:76``. + +## What this asserts + +For every vendored node migration, every column declared by a +``CREATE TABLE IF NOT EXISTS`` must ALSO be covered by a guarded +``ALTER TABLE ADD COLUMN IF NOT EXISTS ...`` in the SAME file. Those +adds are no-ops on the fresh-create path and converge a drifted pre-existing +table on the drifted path, so both paths end at the same schema without +dropping or recreating anything. + +This is the STATIC half of the gate and runs with no database. The EXECUTION +half — RED against the drifted shape, GREEN after, and schema equality between +the two paths — is +``tests/integration/migrations/test_node_migration_shape_drift_omn15376.py``. + +## Fenced ids + +The operator-fenced ids (OMN-14974 / OMN-15313 / OMN-15335) are exempt: they are +never applied and MUST NOT be edited to satisfy a gate. The exemption list is +READ FROM ``scripts/run-forward-migrations.sh`` rather than restated, so this +gate cannot drift from the runner it is protecting. Note that the runner in this +repo fences SIX ids while omninode_infra's k8s Job runner fences SEVEN — that +parity gap is a separate finding (see the execution suite's ``_K8S_ONLY_FENCED`` +note), not something this gate papers over. + +Ticket: OMN-15376 +""" + +from __future__ import annotations + +import re + +import pytest + +from tests.helpers.util_migration_shape import ( + fenced_migration_ids, + guarded_create_tables, + mask_literals, + node_migration_files, + reconciled_columns, +) + +pytestmark = [pytest.mark.unit] + +_BEGIN = "-- ---- BEGIN OMN-15376 shape reconciliation:" +_END = "-- ---- END OMN-15376 shape reconciliation:" + +_FENCED = fenced_migration_ids() +_UNFENCED = [ + (migration_id, path) + for migration_id, path in node_migration_files() + if migration_id not in _FENCED +] +# Only files that actually declare a guarded CREATE TABLE carry the obligation. +# Parametrising over these (rather than skipping the rest) keeps every emitted +# case load-bearing: a skip is not a pass. +_CASES = [ + (migration_id, path) + for migration_id, path in _UNFENCED + if guarded_create_tables(path.read_text(encoding="utf-8")) +] + + +def test_the_corpus_is_non_empty_and_the_fence_resolved() -> None: + """Anti-vacuity: an empty corpus or an unread fence would pass everything. + + The fence COUNT is deliberately not pinned here — that is + ``tests/scripts/test_node_migration_fence_parity.py``'s job, and duplicating + the number would create a second place to update. This only proves the fence + was actually parsed out of the runner and looks like node ids. + """ + assert len(_UNFENCED) >= 60, len(_UNFENCED) + assert len(_CASES) >= 40, len(_CASES) + assert _FENCED, "operator fence parsed as empty — the reader is broken" + assert all(fenced.startswith("node:") for fenced in _FENCED), sorted(_FENCED) + + +@pytest.mark.parametrize( + ("migration_id", "path"), _CASES, ids=[case[0] for case in _CASES] +) +def test_guarded_create_table_reconciles_every_declared_column( + migration_id: str, path: object +) -> None: + """Each declared column must have a guarded ADD COLUMN in the same file.""" + sql = path.read_text(encoding="utf-8") # type: ignore[attr-defined] + tables = guarded_create_tables(sql) + assert tables, migration_id + + missing: dict[str, list[str]] = {} + for table in tables: + covered = reconciled_columns(sql, table) + gaps = [ + column.name.strip('"') + for column in table.columns + if column.name.strip('"').lower() not in covered + ] + if gaps: + missing[table.bare_name] = gaps + + assert not missing, ( + f"{migration_id}: CREATE TABLE IF NOT EXISTS no-ops against a drifted " + f"pre-existing table, and these declared columns are never reconciled, " + f"so the next column-dependent statement fails the whole deploy " + f"(OMN-15376 class): {missing}. Add, immediately after the CREATE TABLE, " + f"one 'ALTER TABLE ADD COLUMN IF NOT EXISTS [DEFAULT ...];' " + f"per declared column." + ) + + +@pytest.mark.parametrize( + ("migration_id", "path"), _UNFENCED, ids=[case[0] for case in _UNFENCED] +) +def test_reconciliation_never_reintroduces_a_drop( + migration_id: str, path: object +) -> None: + """The reconciliation idiom must not smuggle in a data-destroying DROP. + + ``0002_realign_child_tables_to_producer_schema.sql`` legitimately carries a + ``DROP TABLE IF EXISTS`` under an explicit, ticketed zero-rows ruling + (OMN-14513). What must never appear is a DROP inside an OMN-15376 + reconciliation block, because that block runs on tables whose row count is + unknown. + """ + sql = path.read_text(encoding="utf-8") # type: ignore[attr-defined] + for block in re.findall(re.escape(_BEGIN) + r".*?" + re.escape(_END), sql, re.S): + # Comments are masked out first: the block's own header says "no DROP, + # no recreate, no TRUNCATE", and matching that prose would be a + # self-inflicted false positive. + upper = mask_literals(block).upper() + for forbidden in ("DROP TABLE", "DROP COLUMN", "TRUNCATE", "DELETE FROM"): + assert forbidden not in upper, ( + f"{migration_id}: '{forbidden}' inside an OMN-15376 " + f"reconciliation block. That block runs against tables whose row " + f"count is unknown; converge the shape, never destroy the data." + ) diff --git a/tests/ci/test_occ_eval_path_trigger_coverage.py b/tests/ci/test_occ_eval_path_trigger_coverage.py new file mode 100644 index 0000000000..549d536b0c --- /dev/null +++ b/tests/ci/test_occ_eval_path_trigger_coverage.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Regression guard: occ-preflight EVAL-path caller workflows must listen for +the `edited` PR event, or a stamp-only `Evidence-Source:` body edit never +re-validates (OMN-14241). + +Sibling gate to ``test_occ_born_path_trigger_coverage.py`` (OMN-14987), which +guards the born-path publishers (``call-occ-autobind.yml`` / +``call-occ-companion-effect.yml``) against missing ``reopened`` / +``ready_for_review``. This module guards the EVAL-path consumers -- the +workflows whose ``occ-preflight`` job actually gates merge by reading +``Evidence-Source:`` out of the PR body -- against missing ``edited``. + +Live-root-caused 2026-08-09 (mergesweep-0809-infraunblock) on infra#2696 and +#2694: both PRs added their `Evidence-Source: OCC#` line via a body-only +edit *after* opening. ``ci.yml`` (no ``types:`` key -> GitHub's implicit +``[opened, synchronize, reopened]``) and ``hostile-reviewer.yml`` +(``types: [opened, synchronize, reopened]``) each declare a top-level job +literally named ``occ-preflight`` that calls the canonical reusable +``occ-preflight.yml``. Neither reruns on a body-only edit, so the +`occ-preflight / eligibility` check stays FAILURE from the pre-stamp run +forever -- even though the reusable workflow itself already live-fetches the +current PR body via `gh pr view` (a frozen-event-payload read is only a +`gh pr view` API-failure fallback, never the primary path). This is the same +"detection shelf structurally blind" failure mode as OMN-14987: a missing +trigger type produces silence (no new run), not a red check that would +prompt investigation -- `CI Summary` (the required umbrella) just fails +closed on the stale run and never recovers without a manual empty-commit +retrigger. + +Contrast case proving the mechanism: `call-reject-skip.yml` already lists +`edited` and its nested `occ-preflight / eligibility` copy correctly +self-heals on the same body edit (live-observed on #2696: CI's copy stayed +FAILURE from 22:05:03Z; `call-reject-skip-token`'s copy re-ran and passed at +22:07:14Z after the 22:06:20Z body edit). + +Design notes (mirrors OMN-14987's guard): + +* Parses ``on.pull_request.types`` via YAML, not a line regex. +* Handles PyYAML 1.1's ``on:`` -> ``True`` resolution. +* ``test_extraction_catches_a_deliberately_underspecified_fixture`` proves + RED against exists-but-wrong (the pre-fix shape), not merely green-by- + absence (feedback_prove_red_against_exists_but_wrong). +* ``test_eval_path_workflow_set_is_current`` guards against a future third + occ-preflight-caller workflow being added without updating the guarded set + here. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +WORKFLOWS_DIR = REPO_ROOT / ".github" / "workflows" + +# `edited` is the one event this gate cares about: it is what fires when a +# PR's body is changed without a new commit (the Evidence-Source stamp +# case). The born-path gate (OMN-14987) already covers reopened/ +# ready_for_review for the *minting* workflows; this gate is scoped to the +# narrower eval-path defect. +REQUIRED_EVAL_PATH_PR_EVENT_TYPES = frozenset({"edited"}) + +# Every workflow in THIS repo that declares a top-level job literally named +# `occ-preflight` calling the canonical reusable +# `OmniNode-ai/omnibase_core/.github/workflows/occ-preflight.yml`, i.e. every +# workflow whose own `occ-preflight / eligibility` check-run gates merge via +# `CI Summary` or an equivalent required umbrella. `call-reject-skip.yml` is +# deliberately excluded: it does not declare its own `occ-preflight` job (it +# reaches occ-preflight only transitively through the omniclaude-hosted +# reusable, producing a differently-named 3-segment check context), and it +# already lists `edited` today -- it is the working contrast case cited +# above, not a workflow this gate needs to constrain. +EVAL_PATH_WORKFLOWS: tuple[str, ...] = ( + "ci.yml", + "hostile-reviewer.yml", +) + + +def _pull_request_trigger_types(workflow_path: Path) -> set[str]: + """Extract ``on.pull_request.types`` from a workflow file as a set.""" + loaded = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) + assert isinstance(loaded, dict), f"{workflow_path} did not parse to a mapping" + # PyYAML 1.1's default resolver turns an unquoted `on:` key into the + # boolean True -- check both so a formatting change can't blind this. + on_block = loaded.get("on") + if on_block is None: + on_block = loaded.get(True) + assert isinstance(on_block, dict), f"{workflow_path} has no on: mapping" + pull_request_block = on_block.get("pull_request") + assert isinstance(pull_request_block, dict), ( + f"{workflow_path} on.pull_request is not a mapping -- an unqualified " + f"`pull_request` trigger (or a bare list of types) is a different " + f"authoring style this gate does not (yet) understand; update the " + f"gate deliberately, do not let it silently pass." + ) + types = pull_request_block.get("types") + if types is None: + # Omitting `types:` entirely defaults to GitHub's implicit + # [opened, synchronize, reopened] -- still missing `edited`. Treat + # absence as the empty set rather than raising, since (unlike the + # born-path gate) an implicit-default workflow is a real, common, + # currently-broken shape in this repo (ci.yml today) and the + # assertion below must catch it, not skip it. + return set() + assert isinstance(types, list) and types, ( + f"{workflow_path} on.pull_request.types is present but empty/not a " + f"list -- fix the YAML shape." + ) + return set(types) + + +@pytest.mark.unit +def test_extraction_catches_a_deliberately_underspecified_fixture( + tmp_path: Path, +) -> None: + """Prove the check is RED against exists-but-wrong (the pre-fix trigger + shape used by hostile-reviewer.yml today), not just absent + (feedback_prove_red_against_exists_but_wrong).""" + fixture = tmp_path / "fixture.yml" + fixture.write_text( + "on:\n pull_request:\n types: [opened, synchronize, reopened]\n", + encoding="utf-8", + ) + types = _pull_request_trigger_types(fixture) + missing = REQUIRED_EVAL_PATH_PR_EVENT_TYPES - types + assert missing == {"edited"} + + +@pytest.mark.unit +def test_extraction_handles_implicit_default_types_fixture(tmp_path: Path) -> None: + """A workflow with no `types:` key at all relies on GitHub's implicit + default ([opened, synchronize, reopened]) -- still missing `edited`. + This is ci.yml's actual pre-fix shape; the gate must fail loud on it, not + treat "no types: key" as "nothing to check".""" + fixture = tmp_path / "fixture.yml" + fixture.write_text( + "on:\n pull_request:\n branches: [main, dev]\n", encoding="utf-8" + ) + types = _pull_request_trigger_types(fixture) + assert types == set() + missing = REQUIRED_EVAL_PATH_PR_EVENT_TYPES - types + assert missing == {"edited"} + + +@pytest.mark.unit +@pytest.mark.parametrize("workflow_name", EVAL_PATH_WORKFLOWS) +def test_eval_path_workflow_listens_for_edited(workflow_name: str) -> None: + """OMN-14241: a PR body edit that adds `Evidence-Source: OCC#` after + opening must re-trigger the occ-preflight eligibility check. Missing + `edited` here means the check-run from the last commit-triggered run -- + which legitimately failed before the stamp existed -- sits as the + latest, permanently-stale status for `occ-preflight / eligibility`, and + `CI Summary` fails closed on it forever (live case: infra#2696, #2694). + """ + workflow_path = WORKFLOWS_DIR / workflow_name + assert workflow_path.is_file(), f"expected workflow file at {workflow_path}" + types = _pull_request_trigger_types(workflow_path) + missing = REQUIRED_EVAL_PATH_PR_EVENT_TYPES - types + assert not missing, ( + f"{workflow_name} on.pull_request.types is missing {sorted(missing)} -- " + f"a PR body edit (e.g. adding the Evidence-Source stamp) will NEVER " + f"retrigger this workflow's occ-preflight job (OMN-14241 failure " + f"class: the stale pre-stamp FAILURE is never superseded, blocking " + f"merge indefinitely without a manual empty-commit retrigger)." + ) + + +@pytest.mark.unit +def test_eval_path_workflow_set_is_current() -> None: + """Guard against a future third occ-preflight-caller workflow (a top- + level job literally named `occ-preflight`) being added without updating + EVAL_PATH_WORKFLOWS above -- otherwise the new file is silently + unchecked by this module, repeating the OMN-14241 failure class one + layer up.""" + discovered: set[str] = set() + for path in WORKFLOWS_DIR.glob("*.yml"): + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(loaded, dict): + continue + jobs = loaded.get("jobs") + if not isinstance(jobs, dict): + continue + job = jobs.get("occ-preflight") + if not isinstance(job, dict): + continue + uses = job.get("uses", "") + if isinstance(uses, str) and "occ-preflight.yml" in uses: + discovered.add(path.name) + assert discovered == set(EVAL_PATH_WORKFLOWS), ( + f"discovered occ-preflight-caller workflows {sorted(discovered)} do " + f"not match the guarded set {sorted(EVAL_PATH_WORKFLOWS)} -- update " + f"EVAL_PATH_WORKFLOWS (and re-verify edited-trigger coverage) for " + f"the new/removed file before this gate can pass" + ) diff --git a/tests/ci/test_pin_reachability_omn15538.py b/tests/ci/test_pin_reachability_omn15538.py new file mode 100644 index 0000000000..33bf75fe54 --- /dev/null +++ b/tests/ci/test_pin_reachability_omn15538.py @@ -0,0 +1,1152 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Hermetic half of the protected-branch pin-reachability gate (OMN-15538). + +The live half — which resolves the two real incident vectors against the +GitHub API — is ``tests/integration/ci/test_pin_reachability_live_omn15538.py``, +under ``tests/integration`` for the same reasons as its OMN-14941 sibling: the +pre-push selector always ignores that tree, and the gate's enforcement surface +is CI (plus the dedicated ``Pin Reachability (OMN-15538)`` ci.yml job). + +What is pinned here, and why each assertion exists +-------------------------------------------------- +Every test below corresponds to a way one of the two 2026-07-30 incidents got +through, or to a way a naive "fix" would silently un-fix it: + +* ``status_is_reachable`` — the discriminating oracle. ``ahead`` must be + REJECTED. Instance B (``omnimarket@dev`` -> ``omnibase_core@5a907b71``) is + ``ahead`` of dev: it is a descendant commit on a still-open branch, so it + resolves today and dies on merge. Accepting ``ahead`` would make this gate + green on exactly the pin it was built for. +* ``with:`` extraction — Instance A pinned the same dead SHA twice + (``uses:`` and ``vocabulary_ref``). A ``uses:``-only checker leaves the + second one behind. +* ``pyproject`` ``[tool.uv.sources]`` extraction — the pre-existing OMN-14449 + reachability check regexes the uv.lock ``?rev=`` URL form only, so the + ``git = ...`` / ``rev = ...`` split-key form is structurally invisible to + it. That blind spot IS Instance B. +* ``--allow-undetermined`` refusal under CI — an escape hatch that survives + into the enforcing surface converts a fail-closed gate into a silent skip. +* ``--min-pins`` — a gate that extracts nothing passes forever. +""" + +from __future__ import annotations + +import email.message +import http.client +import io +import json +import time +import urllib.error +import urllib.request +from collections.abc import Callable +from pathlib import Path + +import pytest +import yaml + +from scripts.ci.check_pin_reachability import ( + PinRef, + Verdict, + _api_get, + _is_transient_http_status, + _Resolver, + extract_pins, + extract_pyproject_pins, + extract_uv_lock_pins, + extract_workflow_pins, + main, + status_is_reachable, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS_DIR = REPO_ROOT / ".github" / "workflows" + +# The exact pin that wedged omnibase_infra CI for ~2.5h on 2026-07-30 +# (OMN-15536): the head of an omnimarket PR branch, deleted on squash-merge. +INCIDENT_A_DEAD_SHA = "879d6fc6825f876458c6d45ed670c8715de8ac95" +INCIDENT_A_GOOD_SHA = "454c429f328e68e19300f33ffb8121f1bccc7f86" +# The pin live on omnimarket@dev pyproject.toml: the head of the still-open +# jonah/omn-15392-evidence-execution-scope branch. +INCIDENT_B_DEAD_SHA = "5a907b71c5cf321ed6407cd0509ec406afc81ff5" +INCIDENT_B_GOOD_SHA = "3f2998b3337e4050b4758e0dd2a0fe1061ce0d98" + + +# --------------------------------------------------------------------------- +# The oracle +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("status", "expected"), + [ + ("behind", True), # pin is an ancestor of the protected branch + ("identical", True), # pin IS the protected branch head + ("ahead", False), # unlanded descendant — Instance B's shape + ("diverged", False), # no ancestry — Instance A's shape + ("", False), + ("unknown", False), + ], +) +def test_status_is_reachable_maps_compare_status(status: str, expected: bool) -> None: + assert status_is_reachable(status) is expected + + +@pytest.mark.unit +def test_ahead_is_rejected_not_merely_diverged() -> None: + """Falsification control for the single most tempting wrong simplification. + + "Unreachable means diverged" is the intuitive reading, and it is wrong: an + unlanded feature-branch head descends from dev, so it compares ``ahead``. + A gate that only rejects ``diverged`` catches Instance A and waves through + Instance B — which is precisely the half-fix this ticket exists to prevent. + """ + assert status_is_reachable("diverged") is False + assert status_is_reachable("ahead") is False + + +# --------------------------------------------------------------------------- +# Workflow extraction +# --------------------------------------------------------------------------- + + +_WORKFLOW_FIXTURE = f"""\ +name: fixture +on: + pull_request: +# uses: OmniNode-ai/omniclaude/.github/workflows/commented-out.yml@main +jobs: + merge-hold-gate: + uses: OmniNode-ai/omnimarket/.github/workflows/merge-hold-gate-reusable.yml@{INCIDENT_A_DEAD_SHA} + with: + vocabulary_ref: {INCIDENT_A_DEAD_SHA} + context_name: merge-hold-gate / evaluate + dynamic-caller: + uses: OmniNode-ai/omnibase_core/.github/workflows/occ-preflight.yml@dev + with: + core-ref: ${{{{ github.sha }}}} + step-user: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: ./.github/actions/setup-python-uv + - uses: OmniNode-ai/onex_change_control/.github/actions/thing@main +""" + + +@pytest.mark.unit +def test_workflow_extraction_covers_uses_with_and_steps(tmp_path: Path) -> None: + fixture = tmp_path / "fixture.yml" + fixture.write_text(_WORKFLOW_FIXTURE, encoding="utf-8") + pins = extract_workflow_pins(fixture) + + assert [(p.locus, p.kind, p.repo, p.ref) for p in pins] == [ + ( + "jobs.merge-hold-gate.uses", + "workflow-uses", + "omnimarket", + INCIDENT_A_DEAD_SHA, + ), + ( + "jobs.merge-hold-gate.with.vocabulary_ref", + "workflow-with", + "omnimarket", + INCIDENT_A_DEAD_SHA, + ), + ("jobs.dynamic-caller.uses", "workflow-uses", "omnibase_core", "dev"), + ( + "jobs.step-user.steps[2].uses", + "workflow-step-uses", + "onex_change_control", + "main", + ), + ] + + +@pytest.mark.unit +def test_workflow_extraction_catches_the_sibling_ref_incident_a_pinned_twice( + tmp_path: Path, +) -> None: + """OMN-15538 AC-5: repointing only ``uses:`` leaves a second dead ref.""" + fixture = tmp_path / "fixture.yml" + fixture.write_text(_WORKFLOW_FIXTURE, encoding="utf-8") + dead = [p for p in extract_workflow_pins(fixture) if p.ref == INCIDENT_A_DEAD_SHA] + assert len(dead) == 2, "both uses: and vocabulary_ref must be extracted" + assert {p.kind for p in dead} == {"workflow-uses", "workflow-with"} + + +@pytest.mark.unit +def test_workflow_extraction_skips_expressions_commented_lines_and_third_party( + tmp_path: Path, +) -> None: + fixture = tmp_path / "fixture.yml" + fixture.write_text(_WORKFLOW_FIXTURE, encoding="utf-8") + pins = extract_workflow_pins(fixture) + # `core-ref: ${{ github.sha }}` is not a resolvable literal. + assert not any(p.locus.endswith("with.core-ref") for p in pins) + # A commented-out uses: line is not YAML structure. + assert not any("commented-out" in p.ref for p in pins) + # actions/checkout and ./local composites are out of scope. + assert all( + p.repo in {"omnimarket", "omnibase_core", "onex_change_control"} for p in pins + ) + + +@pytest.mark.unit +def test_workflow_extraction_ignores_non_workflow_yaml(tmp_path: Path) -> None: + fixture = tmp_path / "not-a-workflow.yml" + fixture.write_text("just: a mapping\n", encoding="utf-8") + assert extract_workflow_pins(fixture) == [] + + +# --------------------------------------------------------------------------- +# pyproject / uv.lock extraction +# --------------------------------------------------------------------------- + + +_PYPROJECT_FIXTURE = f"""\ +[project] +name = "fixture" +version = "0.0.0" +dependencies = [ + "requests>=2", + "omnibase-spi @ git+https://github.com/OmniNode-ai/omnibase_spi.git@deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", +] + +[project.optional-dependencies] +dev = ["omniclaude @ git+https://github.com/OmniNode-ai/omniclaude@feature/nope"] + +[dependency-groups] +lint = ["omnidash @ git+https://github.com/OmniNode-ai/omnidash.git@v1.2.3"] + +[tool.uv.sources] +omnibase-core = {{ git = "https://github.com/OmniNode-ai/omnibase_core.git", rev = "{INCIDENT_B_DEAD_SHA}" }} +omnibase-infra = {{ git = "https://github.com/OmniNode-ai/omnibase_infra.git", branch = "dev" }} +onex-change-control = {{ git = "https://github.com/OmniNode-ai/onex_change_control.git" }} +third-party = {{ git = "https://github.com/someone-else/thing.git", rev = "0123456789abcdef0123456789abcdef01234567" }} +""" + + +@pytest.mark.unit +def test_pyproject_extraction_covers_uv_sources_and_pep508(tmp_path: Path) -> None: + fixture = tmp_path / "pyproject.toml" + fixture.write_text(_PYPROJECT_FIXTURE, encoding="utf-8") + pins = extract_pyproject_pins(fixture) + + assert {(p.locus, p.repo, p.ref) for p in pins} == { + ( + "project.dependencies[1]", + "omnibase_spi", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + ), + ("project.optional-dependencies.dev[0]", "omniclaude", "feature/nope"), + ("dependency-groups.lint[0]", "omnidash", "v1.2.3"), + ("tool.uv.sources.omnibase-core.rev", "omnibase_core", INCIDENT_B_DEAD_SHA), + ("tool.uv.sources.omnibase-infra.branch", "omnibase_infra", "dev"), + } + + +@pytest.mark.unit +def test_pyproject_extraction_is_the_blind_spot_the_uv_lock_gate_has( + tmp_path: Path, +) -> None: + """Instance B lives in the split-key form the OMN-14449 regex cannot match. + + ``omnimarket/scripts/ci/check_uv_lock_pin_reachability.py`` matches only + ``git = "https://github.com//.git?rev="`` — one string + carrying both URL and rev. A ``[tool.uv.sources]`` table puts the rev in a + separate key, so that regex finds nothing. This asserts the new extractor + does not inherit the blind spot. + """ + fixture = tmp_path / "pyproject.toml" + fixture.write_text(_PYPROJECT_FIXTURE, encoding="utf-8") + revs = {p.ref for p in extract_pyproject_pins(fixture)} + assert INCIDENT_B_DEAD_SHA in revs + + +@pytest.mark.unit +def test_pyproject_extraction_ignores_foreign_org_and_default_branch_sources( + tmp_path: Path, +) -> None: + fixture = tmp_path / "pyproject.toml" + fixture.write_text(_PYPROJECT_FIXTURE, encoding="utf-8") + pins = extract_pyproject_pins(fixture) + # Not an OmniNode-ai repo — out of scope. + assert not any(p.repo == "thing" for p in pins) + # No rev/branch/tag: tracks the default branch, which cannot evaporate. + assert not any( + p.locus.startswith("tool.uv.sources.onex-change-control") for p in pins + ) + + +_UV_LOCK_FIXTURE = f"""\ +[[package]] +name = "omnibase-core" +source = {{ git = "https://github.com/OmniNode-ai/omnibase_core.git?rev={INCIDENT_B_DEAD_SHA}#{INCIDENT_B_DEAD_SHA}" }} + +[[package]] +name = "omnibase-spi" +source = {{ git = "https://github.com/OmniNode-ai/omnibase_spi.git?branch=dev#{INCIDENT_B_GOOD_SHA}" }} + +[[package]] +name = "requests" +source = {{ registry = "https://pypi.org/simple" }} +""" + + +@pytest.mark.unit +def test_uv_lock_extraction_prefers_the_resolved_commit(tmp_path: Path) -> None: + fixture = tmp_path / "uv.lock" + fixture.write_text(_UV_LOCK_FIXTURE, encoding="utf-8") + pins = extract_uv_lock_pins(fixture) + assert {(p.repo, p.ref) for p in pins} == { + ("omnibase_core", INCIDENT_B_DEAD_SHA), + # `?branch=dev` resolves to a concrete commit in the fragment; the + # fragment is the object a build actually fetches, so it is what must + # be reachable — a branch name alone would hide a since-force-pushed ref. + ("omnibase_spi", INCIDENT_B_GOOD_SHA), + } + + +@pytest.mark.unit +def test_extract_pins_dispatches_by_filename(tmp_path: Path) -> None: + (tmp_path / ".github" / "workflows").mkdir(parents=True) + (tmp_path / ".github" / "workflows" / "fixture.yml").write_text( + _WORKFLOW_FIXTURE, encoding="utf-8" + ) + (tmp_path / "pyproject.toml").write_text(_PYPROJECT_FIXTURE, encoding="utf-8") + (tmp_path / "uv.lock").write_text(_UV_LOCK_FIXTURE, encoding="utf-8") + + pins = extract_pins([tmp_path / ".github" / "workflows", tmp_path]) + kinds = {p.kind for p in pins} + assert kinds == { + "workflow-uses", + "workflow-with", + "workflow-step-uses", + "pyproject-source", + "pyproject-pep508", + "uv-lock", + } + + +# --------------------------------------------------------------------------- +# Vacuity + fail-closed posture +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_real_tree_extraction_is_nonempty() -> None: + """Guard the vacuous-green case: a glob typo makes the gate check nothing. + + This repo is known to carry cross-repo workflow pins plus git-sourced + sibling deps; if extraction collapses, the live gate passes on everything. + """ + pins = extract_pins( + [WORKFLOWS_DIR, REPO_ROOT / "pyproject.toml", REPO_ROOT / "uv.lock"] + ) + assert len(pins) >= 10, ( + f"expected >=10 cross-repo pins in this tree, got {len(pins)} — " + "the extractor is more likely broken than the tree is clean" + ) + assert {p.kind for p in pins} >= {"workflow-uses", "pyproject-source", "uv-lock"} + + +@pytest.mark.unit +def test_min_pins_fails_when_extraction_collapses(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text( + '[project]\nname = "x"\nversion = "0"\n', encoding="utf-8" + ) + assert main([str(tmp_path / "pyproject.toml"), "--min-pins", "1"]) == 1 + + +@pytest.mark.unit +def test_allow_undetermined_is_refused_under_ci( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The escape hatch must not survive into the enforcing surface. + + An ``--allow-undetermined`` that CI honours turns a fail-closed gate into + a rate-limit-shaped silent skip — the optional-input trap that makes a + check exist on paper and nowhere else. + """ + monkeypatch.setenv("CI", "true") + (tmp_path / "pyproject.toml").write_text( + '[project]\nname = "x"\nversion = "0"\n', encoding="utf-8" + ) + assert main([str(tmp_path / "pyproject.toml"), "--allow-undetermined"]) == 2 + + +@pytest.mark.unit +def test_no_pins_is_success_without_network( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("CI", raising=False) + (tmp_path / "pyproject.toml").write_text( + '[project]\nname = "x"\nversion = "0"\n', encoding="utf-8" + ) + assert main([str(tmp_path / "pyproject.toml")]) == 0 + + +@pytest.mark.unit +def test_pinref_is_orderable_for_stable_output() -> None: + """Report ordering must be deterministic so CI log diffs are meaningful.""" + a = PinRef("f.yml", "jobs.a.uses", "workflow-uses", "omnimarket", "a" * 40) + b = PinRef("f.yml", "jobs.b.uses", "workflow-uses", "omnimarket", "b" * 40) + assert sorted([b, a]) == [a, b] + + +@pytest.mark.unit +def test_verdict_values_are_stable_strings() -> None: + assert Verdict.REACHABLE.value == "REACHABLE" + assert Verdict.UNREACHABLE.value == "UNREACHABLE" + assert Verdict.UNDETERMINED.value == "UNDETERMINED" + + +# --------------------------------------------------------------------------- +# Transport retry (defect fix, 2026-08-06): one unretried 10s timeout on the +# onex_change_control @ 2dd26ade... compare call was redding essentially every +# open omnibase_infra dev PR, even though the pin is verifiably reachable +# live. ``_api_get`` must absorb a transient hiccup with a bounded retry +# instead of handing the caller a single-shot UNDETERMINED -- while a +# genuinely exhausted retry ceiling must still fail closed (this is a false-RED +# fix, not a weakening of the fail-closed gate), and a definitive 4xx must +# never be retried (it will not change on a second try; retrying it only adds +# latency to a job that runs on every PR). +# --------------------------------------------------------------------------- + + +def _http_error(code: int, message: str = "") -> urllib.error.HTTPError: + body = json.dumps({"message": message}).encode("utf-8") if message else b"{}" + return urllib.error.HTTPError( + url="https://api.github.com/x", + code=code, + msg=message or "error", + hdrs=None, # type: ignore[arg-type] + fp=io.BytesIO(body), + ) + + +class _FakeResponse: + """Minimal stand-in for the ``http.client.HTTPResponse`` context manager.""" + + def __init__(self, status: int, payload: dict[str, object]) -> None: + self.status = status + self._payload = json.dumps(payload).encode("utf-8") + + def read(self) -> bytes: + return self._payload + + def __enter__(self) -> _FakeResponse: + return self + + def __exit__(self, *exc_info: object) -> None: + return None + + +def _scripted_urlopen( + effects: list[Exception | _FakeResponse], +) -> tuple[object, list[int]]: + """Return a fake ``urllib.request.urlopen`` that replays ``effects`` in order. + + ``calls`` records one entry per invocation so tests can assert exactly how + many HTTP attempts were made -- the whole point of the retry-vs-no-retry + split. + """ + calls: list[int] = [] + + def fake_urlopen(request: object, timeout: float | None = None) -> _FakeResponse: + index = len(calls) + calls.append(index) + effect = effects[index] + if isinstance(effect, Exception): + raise effect + return effect + + return fake_urlopen, calls + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("status", "expected"), + [ + (429, True), + (500, True), + (502, True), + (503, True), + (504, True), + (404, False), + (403, False), + (400, False), + (401, False), + (410, False), + ], +) +def test_is_transient_http_status_covers_only_5xx_and_429( + status: int, expected: bool +) -> None: + assert _is_transient_http_status(status) is expected + + +@pytest.mark.unit +def test_api_get_retries_transient_timeout_then_succeeds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """(a) attempt 1 transient, attempt 2 succeeds -> the successful result, no UNDETERMINED.""" + fake_urlopen, calls = _scripted_urlopen( + [TimeoutError("timed out"), _FakeResponse(200, {"status": "behind"})] + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + slept: list[float] = [] + + status, body, detail = _api_get("https://api.github.com/x", sleep=slept.append) + + assert status == 200 + assert body == {"status": "behind"} + assert detail == "HTTP 200" + assert len(calls) == 2, "must retry exactly once after the transient timeout" + assert slept == [2.0], "one bounded backoff sleep before the successful retry" + + +@pytest.mark.unit +def test_api_get_retries_transient_http_5xx_and_429_then_succeeds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Both no-status transport errors AND HTTP 5xx/429 are retried.""" + fake_urlopen, calls = _scripted_urlopen( + [ + _http_error(503, "Service Unavailable"), + _http_error(429, "rate limited"), + _FakeResponse(200, {"status": "identical"}), + ] + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + slept: list[float] = [] + + status, body, _detail = _api_get("https://api.github.com/x", sleep=slept.append) + + assert status == 200 + assert body == {"status": "identical"} + assert len(calls) == 3 + assert slept == [2.0, 4.0], "bounded exponential backoff across two retries" + + +@pytest.mark.unit +def test_api_get_exhausts_retries_and_stays_a_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """(b) fail-closed preserved: exhausting the ceiling is still a FAILURE, never a pass.""" + fake_urlopen, calls = _scripted_urlopen( + [ + TimeoutError("timed out"), + TimeoutError("timed out"), + TimeoutError("timed out"), + ] + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + slept: list[float] = [] + + status, body, detail = _api_get("https://api.github.com/x", sleep=slept.append) + + assert status is None, "exhausted transient failure must not resolve to a pass" + assert body is None + assert "transport error" in detail + assert len(calls) == 3, "retries exactly _API_MAX_ATTEMPTS times, no more" + assert slept == [2.0, 4.0] + + +@pytest.mark.unit +def test_resolver_stays_undetermined_after_retries_exhausted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """(b) end-to-end through ``_Resolver``: exhausted retries -> UNDETERMINED, not REACHABLE. + + This is the assertion that actually matters: a caller consuming + ``_Resolver.resolve`` (as ``main()`` does) must see the same fail-closed + UNDETERMINED verdict it always has -- the retry only removes the FALSE + reds, never the true ones. + """ + fake_urlopen, calls = _scripted_urlopen( + [ + TimeoutError("timed out"), + TimeoutError("timed out"), + TimeoutError("timed out"), + ] + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + + resolution = _Resolver(("dev", "main")).resolve("omnimarket", "a" * 40) + + assert resolution.verdict is Verdict.UNDETERMINED + assert len(calls) == 3, ( + "one compare call, fully retried, is enough to fail closed -- the " + "resolver must not need to exhaust every protected branch" + ) + + +@pytest.mark.unit +def test_api_get_does_not_retry_a_definitive_404( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """(c) a definitive 4xx maps to its existing meaning immediately, no retry.""" + fake_urlopen, calls = _scripted_urlopen([_http_error(404, "Not Found")]) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + slept: list[float] = [] + + status, body, detail = _api_get("https://api.github.com/x", sleep=slept.append) + + assert status == 404 + assert body is None + assert detail == "HTTP 404: Not Found" + assert len(calls) == 1, "a definitive 404 must not be retried" + assert slept == [], "no backoff sleep for a non-retried definitive failure" + + +@pytest.mark.unit +def test_api_get_does_not_retry_a_definitive_403( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """(c) 403 without rate-limit semantics (429) is definitive, not transient.""" + fake_urlopen, calls = _scripted_urlopen([_http_error(403, "Forbidden")]) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + slept: list[float] = [] + + status, _body, _detail = _api_get("https://api.github.com/x", sleep=slept.append) + + assert status == 403 + assert len(calls) == 1, "a definitive 403 must not be retried" + assert slept == [] + + +@pytest.mark.unit +def test_api_get_backoff_is_bounded(monkeypatch: pytest.MonkeyPatch) -> None: + """(d) backoff is bounded -- a persistent transient failure sleeps a fixed, + short, known schedule, never an unbounded or growing-without-limit one.""" + fake_urlopen, _calls = _scripted_urlopen( + [ + TimeoutError("t"), + TimeoutError("t"), + TimeoutError("t"), + ] + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + slept: list[float] = [] + + _api_get("https://api.github.com/x", sleep=slept.append) + + assert len(slept) == 2, "at most _API_MAX_ATTEMPTS - 1 backoff sleeps" + assert all(0 < delay <= 10.0 for delay in slept), ( + f"each backoff sleep must be short and bounded, got {slept}" + ) + assert sum(slept) <= 30.0, f"total backoff time must be bounded, got {slept}" + + +# --------------------------------------------------------------------------- +# Regression (defect fix, 2026-08-06): bounded-worst-case-under-CI-timeout +# --------------------------------------------------------------------------- +# The per-call retry ceiling above (150s, the TRUE worst case -- see the +# rate-limited-class correction below) only bounds ONE call. The +# pre-existing consecutive-transport-failure circuit breaker resets to 0 on +# ANY HTTP-status-bearing response (including a retried-and-still-failing +# 503/429), so an intermittent-transport run can pay the full per-call +# ceiling on every pin without the breaker ever tripping -- the run-wide +# wall time was unbounded. ``_Resolver`` must enforce its own run-wide +# deadline independent of the breaker. + + +def _headers(pairs: dict[str, str] | None = None) -> email.message.Message: + msg = email.message.Message() + for key, value in (pairs or {}).items(): + msg[key] = value + return msg + + +def _http_error_with_headers( + code: int, message: str = "", headers: dict[str, str] | None = None +) -> urllib.error.HTTPError: + body = json.dumps({"message": message}).encode("utf-8") if message else b"{}" + return urllib.error.HTTPError( + url="https://api.github.com/x", + code=code, + msg=message or "error", + hdrs=_headers(headers), + fp=io.BytesIO(body), + ) + + +@pytest.mark.unit +def test_resolver_enforces_a_run_wide_deadline_independent_of_the_breaker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Once the injected clock is already past the run-wide deadline, a fresh + resolution must fail closed to UNDETERMINED WITHOUT making any network + call -- proving the deadline is checked before spending any more wall + time, not merely reported after the fact.""" + from scripts.ci.check_pin_reachability import _RUN_DEADLINE_SECONDS + + def fail_if_called(request: object, timeout: float | None = None) -> None: + raise AssertionError( + "no network call should be attempted once the run-wide deadline " + "is already exceeded" + ) + + monkeypatch.setattr(urllib.request, "urlopen", fail_if_called) + + clock = {"t": _RUN_DEADLINE_SECONDS + 1.0} + resolver = _Resolver(("dev", "main"), now=lambda: clock["t"]) + # Force the deadline to be recorded as already-elapsed regardless of the + # constructor's own start-time read. + resolver._deadline_at = 0.0 + + resolution = resolver.resolve("omnimarket", "b" * 40) + + assert resolution.verdict is Verdict.UNDETERMINED + assert "deadline" in resolution.detail.lower() + + +@pytest.mark.unit +def test_resolver_aggregate_wall_time_is_bounded_under_intermittent_transport( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """End-to-end through ``_Resolver.resolve``, using a pin-boundary-aligned + TRUE-WORST-CASE pattern: a fast ``dev`` compare followed by a ``main`` + compare that fully exhausts the RATE-LIMITED retry ceiling -- 429 with a + server ``Retry-After`` header on the first two attempts (capped at + ``_MAX_RATE_LIMIT_BACKOFF_SECONDS``, and PREFERRED over the fixed 2s/4s + schedule per ``_rate_limit_backoff_seconds``) -- before finally answering + a definitive 404 on the third attempt: + + 3 * _REQUEST_TIMEOUT_SECONDS + 2 * _MAX_RATE_LIMIT_BACKOFF_SECONDS + = 90 + 60 = 150s + + That lands the run-wide deadline crossing exactly inside ``main``'s + still-in-flight compare call, with ``_explain``'s own lookup guarded + against running afterward. + + Corrected 2026-08-06 (terminal adversarial verify round 2, PR #2679 + comment 5209929069): a prior version of this test exercised only the + header-less-503 fixed-schedule backoff class, whose per-call ceiling is + 96s -- cheaper than, and therefore not discriminating against, the + 150s rate-limited ceiling this module's own retry logic actually pays + when GitHub signals a rate limit. A test asserting ``<= 96.0 + 1.0`` on + THIS construction would fail (undercounts the true tail by 54s) -- + proof the old assertion was pinning the wrong number, not merely a + smaller one.""" + from scripts.ci.check_pin_reachability import ( + _API_MAX_ATTEMPTS, + _MAX_RATE_LIMIT_BACKOFF_SECONDS, + ) + + clock = {"t": 0.0} + + def fake_sleep(seconds: float) -> None: + clock["t"] += seconds + + attempt_counts: dict[str, int] = {} + calls: list[str] = [] + + def fake_urlopen(request: object, timeout: float | None = None) -> object: + url = getattr(request, "full_url", "") + calls.append(url) + if "/compare/dev" in url: + # dev-branch compare: fast, definitive "not reachable" -- no + # latency tax. + return _FakeResponse(200, {"status": "diverged"}) + if "/commits/" in url: + # _explain's own lookup: must never be reached once the + # run-wide deadline is already exceeded by main's compare call. + raise AssertionError( + "_explain must not issue its own unguarded network call " + "past the deadline" + ) + # main-branch compare: fully exhaust the RATE-LIMITED retry ceiling + # (429 + Retry-After, capped and preferred over the fixed schedule) + # before finally answering a definitive 404 on the last attempt. + clock["t"] += timeout or 0.0 + attempt_counts[url] = attempt_counts.get(url, 0) + 1 + if attempt_counts[url] < _API_MAX_ATTEMPTS: + raise _http_error_with_headers( + 429, + "rate limited", + {"Retry-After": str(_MAX_RATE_LIMIT_BACKOFF_SECONDS)}, + ) + raise _http_error_with_headers(404, "not found") + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + monkeypatch.setattr(time, "sleep", fake_sleep) + + resolver = _Resolver(("dev", "main"), now=lambda: clock["t"]) + # Land the deadline exactly between "dev's fast compare completes" (~0s) + # and "main's 150s compare completes" (~150s) -- the per-branch loop's + # OWN deadline check (before main's call) must still pass here, so + # main's call is already in flight when the deadline is crossed; it is + # ``_explain``'s guard that must catch the crossing afterward. + deadline_at = 50.0 + resolver._deadline_at = deadline_at + + resolution = resolver.resolve("omnimarket", "b" * 40) + + assert resolution.verdict is Verdict.UNREACHABLE + assert not any("/commits/" in url for url in calls), ( + "the deadline was already exceeded by main's compare call -- " + "_explain must not issue its own unguarded network call past the " + "deadline" + ) + # True bound: the deadline plus at most ONE already-in-flight call's + # REAL 150s rate-limited ceiling -- not the cheaper 96s fixed-schedule + # figure, and not doubled by an unguarded _explain call. + real_ceiling = 3 * 30.0 + 2 * _MAX_RATE_LIMIT_BACKOFF_SECONDS + assert real_ceiling == 150.0, "sanity: this test's own ceiling arithmetic" + assert clock["t"] <= real_ceiling + 1.0, ( + f"aggregate simulated wall time {clock['t']}s must stay bounded by " + f"one already-in-flight call's {real_ceiling}s rate-limited ceiling " + "past the deadline, not double-charged by an unguarded _explain call" + ) + # Post-deadline tail specifically (the portion that runs AFTER the + # deadline was already crossed) must not exceed the real per-call + # ceiling either -- it is bounded by "one already-in-flight call", full + # stop, regardless of when within that call the deadline landed. + post_deadline_tail = clock["t"] - deadline_at + assert post_deadline_tail <= real_ceiling + 1.0, ( + f"post-deadline tail {post_deadline_tail}s must stay within the " + f"real {real_ceiling}s per-call ceiling" + ) + # Discriminate against the old, wrong 96s figure: this construction's + # true cost must exceed it, proving a test that only asserted <= 96s + # would have been fooled by a cheaper backoff class than the one that + # actually governs the rate-limited path. + assert clock["t"] > 96.0, ( + f"aggregate simulated wall time {clock['t']}s must exceed the old " + "(wrong) 96s fixed-schedule ceiling -- otherwise this test cannot " + "discriminate the rate-limited class from the cheaper one" + ) + + +# --------------------------------------------------------------------------- +# Regression (defect fix, 2026-08-06): rate-limit-aware 403/429 handling +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_api_get_retries_a_primary_rate_limited_403_honoring_reset_header( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A 403 carrying GitHub's primary-rate-limit signal + (``x-ratelimit-remaining: 0``) is transient and must be retried, backing + off by the server-provided ``x-ratelimit-reset`` delta rather than a + blind fixed schedule.""" + monkeypatch.setattr(time, "time", lambda: 1000.0) + fake_urlopen, calls = _scripted_urlopen( + [ + _http_error_with_headers( + 403, + "API rate limit exceeded", + headers={"x-ratelimit-remaining": "0", "x-ratelimit-reset": "1005"}, + ), + _FakeResponse(200, {"status": "behind"}), + ] + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + slept: list[float] = [] + + status, body, _detail = _api_get("https://api.github.com/x", sleep=slept.append) + + assert status == 200 + assert body == {"status": "behind"} + assert len(calls) == 2, ( + "a rate-limited 403 must be retried, not treated as definitive" + ) + assert slept == [5.0], "backoff must follow the server-provided reset delta" + + +@pytest.mark.unit +def test_api_get_caps_rate_limit_backoff_instead_of_honoring_it_unbounded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A server-requested backoff longer than the cap must be clamped, never + honored unbounded -- an hour-long primary-rate-limit reset must not turn + one pin into an hour-long CI job.""" + fake_urlopen, _calls = _scripted_urlopen( + [ + _http_error_with_headers( + 429, "rate limited", headers={"Retry-After": "3600"} + ), + _FakeResponse(200, {"status": "identical"}), + ] + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + slept: list[float] = [] + + _api_get("https://api.github.com/x", sleep=slept.append) + + assert slept, "must still back off" + assert slept[0] <= 30.0, f"rate-limit backoff must be capped, got {slept}" + + +@pytest.mark.unit +def test_api_get_does_not_retry_a_plain_403_even_with_unrelated_headers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A 403 with headers present but no rate-limit signal is still + definitive -- discrimination is by signal, not by header presence.""" + fake_urlopen, calls = _scripted_urlopen( + [ + _http_error_with_headers( + 403, "Forbidden", headers={"x-github-request-id": "abc"} + ) + ] + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + slept: list[float] = [] + + status, _body, _detail = _api_get("https://api.github.com/x", sleep=slept.append) + + assert status == 403 + assert len(calls) == 1, "a 403 without rate-limit headers must not be retried" + assert slept == [] + + +# --------------------------------------------------------------------------- +# Regression (defect fix, 2026-08-06): backoff-schedule/max-attempts coupling +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_api_get_does_not_index_error_when_max_attempts_exceeds_backoff_schedule( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``_API_RETRY_BACKOFF_SECONDS`` has 2 entries for the shipped + ``_API_MAX_ATTEMPTS = 3``. Raising ``_API_MAX_ATTEMPTS`` without + extending the tuple must degrade gracefully (reuse the last known + backoff), never raise ``IndexError`` inside a required CI gate.""" + import scripts.ci.check_pin_reachability as module + + monkeypatch.setattr(module, "_API_MAX_ATTEMPTS", 5) + fake_urlopen, calls = _scripted_urlopen( + [TimeoutError("t")] * 5, + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + slept: list[float] = [] + + status, body, detail = module._api_get( + "https://api.github.com/x", sleep=slept.append + ) + + assert status is None + assert body is None + assert "transport error" in detail + assert len(calls) == 5 + assert len(slept) == 4, "sleeps between all 5 attempts, no IndexError" + + +# --------------------------------------------------------------------------- +# Regression (defect fix, 2026-08-06): http.client.HTTPException coverage +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "make_exc", + [ + lambda: http.client.IncompleteRead(b""), + lambda: http.client.BadStatusLine("garbage"), + ], + ids=["IncompleteRead", "BadStatusLine"], +) +def test_api_get_retries_http_client_transport_exceptions( + monkeypatch: pytest.MonkeyPatch, + make_exc: Callable[[], Exception], +) -> None: + """``http.client.IncompleteRead`` (raised by ``response.read()`` on a + truncated/chunked-abort body) and ``http.client.BadStatusLine`` + (re-raised by urllib's ``do_open`` from ``h.getresponse()``) are neither + ``OSError`` nor any of the other pre-fix caught classes -- their MRO is + ``(HTTPException, Exception, BaseException, object)``. Pre-fix, both + escaped ``_api_get`` uncaught: zero retries, an uncaught traceback, on + exactly the transient-transport class this gate exists to retry.""" + fake_urlopen, calls = _scripted_urlopen( + [make_exc(), _FakeResponse(200, {"status": "behind"})] + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + slept: list[float] = [] + + status, body, _detail = _api_get("https://api.github.com/x", sleep=slept.append) + + assert status == 200 + assert body == {"status": "behind"} + assert len(calls) == 2, "a transport HTTPException must be retried, not raised" + + +# --------------------------------------------------------------------------- +# Regression (defect fix, 2026-08-06): negative Retry-After must not crash +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_rate_limit_backoff_floors_a_negative_retry_after_at_zero() -> None: + """A malformed/adversarial ``Retry-After: -5`` header must not produce a + negative delay -- the sibling ``x-ratelimit-reset`` branch already + guards with ``if delta > 0``; this branch must match.""" + from scripts.ci.check_pin_reachability import _rate_limit_backoff_seconds + + delay = _rate_limit_backoff_seconds(_headers({"Retry-After": "-5"})) + + assert delay is not None + assert delay >= 0.0 + + +@pytest.mark.unit +def test_api_get_does_not_crash_on_a_negative_retry_after_header( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """End-to-end, using the REAL ``time.sleep`` (no ``sleep=`` override) -- + exactly where the pre-fix bug crashed: ``effective_sleep(delay)`` at + :640 is OUTSIDE any try/except, so ``time.sleep(-5.0)`` raises + ``ValueError: sleep length must be non-negative`` unconditionally, an + unguarded crash in a required CI gate. Fails closed only in the sense + that a crash halts the job; it must instead retry with a floored, + non-negative backoff.""" + fake_urlopen, calls = _scripted_urlopen( + [ + _http_error_with_headers( + 429, "rate limited", headers={"Retry-After": "-5"} + ), + _FakeResponse(200, {"status": "identical"}), + ] + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + + status, body, _detail = _api_get("https://api.github.com/x") + + assert status == 200 + assert body == {"status": "identical"} + assert len(calls) == 2 + + +# --------------------------------------------------------------------------- +# Regression (workflow-level fix, 2026-08-07, PR #2679 comment 5211687650): +# the script-side deadline alone cannot bound this job -- a 38-run fleet +# measurement found job setup (job start -> script step start) had median +# 317s and max 613s under the old ``cache-enabled: "false"``, with 6/38 runs +# killed by ``timeout-minutes: 10`` (600s) DURING setup, before the script +# ever started. No value of ``_RUN_DEADLINE_SECONDS`` fixes a job whose setup +# alone can exceed the job timeout. This binding test parses ci.yml as DATA +# (not a copied literal) so a future edit to either the job's +# ``timeout-minutes`` or the script's ``_RUN_DEADLINE_SECONDS`` is checked +# against the real measured budget, in both directions: +# +# * too small -- the job timeout no longer covers +# measured-setup + script-deadline + worst-case-tail, so GitHub kills the +# job mid-run exactly like the pre-fix incident; +# * too large -- "raise timeout-minutes" gamed into an absurd value that +# trivially satisfies the lower-bound arithmetic without being a real +# fix, defeating a CI gate's fail-fast purpose. +# --------------------------------------------------------------------------- + +# Measured worst-case job-start -> script-step-start wall time across the +# 38-run fleet (PR #2679 comment 5211687650), under ``cache-enabled: "false"`` +# -- left unchanged for this job (see the ci.yml ``pin-reachability`` job +# comment: this repo's self-hosted CI already replaced the per-job uv cache +# with a shared dependency-environment canary, and +# ``docs/ci/versioned-ci-env-canary.md`` forbids enabling setup-uv cache save +# on jobs that intentionally run ``uv sync --no-cache``, which +# ``test_ci_workflow_resilience.py::test_short_gates_can_disable_uv_cache_cleanup`` +# enforces uniformly). This is the fleet-measured MAX, not a typical/median +# figure, so the arithmetic below holds without assuming any future setup +# speedup. +_MEASURED_SETUP_BUDGET_SECONDS = 613.0 + + +def _job_timeout_seconds(job_id: str) -> float: + """Parse ``timeout-minutes`` for ``job_id`` out of the real ci.yml, as + data -- never a copied/hardcoded literal, so this test tracks the live + workflow file.""" + loaded = yaml.safe_load((WORKFLOWS_DIR / "ci.yml").read_text(encoding="utf-8")) + job = loaded["jobs"][job_id] + timeout_minutes = job["timeout-minutes"] + assert isinstance(timeout_minutes, int | float), ( + f"jobs.{job_id}.timeout-minutes must be a plain number, got {timeout_minutes!r}" + ) + return float(timeout_minutes) * 60.0 + + +def _assert_pin_reachability_timeout_budget_ok(timeout_seconds: float) -> None: + """Both directions of the budget check. + + Lower bound: the job timeout must cover measured setup + the script's + own run-wide deadline + the worst-case post-deadline tail (one + fully-exhausted rate-limited call: + ``_API_MAX_ATTEMPTS * _REQUEST_TIMEOUT_SECONDS + + (_API_MAX_ATTEMPTS - 1) * _MAX_RATE_LIMIT_BACKOFF_SECONDS`` = 150s, per + the derivation in ``check_pin_reachability.py``'s + ``_RUN_DEADLINE_SECONDS`` comment). + + Upper bound: the job timeout must not be absurdly large. This job makes + a handful of bounded, retried GitHub REST calls -- nothing legitimate + ever needs more than 30 minutes. Without this half, "raise + timeout-minutes" could be satisfied by setting it to something enormous, + which trivially passes the lower-bound arithmetic without fixing + anything and defeats the point of having a bounded CI gate at all. + """ + from scripts.ci.check_pin_reachability import ( + _API_MAX_ATTEMPTS, + _MAX_RATE_LIMIT_BACKOFF_SECONDS, + _REQUEST_TIMEOUT_SECONDS, + _RUN_DEADLINE_SECONDS, + ) + + worst_case_tail_seconds = ( + _API_MAX_ATTEMPTS * _REQUEST_TIMEOUT_SECONDS + + (_API_MAX_ATTEMPTS - 1) * _MAX_RATE_LIMIT_BACKOFF_SECONDS + ) + required_seconds = ( + _MEASURED_SETUP_BUDGET_SECONDS + _RUN_DEADLINE_SECONDS + worst_case_tail_seconds + ) + assert required_seconds < timeout_seconds, ( + f"pin-reachability job timeout ({timeout_seconds:.0f}s) does not " + f"cover measured setup ({_MEASURED_SETUP_BUDGET_SECONDS:.0f}s) + " + f"script deadline ({_RUN_DEADLINE_SECONDS:.0f}s) + worst-case tail " + f"({worst_case_tail_seconds:.0f}s) = {required_seconds:.0f}s -- " + "raise jobs.pin-reachability.timeout-minutes in ci.yml" + ) + + upper_bound_seconds = 30 * 60.0 + assert timeout_seconds <= upper_bound_seconds, ( + f"pin-reachability job timeout ({timeout_seconds:.0f}s) exceeds the " + f"sane upper bound ({upper_bound_seconds:.0f}s) for a job that makes " + "a handful of bounded, retried GitHub REST calls -- an absurdly " + "large timeout is not a real fix for the setup-budget problem and " + "defeats CI's fail-fast purpose" + ) + + +@pytest.mark.unit +def test_pin_reachability_job_timeout_covers_measured_budget() -> None: + """Binding test: the real ci.yml ``timeout-minutes`` for the + ``pin-reachability`` job must cover measured setup + the script's own + deadline + the worst-case post-deadline tail, with real margin, and must + not be an absurdly large non-fix. Fails on the pre-fix ``timeout-minutes: + 10`` (600s < 913s required).""" + _assert_pin_reachability_timeout_budget_ok(_job_timeout_seconds("pin-reachability")) + + +@pytest.mark.unit +def test_pin_reachability_job_timeout_budget_kills_too_small_mutant() -> None: + """A too-small job timeout (480s, echoing the old broken ``480.0`` + magnitude that used to be ``_RUN_DEADLINE_SECONDS`` before it was found + to overshoot the job timeout) must fail the lower-bound check -- this is + the exact class of defect this test exists to catch: a job whose setup + alone can exceed 480s gets killed mid-setup, exactly as measured pre-fix.""" + with pytest.raises(AssertionError, match="does not cover"): + _assert_pin_reachability_timeout_budget_ok(480.0) + + +@pytest.mark.unit +def test_pin_reachability_job_timeout_budget_kills_absurd_upper_bound_mutant() -> None: + """An absurdly large job timeout (99999s, ~27.8h) must fail the + upper-bound sanity check even though it trivially satisfies the + lower-bound arithmetic -- proving the test cannot be satisfied by gaming + ``timeout-minutes`` to an unreasonable value instead of a real fix.""" + with pytest.raises(AssertionError, match="exceeds the sane upper bound"): + _assert_pin_reachability_timeout_budget_ok(99999.0) diff --git a/tests/ci/test_prepush_hook_host_identity_guard.py b/tests/ci/test_prepush_hook_host_identity_guard.py index 2762f971af..9c4e55264f 100644 --- a/tests/ci/test_prepush_hook_host_identity_guard.py +++ b/tests/ci/test_prepush_hook_host_identity_guard.py @@ -13,7 +13,7 @@ this exact escalation for 115+ minutes before `.200` was invoked as a rescue rather than having been the execution target from the start. -Two assertion classes: +Three assertion classes: 1. Static wiring -- `guard_full_suite_host` is defined and is the first statement inside EVERY `IS_FULL` (full-suite) branch, so a future edit @@ -24,10 +24,20 @@ real pytest invocation. The override makes this host-independent: it must hold true no matter which host runs the test suite (including `.200` itself), so the test does not rely on the ambient hostname. +3. Heavyweight-SELECTION (OMN-15408) -- the guard must key on the work the + selector actually picked, not on the `is_full_suite` flag. The selector + routinely emits `is_full_suite=False` with `selected_paths=["tests/"]`, and + before OMN-15408 those runs bypassed the guard outright: 13,898 tests / + 506s in omnimarket and 2,429 tests / 245s in omnibase_infra, executed on + `omnibook` through real `git push` runs on 2026-07-29 with the guard never + invoked -- while the identical selected work forced via + `PREPUSH_FULL_SUITE=1` WAS refused. Assertion classes 1 and 2 above were + green that entire time, because both only ever drove the flag-true path. """ from __future__ import annotations +import json import os import re import subprocess @@ -125,3 +135,282 @@ def test_guard_refuses_full_suite_escalation_on_non_200_host() -> None: "the guard must refuse BEFORE pytest ever collects tests -- found a " f"pytest collection banner in stdout: {result.stdout!r}" ) + + +# ============================================================================= +# OMN-15408: the guard must key on the SELECTED WORK, not the is_full_suite flag +# ============================================================================= +# The OMN-15059 guard above was called ONLY from inside the `IS_FULL` branch, so +# it fired on the selector's `is_full_suite` FLAG. The selector routinely emits +# `is_full_suite=False` with `selected_paths=["tests/"]` -- the whole suite +# arriving as an "impacted subset" -- and those runs bypassed the guard +# entirely. Measured on host `omnibook` through real `git push` runs on +# 2026-07-29: omnimarket ran 13,898 tests in 506s and omnibase_infra 2,429 tests +# in 245s, locally, with the guard never invoked; the SAME selected work forced +# via `PREPUSH_FULL_SUITE=1` was refused. Identical cost, opposite outcome, +# decided by a flag. The tests below drive the flag-FALSE path, which is the one +# that reaches production behavior -- the pre-existing tests in this file only +# ever exercised the flag-TRUE path, and were green while the hole was open. +# +# `test_guard_refuses_whole_suite_equivalent_selection_when_flag_is_false` is +# RED against the pre-fix hook (it proceeds to pytest, exit 0) and GREEN after. +# `test_guard_allows_a_genuinely_narrow_selection` is the anti-overreach pin: +# the fix must not brick every push from this Mac. + +_FULL_SUITE_TARGET = "tests/unit/" +# The literal production shape from the OMN-15408 evidence table. +_WHOLE_SUITE_SELECTION = "tests/" +# A real, genuinely-narrow subdirectory of this repo's suite. +_NARROW_SELECTION = "tests/unit/scripts/" + +_PREDICATE_RE = re.compile( + r"^selection_is_whole_suite\(\) \{.*?^\}", + re.DOTALL | re.MULTILINE, +) +_WHOLE_SUITE_GUARD_CALL_RE = re.compile( + r"if selection_is_whole_suite .*?\n(.*?)\n\s*fi", + re.DOTALL, +) + + +def _extract_predicate_source() -> str: + """Return the literal `selection_is_whole_suite` bash function from the hook. + + Extract-and-execute (the pattern already used for the hook's other pure + shell helpers) so these assertions run THE function that ships, never a + Python re-implementation of it -- a re-implementation would pass happily + while the shipped predicate was broken. + """ + match = _PREDICATE_RE.search(HOOK_SCRIPT.read_text(encoding="utf-8")) + assert match is not None, ( + "expected a selection_is_whole_suite() function in " + f"{HOOK_SCRIPT} -- the OMN-15408 heavyweight-selection predicate" + ) + return match.group(0) + + +def _predicate_says_whole_suite(target: str, *paths: str) -> bool: + """Execute the real bash predicate; True == 'this selection is heavyweight'.""" + script = f'{_extract_predicate_source()}\nselection_is_whole_suite "$@"\n' + completed = subprocess.run( + ["bash", "-c", script, "selection_is_whole_suite", target, *paths], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert completed.returncode in (0, 1), ( + "predicate exited abnormally " + f"({completed.returncode}): stderr={completed.stderr!r}" + ) + return completed.returncode == 0 + + +def _run_hook_with_stubbed_selection( + tmp_path: Path, + *, + is_full_suite: bool, + selected_paths: list[str], +) -> subprocess.CompletedProcess[str]: + """Run the REAL hook end-to-end with a stubbed selector + stubbed pytest. + + A shim `uv` earlier on PATH answers the selector invocation with a chosen + selection JSON and turns the pytest invocation into an observable sentinel, + so the test can assert both `did the guard refuse` and `did execution ever + reach pytest` against the actual script rather than a surrogate. Everything + else (`uv run python - ` for JSON parsing) is delegated to the + real interpreter. + + `PREPUSH_BASE_REF=HEAD` keeps the pre-selector preamble deterministic and + offline: it always resolves, `merge-base HEAD HEAD` is HEAD, and the diff is + empty -- the stub supplies the selection regardless. + """ + selection_file = tmp_path / "selection.json" + selection_file.write_text( + json.dumps( + { + "is_full_suite": is_full_suite, + "full_suite_reason": None, + "selected_paths": selected_paths, + } + ), + encoding="utf-8", + ) + + stub_bin = tmp_path / "stub-bin" + stub_bin.mkdir(parents=True, exist_ok=True) + uv_stub = stub_bin / "uv" + uv_stub.write_text( + "#!/usr/bin/env bash\n" + 'args="$*"\n' + 'case "$args" in\n' + " *detect_test_paths*)\n" + ' cat "$PREPUSH_TEST_SELECTION_JSON"\n' + " exit 0\n" + " ;;\n" + " *pytest*)\n" + ' echo "STUB-PYTEST-INVOKED $args"\n' + " exit 0\n" + " ;;\n" + "esac\n" + "shift\n" + 'if [ "$1" = "python" ]; then\n' + " shift\n" + ' exec python3 "$@"\n' + "fi\n" + 'exec "$@"\n', + encoding="utf-8", + ) + uv_stub.chmod(0o755) + + env = dict(os.environ) + env["PATH"] = f"{stub_bin}{os.pathsep}{env['PATH']}" + env["PREPUSH_TEST_SELECTION_JSON"] = str(selection_file) + env["PREPUSH_BASE_REF"] = "HEAD" + env["PREPUSH_200_HOSTNAME"] = _GUARANTEED_NON_MATCHING_HOSTNAME + for leaky in ( + "PREPUSH_FULL_SUITE", + "PREPUSH_ALLOW_LOCAL_FULL_SUITE", + "ENABLE_SMART_TESTS", + "PREPUSH_ADJACENCY", + "PREPUSH_PYTEST_ARGS", + ): + env.pop(leaky, None) + + return subprocess.run( + ["bash", str(HOOK_SCRIPT)], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=300, + check=False, + ) + + +def test_heavyweight_selection_predicate_is_defined() -> None: + assert "selection_is_whole_suite()" in HOOK_SCRIPT.read_text(encoding="utf-8"), ( + "expected a selection_is_whole_suite() predicate so the guard can key " + "on the selected work rather than the is_full_suite flag (OMN-15408)" + ) + + +def test_full_suite_target_is_single_sourced() -> None: + """The predicate and the escalation must read the SAME target. + + If the guard hard-coded its own notion of 'the whole suite' it would drift + from whatever the escalation actually runs -- a second cost model, which is + the thing this fix explicitly avoids. + """ + script_text = HOOK_SCRIPT.read_text(encoding="utf-8") + assert f'FULL_SUITE_TARGET="{_FULL_SUITE_TARGET}"' in script_text, ( + f"expected FULL_SUITE_TARGET to be set to {_FULL_SUITE_TARGET!r} in " + f"{HOOK_SCRIPT}" + ) + assert 'uv run pytest "${FULL_SUITE_TARGET}"' in script_text, ( + "expected the fail-closed escalation to run ${FULL_SUITE_TARGET} " + "itself, so the guard predicate cannot drift from the run it guards" + ) + assert 'selection_is_whole_suite "$FULL_SUITE_TARGET"' in script_text, ( + "expected the predicate to be evaluated against the SAME " + "FULL_SUITE_TARGET the escalation runs" + ) + + +def test_guard_is_called_when_the_selection_is_whole_suite_equivalent() -> None: + """Static wiring: the impacted-subset branch must consult the guard. + + Pairs with the pre-existing IS_FULL-branch assertion above; together they + pin BOTH call sites, so a future edit cannot silently drop either one. + """ + script_text = HOOK_SCRIPT.read_text(encoding="utf-8") + guarded_blocks = _WHOLE_SUITE_GUARD_CALL_RE.findall(script_text) + assert guarded_blocks, ( + "expected an `if selection_is_whole_suite ...` block in the " + "impacted-subset branch (OMN-15408)" + ) + assert any("guard_full_suite_host" in block for block in guarded_blocks), ( + "expected guard_full_suite_host to be called when the selection is " + f"whole-suite-equivalent; found blocks: {guarded_blocks!r}" + ) + + +def test_predicate_flags_a_whole_suite_selection() -> None: + """A selection covering the entire escalation target is heavyweight.""" + assert _predicate_says_whole_suite(_FULL_SUITE_TARGET, _WHOLE_SUITE_SELECTION) + assert _predicate_says_whole_suite(_FULL_SUITE_TARGET, _FULL_SUITE_TARGET) + assert _predicate_says_whole_suite( + _FULL_SUITE_TARGET, _FULL_SUITE_TARGET.rstrip("/") + ), "a trailing-slash-less path must normalize to the same target" + assert _predicate_says_whole_suite( + _FULL_SUITE_TARGET, _NARROW_SELECTION, _WHOLE_SUITE_SELECTION + ), "one whole-suite path anywhere in the selection makes it heavyweight" + + +def test_predicate_allows_a_genuinely_narrow_selection() -> None: + """Real narrowing stays runnable -- the guard is not a blanket push block.""" + assert not _predicate_says_whole_suite(_FULL_SUITE_TARGET, _NARROW_SELECTION) + assert not _predicate_says_whole_suite( + _FULL_SUITE_TARGET, f"{_NARROW_SELECTION}test_something.py" + ) + assert not _predicate_says_whole_suite(_FULL_SUITE_TARGET) + + +def test_guard_refuses_whole_suite_equivalent_selection_when_flag_is_false( + tmp_path: Path, +) -> None: + """THE OMN-15408 REGRESSION. + + `is_full_suite=False` + `selected_paths=["tests/"]` on a non-`.200` host is + the exact shape that ran 13,898 tests locally with the guard never invoked. + RED against the pre-fix hook (it reached pytest and exited 0); GREEN after. + """ + result = _run_hook_with_stubbed_selection( + tmp_path, + is_full_suite=False, + selected_paths=[_WHOLE_SUITE_SELECTION], + ) + assert result.returncode != 0, ( + "expected the host guard to refuse a whole-suite-equivalent selection " + "even though is_full_suite=False; got exit " + f"{result.returncode}. stdout={result.stdout!r} stderr={result.stderr!r}" + ) + assert "not the designated .200 build host" in result.stderr, ( + f"expected the refusal message in stderr, got: {result.stderr!r}" + ) + assert "STUB-PYTEST-INVOKED" not in result.stdout, ( + "the guard must refuse BEFORE pytest is invoked -- found a pytest " + f"invocation in stdout: {result.stdout!r}" + ) + + +def test_guard_allows_a_genuinely_narrow_selection_on_a_local_host( + tmp_path: Path, +) -> None: + """Anti-overreach pin. + + A real narrow selection must still run locally on a non-`.200` host. If + this ever fails, the fix has become a blanket push block and will be + disabled within a week -- which is worse than no guard at all. + """ + result = _run_hook_with_stubbed_selection( + tmp_path, + is_full_suite=False, + selected_paths=[_NARROW_SELECTION], + ) + assert result.returncode == 0, ( + "expected a genuinely narrow selection to be allowed on a non-.200 " + f"host; got exit {result.returncode}. " + f"stdout={result.stdout!r} stderr={result.stderr!r}" + ) + assert "not the designated .200 build host" not in result.stderr, ( + f"the guard must NOT refuse a proper narrowing; stderr: {result.stderr!r}" + ) + assert "STUB-PYTEST-INVOKED" in result.stdout, ( + "expected the narrow selection to actually reach pytest; stdout: " + f"{result.stdout!r}" + ) + assert _NARROW_SELECTION in result.stdout, ( + "expected the narrow selection's own path to be handed to pytest; " + f"stdout: {result.stdout!r}" + ) diff --git a/tests/ci/test_runner_broker_dispatch_wedge_rerun.py b/tests/ci/test_runner_broker_dispatch_wedge_rerun.py new file mode 100644 index 0000000000..d1704ccbb5 --- /dev/null +++ b/tests/ci/test_runner_broker_dispatch_wedge_rerun.py @@ -0,0 +1,468 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Tests for the OMN-15776 broker-dispatch-wedge detection + targeted rerun. + +Proven mechanism (2026-08-09, ``omn15776-wedge`` ledger entry): GitHub's Actions +broker dispatches a job to a self-hosted runner within 2-7s of that same runner +finishing its previous job, exactly while the runner's ``Runner.Listener`` is +mid-reconnect on its broker long-poll (every job completion triggers a +retry/backoff storm on that connection). The new dispatch lands in the +reconnect gap and is never delivered to the runner's local message loop — no +``Runner.Worker`` process is ever spawned (``steps: []``, not a crashed step 1) +— while GitHub's server side records the assignment, sets ``started_at``, and +independently times the orphaned assignment out at a fixed ~10m0-1s. + +No local entrypoint.sh/watchdog fix applies: the drop happens in the GitHub +Actions client/broker protocol path, before any local process (Worker, or the +existing OMN-14564 heartbeat watchdog, which only inspects a process that never +spawned) can observe it. The bounded remediation is a targeted, signature-keyed +rerun: detect jobs matching the exact structural fingerprint (self-hosted +runner assigned, conclusion in {failure, cancelled}, zero steps recorded, +duration within a tight band around the fixed ~10m0-1s server-side timeout) and +reissue only that job — never a blanket retry-on-red policy, which would +launder genuine content failures. + +This test proves the DETECTION GAP first (RED): feeding the exact wedge +signature through the script with no detection logic present must not yet +distinguish it from other failures. Then verifies the fix distinguishes: + - a wedge-signature job -> flagged as a rerun candidate (and, when not + dry-run, the job-rerun endpoint is called exactly once for it) + - a genuine content failure on a self-hosted runner (has steps, short + duration) -> NEVER a candidate + - a GitHub-hosted job (no runner_name) with the same duration/steps shape + -> NEVER a candidate (this fleet's remediation must not touch hosted jobs) + - a successful job -> NEVER a candidate +""" + +from __future__ import annotations + +import http.server +import json +import os +import subprocess +import tempfile +import threading +from pathlib import Path +from urllib.parse import urlparse + +import pytest + +pytestmark = pytest.mark.ci + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "scripts" / "ci" / "runner_broker_dispatch_wedge_rerun.sh" + +ORG = "OmniNode-ai" +REPO = "omnibase_infra" +RUN_ID = 90000000001 + +# The exact proven wedge signature (2026-08-09 ledger evidence): runner +# assigned, conclusion failure, zero steps, duration exactly 600s (10m0s). +WEDGE_JOB = { + "id": 93294479341, + "run_id": RUN_ID, + "name": "Dep Provenance Gate", + "status": "completed", + "conclusion": "failure", + "runner_name": "omninode-runner-17", + "started_at": "2026-08-08T20:06:49Z", + "completed_at": "2026-08-08T20:16:49Z", + "steps": [], +} + +# A genuine content failure on the same fleet: has real steps, short duration. +# Must NEVER be treated as a rerun candidate. +REAL_FAILURE_JOB = { + "id": 93294479999, + "run_id": RUN_ID, + "name": "Unit Tests", + "status": "completed", + "conclusion": "failure", + "runner_name": "omninode-runner-9", + "started_at": "2026-08-08T20:00:00Z", + "completed_at": "2026-08-08T20:00:45Z", + "steps": [{"name": "Run pytest", "conclusion": "failure"}], +} + +# Same duration/steps shape as the wedge signature, but a GitHub-hosted job +# (no runner_name). Must NEVER be treated as a rerun candidate. +HOSTED_JOB_SAME_SHAPE = { + "id": 93294480000, + "run_id": RUN_ID, + "name": "hosted-timeout-job", + "status": "completed", + "conclusion": "cancelled", + "runner_name": None, + "started_at": "2026-08-08T20:06:49Z", + "completed_at": "2026-08-08T20:16:49Z", + "steps": [], +} + +SUCCESS_JOB = { + "id": 93294480001, + "run_id": RUN_ID, + "name": "Lint", + "status": "completed", + "conclusion": "success", + "runner_name": "omninode-runner-3", + "started_at": "2026-08-08T20:00:00Z", + "completed_at": "2026-08-08T20:02:00Z", + "steps": [{"name": "ruff", "conclusion": "success"}], +} + + +class _StubState: + """Records what the stub server observed (thread-shared).""" + + def __init__(self, jobs: list[dict[str, object]]) -> None: + self.jobs = jobs + self.rerun_job_ids: list[int] = [] + self.lock = threading.Lock() + + +def _make_handler(state: _StubState) -> type[http.server.BaseHTTPRequestHandler]: + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args: object) -> None: # silence request logging + pass + + def do_GET(self) -> None: + parsed = urlparse(self.path) + path = parsed.path + if path == f"/repos/{ORG}/{REPO}/actions/runs": + body = { + "workflow_runs": [ + { + "id": RUN_ID, + "status": "completed", + "created_at": "2026-08-08T20:00:00Z", + } + ] + } + elif path == f"/repos/{ORG}/{REPO}/actions/runs/{RUN_ID}/jobs": + body = {"jobs": state.jobs} + else: + self.send_response(404) + self.end_headers() + return + payload = json.dumps(body).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(payload) + + def do_POST(self) -> None: + parsed = urlparse(self.path) + prefix = f"/repos/{ORG}/{REPO}/actions/jobs/" + suffix = "/rerun" + if parsed.path.startswith(prefix) and parsed.path.endswith(suffix): + job_id = int(parsed.path[len(prefix) : -len(suffix)]) + with state.lock: + state.rerun_job_ids.append(job_id) + self.send_response(201) + self.end_headers() + return + self.send_response(404) + self.end_headers() + + return Handler + + +def _run_script( + jobs: list[dict[str, object]], *, dry_run: bool +) -> tuple[subprocess.CompletedProcess[str], _StubState]: + state = _StubState(jobs) + handler = _make_handler(state) + server = http.server.HTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + env = dict(os.environ) + env.update( + { + "GITHUB_API_URL": f"http://127.0.0.1:{server.server_port}", + "RUNNER_GITHUB_TOKEN": "test-token", + "REPOS_CSV": REPO, + "GITHUB_ORG": ORG, + # Fixture timestamps are fixed historical values (2026-08-08); + # use a huge lookback so the test's pass/fail is independent + # of wall-clock time at execution — the lookback window itself + # is not what this test suite is verifying. + "LOOKBACK_HOURS": "87600", + "ONEX_STATE_DIR": "", # forced per-invocation via --state-dir below + } + ) + with tempfile.TemporaryDirectory(prefix="omn15776-test-state-") as state_dir: + args = ["bash", str(SCRIPT), "--state-dir", state_dir] + if dry_run: + args.append("--dry-run") + result = subprocess.run( + args, + check=False, + capture_output=True, + text=True, + env=env, + cwd=REPO_ROOT, + timeout=60, + ) + return result, state + finally: + server.shutdown() + thread.join(timeout=5) + + +class TestBrokerDispatchWedgeSignatureDetection: + """DoD: the exact proven signature is distinguished from every neighbor.""" + + def test_script_exists_and_is_executable_shape(self) -> None: + assert SCRIPT.exists(), ( + f"missing {SCRIPT} — OMN-15776 remediation not implemented" + ) + + def test_wedge_signature_flagged_as_candidate_dry_run(self, tmp_path: Path) -> None: + jobs = [WEDGE_JOB, REAL_FAILURE_JOB, HOSTED_JOB_SAME_SHAPE, SUCCESS_JOB] + result, state = _run_script(jobs, dry_run=True) + assert result.returncode == 0, ( + f"dry-run scan must exit 0: out={result.stdout} err={result.stderr}" + ) + assert str(WEDGE_JOB["id"]) in result.stdout, ( + f"wedge-signature job {WEDGE_JOB['id']} must be flagged as a " + f"candidate: out={result.stdout}" + ) + assert str(REAL_FAILURE_JOB["id"]) not in result.stdout, ( + "a genuine content failure (has steps) must never be a candidate" + ) + assert str(HOSTED_JOB_SAME_SHAPE["id"]) not in result.stdout, ( + "a GitHub-hosted job must never be a candidate regardless of shape" + ) + assert str(SUCCESS_JOB["id"]) not in result.stdout + # dry-run must never call the rerun endpoint + assert state.rerun_job_ids == [] + + def test_wedge_signature_reruns_exactly_the_matched_job(self) -> None: + jobs = [WEDGE_JOB, REAL_FAILURE_JOB, HOSTED_JOB_SAME_SHAPE, SUCCESS_JOB] + result, state = _run_script(jobs, dry_run=False) + assert result.returncode == 0, ( + f"live scan must exit 0: out={result.stdout} err={result.stderr}" + ) + assert state.rerun_job_ids == [WEDGE_JOB["id"]], ( + f"exactly the wedge-signature job must be rerun, got " + f"{state.rerun_job_ids}: out={result.stdout}" + ) + + def test_no_candidates_is_a_clean_noop(self) -> None: + jobs = [REAL_FAILURE_JOB, HOSTED_JOB_SAME_SHAPE, SUCCESS_JOB] + result, state = _run_script(jobs, dry_run=False) + assert result.returncode == 0 + assert state.rerun_job_ids == [] + + def test_off_by_one_duration_below_band_is_not_a_candidate(self) -> None: + """duration=540s (9m) must not match — it is not the proven ~10m0-1s + fixed server-side timeout, so treating it as the same signature would + risk masking a genuine early failure.""" + near_miss = dict(WEDGE_JOB) + near_miss["id"] = 93294481111 + near_miss["completed_at"] = "2026-08-08T20:15:49Z" # 540s after started_at + result, state = _run_script([near_miss], dry_run=True) + assert result.returncode == 0 + assert str(near_miss["id"]) not in result.stdout + assert state.rerun_job_ids == [] + + def test_cancelled_conclusion_also_matches_the_signature(self) -> None: + """Ledger evidence records both failure and cancelled conclusions for + this class (occ#6122/#6161 runs).""" + cancelled = dict(WEDGE_JOB) + cancelled["id"] = 93294482222 + cancelled["conclusion"] = "cancelled" + result, state = _run_script([cancelled], dry_run=False) + assert result.returncode == 0 + assert state.rerun_job_ids == [cancelled["id"]] + + # ------------------------------------------------------------------- + # Isolated single-variable mutation-killers (2026-08-10, + # omn15776-wedge-verify). The mutation matrix found M2 (delete the + # `step_count != 0` skip) and M3 (short-circuit the conclusion-class + # check to always-false) SURVIVING against the fixtures above, because + # every existing negative fixture changes more than one variable at + # once relative to WEDGE_JOB (REAL_FAILURE_JOB changes both steps AND + # duration; HOSTED_JOB_SAME_SHAPE changes runner_name only — that one + # is already isolated and kills M1; SUCCESS_JOB changes conclusion AND + # steps AND duration). These two fixtures hold every other WEDGE_JOB + # property fixed (duration=600s in-band, runner_name set) and flip + # exactly the one property each mutation removes, so only that removed + # check can be the reason the job is excluded. + # ------------------------------------------------------------------- + + def test_in_band_failure_with_steps_present_is_not_a_candidate(self) -> None: + """Isolates the steps==0 check (kills mutation M2: deleting the + `step_count != 0` skip). Same duration (600s, in-band), same + runner_name-set, same conclusion=failure as WEDGE_JOB — the ONLY + difference is steps=[] instead of steps=[]. If the steps + check is removed, this job passes every remaining check and gets + wrongly selected — which would launder a genuine content failure + that happens to also run exactly 600s.""" + one_step_at_600s = dict(WEDGE_JOB) + one_step_at_600s["id"] = 93294483333 + one_step_at_600s["steps"] = [{"name": "Run tests", "conclusion": "failure"}] + result, state = _run_script([one_step_at_600s], dry_run=True) + assert result.returncode == 0 + assert str(one_step_at_600s["id"]) not in result.stdout, ( + "a job with a real step must never be a candidate even at the " + "exact wedge duration — only steps==0 may discriminate this case" + ) + assert state.rerun_job_ids == [] + + def test_in_band_zero_steps_success_is_not_a_candidate(self) -> None: + """Isolates the conclusion-class check (kills mutation M3: + short-circuiting the conclusion check to always-false/no-op). Same + duration (600s, in-band), same runner_name-set, same steps=[] as + WEDGE_JOB — the ONLY difference is conclusion=success instead of + failure/cancelled. If the conclusion check is disabled, this job + passes every remaining check and gets wrongly selected — a + successful job must never be rerun.""" + zero_steps_success_at_600s = dict(WEDGE_JOB) + zero_steps_success_at_600s["id"] = 93294484444 + zero_steps_success_at_600s["conclusion"] = "success" + result, state = _run_script([zero_steps_success_at_600s], dry_run=True) + assert result.returncode == 0 + assert str(zero_steps_success_at_600s["id"]) not in result.stdout, ( + "a successful job must never be a candidate even with steps==0 " + "at the exact wedge duration — only conclusion may discriminate " + "this case" + ) + assert state.rerun_job_ids == [] + + +# --------------------------------------------------------------------------- +# Incident replay (OMN-15547): the guard driven against REAL captured bytes, +# not a hand-typed synthetic fixture. Fetched verbatim via +# gh api repos/OmniNode-ai/onex_change_control/actions/jobs/93294479341 +# on 2026-08-09 (the exact job cited in the omn15776-wedge ledger proof: +# runner=omninode-runner-17, conclusion=failure, steps=[], +# started_at=20:06:56Z, completed_at=20:16:56Z — exactly 600s). Committed +# byte-identical at tests/fixtures/omn15776/job-93294479341.json.captured; +# see tests/incident_replays/registry.yaml (id: omn15776-broker-dispatch-wedge) +# for the sha256 pin and capture provenance. +# --------------------------------------------------------------------------- + +REAL_ORG = "OmniNode-ai" +REAL_REPO = "onex_change_control" +CAPTURED_JOB_FIXTURE = ( + REPO_ROOT / "tests" / "fixtures" / "omn15776" / "job-93294479341.json.captured" +) + + +def _run_script_against_real_org_repo( + jobs: list[dict[str, object]], *, dry_run: bool, run_id: int +) -> tuple[subprocess.CompletedProcess[str], _StubState]: + """Same driver as ``_run_script``, but keyed to the REAL org/repo/run_id + the captured fixture came from — required so the guard's endpoint + construction matches what actually served the real incident.""" + state = _StubState(jobs) + base_handler = _make_handler(state) + + class RealRepoHandler(base_handler): # type: ignore[misc, valid-type] + def do_GET(self) -> None: + parsed = urlparse(self.path) + path = parsed.path + if path == f"/repos/{REAL_ORG}/{REAL_REPO}/actions/runs": + body = { + "workflow_runs": [ + { + "id": run_id, + "status": "completed", + "created_at": "2026-08-09T20:00:00Z", + } + ] + } + elif path == f"/repos/{REAL_ORG}/{REAL_REPO}/actions/runs/{run_id}/jobs": + body = {"jobs": state.jobs} + else: + self.send_response(404) + self.end_headers() + return + payload = json.dumps(body).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(payload) + + def do_POST(self) -> None: + parsed = urlparse(self.path) + prefix = f"/repos/{REAL_ORG}/{REAL_REPO}/actions/jobs/" + suffix = "/rerun" + if parsed.path.startswith(prefix) and parsed.path.endswith(suffix): + job_id = int(parsed.path[len(prefix) : -len(suffix)]) + with state.lock: + state.rerun_job_ids.append(job_id) + self.send_response(201) + self.end_headers() + return + self.send_response(404) + self.end_headers() + + server = http.server.HTTPServer(("127.0.0.1", 0), RealRepoHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + env = dict(os.environ) + env.update( + { + "GITHUB_API_URL": f"http://127.0.0.1:{server.server_port}", + "RUNNER_GITHUB_TOKEN": "test-token", + "REPOS_CSV": REAL_REPO, + "GITHUB_ORG": REAL_ORG, + "LOOKBACK_HOURS": "87600", + } + ) + with tempfile.TemporaryDirectory(prefix="omn15776-replay-state-") as state_dir: + args = ["bash", str(SCRIPT), "--state-dir", state_dir] + if dry_run: + args.append("--dry-run") + result = subprocess.run( + args, + check=False, + capture_output=True, + text=True, + env=env, + cwd=REPO_ROOT, + timeout=60, + ) + return result, state + finally: + server.shutdown() + thread.join(timeout=5) + + +class TestOmn15776IncidentReplay: + """R1-R5 (OMN-15547): the guard driven against the REAL captured job that + motivated it, not a hand-typed reconstruction.""" + + def test_fixture_is_the_real_captured_bytes(self) -> None: + import hashlib + + assert CAPTURED_JOB_FIXTURE.exists(), ( + f"missing captured fixture {CAPTURED_JOB_FIXTURE}" + ) + digest = hashlib.sha256(CAPTURED_JOB_FIXTURE.read_bytes()).hexdigest() + assert digest == ( + "5f4305bd74e2d7ee2ceb9d09e9861d1f8d04cdc198150c64aa2befff8ff0c0bf" + ), "captured fixture bytes drifted from the sha256 pinned in registry.yaml" + + def test_real_wedge_job_is_rejected_ie_flagged_for_rerun(self) -> None: + """The buggy prior state (no guard at all) silently accepted this real + orphaned dispatch as an unremediated red — a human had to notice and + manually rerun it (see the multiple 2026-08-09 ledger passes that + diagnosed but could not act on this exact job). The guard must REJECT + it: flag it as a rerun candidate and reissue it.""" + captured_job = json.loads(CAPTURED_JOB_FIXTURE.read_text(encoding="utf-8")) + run_id = captured_job["run_id"] + result, state = _run_script_against_real_org_repo( + [captured_job], dry_run=False, run_id=run_id + ) + assert result.returncode == 0, ( + f"replay scan must exit 0: out={result.stdout} err={result.stderr}" + ) + assert state.rerun_job_ids == [captured_job["id"]], ( + f"the real captured wedge job must be rejected (flagged + rerun): " + f"out={result.stdout} err={result.stderr}" + ) diff --git a/tests/ci/test_runner_image_node24_floor.py b/tests/ci/test_runner_image_node24_floor.py index 2ff4a6e219..913050b3bc 100644 --- a/tests/ci/test_runner_image_node24_floor.py +++ b/tests/ci/test_runner_image_node24_floor.py @@ -48,10 +48,11 @@ # #3940). Below this, `actions/checkout@v7` (using: node24) fails at "Set up job". NODE24_FLOOR = (2, 327, 0) -# The live self-hosted fleet runs 2.334.0 (verified 2026-06-02, OMN-12585). The -# baked image must never downgrade the fleet's runner. Raise this floor whenever -# the fleet is rolled forward to a newer runner. -LIVE_FLEET_FLOOR = (2, 334, 0) +# The live self-hosted fleet runs 2.336.0 (verified 2026-08-10, OMN-15702 — the +# GitHub-mandated 2.334.0 deprecation cutoff landed 2026-08-10). The baked image +# must never downgrade the fleet's runner. Raise this floor whenever the fleet +# is rolled forward to a newer runner. +LIVE_FLEET_FLOOR = (2, 336, 0) _VERSION_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)$") diff --git a/tests/ci/test_runner_listener_liveness.py b/tests/ci/test_runner_listener_liveness.py index 4b4ead763a..a4f310118d 100644 --- a/tests/ci/test_runner_listener_liveness.py +++ b/tests/ci/test_runner_listener_liveness.py @@ -22,6 +22,7 @@ from __future__ import annotations +import gzip import http.server import json import os @@ -51,12 +52,47 @@ FLEET_CONFIG = REPO_ROOT / "config" / "runner_fleet.yaml" RUNBOOK = REPO_ROOT / "docs" / "runbooks" / "runner-fleet-listener-liveness.md" +# A byte-faithful contiguous tail of a REAL production listener log: +# omninode-runner-10's Runner_20260727-170542-utc.log on omninode-pc, captured +# 2026-07-28 while the container was `Up (healthy)` and the runner was +# registry-ONLINE. Committed gzipped (163 KB -> 8.8 KB) and never edited -- +# the point of the fixture is that it is the input distribution the artifact +# actually sees, not a hand-written approximation of it. +REAL_LISTENER_TAIL = ( + REPO_ROOT / "tests" / "ci" / "fixtures" / "runner_diag_real_tail.log.gz" +) + + +def _run_healthcheck( + runner_home: Path, + max_diag_age: str | None = "900", + window_minutes: str | None = None, + max_starts_per_hour: str | None = None, + max_session_broken: str | None = None, + session_state_check: str | None = None, +) -> subprocess.CompletedProcess[str]: + """Run healthcheck.sh against ``runner_home``. -def _run_healthcheck(runner_home: Path) -> subprocess.CompletedProcess[str]: + Any tunable passed as ``None`` is left UNSET so the script's own default is + what gets exercised (OMN-15233 — the defaults are the thing that was + miscalibrated; a test that always pins a threshold can never catch that). + Ambient values are actively popped, so an operator env on the test host + cannot silently change what is being asserted. + """ env = dict(os.environ) env["RUNNER_HOME"] = str(runner_home) env["RUNNER_HEALTH_EGRESS_CHECK"] = "0" # offline determinism - env["RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS"] = "900" + for name, value in ( + ("RUNNER_HEALTH_MAX_DIAG_AGE_SECONDS", max_diag_age), + ("RUNNER_HEALTH_LOG_RATE_WINDOW_MINUTES", window_minutes), + ("RUNNER_HEALTH_MAX_LOG_STARTS_PER_HOUR", max_starts_per_hour), + ("RUNNER_HEALTH_MAX_SESSION_BROKEN_SECONDS", max_session_broken), + ("RUNNER_HEALTH_SESSION_STATE_CHECK", session_state_check), + ): + if value is None: + env.pop(name, None) + else: + env[name] = value return subprocess.run( ["bash", str(HEALTHCHECK)], check=False, @@ -70,6 +106,11 @@ def _run_healthcheck(runner_home: Path) -> subprocess.CompletedProcess[str]: @pytest.fixture def synthetic_runner_home(tmp_path: Path) -> Path: """A fake RUNNER_HOME with a bin/Runner.Listener sleeper and fresh _diag.""" + if os.getpid() == 1: + pytest.skip( + "pytest is PID 1 in this harness, so every synthetic listener would " + "inherit PPID 1 and be indistinguishable from an OMN-15233 orphan" + ) home = tmp_path / "actions-runner" (home / "bin").mkdir(parents=True) (home / "_diag").mkdir() @@ -158,6 +199,826 @@ def test_healthcheck_pattern_is_runner_home_anchored(self) -> None: ) +def _spawn_orphan(command: Path) -> int: + """Spawn ``command`` detached so this test process is NOT its parent. + + The intermediate shell exits immediately, so the child is reparented (to + PID 1 on macOS/Linux without a subreaper) — the exact orphan shape from the + OMN-15233 incident. Returns the orphan's pid. + """ + spawn = subprocess.run( + ["bash", "-c", f'"{command}" >/dev/null 2>&1 & echo $!'], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + return int(spawn.stdout.strip()) + + +def _ppid_of(pid: int) -> int | None: + result = subprocess.run( + ["ps", "-o", "ppid=", "-p", str(pid)], + check=False, + capture_output=True, + text=True, + ) + raw = result.stdout.strip() + return int(raw) if raw.isdigit() else None + + +def _kill_pid(pid: int) -> None: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +class TestHealthcheckIdleThresholdRecalibration: + """OMN-15233 (a): the default threshold must clear the IDLE _diag cadence. + + When a runner is idle the ONLY thing writing ``_diag`` is the OAuth/AAD + token refresh, on a ~50-minute cadence; the minutes-scale cadence holds + only while jobs run. The shipped 900s default therefore read unhealthy for + ~35 of every 50 idle minutes with nothing degraded — the 2026-07-27 + "13 -> 37 -> 59 unhealthy growth" while the GitHub registry reported 64/64 + online throughout (59 -> 4 resolved with 8 restarts; the untouched control + group self-healed). + """ + + def test_idle_runner_past_old_900s_window_reads_healthy( + self, synthetic_runner_home: Path + ) -> None: + """45 min of idle _diag silence is HEALTHY under the script default.""" + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + diag_log = ( + synthetic_runner_home / "_diag" / "Runner_20260703-000000-utc.log" + ) + idle = time.time() - 2700 # 45 min — inside the idle OAuth cadence + os.utime(diag_log, (idle, idle)) + + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 0, ( + "an idle runner 45 min past its last _diag write must read " + "HEALTHY on the script default (the 900s default flagged it by " + f"arithmetic): rc={result.returncode} out={result.stdout}" + ) + finally: + proc.kill() + + def test_recalibration_did_not_disable_the_check( + self, synthetic_runner_home: Path + ) -> None: + """Past the recalibrated window, silence is still UNHEALTHY.""" + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + diag_log = ( + synthetic_runner_home / "_diag" / "Runner_20260703-000000-utc.log" + ) + dead = time.time() - 9000 # 2.5 h — well past any benign cadence + os.utime(diag_log, (dead, dead)) + + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 1, ( + f"2.5 h of silence must still fail: out={result.stdout}" + ) + assert "heartbeat" in result.stdout + finally: + proc.kill() + + @pytest.mark.parametrize( + ("idle_minutes", "expect_healthy"), + [ + (16, True), # just past the old 900s default — must no longer flag + (50, True), # the observed idle OAuth/AAD refresh cadence ceiling + (70, True), # still inside the recalibrated window + (80, False), # past it — recalibration did not disarm the layer + ], + ) + def test_default_threshold_brackets_the_idle_cadence( + self, + synthetic_runner_home: Path, + idle_minutes: int, + expect_healthy: bool, + ) -> None: + """Behavioral bracket of the SHIPPED default (nothing pinned). + + Replaces a source-text assertion on the literal ``4500`` that a + comment-only edit could satisfy. Driving the real script with the + threshold unset means the calibration itself is what is under test: at + a 900s default the 16/50/70-minute cases all fail, and any "clear the + idle cadence by disabling the check" regression fails the 80-minute + case. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + diag_log = ( + synthetic_runner_home / "_diag" / "Runner_20260703-000000-utc.log" + ) + aged = time.time() - idle_minutes * 60 + os.utime(diag_log, (aged, aged)) + + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + expected_rc = 0 if expect_healthy else 1 + assert result.returncode == expected_rc, ( + f"{idle_minutes} min of idle _diag silence must read " + f"{'HEALTHY' if expect_healthy else 'UNHEALTHY'} on the script " + f"default: rc={result.returncode} out={result.stdout}" + ) + if not expect_healthy: + assert "heartbeat" in result.stdout + finally: + proc.kill() + + +class TestHealthcheckOrphanInversion: + """OMN-15233 (b): the zombie shape #2194 exists to catch scored HEALTHY. + + An orphaned ``Runner.Listener`` reparented to PPID 1 keeps holding the + GitHub session; the watchdog's replacement crash-loops every ~5 min on + ``TaskAgentSessionConflictException``; every crash mints a fresh + ``Runner_*.log``, which keeps the ``_diag`` mtime heartbeat fresh — so the + mtime-only check read HEALTHY forever. Runners 1/43/55/57 sat in this state + with 88-234 log files (vs 3-7 normal) and were found only by process scan. + """ + + def test_duplicate_listeners_read_unhealthy( + self, synthetic_runner_home: Path + ) -> None: + """Two listeners = a contested broker session, whatever _diag says.""" + listener = synthetic_runner_home / "bin" / "Runner.Listener" + first = subprocess.Popen([str(listener)]) + second = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 1, ( + "duplicate Runner.Listener processes with a fresh _diag must " + f"read UNHEALTHY: rc={result.returncode} out={result.stdout}" + ) + assert "duplicate" in result.stdout.lower() + finally: + first.kill() + second.kill() + + def test_orphaned_ppid1_listener_reads_unhealthy( + self, synthetic_runner_home: Path + ) -> None: + """A single listener reparented to PPID 1 is an orphan, not health.""" + listener = synthetic_runner_home / "bin" / "Runner.Listener" + orphan_pid = _spawn_orphan(listener) + try: + ppid = None + for _ in range(50): + ppid = _ppid_of(orphan_pid) + if ppid == 1: + break + time.sleep(0.1) + if ppid != 1: + pytest.skip( + f"host did not reparent the orphan to PID 1 (ppid={ppid}); " + "a subreaper is active, so the incident precondition cannot " + "be reproduced here" + ) + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 1, ( + "a PPID-1 orphan with a fresh _diag must read UNHEALTHY — this " + "is the exact state that scored HEALTHY on runners 1/43/55/57: " + f"rc={result.returncode} out={result.stdout}" + ) + assert "PPID 1" in result.stdout + finally: + _kill_pid(orphan_pid) + + +class TestHealthcheckCrashLoopRate: + """OMN-15233 (b): crash-loop detection must be RATE-based, not cumulative.""" + + def test_crash_loop_rate_flags_but_history_does_not( + self, synthetic_runner_home: Path + ) -> None: + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + diag = synthetic_runner_home / "_diag" + try: + time.sleep(0.5) + + # (1) A long-lived clean runner: 234 historical Runner_*.log files + # (the observed zombie-runner count) all aged 10 days, plus the + # one active log. A CUMULATIVE count would flag this forever. + old = time.time() - 10 * 86400 + for i in range(234): + path = diag / f"Runner_2026070{i % 10}-{i:06d}-utc.log" + path.write_text("historical\n", encoding="utf-8") + os.utime(path, (old, old)) + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 0, ( + "234 historical logs with one active log must read HEALTHY — a " + "cumulative count would red-line every long-lived container: " + f"rc={result.returncode} out={result.stdout}" + ) + + # (2) The crash-loop shape: a replacement listener restarting every + # ~5 min mints ~12 fresh Runner_*.log files per hour. + for i in range(12): + path = diag / f"Runner_20260727-{i:06d}-utc.log" + path.write_text("crash\n", encoding="utf-8") + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 1, ( + "12 listener starts inside the rate window must read UNHEALTHY: " + f"rc={result.returncode} out={result.stdout}" + ) + assert "crash-looping" in result.stdout + finally: + proc.kill() + + def test_identical_logs_flip_the_verdict_purely_by_window_membership( + self, synthetic_runner_home: Path + ) -> None: + """Rate, not cumulative — proven behaviorally on the SAME files. + + Replaces a grep for the string ``NOT CUMULATIVE``. The file set is held + constant and only its mtime moves: 12 logs aged past the window read + healthy, the same 12 touched into the window read unhealthy. A + cumulative implementation cannot produce the first verdict. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + diag = synthetic_runner_home / "_diag" + try: + time.sleep(0.5) + crash_logs = [] + outside = time.time() - 45 * 60 # outside a 30m window + for i in range(12): + path = diag / f"Runner_20260727-{i:06d}-utc.log" + path.write_text("crash\n", encoding="utf-8") + os.utime(path, (outside, outside)) + crash_logs.append(path) + + result = _run_healthcheck( + synthetic_runner_home, max_diag_age=None, window_minutes="30" + ) + assert result.returncode == 0, ( + "12 listener starts that all fall OUTSIDE the rate window must " + f"read HEALTHY: rc={result.returncode} out={result.stdout}" + ) + + now = time.time() + for path in crash_logs: + os.utime(path, (now, now)) + result = _run_healthcheck( + synthetic_runner_home, max_diag_age=None, window_minutes="30" + ) + assert result.returncode == 1, ( + "the same 12 logs touched INSIDE the window must read " + f"UNHEALTHY: rc={result.returncode} out={result.stdout}" + ) + assert "crash-looping" in result.stdout + finally: + proc.kill() + + @pytest.mark.parametrize( + ("window_minutes", "extra_fresh_logs", "expect_healthy"), + [ + # Default 6/hour. Allowance = ceil(6 * window / 60). + ("30", 2, True), # 3 starts in 30m == 6/hour — at the allowance + ("30", 3, False), # 4 starts in 30m == 8/hour — over it + ("120", 11, True), # 12 starts in 120m == 6/hour — at the allowance + ("120", 12, False), # 13 starts in 120m — over it + ], + ) + def test_threshold_is_normalized_to_the_rate_window( + self, + synthetic_runner_home: Path, + window_minutes: str, + extra_fresh_logs: int, + expect_healthy: bool, + ) -> None: + """A per-HOUR threshold must be scaled to the window it is counted over. + + The pre-remediation code compared a window-scoped count directly against + the per-hour threshold, which is only correct at the 60m default: a 30m + window enforced 6-per-30m (= 12/hour, double the intended budget) and a + 120m window enforced 6-per-120m (= 3/hour, half of it). Every case here + is a RED against that arithmetic — ``("30", 3, False)`` reads healthy + unnormalized (4 > 6 is false) and ``("120", 11, True)`` reads unhealthy + unnormalized (12 > 6 is true). + + The fixture already ships one fresh ``Runner_*.log``, so the total start + count is ``extra_fresh_logs + 1``. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + diag = synthetic_runner_home / "_diag" + try: + time.sleep(0.5) + for i in range(extra_fresh_logs): + (diag / f"Runner_20260727-{i:06d}-utc.log").write_text( + "start\n", encoding="utf-8" + ) + + result = _run_healthcheck( + synthetic_runner_home, + max_diag_age=None, + window_minutes=window_minutes, + ) + expected_rc = 0 if expect_healthy else 1 + assert result.returncode == expected_rc, ( + f"{extra_fresh_logs + 1} starts in a {window_minutes}m window " + f"must read {'HEALTHY' if expect_healthy else 'UNHEALTHY'} " + f"against the default 6/hour budget: rc={result.returncode} " + f"out={result.stdout}" + ) + if not expect_healthy: + assert "crash-looping" in result.stdout + finally: + proc.kill() + + @pytest.mark.parametrize( + ("window_minutes", "max_starts_per_hour"), + [("0", None), ("abc", None), (None, "-1"), (None, "six")], + ) + def test_unusable_rate_tunables_fail_closed( + self, + synthetic_runner_home: Path, + window_minutes: str | None, + max_starts_per_hour: str | None, + ) -> None: + """A window/threshold the script cannot normalize must not read healthy. + + Integer normalization on unvalidated input is how a check silently + stops checking: a zero or non-numeric window would otherwise produce a + zero-or-garbage allowance and a permanently green (or permanently red) + layer. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + result = _run_healthcheck( + synthetic_runner_home, + max_diag_age=None, + window_minutes=window_minutes, + max_starts_per_hour=max_starts_per_hour, + ) + assert result.returncode == 1, ( + "an unusable crash-loop tunable must fail closed: " + f"rc={result.returncode} out={result.stdout} err={result.stderr}" + ) + assert "fail closed" in result.stdout + finally: + proc.kill() + + +SESSION_STAMP_NAME = ".session_broken_since" + +_CONNECT_ERROR_LINES = ( + "[2026-07-27 17:26:03Z ERR GitHubActionsRunner] Runner connect error: " + "The HTTP request timed out. Retrying until reconnected.\n" + "[2026-07-27 17:26:24Z ERR GitHubActionsRunner] " + "System.Net.Sockets.SocketException (125): Operation canceled\n" +) +_RECONNECT_LINE = ( + "[2026-07-27 17:31:11Z INFO GitHubActionsRunner] Runner reconnected.\n" +) +_LISTENING_LINE = "[2026-07-27 15:14:35Z INFO GitHubActionsRunner] Listening for Jobs\n" + + +def _write_session_log( + runner_home: Path, + body: str, + name: str = "Runner_20260727-170000-utc.log", +) -> Path: + """Write a Runner_*.log and make it the NEWEST one in _diag. + + The layer reads the newest listener log, so the fixture's own + ``Runner_20260703-000000-utc.log`` is aged back to keep the ordering + unambiguous rather than relying on same-second mtime ties. + """ + diag = runner_home / "_diag" + older = time.time() - 600 + for existing in diag.glob("Runner_*.log"): + os.utime(existing, (older, older)) + path = diag / name + path.write_text(body, encoding="utf-8") + return path + + +def _age_stamp(runner_home: Path, seconds: float) -> Path: + stamp = runner_home / "_diag" / SESSION_STAMP_NAME + stamp.write_text("", encoding="utf-8") + aged = time.time() - seconds + os.utime(stamp, (aged, aged)) + return stamp + + +class TestHealthcheckBrokerSessionState: + """OMN-15311: the FOURTH state — broker session broken, everything else fine. + + Measured live 2026-07-27 during the OMN-15233 fan-out. A transient + host<->GitHub network fault left runners 36/38/56 registry-OFFLINE for ~20 + minutes while the container read exit 0 on every existing layer: one live + non-orphaned listener, ``_diag`` kept FRESH by the listener's own reconnect + RETRY traffic, normal listener start rate, ``github.com`` reachable. Docker + counted them as capacity and ``runner-monitor.sh`` suppressed auto-bounce on + local-listener evidence, so they absorbed zero jobs until restarted. + + Layers 1-5 assert that a listener exists, is singular, is parented, is + writing, and has egress. None of them assert that it HOLDS A LIVE BROKER + SESSION, which is the only property that makes a runner able to take a job. + """ + + def test_persistently_broken_session_reads_unhealthy( + self, synthetic_runner_home: Path + ) -> None: + """THE reproduction. RED against origin/dev, which exits 0 on this input. + + Every input here satisfies every pre-OMN-15311 layer: single live + non-orphaned listener, fresh ``_diag``, one listener start, egress + skipped. The only thing wrong is that the listener's last session marker + is a connect error with no re-establish after it, and it has been that + way past the grace window. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + _write_session_log( + synthetic_runner_home, _LISTENING_LINE + _CONNECT_ERROR_LINES + ) + _age_stamp(synthetic_runner_home, 1800) + + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 1, ( + "a listener whose broker session has been broken for 30 min " + "must read UNHEALTHY even though the process, heartbeat, start " + f"rate and egress layers all pass: rc={result.returncode} " + f"out={result.stdout}" + ) + assert "broker session" in result.stdout.lower() + finally: + proc.kill() + + def test_transient_drop_inside_the_grace_reads_healthy( + self, synthetic_runner_home: Path + ) -> None: + """A blip must not flap 64 runners; the first observation only stamps. + + Reconnects are routine and fast. If a single observation of a dropped + session failed the check, every ordinary network hiccup would red-line + the fleet — the exact false-positive class OMN-15233 spent a rollout + removing. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + stamp = synthetic_runner_home / "_diag" / SESSION_STAMP_NAME + try: + time.sleep(0.5) + _write_session_log( + synthetic_runner_home, _LISTENING_LINE + _CONNECT_ERROR_LINES + ) + assert not stamp.exists() + + # First observation: nothing has been measured yet. + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 0, ( + "the FIRST observation of a broken session must not fail — the " + f"grace has not elapsed: rc={result.returncode} out={result.stdout}" + ) + assert stamp.exists(), ( + "a broken session must be stamped on first observation, or the " + "grace window can never elapse and the layer never fires" + ) + + # Still inside the grace on a later check. + _age_stamp(synthetic_runner_home, 60) + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 0, ( + f"60s of broken session is inside the 900s grace: {result.stdout}" + ) + finally: + proc.kill() + + def test_recovery_clears_the_stamp(self, synthetic_runner_home: Path) -> None: + """Recovery must reset the clock, not leave an ancient stamp armed. + + Without the clear, the next unrelated blip would inherit a stamp older + than the grace and fail on its FIRST observation — turning the + persistence gate into the instant check it exists to avoid. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + stamp = synthetic_runner_home / "_diag" / SESSION_STAMP_NAME + try: + time.sleep(0.5) + _write_session_log( + synthetic_runner_home, + _LISTENING_LINE + _CONNECT_ERROR_LINES + _RECONNECT_LINE, + ) + _age_stamp(synthetic_runner_home, 1800) + + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 0, ( + "a session that RECONNECTED after the errors is healthy however " + f"old the stamp is: rc={result.returncode} out={result.stdout}" + ) + assert not stamp.exists(), ( + "recovery must delete the stamp so the grace clock restarts " + "from the next drop" + ) + finally: + proc.kill() + + @pytest.mark.parametrize( + ("body", "expect_healthy", "case"), + [ + ( + _LISTENING_LINE + _CONNECT_ERROR_LINES + _RECONNECT_LINE, + True, + "errors then reconnect — session is up", + ), + ( + _LISTENING_LINE + _RECONNECT_LINE + _CONNECT_ERROR_LINES, + False, + "reconnect then errors — session is down", + ), + ], + ) + def test_verdict_follows_marker_ORDER_not_presence( + self, + synthetic_runner_home: Path, + body: str, + expect_healthy: bool, + case: str, + ) -> None: + """The same lines, reordered, must flip the verdict. + + This is what separates a session-STATE signal from "grep the log for an + error string". Essentially every long-lived healthy runner has connect + errors somewhere in its log; a presence check would red-line the fleet. + Both parametrizations carry an identical multiset of lines, so any + implementation that keys on presence returns the same verdict for both + and fails one of them. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + _write_session_log(synthetic_runner_home, body) + _age_stamp(synthetic_runner_home, 1800) + + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + expected_rc = 0 if expect_healthy else 1 + assert result.returncode == expected_rc, ( + f"{case}: expected " + f"{'HEALTHY' if expect_healthy else 'UNHEALTHY'}, got " + f"rc={result.returncode} out={result.stdout}" + ) + finally: + proc.kill() + + def test_marker_free_log_reads_healthy(self, synthetic_runner_home: Path) -> None: + """No session markers at all is not evidence of a broken session. + + Absence of a BROKEN marker is the healthy case; the fail-closed posture + applies to a probe that could not run (no listener log at all — see + below), not to a log that simply has not logged a session transition + inside its retained content. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + _write_session_log(synthetic_runner_home, "polling\npolling\n") + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 0, ( + f"a marker-free listener log must read HEALTHY: {result.stdout}" + ) + finally: + proc.kill() + + def test_no_listener_log_fails_closed(self, synthetic_runner_home: Path) -> None: + """A live listener with zero Runner_*.log files is unreadable, not fine. + + A registered listener always mints ``Runner_-utc.log`` at + start, so its absence means the matched process never registered — the + same divergence class the missing-``_diag`` branch already fails on. + ``_diag`` keeps a non-listener log so the heartbeat layer still passes + and this layer is what produces the verdict. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + diag = synthetic_runner_home / "_diag" + try: + time.sleep(0.5) + for existing in diag.glob("Runner_*.log"): + existing.unlink() + (diag / "Worker_20260727-170000-utc.log").write_text( + "worker\n", encoding="utf-8" + ) + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 1, ( + "a live listener with no Runner_*.log must fail closed: " + f"rc={result.returncode} out={result.stdout}" + ) + assert "Runner_*.log" in result.stdout + finally: + proc.kill() + + def test_kill_switch_disables_the_layer(self, synthetic_runner_home: Path) -> None: + """The layer must be disarmable by env without a fleet file swap. + + ``healthcheck.sh`` reaches the 64-runner fleet through a bind mount, so + a misfiring layer would otherwise need a file rollout to disarm. The + input here is the exact one that fails in the first test. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + _write_session_log( + synthetic_runner_home, _LISTENING_LINE + _CONNECT_ERROR_LINES + ) + _age_stamp(synthetic_runner_home, 1800) + + result = _run_healthcheck( + synthetic_runner_home, + max_diag_age=None, + session_state_check="0", + ) + assert result.returncode == 0, ( + "RUNNER_HEALTH_SESSION_STATE_CHECK=0 must skip the layer: " + f"rc={result.returncode} out={result.stdout}" + ) + finally: + proc.kill() + + @pytest.mark.parametrize("grace", ["0", "abc", "-1"]) + def test_unusable_grace_fails_closed( + self, synthetic_runner_home: Path, grace: str + ) -> None: + """A grace the script cannot normalize must not read healthy.""" + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + result = _run_healthcheck( + synthetic_runner_home, + max_diag_age=None, + max_session_broken=grace, + ) + assert result.returncode == 1, ( + "an unusable broker-session grace must fail closed: " + f"rc={result.returncode} out={result.stdout} err={result.stderr}" + ) + assert "fail closed" in result.stdout + finally: + proc.kill() + + def test_default_grace_brackets_the_observed_recovery_window( + self, synthetic_runner_home: Path + ) -> None: + """Behavioral bracket of the SHIPPED default — nothing pinned. + + The 2026-07-27 fault's recoverable drops cleared inside a single poll; + the unrecovered cohort held ~20 min. The default must therefore read + healthy at 10 min of broken session and unhealthy at 20. + """ + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + _write_session_log( + synthetic_runner_home, _LISTENING_LINE + _CONNECT_ERROR_LINES + ) + + _age_stamp(synthetic_runner_home, 600) + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 0, ( + f"10 min broken must be inside the default grace: {result.stdout}" + ) + + _age_stamp(synthetic_runner_home, 1200) + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 1, ( + "20 min broken — the observed unrecovered cohort's dwell — must " + f"be outside the default grace: {result.stdout}" + ) + finally: + proc.kill() + + +class TestHealthcheckAgainstARealListenerLog: + """OMN-15311 regression: the marker VOCABULARY, pinned to real fleet data. + + The synthetic session fixtures above are 3-4 hand-written lines each. They + exercise the artifact that runs; they do not exercise the input distribution + that runs, and that gap shipped a fleet-wide false positive: `SocketException` + was in ``session_broken_patterns``, but `BrokerServer` emits + ``System.Net.Sockets.SocketException (125): Operation canceled`` ~45-150x per + log as ordinary long-poll cancellation, while the connected markers only fire + at session establishment / job assignment. On any runner idle for >15 min the + retry noise is therefore the LAST marker, so marker ORDERING -- the design's + whole defence against a false positive -- did not help. Measured over all 64 + live listeners on omninode-pc, every one Up-healthy and registry-online: + 64/64 classified broken with `SocketException` in the set, 0/64 without it. + + These tests are RED against that vocabulary and GREEN against the corrected + one. Every future marker-vocabulary change has to survive a real log. + """ + + def test_real_healthy_listener_log_reads_healthy( + self, synthetic_runner_home: Path + ) -> None: + """THE regression. A real log from a healthy, registry-ONLINE runner. + + RED with `SocketException` in ``session_broken_patterns`` (the whole + 64-runner fleet flips Docker-unhealthy 15 min after the OMN-15233 + bind-mount swap); GREEN without it. + """ + body = gzip.decompress(REAL_LISTENER_TAIL.read_bytes()).decode("utf-8") + listener = synthetic_runner_home / "bin" / "Runner.Listener" + proc = subprocess.Popen([str(listener)]) + try: + time.sleep(0.5) + _write_session_log(synthetic_runner_home, body) + # Aged well past the 900s grace: if the layer classifies this log as + # broken at all, the grace cannot mask it. + _age_stamp(synthetic_runner_home, 3600) + + result = _run_healthcheck(synthetic_runner_home, max_diag_age=None) + assert result.returncode == 0, ( + "a real listener log from a healthy, registry-ONLINE runner must " + "read HEALTHY -- a marker set that fails this red-lines all 64 " + f"runners on the next healthcheck swap: rc={result.returncode} " + f"out={result.stdout}" + ) + finally: + proc.kill() + + def test_the_real_log_fixture_is_not_vacuous(self) -> None: + """The fixture only proves anything if it CONTAINS the retry noise. + + Guards against someone quietly swapping in a clean log and keeping the + test green: the last connect-class line must come AFTER the last + connected marker, which is exactly the shape that produced the 64/64 + false positive. + """ + lines = ( + gzip.decompress(REAL_LISTENER_TAIL.read_bytes()) + .decode("utf-8") + .splitlines() + ) + connected = re.compile( + r"Listening for Jobs|Runner reconnected|Job message received" + ) + last_connected = max( + (i for i, line in enumerate(lines) if connected.search(line)), + default=None, + ) + assert last_connected is not None, ( + "the fixture must contain a session-established marker, or it proves " + "nothing about ordering" + ) + noise_after = [ + i + for i, line in enumerate(lines) + if "SocketException" in line and i > last_connected + ] + assert len(noise_after) >= 10, ( + "the fixture must carry the long-poll SocketException churn AFTER " + "the last connected marker -- that ordering is the false positive " + f"under test; found {len(noise_after)} such lines" + ) + + def test_socketexception_is_not_a_broken_session_marker(self) -> None: + """Vocabulary guard with the measurement attached, so it cannot re-land. + + The execution test above is the real proof; this one names the offender + so a future edit that re-adds it fails with the reason rather than with + an opaque real-log diff. + """ + content = HEALTHCHECK.read_text(encoding="utf-8") + match = re.search(r"^\s*session_broken_patterns='([^']*)'", content, re.M) + assert match is not None, "session_broken_patterns must be assigned literally" + patterns = match.group(1).split("|") + assert "SocketException" not in patterns, ( + "SocketException is ordinary BrokerServer long-poll cancellation and " + "is the LAST marker on every idle runner: it classified 64/64 live " + "healthy registry-online listeners as broken (OMN-15311)" + ) + # The genuinely-broken markers must survive the narrowing. + assert "Runner connect error" in patterns + assert "TaskAgentSessionConflictException" in patterns + + class TestEntrypointWatchdog: """The entrypoint must supervise the listener, not just the wrapper tree.""" @@ -181,6 +1042,57 @@ def test_entrypoint_pattern_is_runner_home_anchored(self) -> None: "watchdog pgrep pattern must be derived from RUNNER_HOME" ) + def test_recycle_run_helper_pkill_is_runner_home_anchored(self) -> None: + """OMN-15776: an unanchored ``pkill -f "run-helper"`` inside + ``_recycle_runner_tree`` matches ANY process on the host whose cmdline + contains that substring — including the real fleet runner's own + ``run.sh -> run-helper.sh -> Runner.Listener`` wrapper (see + healthcheck.sh's process-tree comment) — when a nested/synthetic + entrypoint.sh runs the recycle path in the same PID namespace (no + docker-in-docker isolation on the self-hosted fleet). This is the + confirmed mechanism behind self-hosted CI jobs receiving an + out-of-band "[HostContext] Runner will be shutdown for UserCancelled" + signal mid-run while TestEntrypointHungListenerWatchdog exercises + this same recycle path elsewhere in the container. + + RED against the unanchored literal; GREEN once the pkill target is + derived from RUNNER_HOME like LISTENER_PGREP_PATTERN/ + WORKER_PGREP_PATTERN already are. + """ + content = ENTRYPOINT.read_text(encoding="utf-8") + recycle = content[content.index("_recycle_runner_tree() {") :] + recycle = recycle[: recycle.index("\n}\n")] + assert 'pkill -TERM -f "run-helper"' not in recycle, ( + "unanchored run-helper pkill would kill the real fleet runner's " + "own wrapper tree from inside a nested/synthetic entrypoint.sh " + "run sharing the same PID namespace" + ) + assert 'pkill -KILL -f "run-helper"' not in recycle, ( + "unanchored run-helper pkill would kill the real fleet runner's " + "own wrapper tree from inside a nested/synthetic entrypoint.sh " + "run sharing the same PID namespace" + ) + assert "RUN_HELPER_PGREP_PATTERN" in recycle, ( + "_recycle_runner_tree must pkill the run-helper wrapper via a " + "RUNNER_HOME-anchored pattern variable, not a bare substring" + ) + pattern_line = next( + ( + line + for line in content.splitlines() + if line.strip().startswith('RUN_HELPER_PGREP_PATTERN="') + ), + None, + ) + assert pattern_line is not None, ( + "RUN_HELPER_PGREP_PATTERN must have a default assignment" + ) + assert "${RUNNER_HOME//." in pattern_line, ( + "RUN_HELPER_PGREP_PATTERN default must be anchored to RUNNER_HOME, " + f"matching the LISTENER_PGREP_PATTERN/WORKER_PGREP_PATTERN convention: " + f"{pattern_line}" + ) + def test_entrypoint_bash_syntax(self) -> None: result = subprocess.run( ["bash", "-n", str(ENTRYPOINT)], check=False, capture_output=True, text=True @@ -274,13 +1186,20 @@ def test_entrypoint_has_heartbeat_watchdog(self) -> None: assert "LISTENER_HEARTBEAT_MISSES" in content assert "_listener_heartbeat_stale" in content - def test_kill_threshold_decoupled_and_above_alert_threshold(self) -> None: - """The watchdog KILL threshold must exceed the healthcheck ALERT - threshold (900s). Live 2026-07-23T05:25-06:02Z readback: a fleet-wide - broker-quiet window silenced _diag on 53/64 listeners for 35-50 min - while GitHub kept them online and docker-"unhealthy" runners were - actively executing jobs — killing at the 900s alert threshold would - have mass-recycled ~50 healthy-but-quiet listeners mid-window.""" + def test_kill_threshold_clears_the_benign_broker_quiet_ceiling(self) -> None: + """The watchdog KILL threshold must clear the observed benign + broker-quiet ceiling (~50 min). Live 2026-07-23T05:25-06:02Z readback: + a fleet-wide broker-quiet window silenced _diag on 53/64 listeners for + 35-50 min while GitHub kept them online and docker-"unhealthy" runners + were actively executing jobs — killing at the then-current 900s alert + threshold would have mass-recycled ~50 healthy-but-quiet listeners + mid-window. + + OMN-15233 note: the alert threshold is now 4500s, i.e. ABOVE this kill + threshold. The ordering flipped on purpose — this watchdog additionally + requires LISTENER_HEARTBEAT_MISSES consecutive ticks and never fires + while a Runner.Worker runs, so it is the narrower signal. What must + hold is the >= 3600s floor, not an ordering between the two.""" content = ENTRYPOINT.read_text(encoding="utf-8") match = re.search(r"LISTENER_HEARTBEAT_MAX_AGE_SECONDS:-(\d+)", content) assert match, "LISTENER_HEARTBEAT_MAX_AGE_SECONDS default missing" @@ -381,6 +1300,139 @@ def test_entrypoint_never_recycles_while_worker_running( subprocess.run(["pkill", "-KILL", "-f", str(home)], check=False) +@pytest.fixture +def synthetic_reap_home(tmp_path: Path) -> Path: + """A RUNNER_HOME whose ``run.sh`` reproduces the session-conflict failure. + + The real ``run.sh`` cannot register while a surviving listener still owns + this runner's GitHub broker session — it dies on + ``TaskAgentSessionConflictException``. This stub asserts the same + precondition: if a listener for this home is already running when run.sh + starts, it prints that exception and exits non-zero. So the ONLY way the + entrypoint gets a clean start is by reaping first. + """ + home = tmp_path / "actions-runner" + (home / "bin").mkdir(parents=True) + (home / "_diag").mkdir() + listener = home / "bin" / "Runner.Listener" + _make_executable(listener, "#!/bin/bash\nsleep 300\n") + _make_executable( + home / "run.sh", + "#!/bin/bash\n" + f'if pgrep -f "{listener}" >/dev/null 2>&1; then\n' + ' echo "TaskAgentSessionConflictException: session held by another listener"\n' + " exit 1\n" + "fi\n" + f'"{listener}" &\n' + "wait $!\n", + ) + (home / ".runner").write_text("{}\n", encoding="utf-8") + (home / ".credentials").write_text("{}\n", encoding="utf-8") + (home / "_diag" / "Runner_20260727-000000-utc.log").write_text( + "heartbeat\n", encoding="utf-8" + ) + return home + + +class TestEntrypointOrphanReap: + """OMN-15233: reap the orphan BEFORE spawning a replacement. + + Spawn-without-reap is what manufactures ``TaskAgentSessionConflictException`` + in the first place: the orphaned listener still owns the session, so every + replacement dies within ~5 min and mints a fresh ``Runner_*.log`` that keeps + the ``_diag`` heartbeat looking healthy. + + Reap-BEFORE-spawn ordering is asserted behaviorally, not by source position: + ``test_entrypoint_reaps_orphan_and_replacement_sees_no_session_conflict`` + starts the entrypoint with an orphan already holding the session against a + ``run.sh`` that fails with ``TaskAgentSessionConflictException`` whenever a + listener is alive at spawn time. Spawning first is therefore observable in + ``LOG_FILE``; a source-order grep (which a comment-only edit could satisfy) + would prove strictly less and has been removed rather than kept. + """ + + def test_reap_escalates_term_to_kill(self) -> None: + """A listener deadlocked in token refresh ignores TERM (OMN-14564).""" + content = ENTRYPOINT.read_text(encoding="utf-8") + reap = content[content.index("_reap_orphaned_listeners() {") :] + reap = reap[: reap.index("\n}\n")] + assert 'pkill -TERM -f "${LISTENER_PGREP_PATTERN}"' in reap + assert 'pkill -KILL -f "${LISTENER_PGREP_PATTERN}"' in reap + assert "LISTENER_REAP_TIMEOUT_SECONDS" in reap + + @pytest.mark.skipif( + os.getuid() == 0, reason="entrypoint takes root-only paths (gosu/groupmod)" + ) + def test_entrypoint_reaps_orphan_and_replacement_sees_no_session_conflict( + self, synthetic_reap_home: Path, tmp_path: Path + ) -> None: + """Functional: orphan present at start → reaped → clean replacement. + + DoD: "kill the parent and confirm no TaskAgentSessionConflictException + in the replacement's logs." + """ + home = synthetic_reap_home + orphan_pid = _spawn_orphan(home / "bin" / "Runner.Listener") + time.sleep(0.5) + assert _ppid_of(orphan_pid) is not None, "orphan failed to start" + + env = dict(os.environ) + env.update( + { + "RUNNER_NAME": "synthetic-runner", + "RUNNER_LABELS": "synthetic", + "GITHUB_ORG_URL": "https://github.com/OmniNode-ai", + "RUNNER_HOME": str(home), + "LOG_FILE": str(tmp_path / "listener.log"), + "LISTENER_SUPERVISE_INTERVAL": "1", + "LISTENER_REAP_TIMEOUT_SECONDS": "5", + } + ) + stdout_path = tmp_path / "entrypoint-stdout.log" + proc = None + try: + with stdout_path.open("wb") as stdout_file: + proc = subprocess.Popen( + ["bash", str(ENTRYPOINT)], + stdout=stdout_file, + stderr=subprocess.STDOUT, + env=env, + start_new_session=True, + ) + deadline = time.time() + 60 + output = "" + while time.time() < deadline: + output = stdout_path.read_text(encoding="utf-8") + if "REAP: no Runner.Listener remains" in output: + break + if proc.poll() is not None: + break + time.sleep(0.5) + output = stdout_path.read_text(encoding="utf-8") + assert "REAP: no Runner.Listener remains" in output, ( + f"entrypoint never reaped the orphan before spawning: {output}" + ) + # Give run.sh time to clear its session-conflict guard and spawn. + # Assert against LOG_FILE (the tee'd run.sh output), NOT the + # entrypoint stdout — the entrypoint's own REAP banner names the + # exception, so asserting on stdout would be vacuous. + time.sleep(2) + replacement_log = (tmp_path / "listener.log").read_text(encoding="utf-8") + assert "TaskAgentSessionConflictException" not in replacement_log, ( + f"replacement spawned into a contested session: {replacement_log}" + ) + output = stdout_path.read_text(encoding="utf-8") + assert proc.poll() is None, ( + f"entrypoint exited instead of running the replacement: {output}" + ) + assert _ppid_of(orphan_pid) is None, "the orphan survived the reap" + finally: + _kill_pid(orphan_pid) + if proc is not None and proc.poll() is None: + os.killpg(proc.pid, signal.SIGKILL) + subprocess.run(["pkill", "-KILL", "-f", str(home)], check=False) + + class TestComposeHealthcheckWiring: """The compose healthcheck stanza must invoke the mounted script.""" @@ -565,3 +1617,15 @@ def test_runbook_exists_with_authoritative_signal_note(self) -> None: assert "NOT sufficient" in content assert "canary" in content.lower() assert "OMN-13915" in content + + def test_runbook_records_registry_crosscheck_before_restart_sweep(self) -> None: + """OMN-15233 interim rule: the Docker-unhealthy count is not a + degradation metric on this fleet. An operator who restart-sweeps off it + alone repeats the 2026-07-27 false-positive sweep.""" + content = RUNBOOK.read_text(encoding="utf-8") + assert "OMN-15233" in content + assert "before ANY restart sweep" in content + assert "actions/runners" in content, ( + "the runbook must name the registry probe the operator runs, not " + "just tell them to 'cross-check'" + ) diff --git a/tests/ci/test_runner_routing_audit.py b/tests/ci/test_runner_routing_audit.py index 0606d10ec7..27be2c6d78 100644 --- a/tests/ci/test_runner_routing_audit.py +++ b/tests/ci/test_runner_routing_audit.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT from __future__ import annotations @@ -60,6 +60,123 @@ def test_local_workflow_audit_honors_explicit_allowlist(tmp_path: Path) -> None: assert module.audit_local_workflows(policy, tmp_path) == [] +def test_local_workflow_audit_rejects_dev_base_shortcut(tmp_path: Path) -> None: + module = _load_script() + workflow_dir = tmp_path / ".github" / "workflows" + workflow_dir.mkdir(parents=True) + (workflow_dir / "bad.yml").write_text( + """name: bad +jobs: + test: + runs-on: >- + ${{ + (github.event_name == 'pull_request' && github.base_ref == 'dev') + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON) + || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON) + }} +""", + encoding="utf-8", + ) + + findings = module.audit_local_workflows({"hosted_runner_allowlist": []}, tmp_path) + + assert len(findings) == 2 + assert "dev-base shortcut" in findings[0].message + assert "head repository differs" in findings[1].message + + +def test_local_workflow_audit_rejects_public_runner_for_every_pr( + tmp_path: Path, +) -> None: + module = _load_script() + workflow_dir = tmp_path / ".github" / "workflows" + workflow_dir.mkdir(parents=True) + (workflow_dir / "bad.yml").write_text( + """name: bad +jobs: + test: + runs-on: >- + ${{ + github.event_name == 'pull_request' + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON) + || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON) + }} +""", + encoding="utf-8", + ) + + findings = module.audit_local_workflows({"hosted_runner_allowlist": []}, tmp_path) + + assert len(findings) == 1 + assert "head repository differs" in findings[0].message + + +def test_runner_variable_selection_is_fork_aware() -> None: + module = _load_script() + + assert ( + module.runner_variable_for_event( + "pull_request", "OmniNode-ai/omnibase_infra", "OmniNode-ai/omnibase_infra" + ) + == "OMNI_TRUSTED_CI_RUNS_ON_JSON" + ) + assert ( + module.runner_variable_for_event( + "pull_request", "contributor/omnibase_infra", "OmniNode-ai/omnibase_infra" + ) + == "OMNI_PUBLIC_PR_RUNS_ON_JSON" + ) + assert ( + module.runner_variable_for_event("push", None, "OmniNode-ai/omnibase_infra") + == "OMNI_TRUSTED_CI_RUNS_ON_JSON" + ) + assert ( + module.runner_variable_for_event( + "merge_group", + None, + "OmniNode-ai/omnibase_infra", + merge_group_variable="OMNI_REQUIRED_CI_RUNS_ON_JSON", + ) + == "OMNI_REQUIRED_CI_RUNS_ON_JSON" + ) + + +def test_repository_workflows_follow_fork_aware_runner_policy() -> None: + module = _load_script() + import yaml + + policy = yaml.safe_load(POLICY.read_text(encoding="utf-8")) + + assert module.audit_local_workflows(policy, REPO_ROOT) == [] + + +def test_local_workflow_audit_rejects_pull_request_target(tmp_path: Path) -> None: + module = _load_script() + workflow_dir = tmp_path / ".github" / "workflows" + workflow_dir.mkdir(parents=True) + (workflow_dir / "bad.yml").write_text( + """name: bad +on: pull_request_target +jobs: + test: + runs-on: ubuntu-latest +""", + encoding="utf-8", + ) + + findings = module.audit_local_workflows( + { + "hosted_runner_allowlist": [ + {"path": ".github/workflows/bad.yml", "reason": "test"} + ] + }, + tmp_path, + ) + + assert len(findings) == 1 + assert "pull_request_target is prohibited" in findings[0].message + + def test_policy_tracks_repos_that_drifted_to_hosted_minutes() -> None: import yaml diff --git a/tests/ci/test_runtime_env_anchor.py b/tests/ci/test_runtime_env_anchor.py index 5098da06b3..848863a6d7 100644 --- a/tests/ci/test_runtime_env_anchor.py +++ b/tests/ci/test_runtime_env_anchor.py @@ -51,6 +51,13 @@ "USE_EVENT_ROUTING", "GITHUB_TOKEN", "GH_TOKEN", + # OMN-15529: OnexBot-OCC-Writer App identity for the OCC companion + # producer. Absent from the anchor, the credential on the host is + # invisible to the container and the OMN-15362 cutover is a no-op. + "ONEXBOT_OCC_APP_ID", + "ONEXBOT_OCC_PRIVATE_KEY", + "OMNI_OCC_GITHUB_AUTH_MODE", + "DEPLOY_AGENT_HMAC_SECRET", "LLM_GLM_URL", "LLM_GLM_MODEL_NAME", "LLM_GLM_API_KEY", @@ -59,6 +66,19 @@ "LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST", "VALKEY_HOST", "VALKEY_PORT", + # OMN-15645: omnimarket#2000 (OMN-15628) removed the packaged-default + # fallback for this key in the delegation routing reducer + # (resolve_required_path_config("DELEGATION_ROUTING_TIERS_PATH"), + # omnimarket src/omnimarket/nodes/node_delegation_routing_reducer/ + # handlers/handler_delegation_routing.py:392-393). An unbound key now + # raises ProtocolConfigurationError at first config read instead of + # silently defaulting — absent from the anchor, this key is a boot + # landmine, not a missing-optional-feature. This entry is the + # code-declared-required-key -> compose-anchor-coverage check AC4 of + # OMN-15645 asks for, scoped to this one key (registry-based, not a + # generic code scanner — see the OMN-15645 PR body for the deferral of + # the fully generic mechanism to OMN-14951). + "DELEGATION_ROUTING_TIERS_PATH", } ) @@ -111,6 +131,88 @@ def test_anchor_contains_required_keys(self) -> None: ) +class TestOccAppIdentityReachesRuntimeServices: + """OMN-15529: the OCC App credential must reach the container, not just the anchor. + + Asserting anchor membership alone is not the seam — a service only sees a + var if the anchor is merged into *its* environment. These tests drive the + resolved ``services..environment`` mapping (PyYAML resolves the + ``!!merge <<: *runtime-env`` keys), which is the same merge Docker Compose + performs before interpolation. + + Failure mode this closes (OMN-15362 defect D2): the OnexBot-OCC-Writer key + was provisioned into ``~/.omnibase/.env`` on the runtime host and was still + invisible to ``node_occ_companion_effect`` / ``OccCompanionEmitter``, + because ``x-runtime-env`` is an allowlist rather than a pass-through. + ``OMNI_OCC_GITHUB_AUTH_MODE=app`` would then fail with + ``GitHubAppCredentialMissingError`` against a dead runtime path. + """ + + OCC_APP_IDENTITY_KEYS: frozenset[str] = frozenset( + { + "ONEXBOT_OCC_APP_ID", + "ONEXBOT_OCC_PRIVATE_KEY", + "OMNI_OCC_GITHUB_AUTH_MODE", + } + ) + + # runtime-effects hosts the OCC companion EFFECT — the producer whose + # GitHub identity this credential changes. The other two runtime services + # inherit the same anchor and are asserted for consistency. + OCC_PRODUCER_SERVICE = "runtime-effects" + RUNTIME_SERVICES = ("omninode-runtime", "runtime-effects", "runtime-worker") + + @pytest.mark.unit + def test_occ_app_identity_reaches_occ_producer_service(self) -> None: + """The OCC producer container's env exposes all three App identity vars.""" + data = _load_compose() + service = data["services"][self.OCC_PRODUCER_SERVICE] + environment = service["environment"] + assert isinstance(environment, dict) + + missing = self.OCC_APP_IDENTITY_KEYS - set(environment) + assert not missing, ( + f"Service '{self.OCC_PRODUCER_SERVICE}' does not expose " + f"{sorted(missing)}. The host credential is invisible to the " + "containerized OCC companion producer — add the name(s) to " + "x-runtime-env in docker/docker-compose.infra.yml (OMN-15529)." + ) + + @pytest.mark.unit + def test_occ_app_identity_reaches_every_runtime_service(self) -> None: + """Every anchor-inheriting runtime service sees the same App identity vars.""" + data = _load_compose() + for service_name in self.RUNTIME_SERVICES: + environment = data["services"][service_name]["environment"] + assert isinstance(environment, dict) + missing = self.OCC_APP_IDENTITY_KEYS - set(environment) + assert not missing, ( + f"Service '{service_name}' is missing {sorted(missing)} from its " + "resolved environment (x-runtime-env merge)." + ) + + @pytest.mark.unit + def test_occ_app_identity_is_optional_not_fail_closed(self) -> None: + """The three vars use the optional form, so lanes without the key still render. + + ``${VAR:?...}`` here would abort ``docker compose config`` on every lane + that has not been provisioned with the App key. Empty is safe on both + consumers: the auth-mode readers coalesce ``""`` to ``pat``, and + ``resolve_api_key(..., required=False)`` treats an empty credential as + absent (raising ``GitHubAppCredentialMissingError`` in app mode rather + than silently falling back to the shared PAT). + """ + raw = COMPOSE_PATH.read_text() + for key in sorted(self.OCC_APP_IDENTITY_KEYS): + declaration = f"{key}: ${{{key}:-}}" + assert declaration in raw, ( + f"{key} must be declared as '{declaration}' in x-runtime-env. " + "A required (:?) or defaulted (:-value) form would either wedge " + "unprovisioned lanes or pin a default that belongs to the " + "consuming code, not to compose." + ) + + class TestRuntimeEnvAnchorSyntaxValid: """Test 2: docker-compose.infra.yml parses without YAML errors.""" diff --git a/tests/ci/test_runtime_policy_contract.py b/tests/ci/test_runtime_policy_contract.py index be8c946a53..e4d8a68d80 100644 --- a/tests/ci/test_runtime_policy_contract.py +++ b/tests/ci/test_runtime_policy_contract.py @@ -183,9 +183,11 @@ def test_compose_consumes_policy_env_instead_of_hardcoded_policy_values() -> Non def test_worker_replicas_pinned_in_contract_for_every_lane() -> None: """OMN-12990: each lane's worker process declares an explicit replica pin. - The base compose default is ``${WORKER_REPLICAS:-0}`` (silent drop). The - contract worker process must carry a contract-declared replica count >= 1 so - the rendered policy env preserves the worker on a lane recreate. + The contract worker process must carry a contract-declared replica count + >= 1 so the rendered policy env preserves the worker on a lane recreate. + Every lane's compose surface resolves that pin fail-closed (OMN-14968); a + lane whose contract dropped below 1 would push a zero-container lane through + a render that cannot detect it. """ contract = _load_contract() @@ -208,18 +210,34 @@ def test_worker_replicas_rendered_into_policy_env_for_every_lane() -> None: def test_runtime_worker_replicas_are_fail_fast_not_silent_default() -> None: - """OMN-12990: lane overrides reference the policy value with fail-fast ``:?``. - - A soft ``:-1`` / ``:-0`` default would silently re-introduce the silent-drop - hole on any recreate that omitted the policy env. The base compose keeps its - ``${WORKER_REPLICAS:-0}`` default (that is the gap the overrides close), so a - plain recreate against the base alone still scales to zero — by design the - lane overrides MUST be applied, and they now abort loudly when the policy env - is missing rather than dropping the worker with no signal. + """OMN-12990 / OMN-14968: EVERY lane surface resolves replicas fail-fast. + + A soft ``:-1`` / ``:-0`` default silently re-introduces the silent-drop hole + on any recreate that omits the policy env. + + OMN-12990 converted the stability-test and prod overlays but left the base + infra compose on a BARE ``${WORKER_REPLICAS:-0}``. That bare name is exported + by no surface in this repo, so it always took the ``0`` branch — and the base + file with no overlay IS the dev lane, so the dev lane rendered a + zero-container worker, `up` created nothing, and the RT-6 deploy readback in + ``scripts/deploy-runtime.sh`` (whose ``RUNTIME_SERVICES`` includes + ``runtime-worker``) failed closed on every dev-lane deploy. OMN-14968 closed + it with the lane-prefixed ``${DEV_WORKER_REPLICAS:?...}`` form used by the + sibling ``DEV_RUNTIME_WORKER_*`` vars in the same service block. """ + base_text = COMPOSE_PATH.read_text(encoding="utf-8") stability_text = STABILITY_COMPOSE_PATH.read_text(encoding="utf-8") prod_text = PROD_COMPOSE_PATH.read_text(encoding="utf-8") + assert "${DEV_WORKER_REPLICAS:?" in base_text, ( + "the base infra compose (the dev lane's own compose file) must resolve " + "worker replicas fail-fast on the ledgered DEV_WORKER_REPLICAS" + ) + assert "${DEV_WORKER_REPLICAS:-" not in base_text + assert "replicas: ${WORKER_REPLICAS" not in base_text, ( + "the bare WORKER_REPLICAS name is exported by no surface in this repo; " + "it always resolved to the silent 0 default (OMN-14968)" + ) assert "${STABILITY_TEST_WORKER_REPLICAS:?" in stability_text, ( "stability worker replicas must be fail-fast on the ledgered policy value" ) @@ -239,7 +257,9 @@ def test_boundary_dlq_enabled_declared_explicitly_for_every_lane() -> None: """ contract = _load_contract() - assert contract.profiles["dev"].boundary_dlq_enabled is False + # OMN-14551: flipped ON 2026-08-05 after a dedicated dev-lane live proof + # (see the contract's boundary_dlq_enabled comment on the dev profile). + assert contract.profiles["dev"].boundary_dlq_enabled is True assert contract.profiles["stability-test"].boundary_dlq_enabled is True assert contract.profiles["judge"].boundary_dlq_enabled is False assert contract.profiles["prod"].boundary_dlq_enabled is False @@ -249,7 +269,7 @@ def test_boundary_dlq_enabled_rendered_into_policy_env_for_every_lane() -> None: """OMN-14551: the renderer emits ``{PROFILE}_BOUNDARY_DLQ_ENABLED`` per lane.""" env = _load_dotenv(POLICY_ENV_PATH) - assert env["DEV_BOUNDARY_DLQ_ENABLED"] == "false" + assert env["DEV_BOUNDARY_DLQ_ENABLED"] == "true" assert env["STABILITY_TEST_BOUNDARY_DLQ_ENABLED"] == "true" assert env["JUDGE_BOUNDARY_DLQ_ENABLED"] == "false" assert env["PROD_BOUNDARY_DLQ_ENABLED"] == "false" @@ -277,3 +297,16 @@ def test_stability_test_boundary_dlq_wired_fail_fast_prod_and_judge_untouched() assert "${STABILITY_TEST_BOUNDARY_DLQ_ENABLED:-" not in stability_text assert "ONEX_BOUNDARY_DLQ_ENABLED" not in prod_text assert "ONEX_BOUNDARY_DLQ_ENABLED" not in judge_text + + +def test_dev_boundary_dlq_wired_fail_fast() -> None: + """OMN-14551 dev-lane flip (2026-08-05): dev's runtime containers reference + the ledgered ``DEV_BOUNDARY_DLQ_ENABLED`` value fail-fast (``:?``, no + silent ``:-false`` default) via the shared ``x-runtime-env`` anchor. + """ + dev_text = COMPOSE_PATH.read_text(encoding="utf-8") + + assert "ONEX_BOUNDARY_DLQ_ENABLED: ${DEV_BOUNDARY_DLQ_ENABLED:?" in dev_text, ( + "expected the flag wired fail-fast on the shared runtime-env anchor" + ) + assert "${DEV_BOUNDARY_DLQ_ENABLED:-" not in dev_text diff --git a/tests/ci/test_validate_test_root_collection.py b/tests/ci/test_validate_test_root_collection.py new file mode 100644 index 0000000000..0e97643cdc --- /dev/null +++ b/tests/ci/test_validate_test_root_collection.py @@ -0,0 +1,466 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Tests for the OMN-15378/OMN-15410 uncollected-pytest-root guard. + +``scripts/deploy-agent/tests/`` sat uncollected by any CI job for ~5 weeks; a +RED test inside it (superseded OMN-12988 literal) went unnoticed the entire +time. OMN-15410 then collected the four remaining roots of the same class and +closed the two seams that made the class possible. This module proves all of +it: + + 1. The live repo passes today (this IS the CI gate assertion — it runs + inside the required full-suite / smart-selection pytest job). + 2. A synthetic stray ``tests/`` directory outside every collected root is + detected and fails (RED-proof per OMN-15378 acceptance criterion 3, not + just green-proof). + 3. Every ``KNOWN_UNCOLLECTED_DEBT`` entry still exists and is still + genuinely uncollected — an allowlist that silently drifts from reality + (a debt entry that got fixed, or a debt entry that no longer exists) is + itself a defect the guard should not paper over. + 4. Every ``STANDALONE_PROJECT_ROOTS`` entry resolves to a real + ``pyproject.toml`` and a real, live CI workflow file. + 5. That workflow is actually *reachable on a pull request* and actually + *references the root* (OMN-15378 AC3). File existence alone was never + proof of wiring: the deploy-agent workflow is now ``workflow_call``-only, + reachable solely through ci.yml's caller job, which is what puts its + result under the required "CI Summary" context. + 6. OMN-15410: collected roots come from ``pyproject.toml`` ``testpaths``, + ci.yml's full suite passes no positional path that would override them, + and every collocated root is selectable by the change-aware selector. + Each has a synthetic RED-proof alongside the live-repo green assertion. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.ci.detect_test_paths import COLLOCATED_TEST_ROOTS +from scripts.validation.validate_test_root_collection import ( + FULL_SUITE_STEP_NAME, + KNOWN_UNCOLLECTED_DEBT, + REPO_ROOT, + STANDALONE_PROJECT_ROOTS, + check_collocated_selector_coverage, + check_full_suite_invocation, + collected_roots, + find_test_dirs, + find_violations, + positional_pytest_args, +) + +pytestmark = pytest.mark.unit + +# The four roots OMN-15410 moved out of KNOWN_UNCOLLECTED_DEBT and into +# collection. Named literally so a silent removal from testpaths reddens here +# rather than quietly dropping 366 tests again. +OMN_15410_COLLECTED_ROOTS = ( + "scripts/ci/tests/", + "scripts/tests/", + "scripts/runtime_build/tests/", + "src/omnibase_infra/services/observability/agent_actions/tests/", +) + + +def _synthetic_repo(tmp_path: Path, testpaths: str = '["tests"]') -> Path: + """A tmp_path repo root with just enough config for the guard to run. + + The guard reads ``testpaths`` from pyproject.toml rather than assuming + ``tests/`` (OMN-15410), so a synthetic fixture must declare its own — + fail-closed by design: no pyproject means no answer, not a guessed default. + """ + (tmp_path / "pyproject.toml").write_text( + f"[tool.pytest.ini_options]\ntestpaths = {testpaths}\n" + ) + return tmp_path + + +def test_live_repo_has_no_uncollected_test_roots() -> None: + """The actual CI gate: every tests/ dir in THIS repo is wired somewhere.""" + violations = find_violations(REPO_ROOT) + assert violations == [], ( + "Uncollected pytest root(s) detected (OMN-15378 class):\n" + + "\n".join(f" - {v}" for v in violations) + ) + + +def test_synthetic_stray_tests_dir_is_a_violation(tmp_path: Path) -> None: + """RED-proof: a stray tests/ dir with no wiring anywhere must fail.""" + repo = _synthetic_repo(tmp_path) + (repo / "tests").mkdir() + stray = repo / "scripts" / "widget" / "tests" + stray.mkdir(parents=True) + (stray / "test_widget.py").write_text("def test_ok() -> None:\n pass\n") + + violations = find_violations(repo) + + assert len(violations) == 1 + assert violations[0].startswith("scripts/widget/tests/:") + assert "unregistered" not in violations[0] # human message, not the raw sentinel + assert "no pytest invocation in CI can ever run these tests" in violations[0] + + +def test_synthetic_collected_root_tests_dir_is_not_a_violation( + tmp_path: Path, +) -> None: + """A tests/ dir under the root-collected tree is never flagged.""" + repo = _synthetic_repo(tmp_path) + collected = repo / "tests" / "unit" / "widget" + collected.mkdir(parents=True) + (collected / "test_widget.py").write_text("def test_ok() -> None:\n pass\n") + + assert find_violations(repo) == [] + + +def test_synthetic_root_named_in_testpaths_is_collected(tmp_path: Path) -> None: + """OMN-15410: a collocated root becomes collected by declaring it in + testpaths — the mechanism the four real roots now use.""" + repo = _synthetic_repo(tmp_path, testpaths='["tests", "scripts/widget/tests"]') + (repo / "tests").mkdir() + widget = repo / "scripts" / "widget" / "tests" + widget.mkdir(parents=True) + (widget / "test_widget.py").write_text("def test_ok() -> None:\n pass\n") + + assert find_violations(repo) == [] + + +def test_synthetic_missing_testpaths_entry_is_a_violation(tmp_path: Path) -> None: + """RED-proof: a testpaths entry with no directory behind it would abort + pytest collection with exit 5, so the guard rejects it.""" + repo = _synthetic_repo(tmp_path, testpaths='["tests", "scripts/gone/tests"]') + (repo / "tests").mkdir() + + violations = find_violations(repo) + + assert len(violations) == 1 + assert violations[0].startswith("scripts/gone/tests/:") + assert "exit 5" in violations[0] + + +def test_collected_roots_fails_closed_on_empty_testpaths(tmp_path: Path) -> None: + """RED-proof: an empty testpaths would make bare `pytest` collect the whole + repository (including .venv), so reading it must raise, not return ().""" + repo = _synthetic_repo(tmp_path, testpaths="[]") + + with pytest.raises(ValueError, match="declares no"): + collected_roots(repo) + + +def test_collected_roots_matches_live_testpaths() -> None: + """The live repo collects tests/ plus the four OMN-15410 roots.""" + roots = collected_roots(REPO_ROOT) + + assert "tests/" in roots + for root in OMN_15410_COLLECTED_ROOTS: + assert root in roots, ( + f"{root} dropped out of pyproject.toml testpaths — the OMN-15410 " + "roots would silently stop being collected again." + ) + + +def test_synthetic_unregistered_standalone_project_fails_closed( + tmp_path: Path, +) -> None: + """A registered STANDALONE_PROJECT_ROOTS entry with no real workflow file + behind it must still fail — the allowlist cannot be satisfied by adding a + dict entry alone.""" + from scripts.validation import validate_test_root_collection as module + + repo = _synthetic_repo(tmp_path) + (repo / "tests").mkdir() + root = repo / "scripts" / "unwired-agent" + tests_dir = root / "tests" + tests_dir.mkdir(parents=True) + (tests_dir / "test_thing.py").write_text("def test_ok() -> None:\n pass\n") + (root / "pyproject.toml").write_text("[project]\nname = 'unwired-agent'\n") + # Deliberately do NOT create the workflow file the registration claims. + + original = dict(module.STANDALONE_PROJECT_ROOTS) + module.STANDALONE_PROJECT_ROOTS.clear() + module.STANDALONE_PROJECT_ROOTS["scripts/unwired-agent"] = ( + ".github/workflows/does-not-exist.yml" + ) + try: + violations = module.find_violations(repo) + finally: + module.STANDALONE_PROJECT_ROOTS.clear() + module.STANDALONE_PROJECT_ROOTS.update(original) + + assert len(violations) == 1 + assert "does-not-exist.yml does not exist" in violations[0] + + +def _synthetic_standalone_root(tmp_path: Path) -> Path: + """A registered-shaped standalone project: pyproject.toml + its own tests/. + + The synthetic repo also gets a ROOT ``pyproject.toml`` declaring + ``testpaths``. Since OMN-15410 made ``testpaths`` the single source of + truth, :func:`collected_roots` fails closed when it is missing, and a + fixture repo without one is not a faithful stand-in for any real repo -- + merging #2553 with OMN-15410 surfaced exactly this. ``testpaths`` + deliberately lists only ``tests`` so ``scripts/widget-agent/tests`` stays + UNcollected and the standalone-registration path under test is the thing + actually exercised. + """ + root = tmp_path / "scripts" / "widget-agent" + tests_dir = root / "tests" + tests_dir.mkdir(parents=True) + (tests_dir / "test_thing.py").write_text("def test_ok() -> None:\n pass\n") + (root / "pyproject.toml").write_text("[project]\nname = 'widget-agent'\n") + (tmp_path / "pyproject.toml").write_text( + '[tool.pytest.ini_options]\ntestpaths = ["tests"]\n' + ) + (tmp_path / "tests").mkdir() + (tmp_path / ".github" / "workflows").mkdir(parents=True) + return root + + +def _with_registration( + tmp_path: Path, workflow_rel: str +) -> list[str]: # pragma: no cover - helper + from scripts.validation import validate_test_root_collection as module + + original = dict(module.STANDALONE_PROJECT_ROOTS) + module.STANDALONE_PROJECT_ROOTS.clear() + module.STANDALONE_PROJECT_ROOTS["scripts/widget-agent"] = workflow_rel + try: + return module.find_violations(tmp_path) + finally: + module.STANDALONE_PROJECT_ROOTS.clear() + module.STANDALONE_PROJECT_ROOTS.update(original) + + +def test_registered_workflow_that_never_runs_on_a_pr_fails_closed( + tmp_path: Path, +) -> None: + """RED-proof for the OMN-15378 AC3 hardening: a `workflow_call`-only wiring + workflow that NO workflow invokes runs zero tests, so registering it must + still fail — file existence alone was never proof of wiring.""" + _synthetic_standalone_root(tmp_path) + (tmp_path / ".github" / "workflows" / "widget-agent-tests.yml").write_text( + "name: Widget Agent Tests\n" + "on:\n" + " workflow_call:\n" + "jobs:\n" + " widget-agent-tests:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: pytest scripts/widget-agent/tests\n" + ) + + violations = _with_registration( + tmp_path, ".github/workflows/widget-agent-tests.yml" + ) + + assert len(violations) == 1 + assert "never runs on a pull request" in violations[0] + + +def test_registered_workflow_called_by_a_pr_workflow_is_accepted( + tmp_path: Path, +) -> None: + """The shape this repo now uses: the reusable is invoked by a PR-triggered + caller (ci.yml), which is what puts its result under a required context.""" + _synthetic_standalone_root(tmp_path) + workflows = tmp_path / ".github" / "workflows" + (workflows / "widget-agent-tests.yml").write_text( + "name: Widget Agent Tests\n" + "on:\n" + " workflow_call:\n" + "jobs:\n" + " widget-agent-tests:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: pytest scripts/widget-agent/tests\n" + ) + (workflows / "ci.yml").write_text( + "name: CI\n" + "on:\n" + " pull_request:\n" + " branches: [dev]\n" + "jobs:\n" + " widget-agent-tests:\n" + " uses: ./.github/workflows/widget-agent-tests.yml\n" + ) + + assert ( + _with_registration(tmp_path, ".github/workflows/widget-agent-tests.yml") == [] + ) + + +def test_registered_workflow_that_does_not_reference_the_root_fails_closed( + tmp_path: Path, +) -> None: + """A PR-triggered workflow that never mentions the root cannot be running + its tests — registration must not be satisfiable by pointing at any old + workflow file (e.g. re-pointing an entry at ci.yml).""" + _synthetic_standalone_root(tmp_path) + (tmp_path / ".github" / "workflows" / "unrelated.yml").write_text( + "name: Unrelated\n" + "on:\n" + " pull_request:\n" + " branches: [dev]\n" + "jobs:\n" + " lint:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: echo lint\n" + ) + + violations = _with_registration(tmp_path, ".github/workflows/unrelated.yml") + + assert len(violations) == 1 + assert "never references scripts/widget-agent" in violations[0] + + +def test_deploy_agent_wiring_workflow_is_pr_reachable_in_this_repo() -> None: + """Live assertion for the registration this repo actually ships: the + deploy-agent reusable is `workflow_call`-only, so its PR reachability comes + entirely from ci.yml's caller job. If that caller is removed, this fails.""" + from scripts.validation.validate_test_root_collection import ( + _workflow_runs_on_pull_request, + ) + + for root, workflow in STANDALONE_PROJECT_ROOTS.items(): + assert _workflow_runs_on_pull_request(workflow, REPO_ROOT), ( + f"{root}'s wiring workflow {workflow} is not reachable on a pull " + "request — its tests would be uncollected in practice" + ) + + +def test_known_uncollected_debt_entries_still_exist_and_are_still_uncollected() -> None: + """The debt allowlist must track reality: an entry pointing at a directory + that no longer exists (or that got wired up) is stale and should be + removed, not left as permanent cover for an unrelated future violation.""" + live_test_dirs = {d.rstrip("/") for d in find_test_dirs(REPO_ROOT)} + collected = collected_roots(REPO_ROOT) + for debt_entry in KNOWN_UNCOLLECTED_DEBT: + assert debt_entry in live_test_dirs, ( + f"KNOWN_UNCOLLECTED_DEBT entry {debt_entry!r} no longer exists as " + "a tests/ directory with test_*.py files — remove it from the " + "allowlist (OMN-15378 guard)." + ) + assert not any(f"{debt_entry}/".startswith(root) for root in collected), ( + f"KNOWN_UNCOLLECTED_DEBT entry {debt_entry!r} IS collected now — " + "remove it from the allowlist rather than leaving dead cover " + "(OMN-15410)." + ) + + +def test_omn15410_roots_are_no_longer_uncollected_debt() -> None: + """The OMN-15410 deliverable: the baseline shrank by exactly these four.""" + for root in OMN_15410_COLLECTED_ROOTS: + assert root.rstrip("/") not in KNOWN_UNCOLLECTED_DEBT, ( + f"{root} is collected via testpaths but still listed as " + "KNOWN_UNCOLLECTED_DEBT — the two cannot both be true." + ) + + +def test_standalone_project_roots_are_all_real() -> None: + """Every STANDALONE_PROJECT_ROOTS registration has a real pyproject.toml + and a real, live workflow file behind it (belt-and-suspenders on top of + find_violations itself already asserting this for the live repo).""" + for root, workflow in STANDALONE_PROJECT_ROOTS.items(): + assert (REPO_ROOT / root / "pyproject.toml").is_file(), ( + f"{root} is registered as a standalone project but has no pyproject.toml" + ) + assert (REPO_ROOT / workflow).is_file(), ( + f"{root} is registered as a standalone project but its wiring " + f"workflow {workflow} does not exist" + ) + + +# ============================================================================= +# OMN-15410 seam 2: ci.yml must not override testpaths with a positional path +# ============================================================================= + + +def test_live_full_suite_step_passes_no_positional_path() -> None: + """The live gate: ci.yml's full suite inherits testpaths verbatim.""" + violations = check_full_suite_invocation(REPO_ROOT) + assert violations == [], "\n".join(violations) + + +def test_positional_pytest_args_ignores_options_and_gh_expressions() -> None: + """The real full-suite command shape yields zero positional paths. + + Guards the parser itself: ``--splits ${{ ... }}``, a quoted ``-m`` marker + expression, ``-n 2 --dist loadgroup`` and + ``--junitxml=junit-${{ matrix.split }}.xml`` must not be read as paths. + """ + run_block = ( + "uv run pytest \\\n" + " --ignore=tests/integration/docker \\\n" + ' -m "not slow and not chaos and not kafka and not performance" \\\n' + " --splits ${{ needs.detect-changes.outputs.split_count }} \\\n" + " --group ${{ matrix.split }} \\\n" + " -n 2 --dist loadgroup \\\n" + " --timeout=60 \\\n" + " --timeout-method=thread \\\n" + " --tb=short \\\n" + " --store-durations \\\n" + " --junitxml=junit-${{ matrix.split }}.xml\n" + ) + + assert positional_pytest_args(run_block) == [] + + +def test_positional_pytest_args_detects_reintroduced_path() -> None: + """RED-proof: re-adding `tests/` is exactly the OMN-15410 regression.""" + run_block = "uv run pytest tests/ --ignore=tests/integration/docker -n 2\n" + + assert positional_pytest_args(run_block) == ["tests/"] + + +def test_full_suite_check_reports_reintroduced_positional_path( + tmp_path: Path, +) -> None: + """RED-proof end to end: a synthetic ci.yml with `pytest tests/` fails.""" + repo = _synthetic_repo(tmp_path) + (repo / "tests").mkdir() + workflow_dir = repo / ".github" / "workflows" + workflow_dir.mkdir(parents=True) + (workflow_dir / "ci.yml").write_text( + "jobs:\n" + " test:\n" + " steps:\n" + f" - name: {FULL_SUITE_STEP_NAME}\n" + " run: |\n" + " uv run pytest tests/ --tb=short\n" + ) + + violations = check_full_suite_invocation(repo) + + assert len(violations) == 1 + assert "overriding pyproject.toml testpaths" in violations[0] + assert "tests/" in violations[0] + + +# ============================================================================= +# OMN-15410 seam 3: collocated roots must be selectable by the selector +# ============================================================================= + + +def test_live_collocated_roots_are_selector_reachable() -> None: + """The live gate: testpaths <-> COLLOCATED_TEST_ROOTS parity holds.""" + violations = check_collocated_selector_coverage(REPO_ROOT) + assert violations == [], "\n".join(violations) + + +def test_every_omn15410_root_is_mapped_from_its_own_source_prefix() -> None: + """Each collocated root is reachable from a diff touching its own code, not + only from an unrelated full-suite escalation.""" + mapped = set(COLLOCATED_TEST_ROOTS.values()) + for root in OMN_15410_COLLECTED_ROOTS: + assert root in mapped, ( + f"{root} has no COLLOCATED_TEST_ROOTS mapping — a narrowed " + "smart-selection run could never select it (OMN-15410)." + ) + for source_prefix, root in COLLOCATED_TEST_ROOTS.items(): + assert root.startswith(source_prefix), ( + f"COLLOCATED_TEST_ROOTS maps {source_prefix!r} -> {root!r}, but the " + "root does not live under that prefix; the mapping would not fire " + "for a change to the code it covers." + ) diff --git a/tests/ci/test_verify_pypi_pin_resolvability.py b/tests/ci/test_verify_pypi_pin_resolvability.py index 128fb70b39..2959505e98 100644 --- a/tests/ci/test_verify_pypi_pin_resolvability.py +++ b/tests/ci/test_verify_pypi_pin_resolvability.py @@ -12,17 +12,27 @@ * fails RED (integration, real PyPI) on a deliberately-broken pin -- the exact OMN-14064 failure shape (a version that does not exist on PyPI), and * passes GREEN (integration, real PyPI) once the pin is a resolvable range. + +OMN-16047 adds the timeout dimension. The gate installs the whole transitive +closure with ``--no-cache``, so on a saturated fleet it can exceed its wall-clock +budget without any pin being wrong. The original code let +``subprocess.TimeoutExpired`` escape, which (a) discarded the partial ``uv`` +output and (b) presented a throughput failure in language reserved for an +unresolvable pin. These tests pin the distinction. """ from __future__ import annotations +import importlib import shutil import subprocess # nosec B404 - invokes `uv build` with a fixed argv in tests from pathlib import Path import pytest +import scripts.ci.verify_pypi_pin_resolvability as pin_gate from scripts.ci.verify_pypi_pin_resolvability import ( + PinResolveTimeoutError, find_single_wheel, verify_pin_resolvability, ) @@ -121,3 +131,136 @@ def test_resolvable_pin_passes_green(tmp_path: Path) -> None: ok, log = verify_pin_resolvability(wheel) assert ok is True, log + + +# --------------------------------------------------------------------------- +# Timeout handling (OMN-16047) -- unit, no network +# --------------------------------------------------------------------------- + + +def _stub_runs( + monkeypatch: pytest.MonkeyPatch, + *, + install_raises: subprocess.TimeoutExpired | None, +) -> None: + """Replace the module's ``subprocess.run`` so no real ``uv`` is invoked. + + ``uv venv`` always succeeds (and its side effect of creating the venv + directory is irrelevant to these assertions); the ``uv pip install`` call + raises whatever the caller supplies. + """ + + def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + if "venv" in argv: + return subprocess.CompletedProcess(argv, 0, "venv-created\n", "") + if install_raises is not None: + raise install_raises + return subprocess.CompletedProcess(argv, 0, "installed\n", "") + + monkeypatch.setattr(pin_gate.subprocess, "run", fake_run) + + +@pytest.mark.unit +def test_install_timeout_raises_typed_error_and_keeps_partial_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A slow install must surface as ``PinResolveTimeoutError`` carrying whatever + ``uv`` had already emitted -- not a bare ``TimeoutExpired`` traceback that + throws the diagnostic output away. This is the v0.38.4 failure shape. + """ + wheel = tmp_path / "pkg-1.0-py3-none-any.whl" + wheel.write_bytes(b"") + _stub_runs( + monkeypatch, + install_raises=subprocess.TimeoutExpired( + cmd=["uv", "pip", "install"], + timeout=pin_gate._INSTALL_TIMEOUT_SECONDS, + output=b"Resolved 135 packages\nDownloading grpcio\n", + ), + ) + + with pytest.raises(PinResolveTimeoutError) as caught: + verify_pin_resolvability(wheel) + + assert caught.value.step == "uv pip install" + assert caught.value.budget_seconds == pin_gate._INSTALL_TIMEOUT_SECONDS + assert "Downloading grpcio" in caught.value.partial_output + assert "venv-created" in caught.value.prior_output + + +@pytest.mark.unit +def test_timeout_is_not_reported_as_an_unresolvable_pin( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """The whole point of this gate is to name a bad pin. A timeout says nothing + about the pins, so the report must not borrow that language -- otherwise a + saturated runner reads as a broken release, which is what sent v0.38.4's + diagnosis down the wrong path. + """ + (tmp_path / "pkg-1.0-py3-none-any.whl").write_bytes(b"") + _stub_runs( + monkeypatch, + install_raises=subprocess.TimeoutExpired( + cmd=["uv", "pip", "install"], timeout=1800 + ), + ) + + assert pin_gate.main([str(tmp_path)]) == 1 + + out = capsys.readouterr().out + assert "THROUGHPUT failure" in out + assert "do not resolve" not in out + assert pin_gate._INSTALL_TIMEOUT_ENV_VAR in out + + +@pytest.mark.unit +def test_venv_creation_has_its_own_smaller_budget( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """``uv venv`` is purely local work. It must not be able to consume the + install's (much larger) network budget before the install even starts. + """ + assert pin_gate._VENV_TIMEOUT_SECONDS < pin_gate._INSTALL_TIMEOUT_SECONDS + + wheel = tmp_path / "pkg-1.0-py3-none-any.whl" + wheel.write_bytes(b"") + + def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + assert kwargs["timeout"] == pin_gate._VENV_TIMEOUT_SECONDS + raise subprocess.TimeoutExpired( + cmd=argv, timeout=pin_gate._VENV_TIMEOUT_SECONDS + ) + + monkeypatch.setattr(pin_gate.subprocess, "run", fake_run) + + with pytest.raises(PinResolveTimeoutError) as caught: + verify_pin_resolvability(wheel) + + assert caught.value.step == "uv venv" + + +@pytest.mark.unit +def test_install_budget_is_overridable_from_the_environment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The fleet's throughput is not a property of this repo, so the budget has + to be tunable from the workflow without editing the script. + """ + monkeypatch.setenv(pin_gate._INSTALL_TIMEOUT_ENV_VAR, "2400") + reloaded = importlib.reload(pin_gate) + try: + assert reloaded._INSTALL_TIMEOUT_SECONDS == 2400 + finally: + monkeypatch.delenv(pin_gate._INSTALL_TIMEOUT_ENV_VAR, raising=False) + importlib.reload(pin_gate) + + +@pytest.mark.unit +def test_default_install_budget_clears_the_measured_fleet_floor() -> None: + """A *cached* uv sync on the omnibase-ci fleet was measured at 110-402s + (OMN-16047). An uncached 242 MB / 135-package install cannot fit inside the + old 300s ceiling, so the default must stay well clear of it. + """ + assert pin_gate._INSTALL_TIMEOUT_SECONDS >= 1800 diff --git a/tests/ci/test_workspace_candidate_build.py b/tests/ci/test_workspace_candidate_build.py index 91c1bc8b21..dcf3ed12bf 100644 --- a/tests/ci/test_workspace_candidate_build.py +++ b/tests/ci/test_workspace_candidate_build.py @@ -84,6 +84,18 @@ def test_workflow_is_dispatch_only(self, workflow: dict[object, object]) -> None assert isinstance(on, dict) assert set(on.keys()) == {"workflow_dispatch"} + def test_workflow_uses_clean_cloud_egress_for_ecr_push( + self, workflow: dict[object, object] + ) -> None: + jobs = workflow.get("jobs") + assert isinstance(jobs, dict) + build_job = jobs.get("build-workspace-candidate") + assert isinstance(build_job, dict) + assert build_job.get("runs-on") == "ubuntu-latest", ( + "workspace candidate builds must use hosted clean-cloud egress; " + "the .201 self-hosted Docker daemon is a known unreliable ECR push lane" + ) + def test_workflow_builds_workspace_mode_with_candidate_stamp( self, workflow_text: str ) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index a8a44a0a88..1854a268c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,6 +76,35 @@ import pytest from dotenv import load_dotenv + +def _strip_inherited_git_environment() -> None: + """Remove caller-owned Git process state before tests are collected. + + Git hooks export repository and command-scoped ``GIT_*`` variables. Those + variables override both ``cwd`` and ``git -C`` in subprocesses, so a test + constructing a disposable repository can otherwise mutate the repository + whose hook launched pytest. Run this during pytest configuration so module- + level environment snapshots are also hermetic. + """ + for key in tuple(os.environ): + if key.startswith("GIT_"): + os.environ.pop(key) + + +def pytest_configure(config: pytest.Config) -> None: + """Make every test root independent of an invoking Git hook.""" + del config + _strip_inherited_git_environment() + + +@pytest.fixture(autouse=True) +def _strip_test_local_git_environment(monkeypatch: pytest.MonkeyPatch) -> None: + """Prevent one test's Git process state from leaking into the next test.""" + for key in tuple(os.environ): + if key.startswith("GIT_"): + monkeypatch.delenv(key, raising=False) + + # Load environment variables from .env file at test session start # This enables tests to use infrastructure config (KAFKA_BOOTSTRAP_SERVERS, etc.) # without needing to set env vars on command line diff --git a/tests/fixtures/application_database_acl/acl-policy-postgres16.yaml b/tests/fixtures/application_database_acl/acl-policy-postgres16.yaml new file mode 100644 index 0000000000..25ff4d7cf2 --- /dev/null +++ b/tests/fixtures/application_database_acl/acl-policy-postgres16.yaml @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +database_ref: application +physical_database: omnidash_analytics +completion_status: verified +source_kind: synthetic_fixture +principal_domains: + onex_api: [TENANT, PLATFORM_CATALOG] + tenant_projection_writer: [TENANT] + app_dashboard: [TENANT, PLATFORM_CATALOG] + omninode_runtime: [OMNINODE_INTERNAL] +database_owner_role: owner_platform_catalog +migration_principal: db_migrator +migration_owner_roles: + - owner_onex_tenant + - owner_omninode_internal + - owner_platform_catalog +governed_role_states: + - {role: app_dashboard, role_kind: workload, login: true} + - {role: db_migrator, role_kind: migration, login: false} + - {role: keycloak_service, role_kind: external_connect, login: true, manage_attributes: false} + - {role: omninode_runtime, role_kind: workload, login: true} + - {role: omnibase_infra_service, role_kind: external_connect, login: true, manage_attributes: false} + - {role: omniclaude_service, role_kind: external_connect, login: true, manage_attributes: false} + - {role: omniintelligence_service, role_kind: external_connect, login: true, manage_attributes: false} + - {role: omnimemory_service, role_kind: external_connect, login: true, manage_attributes: false} + - {role: omninode_cloud_service, role_kind: external_connect, login: true, manage_attributes: false} + - {role: onex_api, role_kind: workload, login: true} + - {role: owner_omninode_internal, role_kind: owner, login: false} + - {role: owner_onex_tenant, role_kind: owner, login: false} + - {role: owner_platform_catalog, role_kind: owner, login: false} + - {role: tenant_projection_writer, role_kind: workload, login: true} + - {role: umami_service, role_kind: external_connect, login: true, manage_attributes: false} +retained_administrative_principals: [rls_admin] +connection_policies: + - database_ref: keycloak + physical_database: keycloak + observed_database_owner_role: postgres + allowed_principals: [keycloak_service] + - database_ref: omnibase_infra + physical_database: omnibase_infra + observed_database_owner_role: postgres + allowed_principals: [omnibase_infra_service] + - database_ref: application + physical_database: omnidash_analytics + observed_database_owner_role: app_dashboard + allowed_principals: + - app_dashboard + - omninode_runtime + - onex_api + - tenant_projection_writer + - database_ref: omninode_cloud + physical_database: omninode_cloud + observed_database_owner_role: postgres + allowed_principals: [omninode_cloud_service] + - database_ref: omniclaude + physical_database: omniclaude + observed_database_owner_role: postgres + allowed_principals: [omniclaude_service] + - database_ref: omniintelligence + physical_database: omniintelligence + observed_database_owner_role: postgres + allowed_principals: [omniintelligence_service] + - database_ref: omnimemory + physical_database: omnimemory + observed_database_owner_role: postgres + allowed_principals: [omnimemory_service] + - database_ref: umami + physical_database: umami + observed_database_owner_role: postgres + allowed_principals: [umami_service] +reason: Independent eight-database CONNECT and application-domain policy. diff --git a/tests/fixtures/application_database_acl/acl-policy.yaml b/tests/fixtures/application_database_acl/acl-policy.yaml new file mode 100644 index 0000000000..afc0610c11 --- /dev/null +++ b/tests/fixtures/application_database_acl/acl-policy.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +database_ref: application +physical_database: omnidash_analytics +completion_status: verified +source_kind: synthetic_fixture +principal_domains: + onex_api: [TENANT, PLATFORM_CATALOG] + tenant_projection_writer: [TENANT] + app_dashboard: [TENANT, PLATFORM_CATALOG] + omninode_runtime: [OMNINODE_INTERNAL] +database_owner_role: owner_platform_catalog +migration_principal: db_migrator +migration_owner_roles: + - owner_onex_tenant + - owner_omninode_internal + - owner_platform_catalog +governed_role_states: + - {role: app_dashboard, role_kind: workload, login: true} + - {role: db_migrator, role_kind: migration, login: false} + - {role: omninode_runtime, role_kind: workload, login: true} + - {role: onex_api, role_kind: workload, login: true} + - {role: owner_omninode_internal, role_kind: owner, login: false} + - {role: owner_onex_tenant, role_kind: owner, login: false} + - {role: owner_platform_catalog, role_kind: owner, login: false} + - {role: tenant_projection_writer, role_kind: workload, login: true} +retained_administrative_principals: [rls_admin] +connection_policies: + - database_ref: application + physical_database: omnidash_analytics + observed_database_owner_role: app_dashboard + allowed_principals: + - app_dashboard + - omninode_runtime + - onex_api + - tenant_projection_writer +reason: Independent domain policy for the wholly synthetic ACL proof. diff --git a/tests/fixtures/application_database_acl/inventory.yaml b/tests/fixtures/application_database_acl/inventory.yaml new file mode 100644 index 0000000000..a3d9c07703 --- /dev/null +++ b/tests/fixtures/application_database_acl/inventory.yaml @@ -0,0 +1,257 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +ticket: synthetic-proof +database_ref: application +physical_seed_database: omnidash_analytics +ownership_authority: typed synthetic ACL proof inventory +inventory_projection: generated; never an ownership allowlist +completion_status: verified +relation_counts: + table: 5 + view: 1 + materialized_view: 1 + function: 7 + procedure: 1 + sequence: 1 + extension: 0 + type: 3 +relations: + - name: tenant_accounts + kind: table + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: service:onex_api + readers: [onex-api] + writers: [onex-api] + classification_evidence: synthetic control-plane fixture + classification_status: classified + migration_root: proof + - name: delegation_events + kind: table + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: node_projection_delegation + readers: [app-dashboard] + writers: [node_projection_delegation] + classification_evidence: synthetic tenant projection fixture + classification_status: classified + migration_root: proof + - name: partitioned_events + kind: table + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: node_projection_delegation + readers: [] + writers: [] + classification_evidence: synthetic partitioned-table fixture + classification_status: classified + migration_root: proof + - name: runtime_state + kind: table + target_schema: omninode_internal + current_schema: [omninode_internal] + domain: OMNINODE_INTERNAL + owner_declaration: node_runtime_state_projection + readers: [node_runtime_state_projection] + writers: [node_runtime_state_projection] + classification_evidence: synthetic internal fixture + classification_status: classified + migration_root: proof + - name: plan_tiers + kind: table + target_schema: platform_catalog + current_schema: [platform_catalog] + domain: PLATFORM_CATALOG + owner_declaration: service:onex_api + readers: [onex-api, app-dashboard] + writers: [] + classification_evidence: synthetic catalog fixture + classification_status: classified + migration_root: proof + - name: delegation_events_id_seq + kind: sequence + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: node_projection_delegation + writers: [node_projection_delegation] + classification_evidence: synthetic sequence fixture + classification_status: classified + migration_root: proof + - name: delegation_event_count + kind: function + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: node_projection_delegation + readers: [app-dashboard] + function_signature: "()" + classification_evidence: synthetic function fixture + classification_status: classified + migration_root: proof + - name: hostile_signature + kind: function + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: service:onex_api + function_signature: '("arg''name%" integer)' + classification_evidence: synthetic quoted routine-identity fixture + classification_status: classified + migration_root: proof + - name: record_delegation + kind: procedure + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: node_projection_delegation + writers: [node_projection_delegation] + function_signature: "(IN p_payload text)" + classification_evidence: synthetic procedure fixture + classification_status: classified + migration_root: proof + - name: tenant_account_names + kind: view + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: service:onex_api + readers: [app-dashboard] + classification_evidence: synthetic view fixture + classification_status: classified + migration_root: proof + - name: plan_tier_snapshot + kind: materialized_view + target_schema: platform_catalog + current_schema: [platform_catalog] + domain: PLATFORM_CATALOG + owner_declaration: service:onex_api + readers: [app-dashboard] + classification_evidence: synthetic materialized-view fixture + classification_status: classified + migration_root: proof + - name: account_ref + kind: type + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: service:onex_api + readers: [onex-api] + classification_evidence: synthetic composite-type fixture + classification_status: classified + migration_root: proof + - name: account_id_span + kind: range_type + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: service:onex_api + readers: [onex-api] + classification_evidence: synthetic range-type fixture + classification_status: classified + migration_root: proof + - name: account_id_span_set + kind: multirange_type + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: service:onex_api + readers: [onex-api] + classification_evidence: synthetic multirange-type fixture + classification_status: classified + migration_root: proof + - name: account_id_span + kind: function + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: service:onex_api + function_signature: "(bigint, bigint)" + classification_evidence: synthetic range-constructor fixture + classification_status: classified + migration_root: proof + - name: account_id_span + kind: function + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: service:onex_api + function_signature: "(bigint, bigint, text)" + classification_evidence: synthetic range-constructor fixture + classification_status: classified + migration_root: proof + - name: account_id_span_set + kind: function + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: service:onex_api + function_signature: "()" + classification_evidence: synthetic multirange-constructor fixture + classification_status: classified + migration_root: proof + - name: account_id_span_set + kind: function + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: service:onex_api + function_signature: "(VARIADIC tenant.account_id_span[])" + classification_evidence: synthetic multirange-constructor fixture + classification_status: classified + migration_root: proof + - name: account_id_span_set + kind: function + target_schema: tenant + current_schema: [tenant] + domain: TENANT + owner_declaration: service:onex_api + function_signature: "(tenant.account_id_span)" + classification_evidence: synthetic multirange-constructor fixture + classification_status: classified + migration_root: proof + - name: runtime_status + kind: type + target_schema: omninode_internal + current_schema: [omninode_internal] + domain: OMNINODE_INTERNAL + owner_declaration: node_runtime_state_projection + readers: [node_runtime_state_projection] + classification_evidence: synthetic type fixture + classification_status: classified + migration_root: proof + - name: runtime_code + kind: type + target_schema: omninode_internal + current_schema: [omninode_internal] + domain: OMNINODE_INTERNAL + owner_declaration: node_runtime_state_projection + readers: [node_runtime_state_projection] + classification_evidence: synthetic domain-type fixture + classification_status: classified + migration_root: proof +blocked_relations: [] +retained_live_census: + observed_base_tables: 5 + observed_views_and_materialized_views: 2 + observed_sequences: 1 + observed_functions: 7 + observed_procedures: 1 + observed_types: 5 + observed_extensions: 0 + parity_status: verified + reason: synthetic inventory is exact +runtime_evidence: + dsn_key_provenance: {} + full_day_datname_usename_activity: + status: verified + reason: synthetic proof has no live activity claim + credentials_captured: false + live_catalog_parity: + status: verified + reason: synthetic object seed is exact diff --git a/tests/fixtures/application_database_acl/principal-inventories-external.yaml b/tests/fixtures/application_database_acl/principal-inventories-external.yaml new file mode 100644 index 0000000000..3b937d80e7 --- /dev/null +++ b/tests/fixtures/application_database_acl/principal-inventories-external.yaml @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +- schema_version: "1.0" + database_ref: keycloak + physical_database: keycloak + completion_status: verified + catalog_parity_status: verified + source_kind: synthetic_fixture + database_owner_role: postgres + principal_refs: &all_external_principals + - app_dashboard + - external_connect_parent + - keycloak_service + - omnibase_infra_service + - omniclaude_service + - omniintelligence_service + - omnimemory_service + - omninode_cloud_service + - omninode_runtime + - onex_api + - rls_admin + - tenant_projection_writer + - umami_service + - untrusted_login + observed_role_states: &all_external_role_states + - {role: app_dashboard, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: true, replication: false, inherit: true} + - {role: external_connect_parent, login: false, superuser: false, bypass_rls: true, create_database: false, create_role: false, replication: false, inherit: true} + - {role: keycloak_service, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: omnibase_infra_service, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: omniclaude_service, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: omniintelligence_service, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: omnimemory_service, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: omninode_cloud_service, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: omninode_runtime, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: true, inherit: true} + - {role: onex_api, login: true, superuser: false, bypass_rls: false, create_database: true, create_role: false, replication: false, inherit: true} + - {role: rls_admin, login: false, superuser: false, bypass_rls: true, create_database: false, create_role: false, replication: false, inherit: true} + - {role: tenant_projection_writer, login: true, superuser: false, bypass_rls: true, create_database: false, create_role: false, replication: false, inherit: true} + - {role: umami_service, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: untrusted_login, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: true} + live_database_read: false + reason: Exact synthetic cluster-role census for Keycloak CONNECT isolation. +- schema_version: "1.0" + database_ref: omnibase_infra + physical_database: omnibase_infra + completion_status: verified + catalog_parity_status: verified + source_kind: synthetic_fixture + database_owner_role: postgres + principal_refs: *all_external_principals + observed_role_states: *all_external_role_states + live_database_read: false + reason: Exact synthetic cluster-role census for omnibase_infra CONNECT isolation. +- schema_version: "1.0" + database_ref: omninode_cloud + physical_database: omninode_cloud + completion_status: verified + catalog_parity_status: verified + source_kind: synthetic_fixture + database_owner_role: postgres + principal_refs: *all_external_principals + observed_role_states: *all_external_role_states + live_database_read: false + reason: Exact synthetic cluster-role census for omninode_cloud CONNECT isolation. +- schema_version: "1.0" + database_ref: omniclaude + physical_database: omniclaude + completion_status: verified + catalog_parity_status: verified + source_kind: synthetic_fixture + database_owner_role: postgres + principal_refs: *all_external_principals + observed_role_states: *all_external_role_states + live_database_read: false + reason: Exact synthetic cluster-role census for omniclaude CONNECT isolation. +- schema_version: "1.0" + database_ref: omniintelligence + physical_database: omniintelligence + completion_status: verified + catalog_parity_status: verified + source_kind: synthetic_fixture + database_owner_role: postgres + principal_refs: *all_external_principals + observed_role_states: *all_external_role_states + live_database_read: false + reason: Exact synthetic cluster-role census for omniintelligence CONNECT isolation. +- schema_version: "1.0" + database_ref: omnimemory + physical_database: omnimemory + completion_status: verified + catalog_parity_status: verified + source_kind: synthetic_fixture + database_owner_role: postgres + principal_refs: *all_external_principals + observed_role_states: *all_external_role_states + live_database_read: false + reason: Exact synthetic cluster-role census for omnimemory CONNECT isolation. +- schema_version: "1.0" + database_ref: umami + physical_database: umami + completion_status: verified + catalog_parity_status: verified + source_kind: synthetic_fixture + database_owner_role: postgres + principal_refs: *all_external_principals + observed_role_states: *all_external_role_states + live_database_read: false + reason: Exact synthetic cluster-role census for umami CONNECT isolation. diff --git a/tests/fixtures/application_database_acl/principal-inventory-postgres16.yaml b/tests/fixtures/application_database_acl/principal-inventory-postgres16.yaml new file mode 100644 index 0000000000..10c084774b --- /dev/null +++ b/tests/fixtures/application_database_acl/principal-inventory-postgres16.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +database_ref: application +physical_database: omnidash_analytics +completion_status: verified +catalog_parity_status: verified +source_kind: synthetic_fixture +database_owner_role: app_dashboard +principal_refs: + - app_dashboard + - db_migrator + - omninode_runtime + - onex_api + - rls_admin + - tenant_projection_writer + - untrusted_login +owner_refs: + - owner_omninode_internal + - owner_onex_tenant + - owner_platform_catalog +observed_role_states: + - {role: app_dashboard, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: true, replication: false, inherit: true} + - {role: db_migrator, login: false, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: true} + - {role: omninode_runtime, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: true, inherit: true} + - {role: onex_api, login: true, superuser: false, bypass_rls: false, create_database: true, create_role: false, replication: false, inherit: true} + - {role: rls_admin, login: false, superuser: false, bypass_rls: true, create_database: false, create_role: false, replication: false, inherit: true} + - {role: tenant_projection_writer, login: true, superuser: false, bypass_rls: true, create_database: false, create_role: false, replication: false, inherit: true} + - {role: untrusted_login, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: true} + - {role: owner_omninode_internal, login: false, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: true} + - {role: owner_onex_tenant, login: false, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: true} + - {role: owner_platform_catalog, login: false, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: true} +observed_schema_owners: + tenant: app_dashboard + omninode_internal: tenant_projection_writer + platform_catalog: onex_api +observed_objects: + - {schema_ref: tenant, catalog_kind: table, object_ref: tenant_accounts, owner: onex_api} + - {schema_ref: tenant, catalog_kind: table, object_ref: delegation_events, owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: table, object_ref: partitioned_events, owner: onex_api} + - {schema_ref: tenant, catalog_kind: sequence, object_ref: delegation_events_id_seq, owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: function, object_ref: delegation_event_count, function_signature: "()", owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: function, object_ref: hostile_signature, function_signature: '("arg''name%" integer)', owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: procedure, object_ref: record_delegation, function_signature: "(IN p_payload text)", owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: view, object_ref: tenant_account_names, owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: type, object_ref: account_ref, owner: onex_api} + - {schema_ref: tenant, catalog_kind: range_type, object_ref: account_id_span, owner: onex_api} + - {schema_ref: tenant, catalog_kind: multirange_type, object_ref: account_id_span_set, owner: onex_api} + - {schema_ref: tenant, catalog_kind: function, object_ref: account_id_span, function_signature: "(bigint, bigint)", owner: onex_api} + - {schema_ref: tenant, catalog_kind: function, object_ref: account_id_span, function_signature: "(bigint, bigint, text)", owner: onex_api} + - {schema_ref: tenant, catalog_kind: function, object_ref: account_id_span_set, function_signature: "()", owner: onex_api} + - {schema_ref: tenant, catalog_kind: function, object_ref: account_id_span_set, function_signature: "(VARIADIC tenant.account_id_span[])", owner: onex_api} + - {schema_ref: tenant, catalog_kind: function, object_ref: account_id_span_set, function_signature: "(tenant.account_id_span)", owner: onex_api} + - {schema_ref: omninode_internal, catalog_kind: type, object_ref: runtime_status, owner: omninode_runtime} + - {schema_ref: omninode_internal, catalog_kind: type, object_ref: runtime_code, owner: omninode_runtime} + - {schema_ref: omninode_internal, catalog_kind: table, object_ref: runtime_state, owner: tenant_projection_writer} + - {schema_ref: platform_catalog, catalog_kind: table, object_ref: plan_tiers, owner: onex_api} + - {schema_ref: platform_catalog, catalog_kind: materialized_view, object_ref: plan_tier_snapshot, owner: onex_api} +live_database_read: false +reason: Exact relevant-principal census for the synthetic application database. diff --git a/tests/fixtures/application_database_acl/principal-inventory.yaml b/tests/fixtures/application_database_acl/principal-inventory.yaml new file mode 100644 index 0000000000..4a4be03aa8 --- /dev/null +++ b/tests/fixtures/application_database_acl/principal-inventory.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +database_ref: application +physical_database: omnidash_analytics +completion_status: verified +catalog_parity_status: verified +source_kind: synthetic_fixture +database_owner_role: app_dashboard +principal_refs: + - app_dashboard + - db_migrator + - omninode_runtime + - onex_api + - rls_admin + - tenant_projection_writer + - untrusted_login +owner_refs: + - owner_omninode_internal + - owner_onex_tenant + - owner_platform_catalog +observed_role_states: + - {role: app_dashboard, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: db_migrator, login: false, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: omninode_runtime, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: onex_api, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: rls_admin, login: false, superuser: false, bypass_rls: true, create_database: false, create_role: false, replication: false, inherit: true} + - {role: tenant_projection_writer, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: untrusted_login, login: true, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: true} + - {role: owner_omninode_internal, login: false, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: owner_onex_tenant, login: false, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} + - {role: owner_platform_catalog, login: false, superuser: false, bypass_rls: false, create_database: false, create_role: false, replication: false, inherit: false} +observed_schema_owners: + tenant: app_dashboard + omninode_internal: tenant_projection_writer + platform_catalog: onex_api +observed_objects: + - {schema_ref: tenant, catalog_kind: table, object_ref: tenant_accounts, owner: onex_api} + - {schema_ref: tenant, catalog_kind: table, object_ref: delegation_events, owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: table, object_ref: partitioned_events, owner: onex_api} + - {schema_ref: tenant, catalog_kind: sequence, object_ref: delegation_events_id_seq, owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: function, object_ref: delegation_event_count, function_signature: "()", owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: function, object_ref: hostile_signature, function_signature: '("arg''name%" integer)', owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: procedure, object_ref: record_delegation, function_signature: "(IN p_payload text)", owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: view, object_ref: tenant_account_names, owner: app_dashboard} + - {schema_ref: tenant, catalog_kind: type, object_ref: account_ref, owner: onex_api} + - {schema_ref: tenant, catalog_kind: range_type, object_ref: account_id_span, owner: onex_api} + - {schema_ref: tenant, catalog_kind: multirange_type, object_ref: account_id_span_set, owner: onex_api} + - {schema_ref: tenant, catalog_kind: function, object_ref: account_id_span, function_signature: "(bigint, bigint)", owner: onex_api} + - {schema_ref: tenant, catalog_kind: function, object_ref: account_id_span, function_signature: "(bigint, bigint, text)", owner: onex_api} + - {schema_ref: tenant, catalog_kind: function, object_ref: account_id_span_set, function_signature: "()", owner: onex_api} + - {schema_ref: tenant, catalog_kind: function, object_ref: account_id_span_set, function_signature: "(VARIADIC tenant.account_id_span[])", owner: onex_api} + - {schema_ref: tenant, catalog_kind: function, object_ref: account_id_span_set, function_signature: "(tenant.account_id_span)", owner: onex_api} + - {schema_ref: omninode_internal, catalog_kind: type, object_ref: runtime_status, owner: omninode_runtime} + - {schema_ref: omninode_internal, catalog_kind: type, object_ref: runtime_code, owner: omninode_runtime} + - {schema_ref: omninode_internal, catalog_kind: table, object_ref: runtime_state, owner: tenant_projection_writer} + - {schema_ref: platform_catalog, catalog_kind: table, object_ref: plan_tiers, owner: onex_api} + - {schema_ref: platform_catalog, catalog_kind: materialized_view, object_ref: plan_tier_snapshot, owner: onex_api} +live_database_read: false +reason: Exact role census for the wholly synthetic PostgreSQL 16 ACL proof. diff --git a/tests/fixtures/application_database_acl/topology.yaml b/tests/fixtures/application_database_acl/topology.yaml new file mode 100644 index 0000000000..ea63e9e0d0 --- /dev/null +++ b/tests/fixtures/application_database_acl/topology.yaml @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "2.0" +services: {} +presets: {} +active_preset: null +databases: + application: + physical_name: omnidash_analytics + schemas: + tenant: {domain: TENANT, owner: owner_onex_tenant} + omninode_internal: {domain: OMNINODE_INTERNAL, owner: owner_omninode_internal} + platform_catalog: {domain: PLATFORM_CATALOG, owner: owner_platform_catalog} + owners: + owner_onex_tenant: {login: false} + owner_omninode_internal: {login: false} + owner_platform_catalog: {login: false} + principals: + onex_api: + login: true + bypass_rls: false + grants: + - {object_type: DATABASE, privileges: [CONNECT]} + - {object_type: SCHEMA, schema: tenant, privileges: [USAGE]} + - {object_type: SCHEMA, schema: platform_catalog, privileges: [USAGE]} + - object_type: TABLE + schema: tenant + objects: [tenant_accounts] + privileges: [SELECT, INSERT, UPDATE] + - object_type: TYPE + schema: tenant + objects: [account_id_span, account_id_span_set, account_ref] + privileges: [USAGE] + - object_type: TABLE + schema: platform_catalog + objects: [plan_tiers] + privileges: [SELECT] + tenant_projection_writer: + login: true + bypass_rls: false + grants: + - {object_type: DATABASE, privileges: [CONNECT]} + - {object_type: SCHEMA, schema: tenant, privileges: [USAGE]} + - object_type: TABLE + schema: tenant + objects: [delegation_events] + privileges: [SELECT, INSERT, UPDATE] + - object_type: SEQUENCE + schema: tenant + objects: [delegation_events_id_seq] + privileges: [USAGE] + - object_type: FUNCTION + schema: tenant + objects: [record_delegation] + privileges: [EXECUTE] + app_dashboard: + login: true + bypass_rls: false + grants: + - {object_type: DATABASE, privileges: [CONNECT]} + - {object_type: SCHEMA, schema: tenant, privileges: [USAGE]} + - {object_type: SCHEMA, schema: platform_catalog, privileges: [USAGE]} + - object_type: TABLE + schema: tenant + objects: [delegation_events, tenant_account_names] + privileges: [SELECT] + - object_type: TABLE + schema: platform_catalog + objects: [plan_tier_snapshot, plan_tiers] + privileges: [SELECT] + - object_type: FUNCTION + schema: tenant + objects: [delegation_event_count] + privileges: [EXECUTE] + omninode_runtime: + login: true + bypass_rls: false + grants: + - {object_type: DATABASE, privileges: [CONNECT]} + - {object_type: SCHEMA, schema: omninode_internal, privileges: [USAGE]} + - object_type: TABLE + schema: omninode_internal + objects: [runtime_state] + privileges: [SELECT, INSERT, UPDATE] + - object_type: TYPE + schema: omninode_internal + objects: [runtime_code, runtime_status] + privileges: [USAGE] + bindings: + onex_api: + database_ref: application + principal: onex_api + dsn_env: OMNINODE_CLOUD_DB_URL + tenant_projection: + database_ref: application + principal: tenant_projection_writer + dsn_env: OMNIDASH_ANALYTICS_DB_URL + app_dashboard: + database_ref: application + principal: app_dashboard + dsn_env: DATABASE_URL + omninode_runtime_service: + database_ref: application + principal: omninode_runtime + dsn_env: OMNINODE_INTERNAL_DB_URL + migration_stream: omnibase_infra.application + checksum_ledgers: + canonical: + schema: platform_catalog + relation: schema_migrations + stream_column: migration_stream + domain_column: domain + version_column: version + checksum_column: checksum + checksum_ledger: canonical diff --git a/tests/fixtures/application_relation_ownership/blocked-service.yaml b/tests/fixtures/application_relation_ownership/blocked-service.yaml new file mode 100644 index 0000000000..ed2b718628 --- /dev/null +++ b/tests/fixtures/application_relation_ownership/blocked-service.yaml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +service: blocked_migration_runner +owner_declaration: service:blocked_migration_runner +target_database_ref: application +completion_status: blocked_pending_live_catalog_and_activity_evidence +db_io: + db_tables: + - name: schema_migrations + database_ref: application + schema: omninode_internal + migration: run-migrations.py + access: read_write + role: canonical_migration_ledger +relation_evidence: + - name: schema_migrations + kind: table + domain: OMNINODE_INTERNAL + owner_declaration: service:blocked_migration_runner + readers: [] +blocked_relations: + - name: unresolved_live_relation + kind: table + reason: no authoritative repository DDL or current owner declaration was found diff --git a/tests/fixtures/application_relation_ownership/conflicting-location.yaml b/tests/fixtures/application_relation_ownership/conflicting-location.yaml new file mode 100644 index 0000000000..d91a1fb384 --- /dev/null +++ b/tests/fixtures/application_relation_ownership/conflicting-location.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +name: node_conflicting_delegation_owner +node_type: EFFECT_GENERIC +db_io: + db_tables: + - name: delegation_events + database_ref: application + schema: omninode_internal + migration: conflict.sql + access: read_write + role: conflicting_events diff --git a/tests/fixtures/application_relation_ownership/duplicate-owner.yaml b/tests/fixtures/application_relation_ownership/duplicate-owner.yaml new file mode 100644 index 0000000000..1adae0ed3c --- /dev/null +++ b/tests/fixtures/application_relation_ownership/duplicate-owner.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +service: duplicate_delegation_owner +owner_declaration: service:duplicate_delegation_owner +target_database_ref: application +db_io: + db_tables: + - name: delegation_events + database_ref: application + schema: tenant + migration: duplicate.sql + access: read_write + role: duplicate_events +relation_evidence: + - name: delegation_events + kind: table + domain: TENANT + owner_declaration: service:duplicate_delegation_owner + readers: [] diff --git a/tests/fixtures/application_relation_ownership/false-complete-census.yaml b/tests/fixtures/application_relation_ownership/false-complete-census.yaml new file mode 100644 index 0000000000..470aa0894a --- /dev/null +++ b/tests/fixtures/application_relation_ownership/false-complete-census.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +ticket: OMN-15423 +database_ref: application +physical_seed_database: omnidash_analytics +ownership_authority: distributed node contract.yaml -> db_io.db_tables +inventory_projection: generated; never an ownership allowlist +completion_status: passed +relation_counts: + table: 0 + view: 0 + materialized_view: 0 + function: 0 + sequence: 0 + extension: 0 +relations: [] +blocked_relations: [] +retained_live_census: + observed_base_tables: 86 + observed_views_and_materialized_views: 9 + parity_status: passed +runtime_evidence: + dsn_key_provenance: {} + full_day_datname_usename_activity: + status: passed + live_catalog_parity: + status: passed diff --git a/tests/fixtures/application_relation_ownership/incomplete-census.yaml b/tests/fixtures/application_relation_ownership/incomplete-census.yaml new file mode 100644 index 0000000000..02bc281c04 --- /dev/null +++ b/tests/fixtures/application_relation_ownership/incomplete-census.yaml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +service: incomplete_census_runner +owner_declaration: service:incomplete_census_runner +target_database_ref: application +db_io: + db_tables: + - name: schema_migrations + database_ref: application + schema: omninode_internal + migration: run-migrations.py + access: read_write + role: canonical_migration_ledger +retained_live_census: + observed_base_tables: 3 + observed_views_and_materialized_views: 2 + parity_status: blocked + reason: two retained live table names and two view names remain unresolved +relation_evidence: + - name: schema_migrations + kind: table + domain: OMNINODE_INTERNAL + owner_declaration: service:incomplete_census_runner + readers: [] diff --git a/tests/fixtures/application_relation_ownership/inventory-unknown-schema.yaml b/tests/fixtures/application_relation_ownership/inventory-unknown-schema.yaml new file mode 100644 index 0000000000..06a1b02e5b --- /dev/null +++ b/tests/fixtures/application_relation_ownership/inventory-unknown-schema.yaml @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +relations: + - {name: unknown_schema_table, database_ref: application, schema: not_declared, kind: table, purpose: data} diff --git a/tests/fixtures/application_relation_ownership/inventory-with-missing.yaml b/tests/fixtures/application_relation_ownership/inventory-with-missing.yaml new file mode 100644 index 0000000000..77424fec9f --- /dev/null +++ b/tests/fixtures/application_relation_ownership/inventory-with-missing.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +relations: + - {name: delegation_events, database_ref: application, schema: tenant, kind: table, purpose: data} + - {name: undeclared_live_table, database_ref: application, schema: tenant, kind: table, purpose: data} diff --git a/tests/fixtures/application_relation_ownership/inventory-without-delegation.yaml b/tests/fixtures/application_relation_ownership/inventory-without-delegation.yaml new file mode 100644 index 0000000000..2c85671a47 --- /dev/null +++ b/tests/fixtures/application_relation_ownership/inventory-without-delegation.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +relations: + - {name: schema_migrations, database_ref: application, schema: omninode_internal, kind: table, purpose: migration_ledger} + - {name: projection_delegation_summary, database_ref: application, schema: tenant, kind: view, purpose: data} + - {name: projection_delegation_rollup, database_ref: application, schema: omninode_internal, kind: materialized_view, purpose: data} + - {name: resolve_delegation_state, database_ref: application, schema: platform_catalog, kind: function, purpose: data} diff --git a/tests/fixtures/application_relation_ownership/inventory.yaml b/tests/fixtures/application_relation_ownership/inventory.yaml new file mode 100644 index 0000000000..d4ebb4201b --- /dev/null +++ b/tests/fixtures/application_relation_ownership/inventory.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +relations: + - {name: delegation_events, database_ref: application, schema: tenant, kind: table, purpose: data} + - {name: schema_migrations, database_ref: application, schema: omninode_internal, kind: table, purpose: migration_ledger} + - {name: projection_delegation_summary, database_ref: application, schema: tenant, kind: view, purpose: data} + - {name: projection_delegation_rollup, database_ref: application, schema: omninode_internal, kind: materialized_view, purpose: data} + - {name: resolve_delegation_state, database_ref: application, schema: platform_catalog, kind: function, purpose: data} diff --git a/tests/fixtures/application_relation_ownership/node-owner.yaml b/tests/fixtures/application_relation_ownership/node-owner.yaml new file mode 100644 index 0000000000..6406804e63 --- /dev/null +++ b/tests/fixtures/application_relation_ownership/node-owner.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +name: node_projection_delegation +node_type: EFFECT_GENERIC +db_io: + db_tables: + - name: delegation_events + database_ref: application + schema: tenant + migration: nodes/node_projection_delegation/0001.sql + access: read_write + role: events diff --git a/tests/fixtures/application_relation_ownership/node-reader.yaml b/tests/fixtures/application_relation_ownership/node-reader.yaml new file mode 100644 index 0000000000..d921081adc --- /dev/null +++ b/tests/fixtures/application_relation_ownership/node-reader.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +name: node_delegation_reader +node_type: COMPUTE_GENERIC +db_io: + db_tables: + - name: delegation_events + database_ref: application + schema: tenant + migration: nodes/node_delegation_reader/0001.sql + access: read + role: events_reader diff --git a/tests/fixtures/application_relation_ownership/parallel-location.yaml b/tests/fixtures/application_relation_ownership/parallel-location.yaml new file mode 100644 index 0000000000..a354b2eaec --- /dev/null +++ b/tests/fixtures/application_relation_ownership/parallel-location.yaml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +service: invalid_parallel_location +owner_declaration: service:invalid_parallel_location +target_database_ref: application +db_io: + db_tables: + - name: schema_migrations + database: omnidash_analytics + database_ref: application + schema: omninode_internal + migration: run-migrations.py + access: read_write + role: migration_ledger diff --git a/tests/fixtures/application_relation_ownership/rich-inventory.yaml b/tests/fixtures/application_relation_ownership/rich-inventory.yaml new file mode 100644 index 0000000000..b876f07246 --- /dev/null +++ b/tests/fixtures/application_relation_ownership/rich-inventory.yaml @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +ticket: OMN-15423 +database_ref: application +physical_seed_database: omnidash_analytics +ownership_authority: distributed node contract.yaml -> db_io.db_tables +inventory_projection: generated; never an ownership allowlist +completion_status: blocked_pending_live_catalog_and_activity_evidence +relation_counts: + table: 1 + view: 1 + materialized_view: 0 + function: 1 + sequence: 1 + extension: 1 +relations: + - name: delegation_events + kind: table + target_schema: tenant + domain: TENANT + owner_declaration: node:node_projection_delegation + producer: node_projection_delegation + classification_evidence: owning node migration + classification_status: classified + migration_root: src/omnimarket/nodes + migration_stream: omnimarket_node_migrations + - name: delegation_summary + kind: view + target_schema: tenant + domain: TENANT + owner_declaration: node:node_projection_delegation + producer: node_projection_delegation + classification_evidence: owning view migration + classification_status: classified + migration_root: src/omnimarket/nodes + migration_stream: omnimarket_node_migrations + - name: resolve_delegation + kind: function + function_signature: "()" + target_schema: platform_catalog + domain: PLATFORM_CATALOG + owner_declaration: node:node_projection_delegation + producer: node_projection_delegation + classification_evidence: owning function migration + classification_status: classified + migration_root: src/omnimarket/nodes + migration_stream: omnimarket_node_migrations + - name: delegation_events_id_seq + kind: sequence + target_schema: tenant + domain: TENANT + owner_declaration: node:node_projection_delegation + producer: node_projection_delegation + classification_evidence: table-owned sequence + classification_status: classified + migration_root: src/omnimarket/nodes + migration_stream: omnimarket_node_migrations + - name: pgcrypto + kind: extension + target_schema: platform_catalog + domain: PLATFORM_CATALOG + owner_declaration: omnimarket_node_migration_stream + producer: omnimarket_node_migration_stream + classification_evidence: migration extension dependency + classification_status: classified + migration_root: src/omnimarket/nodes + migration_stream: omnimarket_node_migrations +blocked_relations: + - name: delegation_judge_verdict_events + kind: table + reason: schema 'unresolved' does not resolve through deployment topology +retained_live_census: + observed_base_tables: 86 + observed_views_and_materialized_views: 9 + parity_status: blocked + reason: fresh authorized catalog read required +runtime_evidence: + dsn_key_provenance: + OMNIDASH_ANALYTICS_DB_URL: + - src/omnimarket/projection/runner.py + full_day_datname_usename_activity: + status: blocked + reason: live database access was outside this build lane's authorization + credentials_captured: false + live_catalog_parity: + status: blocked + reason: current parity is not proven diff --git a/tests/fixtures/application_relation_ownership/service-owner.yaml b/tests/fixtures/application_relation_ownership/service-owner.yaml new file mode 100644 index 0000000000..7691a06d5e --- /dev/null +++ b/tests/fixtures/application_relation_ownership/service-owner.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "1.0" +service: fixture_migration_runner +target_database_ref: application +db_io: + db_tables: + - name: schema_migrations + database_ref: application + schema: omninode_internal + migration: run-migrations.py + access: read_write + role: canonical_migration_ledger +relation_evidence: + - name: schema_migrations + kind: table + domain: OMNINODE_INTERNAL + owner_declaration: service:fixture_migration_runner + readers: [fixture_migration_runner] + - name: projection_delegation_summary + kind: view + database_ref: application + schema: tenant + domain: TENANT + owner_declaration: service:fixture_migration_runner + readers: [onex_api] + - name: projection_delegation_rollup + kind: materialized_view + database_ref: application + schema: omninode_internal + domain: OMNINODE_INTERNAL + owner_declaration: service:fixture_migration_runner + readers: [omninode_runtime] +database_objects: + - name: resolve_delegation_state + kind: function + database_ref: application + schema: platform_catalog + domain: PLATFORM_CATALOG + owner_declaration: service:fixture_migration_runner + readers: [onex_api] diff --git a/tests/fixtures/application_relation_ownership/topology.yaml b/tests/fixtures/application_relation_ownership/topology.yaml new file mode 100644 index 0000000000..0021098457 --- /dev/null +++ b/tests/fixtures/application_relation_ownership/topology.yaml @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +schema_version: "2.0" +services: {} +presets: {} +active_preset: null +databases: + application: + physical_name: omnidash_analytics + schemas: + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + owners: + owner_onex_tenant: {login: false} + owner_omninode_internal: {login: false} + owner_platform_catalog: {login: false} + principals: + app_dashboard: + login: true + bypass_rls: false + grants: + - object_type: SCHEMA + schema: platform_catalog + privileges: [USAGE] + - object_type: TABLE + schema: platform_catalog + objects: [plan_tiers] + privileges: [SELECT] + onex_api: + login: true + bypass_rls: false + grants: + - object_type: SCHEMA + schema: platform_catalog + privileges: [USAGE] + omninode_runtime: + login: true + bypass_rls: false + grants: + - object_type: SCHEMA + schema: omninode_internal + privileges: [USAGE] + - object_type: TABLE + schema: omninode_internal + objects: [future_internal_projection] + privileges: [SELECT, INSERT, UPDATE] + tenant_projection_writer: + login: true + bypass_rls: false + grants: + - object_type: SCHEMA + schema: public + privileges: [USAGE] + - object_type: SCHEMA + schema: tenant + privileges: [USAGE] + - object_type: TABLE + schema: public + objects: [delegation_events] + privileges: [SELECT, INSERT, UPDATE] + bindings: + app_dashboard: + database_ref: application + principal: app_dashboard + dsn_env: OMNIDASH_ANALYTICS_DB_URL + onex_api: + database_ref: application + principal: onex_api + dsn_env: OMNINODE_CLOUD_DB_URL + omninode_runtime_service: + database_ref: application + principal: omninode_runtime + dsn_env: OMNINODE_INTERNAL_DB_URL + tenant_projection: + database_ref: application + principal: tenant_projection_writer + dsn_env: OMNIDASH_ANALYTICS_DB_URL + migration_stream: omnibase_infra.application + checksum_ledgers: + canonical: + schema: omninode_internal + relation: schema_migrations + stream_column: migration_stream + domain_column: domain + version_column: version + checksum_column: checksum + checksum_ledger: canonical diff --git a/tests/fixtures/application_relation_ownership/unknown-schema.yaml b/tests/fixtures/application_relation_ownership/unknown-schema.yaml new file mode 100644 index 0000000000..83a90f5efa --- /dev/null +++ b/tests/fixtures/application_relation_ownership/unknown-schema.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +name: node_unknown_schema +node_type: EFFECT_GENERIC +db_io: + db_tables: + - name: unknown_schema_table + database_ref: application + schema: not_declared + migration: unknown.sql + access: read_write + role: unknown_schema diff --git a/tests/fixtures/dispatch_parity/baseline-selection-v2.json b/tests/fixtures/dispatch_parity/baseline-selection-v2.json index 6e6e385801..97ff8a9136 100644 --- a/tests/fixtures/dispatch_parity/baseline-selection-v2.json +++ b/tests/fixtures/dispatch_parity/baseline-selection-v2.json @@ -404,6 +404,63 @@ "package": "omnibase_infra", "type_scoped": false }, + "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayAttach.gateway_attach_c72af942": { + "category": "command", + "contract_name": "node_gateway_attach_effect", + "event_model_module": null, + "event_model_name": null, + "handler_id": "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayAttach.gateway_attach_c72af942", + "handler_module": "omnibase_infra.nodes.node_gateway_attach_effect.handlers.handler_gateway_attach", + "handler_name": "HandlerGatewayAttach", + "message_types": [ + "omnibase-infra.gateway-attach-request", + "omnibase-infra.gateway-detach-request", + "omnibase-infra.gateway-heartbeat-request", + "onex.cmd.omnibase-infra.gateway-attach-request.v1", + "onex.cmd.omnibase-infra.gateway-detach-request.v1", + "onex.cmd.omnibase-infra.gateway-heartbeat-request.v1" + ], + "package": "omnibase_infra", + "type_scoped": false + }, + "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayDetach.gateway_detach_e5dfdba5": { + "category": "command", + "contract_name": "node_gateway_attach_effect", + "event_model_module": null, + "event_model_name": null, + "handler_id": "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayDetach.gateway_detach_e5dfdba5", + "handler_module": "omnibase_infra.nodes.node_gateway_attach_effect.handlers.handler_gateway_detach", + "handler_name": "HandlerGatewayDetach", + "message_types": [ + "omnibase-infra.gateway-attach-request", + "omnibase-infra.gateway-detach-request", + "omnibase-infra.gateway-heartbeat-request", + "onex.cmd.omnibase-infra.gateway-attach-request.v1", + "onex.cmd.omnibase-infra.gateway-detach-request.v1", + "onex.cmd.omnibase-infra.gateway-heartbeat-request.v1" + ], + "package": "omnibase_infra", + "type_scoped": false + }, + "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayHeartbeat.gateway_heartbeat_9ace454d": { + "category": "command", + "contract_name": "node_gateway_attach_effect", + "event_model_module": null, + "event_model_name": null, + "handler_id": "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayHeartbeat.gateway_heartbeat_9ace454d", + "handler_module": "omnibase_infra.nodes.node_gateway_attach_effect.handlers.handler_gateway_heartbeat", + "handler_name": "HandlerGatewayHeartbeat", + "message_types": [ + "omnibase-infra.gateway-attach-request", + "omnibase-infra.gateway-detach-request", + "omnibase-infra.gateway-heartbeat-request", + "onex.cmd.omnibase-infra.gateway-attach-request.v1", + "onex.cmd.omnibase-infra.gateway-detach-request.v1", + "onex.cmd.omnibase-infra.gateway-heartbeat-request.v1" + ], + "package": "omnibase_infra", + "type_scoped": false + }, "dispatcher.auto.node_github_pr_poller_effect.HandlerGitHubApiPoll.github_poll_prs_7786448a": { "category": "intent", "contract_name": "node_github_pr_poller_effect", @@ -3173,15 +3230,15 @@ "corpus": { "contracts_by_package": { "omnibase_core": 15, - "omnibase_infra": 112, + "omnibase_infra": 113, "onex-change-control": 4 }, - "contracts_discovered": 131, + "contracts_discovered": 132, "discovery_errors": 0, - "distinct_subscribe_topics": 105, - "probe_count": 1040, - "registered_dispatchers": 111, - "registered_routes": 121, + "distinct_subscribe_topics": 108, + "probe_count": 1046, + "registered_dispatchers": 114, + "registered_routes": 124, "type_scoped_dispatchers": 69 }, "duplicate_winners": [], @@ -3222,7 +3279,7 @@ "omnimemory" ], "fixture_version": "v2", - "generated_at_utc": "2026-07-26T13:21:53.606701+00:00", + "generated_at_utc": "2026-08-13T00:51:20.726285+00:00", "installed_package_versions": { "omnibase-core": "0.46.8", "omnibase-infra": "0.38.4", @@ -3406,6 +3463,57 @@ }, "topic": "onex.cmd.omnibase-infra.coding-agent-workspace-validate.v1" }, + "P1a::onex.cmd.omnibase-infra.gateway-attach-request.v1": { + "family": "P1_topic_event_type_matrix", + "input": { + "event_type": "omnibase-infra.gateway-attach-request", + "payload": "_DictPayload" + }, + "selection": { + "dispatcher_ids": [ + "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayAttach.gateway_attach_c72af942" + ], + "dlq_topic": null, + "message_category": "command", + "message_type": "omnibase-infra.gateway-attach-request", + "status": "success" + }, + "topic": "onex.cmd.omnibase-infra.gateway-attach-request.v1" + }, + "P1a::onex.cmd.omnibase-infra.gateway-detach-request.v1": { + "family": "P1_topic_event_type_matrix", + "input": { + "event_type": "omnibase-infra.gateway-detach-request", + "payload": "_DictPayload" + }, + "selection": { + "dispatcher_ids": [ + "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayDetach.gateway_detach_e5dfdba5" + ], + "dlq_topic": null, + "message_category": "command", + "message_type": "omnibase-infra.gateway-detach-request", + "status": "success" + }, + "topic": "onex.cmd.omnibase-infra.gateway-detach-request.v1" + }, + "P1a::onex.cmd.omnibase-infra.gateway-heartbeat-request.v1": { + "family": "P1_topic_event_type_matrix", + "input": { + "event_type": "omnibase-infra.gateway-heartbeat-request", + "payload": "_DictPayload" + }, + "selection": { + "dispatcher_ids": [ + "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayHeartbeat.gateway_heartbeat_9ace454d" + ], + "dlq_topic": null, + "message_category": "command", + "message_type": "omnibase-infra.gateway-heartbeat-request", + "status": "success" + }, + "topic": "onex.cmd.omnibase-infra.gateway-heartbeat-request.v1" + }, "P1a::onex.cmd.omnibase-infra.llm-completion-request.v1": { "family": "P1_topic_event_type_matrix", "input": { @@ -5052,6 +5160,51 @@ }, "topic": "onex.cmd.omnibase-infra.coding-agent-workspace-validate.v1" }, + "P1b::onex.cmd.omnibase-infra.gateway-attach-request.v1": { + "family": "P1_topic_event_type_matrix", + "input": { + "event_type": null, + "payload": "_DictPayload" + }, + "selection": { + "dispatcher_ids": [], + "dlq_topic": "onex.dlq.omnibase-infra.commands.v1", + "message_category": "command", + "message_type": "_DictPayload", + "status": "no_dispatcher" + }, + "topic": "onex.cmd.omnibase-infra.gateway-attach-request.v1" + }, + "P1b::onex.cmd.omnibase-infra.gateway-detach-request.v1": { + "family": "P1_topic_event_type_matrix", + "input": { + "event_type": null, + "payload": "_DictPayload" + }, + "selection": { + "dispatcher_ids": [], + "dlq_topic": "onex.dlq.omnibase-infra.commands.v1", + "message_category": "command", + "message_type": "_DictPayload", + "status": "no_dispatcher" + }, + "topic": "onex.cmd.omnibase-infra.gateway-detach-request.v1" + }, + "P1b::onex.cmd.omnibase-infra.gateway-heartbeat-request.v1": { + "family": "P1_topic_event_type_matrix", + "input": { + "event_type": null, + "payload": "_DictPayload" + }, + "selection": { + "dispatcher_ids": [], + "dlq_topic": "onex.dlq.omnibase-infra.commands.v1", + "message_category": "command", + "message_type": "_DictPayload", + "status": "no_dispatcher" + }, + "topic": "onex.cmd.omnibase-infra.gateway-heartbeat-request.v1" + }, "P1b::onex.cmd.omnibase-infra.llm-completion-request.v1": { "family": "P1_topic_event_type_matrix", "input": { @@ -22892,6 +23045,36 @@ "topic_pattern": "*.evt.omnibase-infra.tool-update.*", "type_scoped": false }, + { + "contract_name": "node_gateway_attach_effect", + "dispatcher_id": "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayAttach.gateway_attach_c72af942", + "handler_id": "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayAttach.gateway_attach_c72af942", + "message_category": "command", + "route_id": "route.auto.node_gateway_attach_effect.HandlerGatewayAttach.gateway_attach_c72af942.onex_cmd_omnibase_infra_gateway_attach_request_v1", + "topic": "onex.cmd.omnibase-infra.gateway-attach-request.v1", + "topic_pattern": "*.cmd.omnibase-infra.gateway-attach-request.*", + "type_scoped": false + }, + { + "contract_name": "node_gateway_attach_effect", + "dispatcher_id": "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayDetach.gateway_detach_e5dfdba5", + "handler_id": "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayDetach.gateway_detach_e5dfdba5", + "message_category": "command", + "route_id": "route.auto.node_gateway_attach_effect.HandlerGatewayDetach.gateway_detach_e5dfdba5.onex_cmd_omnibase_infra_gateway_detach_request_v1", + "topic": "onex.cmd.omnibase-infra.gateway-detach-request.v1", + "topic_pattern": "*.cmd.omnibase-infra.gateway-detach-request.*", + "type_scoped": false + }, + { + "contract_name": "node_gateway_attach_effect", + "dispatcher_id": "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayHeartbeat.gateway_heartbeat_9ace454d", + "handler_id": "dispatcher.auto.node_gateway_attach_effect.HandlerGatewayHeartbeat.gateway_heartbeat_9ace454d", + "message_category": "command", + "route_id": "route.auto.node_gateway_attach_effect.HandlerGatewayHeartbeat.gateway_heartbeat_9ace454d.onex_cmd_omnibase_infra_gateway_heartbeat_request_v1", + "topic": "onex.cmd.omnibase-infra.gateway-heartbeat-request.v1", + "topic_pattern": "*.cmd.omnibase-infra.gateway-heartbeat-request.*", + "type_scoped": false + }, { "contract_name": "node_github_pr_poller_effect", "dispatcher_id": "dispatcher.auto.node_github_pr_poller_effect.HandlerGitHubApiPoll.github_poll_prs_7786448a", diff --git a/tests/fixtures/golden_chains/steel_dispatch_ledger_negative_evidence.json b/tests/fixtures/golden_chains/steel_dispatch_ledger_negative_evidence.json new file mode 100644 index 0000000000..0cba822691 --- /dev/null +++ b/tests/fixtures/golden_chains/steel_dispatch_ledger_negative_evidence.json @@ -0,0 +1,17 @@ +{ + "ticket": "OMN-15169", + "precedent": "OMN-15002", + "claim": "event_ledger had zero rows for the steel topic while the deployed stability-test contract predated the OMN-15168 paired allowlist diff, proving the allowlist gate (not automatic-for-all-bus-traffic dispatch).", + "topic": "onex.evt.steel-onslaught.match-terminal.v1", + "lane": "stability-test", + "verified_date": "2026-07-26", + "verified_before_refresh_ran_at_utc": "2026-07-26T21:45:00Z", + "verified_via": "ssh omni-201-ts + docker exec omninode-stability-test-runtime-effects cat contract.yaml (deployed contract_version 1.1.0, no steel topic present) + psql -h 100.109.203.94 -p 15436 -U postgres -d omnibase_infra -c \"SELECT count(*) FROM event_ledger WHERE topic = 'onex.evt.steel-onslaught.match-terminal.v1'\"", + "deployed_contract_version_before_refresh": {"major": 1, "minor": 1, "patch": 0}, + "steel_topic_present_in_deployed_contract_before_refresh": false, + "event_ledger_row_count_before_refresh": 0, + "topic_high_watermark_at_check_time": 14, + "offsets_present_on_topic_with_zero_ledger_rows": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], + "offsets_source": "steel_onslaught's own OMN-15170 live-driver test runs (a separate repo/ticket) produced these offsets; this ticket did not publish them and only observed the resulting absence of ledger rows", + "narrative_evidence": "full verbatim command transcript recorded in this PR's body (OMN-15169), not duplicated here to avoid an unfollowable cross-repo doc pointer for PR reviewers" +} diff --git a/tests/fixtures/golden_chains/steel_dispatch_ledger_success.json b/tests/fixtures/golden_chains/steel_dispatch_ledger_success.json new file mode 100644 index 0000000000..279766bc98 --- /dev/null +++ b/tests/fixtures/golden_chains/steel_dispatch_ledger_success.json @@ -0,0 +1,33 @@ +[ + { + "sequence": 0, + "event_type": "topic_event_published", + "topic": "onex.evt.steel-onslaught.match-terminal.v1", + "source_node": "steel_onslaught.bus.kafka_forwarder.KafkaTerminalEventForwarder" + }, + { + "sequence": 1, + "event_type": "contract_subscribes_topic", + "source_node": "node_ledger_projection_compute", + "ticket": "OMN-15168" + }, + { + "sequence": 2, + "event_type": "dispatch_routed", + "source_node": "message_dispatch_engine", + "handler": "HandlerLedgerProjection", + "operation": "ledger.project" + }, + { + "sequence": 3, + "event_type": "ledger_append_intent_emitted", + "source_node": "node_ledger_projection_compute", + "intent_type": "ledger.append" + }, + { + "sequence": 4, + "event_type": "ledger_row_persisted", + "source_node": "node_ledger_write_effect", + "table": "event_ledger" + } +] diff --git a/tests/fixtures/omn15509/omninode-system-slack-report.as-deployed-20260730.sh.captured b/tests/fixtures/omn15509/omninode-system-slack-report.as-deployed-20260730.sh.captured new file mode 100755 index 0000000000..743b59e03f --- /dev/null +++ b/tests/fixtures/omn15509/omninode-system-slack-report.as-deployed-20260730.sh.captured @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +set -euo pipefail + +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV_FILE=${OMNINODE_ALERT_ENV_FILE:-/data/omninode/omnibase_infra/.env} +STATE_DIR=/data/maintenance/state +LOG_DIR=/data/maintenance/logs +LOCK_FILE=/run/omninode-system-slack-report.lock +MODE=digest + +if [[ "${1:-}" == "--mode" ]]; then + MODE="${2:-digest}" +elif [[ -n "${1:-}" ]]; then + MODE="$1" +fi + +mkdir -p "$STATE_DIR" "$LOG_DIR" +LOG_FILE="$LOG_DIR/omninode-system-slack-report-$(date -u +%Y%m%dT%H%M%SZ).log" +exec >>"$LOG_FILE" 2>&1 +exec 9>"$LOCK_FILE" +if ! flock -n 9; then + echo "$(date -Is) another system report is already running" + exit 0 +fi + +if [[ -f "$ENV_FILE" ]]; then + set -a + set +u + # shellcheck disable=SC1090 + . "$ENV_FILE" + set -u + set +a +fi + +: "${SLACK_BOT_TOKEN:?SLACK_BOT_TOKEN must be set in $ENV_FILE}" +SLACK_CHANNEL_ID="${SLACK_CHANNEL_ID:-${SLACK_DEFAULT_CHANNEL:-}}" +: "${SLACK_CHANNEL_ID:?SLACK_CHANNEL_ID or SLACK_DEFAULT_CHANNEL must be set in $ENV_FILE}" + +DATA_WARN_PCT=${DATA_WARN_PCT:-95} +DATA_CRIT_PCT=${DATA_CRIT_PCT:-98} +DATA_WARN_FREE_GB=${DATA_WARN_FREE_GB:-200} +DATA_CRIT_FREE_GB=${DATA_CRIT_FREE_GB:-50} +ROOT_WARN_PCT=${ROOT_WARN_PCT:-85} +ROOT_CRIT_PCT=${ROOT_CRIT_PCT:-92} +ROOT_WARN_FREE_GB=${ROOT_WARN_FREE_GB:-50} +ROOT_CRIT_FREE_GB=${ROOT_CRIT_FREE_GB:-20} + +post_slack() { + local text="$1" + local color="${2:-#439FE0}" + local payload + payload=$(jq -n \ + --arg channel "$SLACK_CHANNEL_ID" \ + --arg text "$text" \ + --arg color "$color" \ + '{channel:$channel,text:$text,attachments:[{color:$color,text:$text,mrkdwn_in:["text"]}]}') + curl -fsS --retry 2 --max-time 10 \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + -H 'Content-Type: application/json; charset=utf-8' \ + -d "$payload" \ + https://slack.com/api/chat.postMessage | jq -e '.ok == true' >/dev/null +} + +df_line() { + local mount="$1" + df -BG --output=target,size,used,avail,pcent "$mount" | awk 'NR==2 {gsub("G","",$4); gsub("%","",$5); print $1"|"$2"|"$3"|"$4"|"$5}' +} + +classify_disk() { + local mount="$1" pct="$2" avail_gb="$3" warn_pct="$4" crit_pct="$5" warn_free="$6" crit_free="$7" + if (( pct >= crit_pct || avail_gb <= crit_free )); then + echo "CRITICAL" + elif (( pct >= warn_pct || avail_gb <= warn_free )); then + echo "WARNING" + else + echo "OK" + fi +} + +check_http() { + local label="$1" url="$2" expect_regex="${3:-}" + local tmp code status body + tmp=$(mktemp) + code=$(curl -sS --max-time 4 -o "$tmp" -w '%{http_code}' "$url" 2>/dev/null || true) + body=$(tr '\n' ' ' <"$tmp" | head -c 180) + rm -f "$tmp" + status="OK" + if [[ ! "$code" =~ ^2 ]]; then + status="CRITICAL" + elif [[ -n "$expect_regex" ]] && ! grep -Eiq "$expect_regex" <<<"$body"; then + status="WARNING" + fi + printf '%s|%s|%s|%s\n' "$status" "$label" "${code:-000}" "$body" +} + +collect() { + local now host root data root_status data_status running unhealthy restarting dead created exited dangling named_dangling anonymous_dangling + now=$(date -Is) + host=$(hostname) + root=$(df_line /) + data=$(df_line /data) + IFS='|' read -r _ root_size root_used root_avail root_pct <<<"$root" + IFS='|' read -r _ data_size data_used data_avail data_pct <<<"$data" + root_status=$(classify_disk / "$root_pct" "$root_avail" "$ROOT_WARN_PCT" "$ROOT_CRIT_PCT" "$ROOT_WARN_FREE_GB" "$ROOT_CRIT_FREE_GB") + data_status=$(classify_disk /data "$data_pct" "$data_avail" "$DATA_WARN_PCT" "$DATA_CRIT_PCT" "$DATA_WARN_FREE_GB" "$DATA_CRIT_FREE_GB") + + running=$(docker ps --format '{{.Names}}' | wc -l | tr -d ' ') + unhealthy=$(docker ps -a --format '{{.Names}}\t{{.Status}}' | grep -Eci 'unhealthy' || true) + restarting=$(docker ps -a --format '{{.Names}}\t{{.Status}}' | grep -Eci 'Restarting' || true) + dead=$(docker ps -a --format '{{.Names}}\t{{.Status}}' | grep -Eci 'Dead' || true) + created=$(docker ps -a --format '{{.Names}}\t{{.Status}}' | grep -Eci 'Created' || true) + exited=$(docker ps -a --format '{{.Names}}\t{{.Status}}' | grep -Eci 'Exited' || true) + dangling=$(docker volume ls -qf dangling=true | wc -l | tr -d ' ') + anonymous_dangling=$(docker volume ls -qf dangling=true | grep -Ec '^[0-9a-f]{64}$' || true) + named_dangling=$(docker volume ls -qf dangling=true | grep -Evc '^[0-9a-f]{64}$' || true) + + { + echo "timestamp|$now" + echo "host|$host" + echo "disk|$root_status|/|${root_used}/${root_size}|${root_avail}G free|${root_pct}%" + echo "disk|$data_status|/data|${data_used}/${data_size}|${data_avail}G free|${data_pct}%" + echo "docker|OK|running_containers|$running" + echo "docker|$([[ "$unhealthy" == 0 && "$restarting" == 0 && "$dead" == 0 && "$created" == 0 ]] && echo OK || echo CRITICAL)|container_issues|unhealthy=$unhealthy restarting=$restarting dead=$dead created=$created exited=$exited" + echo "docker|OK|dangling_volumes|total=$dangling anonymous=$anonymous_dangling named=$named_dangling" + check_http runtime-18085 http://127.0.0.1:18085/health 'healthy|ok' + check_http runtime-28085 http://127.0.0.1:28085/health 'healthy|ok' + check_http projection-api-13002 http://127.0.0.1:13002/health 'ok|healthy' + check_http deploy-agent-8099 http://127.0.0.1:8099/health 'idle|running|state|ok' + check_http web-3003 http://127.0.0.1:3003/ '' + } +} + +snapshot=$(collect) +echo "$snapshot" + +host=$(awk -F'|' '$1=="host"{print $2}' <<<"$snapshot") +issues=$(awk -F'|' '$2=="WARNING" || $2=="CRITICAL" {print}' <<<"$snapshot" || true) +issue_keys=$(awk -F'|' '$2=="WARNING" || $2=="CRITICAL" {print $1 "|" $2 "|" $3}' <<<"$snapshot" || true) +critical_count=$(awk -F'|' '$2=="CRITICAL" {c++} END {print c+0}' <<<"$snapshot") +warning_count=$(awk -F'|' '$2=="WARNING" {c++} END {print c+0}' <<<"$snapshot") +issue_hash=$(printf '%s\n' "$issue_keys" | sha256sum | awk '{print $1}') +state_file="$STATE_DIR/omninode-system-alert.hash" +prev_hash=$(cat "$state_file" 2>/dev/null || true) + +format_digest() { + local title="$1" + local lines endpoint_lines issue_lines + lines=$(awk -F'|' '$1=="disk" {printf "- `%s`: %s, %s, %s (%s)\n", $3, $4, $5, $6, $2} $1=="docker" {printf "- Docker `%s`: %s (%s)\n", $3, $4, $2}' <<<"$snapshot") + endpoint_lines=$(awk -F'|' '$1=="OK" || $1=="WARNING" || $1=="CRITICAL" {printf "- `%s`: HTTP %s (%s)\n", $2, $3, $1}' <<<"$snapshot") + issue_lines=$(awk -F'|' '$2=="WARNING" || $2=="CRITICAL" {printf "- %s `%s`: %s %s %s\n", $2, $3, $4, $5, $6}' <<<"$snapshot") + if [[ -z "$issue_lines" ]]; then + issue_lines="- No active warning/critical checks" + fi + cat <"$state_file" + elif [[ "$issue_hash" != "$prev_hash" ]]; then + color='warning' + [[ "$critical_count" != 0 ]] && color='danger' + post_slack "$(format_digest '*OmniNode system alert*')" "$color" + echo "$issue_hash" >"$state_file" + else + echo "$(date -Is) issues unchanged; Slack suppressed" + fi + ;; + dry-run) + format_digest '*OmniNode system dry run*' + ;; + *) + echo "unknown mode: $MODE" >&2 + exit 2 + ;; +esac diff --git a/tests/fixtures/omn15547/application-acl-prechange-fixture.json.captured b/tests/fixtures/omn15547/application-acl-prechange-fixture.json.captured new file mode 100644 index 0000000000..84a853e2f4 --- /dev/null +++ b/tests/fixtures/omn15547/application-acl-prechange-fixture.json.captured @@ -0,0 +1,1361 @@ +{ + "column_acl": [ + { + "catalog_kind": "table", + "column_name": "display_name", + "grantee": "rls_admin", + "grantor": "onex_api", + "is_grantable": true, + "object_name": "tenant_accounts", + "privilege_type": "UPDATE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "column_name": "display_name", + "grantee": "untrusted_login", + "grantor": "onex_api", + "is_grantable": true, + "object_name": "tenant_accounts", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "column_name": "display_name", + "grantee": "untrusted_login", + "grantor": "rls_admin", + "is_grantable": false, + "object_name": "tenant_accounts", + "privilege_type": "UPDATE", + "schema_name": "tenant" + } + ], + "database_acl": [ + { + "database_name": "keycloak", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "keycloak", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "keycloak", + "grantee": "app_dashboard", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CREATE" + }, + { + "database_name": "keycloak", + "grantee": "external_connect_parent", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnibase_infra", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnibase_infra", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omnibase_infra", + "grantee": "keycloak_service", + "grantor": "rls_admin", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnibase_infra", + "grantee": "onex_api", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CREATE" + }, + { + "database_name": "omnibase_infra", + "grantee": "onex_api", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omnibase_infra", + "grantee": "rls_admin", + "grantor": "postgres", + "is_grantable": true, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnibase_infra", + "grantee": "rls_admin", + "grantor": "postgres", + "is_grantable": true, + "privilege_type": "CREATE" + }, + { + "database_name": "omniclaude", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omniclaude", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omnidash_analytics", + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnidash_analytics", + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omnidash_analytics", + "grantee": "untrusted_login", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnidash_analytics", + "grantee": "untrusted_login", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omniintelligence", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omniintelligence", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omnimemory", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omnimemory", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "omninode_cloud", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "omninode_cloud", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + }, + { + "database_name": "umami", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "CONNECT" + }, + { + "database_name": "umami", + "grantee": "PUBLIC", + "grantor": "postgres", + "is_grantable": false, + "privilege_type": "TEMPORARY" + } + ], + "database_owner": [ + { + "database_name": "keycloak", + "owner": "postgres" + }, + { + "database_name": "omnibase_infra", + "owner": "postgres" + }, + { + "database_name": "omniclaude", + "owner": "postgres" + }, + { + "database_name": "omnidash_analytics", + "owner": "app_dashboard" + }, + { + "database_name": "omniintelligence", + "owner": "postgres" + }, + { + "database_name": "omnimemory", + "owner": "postgres" + }, + { + "database_name": "omninode_cloud", + "owner": "postgres" + }, + { + "database_name": "umami", + "owner": "postgres" + } + ], + "default_acl": [ + { + "grantee": "PUBLIC", + "grantor": "owner_onex_tenant", + "is_grantable": false, + "object_type": "FUNCTION", + "owner": "owner_onex_tenant", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "grantee": "app_dashboard", + "grantor": "owner_onex_tenant", + "is_grantable": false, + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "privilege_type": "SELECT", + "schema_name": null + }, + { + "grantee": "app_dashboard", + "grantor": "owner_onex_tenant", + "is_grantable": true, + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "grantee": "keycloak_service", + "grantor": "owner_onex_tenant", + "is_grantable": true, + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "grantee": "untrusted_login", + "grantor": "owner_onex_tenant", + "is_grantable": false, + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "privilege_type": "SELECT", + "schema_name": null + } + ], + "default_acl_catalog_rows": [ + { + "object_type": "FUNCTION", + "owner": "owner_onex_tenant", + "raw_acl": "=X/owner_onex_tenant", + "schema_name": "tenant" + }, + { + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "raw_acl": "app_dashboard=r*/owner_onex_tenant,keycloak_service=r*/owner_onex_tenant", + "schema_name": "tenant" + }, + { + "object_type": "TABLE", + "owner": "owner_onex_tenant", + "raw_acl": "app_dashboard=r/owner_onex_tenant,owner_onex_tenant=arwdDxt/owner_onex_tenant,untrusted_login=r/owner_onex_tenant", + "schema_name": null + } + ], + "memberships": [ + { + "admin_option": false, + "grantor": "postgres", + "inherit_option": false, + "member_role": "owner_onex_tenant", + "parent_role": "rls_admin", + "set_option": true + }, + { + "admin_option": false, + "grantor": "postgres", + "inherit_option": true, + "member_role": "omninode_runtime", + "parent_role": "owner_omninode_internal", + "set_option": true + }, + { + "admin_option": false, + "grantor": "postgres", + "inherit_option": true, + "member_role": "shadow_login", + "parent_role": "keycloak_service", + "set_option": true + }, + { + "admin_option": false, + "grantor": "postgres", + "inherit_option": true, + "member_role": "tenant_projection_writer", + "parent_role": "owner_onex_tenant", + "set_option": true + }, + { + "admin_option": false, + "grantor": "rls_admin", + "inherit_option": false, + "member_role": "keycloak_service", + "parent_role": "external_connect_parent", + "set_option": true + }, + { + "admin_option": true, + "grantor": "postgres", + "inherit_option": true, + "member_role": "app_dashboard", + "parent_role": "rls_admin", + "set_option": true + }, + { + "admin_option": true, + "grantor": "postgres", + "inherit_option": true, + "member_role": "db_migrator", + "parent_role": "owner_onex_tenant", + "set_option": false + } + ], + "object_acl": [ + { + "catalog_kind": "function", + "function_signature": "()", + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "object_name": "delegation_event_count", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "()", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(VARIADIC tenant.account_id_span[])", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(\"arg'name%\" integer)", + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "object_name": "hostile_signature", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(bigint, bigint)", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(bigint, bigint, text)", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(tenant.account_id_span)", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "DELETE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "INSERT", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "REFERENCES", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "TRIGGER", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "TRUNCATE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "materialized_view", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "privilege_type": "UPDATE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "procedure", + "function_signature": "(IN p_payload text)", + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "object_name": "record_delegation", + "object_type": "PROCEDURE", + "privilege_type": "EXECUTE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "DELETE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "INSERT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "REFERENCES", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "TRIGGER", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "TRUNCATE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "UPDATE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "DELETE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "INSERT", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "REFERENCES", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "TRIGGER", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "TRUNCATE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "plan_tiers", + "object_type": "TABLE", + "privilege_type": "UPDATE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "DELETE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "INSERT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "REFERENCES", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "TRIGGER", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "TRUNCATE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "UPDATE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "onex_api", + "is_grantable": true, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "DELETE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "INSERT", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "REFERENCES", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "TRIGGER", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "TRUNCATE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "object_name": "runtime_state", + "object_type": "TABLE", + "privilege_type": "UPDATE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "grantee": "rls_admin", + "grantor": "onex_api", + "is_grantable": true, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "rls_admin", + "grantor": "onex_api", + "is_grantable": true, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "untrusted_login", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "untrusted_login", + "grantor": "rls_admin", + "is_grantable": false, + "object_name": "partitioned_events", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "grantee": "untrusted_login", + "grantor": "rls_admin", + "is_grantable": false, + "object_name": "tenant_accounts", + "object_type": "TABLE", + "privilege_type": "SELECT", + "schema_name": "tenant" + }, + { + "catalog_kind": "type", + "grantee": "PUBLIC", + "grantor": "omninode_runtime", + "is_grantable": false, + "object_name": "runtime_code", + "object_type": "TYPE", + "privilege_type": "USAGE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "type", + "grantee": "PUBLIC", + "grantor": "omninode_runtime", + "is_grantable": false, + "object_name": "runtime_status", + "object_type": "TYPE", + "privilege_type": "USAGE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "type", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span", + "object_type": "TYPE", + "privilege_type": "USAGE", + "schema_name": "tenant" + }, + { + "catalog_kind": "type", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_id_span_set", + "object_type": "TYPE", + "privilege_type": "USAGE", + "schema_name": "tenant" + }, + { + "catalog_kind": "type", + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "object_name": "account_ref", + "object_type": "TYPE", + "privilege_type": "USAGE", + "schema_name": "tenant" + } + ], + "object_owners": [ + { + "catalog_kind": "function", + "function_signature": "()", + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "owner": "onex_api", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "()", + "object_name": "delegation_event_count", + "object_type": "FUNCTION", + "owner": "app_dashboard", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(VARIADIC tenant.account_id_span[])", + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "owner": "onex_api", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(\"arg'name%\" integer)", + "object_name": "hostile_signature", + "object_type": "FUNCTION", + "owner": "app_dashboard", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(bigint, bigint)", + "object_name": "account_id_span", + "object_type": "FUNCTION", + "owner": "onex_api", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(bigint, bigint, text)", + "object_name": "account_id_span", + "object_type": "FUNCTION", + "owner": "onex_api", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "function", + "function_signature": "(tenant.account_id_span)", + "object_name": "account_id_span_set", + "object_type": "FUNCTION", + "owner": "onex_api", + "owner_keyword": "FUNCTION", + "schema_name": "tenant" + }, + { + "catalog_kind": "materialized_view", + "object_name": "plan_tier_snapshot", + "object_type": "TABLE", + "owner": "onex_api", + "owner_keyword": "MATERIALIZED VIEW", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "procedure", + "function_signature": "(IN p_payload text)", + "object_name": "record_delegation", + "object_type": "PROCEDURE", + "owner": "app_dashboard", + "owner_keyword": "PROCEDURE", + "schema_name": "tenant" + }, + { + "catalog_kind": "sequence", + "object_name": "delegation_events_id_seq", + "object_type": "SEQUENCE", + "owner": "app_dashboard", + "owner_keyword": "SEQUENCE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "object_name": "delegation_events", + "object_type": "TABLE", + "owner": "app_dashboard", + "owner_keyword": "TABLE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "object_name": "partitioned_events", + "object_type": "TABLE", + "owner": "onex_api", + "owner_keyword": "TABLE", + "schema_name": "tenant" + }, + { + "catalog_kind": "table", + "object_name": "plan_tiers", + "object_type": "TABLE", + "owner": "onex_api", + "owner_keyword": "TABLE", + "schema_name": "platform_catalog" + }, + { + "catalog_kind": "table", + "object_name": "runtime_state", + "object_type": "TABLE", + "owner": "tenant_projection_writer", + "owner_keyword": "TABLE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "table", + "object_name": "tenant_accounts", + "object_type": "TABLE", + "owner": "onex_api", + "owner_keyword": "TABLE", + "schema_name": "tenant" + }, + { + "catalog_kind": "type", + "object_name": "account_id_span", + "object_type": "TYPE", + "owner": "onex_api", + "owner_keyword": "TYPE", + "schema_name": "tenant" + }, + { + "catalog_kind": "type", + "object_name": "account_id_span_set", + "object_type": "TYPE", + "owner": "onex_api", + "owner_keyword": "TYPE", + "schema_name": "tenant" + }, + { + "catalog_kind": "type", + "object_name": "account_ref", + "object_type": "TYPE", + "owner": "onex_api", + "owner_keyword": "TYPE", + "schema_name": "tenant" + }, + { + "catalog_kind": "type", + "object_name": "runtime_code", + "object_type": "TYPE", + "owner": "omninode_runtime", + "owner_keyword": "TYPE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "type", + "object_name": "runtime_status", + "object_type": "TYPE", + "owner": "omninode_runtime", + "owner_keyword": "TYPE", + "schema_name": "omninode_internal" + }, + { + "catalog_kind": "view", + "object_name": "tenant_account_names", + "object_type": "TABLE", + "owner": "app_dashboard", + "owner_keyword": "VIEW", + "schema_name": "tenant" + } + ], + "provenance": { + "authorization_scope": "synthetic_proof", + "dump_derived": false, + "live_database_read": false, + "postgres_major": 16, + "source": "sanitized_postgresql_16_fixture" + }, + "roles": [ + { + "rolbypassrls": false, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "db_migrator", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "owner_omninode_internal", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "owner_onex_tenant", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "owner_platform_catalog", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "keycloak_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "omnibase_infra_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "omniclaude_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "omniintelligence_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "omnimemory_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "omninode_cloud_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": false, + "rolname": "umami_service", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "omninode_runtime", + "rolreplication": true, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "untrusted_login", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": true, + "rolinherit": true, + "rolname": "app_dashboard", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": false, + "rolcanlogin": true, + "rolcreatedb": true, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "onex_api", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": true, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "external_connect_parent", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": true, + "rolcanlogin": false, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "rls_admin", + "rolreplication": false, + "rolsuper": false + }, + { + "rolbypassrls": true, + "rolcanlogin": true, + "rolcreatedb": false, + "rolcreaterole": false, + "rolinherit": true, + "rolname": "tenant_projection_writer", + "rolreplication": false, + "rolsuper": false + } + ], + "schema_acl": [ + { + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "CREATE", + "schema_name": "tenant" + }, + { + "grantee": "PUBLIC", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "tenant" + }, + { + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "privilege_type": "CREATE", + "schema_name": "platform_catalog" + }, + { + "grantee": "PUBLIC", + "grantor": "onex_api", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "platform_catalog" + }, + { + "grantee": "PUBLIC", + "grantor": "pg_database_owner", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "public" + }, + { + "grantee": "PUBLIC", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "privilege_type": "CREATE", + "schema_name": "omninode_internal" + }, + { + "grantee": "PUBLIC", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "omninode_internal" + }, + { + "grantee": "app_dashboard", + "grantor": "tenant_projection_writer", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "omninode_internal" + }, + { + "grantee": "untrusted_login", + "grantor": "app_dashboard", + "is_grantable": false, + "privilege_type": "USAGE", + "schema_name": "tenant" + } + ], + "schema_owners": [ + { + "owner": "app_dashboard", + "schema_name": "tenant" + }, + { + "owner": "onex_api", + "schema_name": "platform_catalog" + }, + { + "owner": "pg_database_owner", + "schema_name": "public" + }, + { + "owner": "tenant_projection_writer", + "schema_name": "omninode_internal" + } + ], + "schema_version": "3.0" +} diff --git a/tests/fixtures/omn15547/application-acl-source-lock.yaml.captured b/tests/fixtures/omn15547/application-acl-source-lock.yaml.captured new file mode 100644 index 0000000000..8ed72bfacd --- /dev/null +++ b/tests/fixtures/omn15547/application-acl-source-lock.yaml.captured @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +schema_version: "1.0" +required_connect_databases: + - keycloak + - omnibase_infra + - omnidash_analytics + - omninode_cloud + - omniclaude + - omniintelligence + - omnimemory + - umami +sources: + - source_key: topology_owner + repository: OmniNode-ai/omnibase_infra + revision: d7f1ac464a54696ac9ed05fd8f3766463a0a7eea + path: src/omnibase_infra/topology/instances/local.yaml + sha256: e4170e97e26fb9c63a53f5f660cedacd15bf7e99609309c54bff15c3d026ac8d + purpose: topology + - source_key: typed_ownership_loader + repository: OmniNode-ai/omnibase_infra + revision: 9e9e62c3177f568b7e5110f653ee4be6dcda965c + path: src/omnibase_infra/validation/application_relation_ownership.py + sha256: be712ed5d6372dc9bc0720742291b2739f7b815c61abbab3d87157dfb80216c4 + purpose: typed_loader + - source_key: kubernetes_topology_projection + repository: OmniNode-ai/omninode_infra + revision: 1a1fff92cbf8e02e488ac6e6e89b1087ca396657 + path: topology/kubernetes/source-lock.yaml + sha256: aa6b4502f79244160f07769da788f1826f2b9d5e756b477508eddf5af929226f + purpose: rendered_topology + - source_key: omnimarket_relation_inventory + repository: OmniNode-ai/omnimarket + revision: a0f65e9471da06b88eecb91bbc1c847989cb10d8 + path: docs/evidence/OMN-15423-relation-inventory.json + sha256: c7ff16080dc553ae204da405519f51b5b631847b129f5876db136c048a4c63da + purpose: relation_inventory + - source_key: onex_api_service_ownership + repository: OmniNode-ai/omninode_infra + revision: 39033d55147ef22a061b665345b506246d3aa543 + path: db/migrations/application-relation-ownership.yaml + sha256: 4a47bb42a8e724ffc44453cb649b754568f3ffa83dabaf831ccfaebaa0723712 + purpose: service_ownership + - source_key: node_migration_service_ownership + repository: OmniNode-ai/omninode_infra + revision: 39033d55147ef22a061b665345b506246d3aa543 + path: k8s/migrations/application-relation-ownership.yaml + sha256: 8ca701c4804d2f7d749dc1ef2a26fc383dcd3806bc42c004ed8851a5ff5d25e0 + purpose: service_ownership + - source_key: sanitized_legacy_fixture + repository: OmniNode-ai/omnibase_infra + revision: 18df1728bdbcc7a773c790f64b6569f58b524841 + path: docker/legacy-rds-fixture/fixture-manifest.json + sha256: 1cdc1db7d707bc2daa72ad8f1521bb589cf87594e5b48f6480893b08875e3f26 + purpose: legacy_fixture diff --git a/tests/fixtures/omn15547/health-dev-8085.json.captured b/tests/fixtures/omn15547/health-dev-8085.json.captured new file mode 100644 index 0000000000..abfce7f1ca --- /dev/null +++ b/tests/fixtures/omn15547/health-dev-8085.json.captured @@ -0,0 +1 @@ +{"status":"healthy","version":"0.38.4","details":{"healthy":true,"degraded":false,"startup_in_progress":false,"is_running":true,"is_draining":false,"pending_message_count":0,"max_concurrent_handlers":10,"handler_pool_size":4,"in_flight_tasks":0,"batch_response_enabled":false,"batch_response_pending":0,"event_bus":{"healthy":true,"started":true,"environment":"local","bootstrap_servers":"redpanda:9092","circuit_state":"closed","subscriber_count":359,"topic_count":306,"consumer_count":359},"event_bus_healthy":true,"failed_handlers":{},"skipped_handlers":{},"registered_handlers":["db","http"],"handlers":{"db":{"healthy":true,"note":"no health_check method"},"http":{"healthy":true,"note":"no health_check method"}},"handler_pools":{"db":{"healthy":true,"handler_type":"db","pool_size":4,"available":4,"total_instances":4,"checkout_count":0,"checkin_count":0,"recycle_count":0,"avg_checkout_wait_ms":0.0},"http":{"healthy":true,"handler_type":"http","pool_size":4,"available":4,"total_instances":4,"checkout_count":0,"checkin_count":0,"recycle_count":0,"avg_checkout_wait_ms":0.0}},"no_handlers_registered":false,"config_prefetch_status":"skipped","local_ingress":{"enabled":true,"running":true,"socket_path":"/run/onex-runtime/onex-runtime.sock","route_count":2935,"active_packages":["omnibase_infra","omnimarket","omniclaude","omniintelligence"],"request_model":"ModelLocalRuntimeIngressRequest","response_model":"ModelLocalRuntimeIngressResponse"},"components":{"event_bus":{"name":"event_bus","status":"healthy","last_healthy":"2026-07-31T00:03:08.520792+00:00","details":{"healthy":true,"started":true,"environment":"local","bootstrap_servers":"redpanda:9092","circuit_state":"closed","subscriber_count":359,"topic_count":306,"consumer_count":359}},"db":{"name":"db","status":"healthy","last_healthy":"2026-07-31T00:03:08.520792+00:00","details":{"healthy":true,"note":"no health_check method"}},"http":{"name":"http","status":"healthy","last_healthy":"2026-07-31T00:03:08.520792+00:00","details":{"healthy":true,"note":"no health_check method"}}},"runtime_health":{"status":"HEALTHY","observed_at":"2026-07-30T23:58:48.379221+00:00","age_seconds":260.142,"contract_count":296,"discovery_error_count":0,"consumer_group_count":606,"empty_consumer_group_count":0,"subscribe_topic_count":306,"uncovered_topic_count":0,"dimensions":[{"name":"discovery_errors","status":"HEALTHY","detail":"296 contracts loaded cleanly"},{"name":"empty_consumer_groups","status":"HEALTHY","detail":"All 606 consumer groups active"},{"name":"topic_coverage","status":"HEALTHY","detail":"All 359 expected consumer group(s) covered"}]},"runtime_attached":true}} \ No newline at end of file diff --git a/tests/fixtures/omn15547/legacy-rds-fixture-prove.sh.captured b/tests/fixtures/omn15547/legacy-rds-fixture-prove.sh.captured new file mode 100644 index 0000000000..4ac7b59ba4 --- /dev/null +++ b/tests/fixtures/omn15547/legacy-rds-fixture-prove.sh.captured @@ -0,0 +1,336 @@ +#!/bin/sh +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +set -eu + +FRESH_HOST="${FRESH_HOST:-fresh-postgres}" +LEGACY_HOST="${LEGACY_HOST:-legacy-postgres}" +FRESH_PORT="${FRESH_PORT:-5432}" +LEGACY_PORT="${LEGACY_PORT:-5432}" +MIGRATIONS_DIR="${MIGRATIONS_DIR:-/migrations/forward}" +RUNNER="${RUNNER:-/opt/omn15422/run-forward-migrations.sh}" +CONTROL_MIGRATIONS_DIR="${CONTROL_MIGRATIONS_DIR:-/opt/omn15422/ledger-control/forward}" +LEDGER_BLOCKER='unresolved migration domain for node:node_projection_delegation:0016_delegation_judge_verdict_events.sql (OMN-15423: delegation_judge_verdict_events domain unresolved)' + +fail() { + echo "fixture_status=FAIL detail=$1" >&2 + exit 1 +} + +sql_value() { + host="$1" + database="$2" + statement="$3" + if [ "$host" = "$FRESH_HOST" ]; then + port="$FRESH_PORT" + else + port="$LEGACY_PORT" + fi + psql -X -qAt -h "$host" -p "$port" -U postgres -d "$database" \ + -v ON_ERROR_STOP=1 -c "$statement" +} + +assert_pair() { + case_id="$1" + positive_sql="$2" + red_sql="$3" + red_signature="$4" + + positive_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "$positive_sql")" + [ "$positive_count" = "0" ] || fail "$case_id positive control reported $positive_count defect(s)" + + red_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "$red_sql")" + [ "$red_count" -gt 0 ] || fail "$case_id RED control was not discriminated" + echo "fixture_case=$case_id positive=PASS red=DETECTED red_signature=$red_signature red_count=$red_count" +} + +assert_pair \ + mapping_ambiguity \ + "SELECT count(*) FROM (SELECT legacy_tenant_value FROM omn15422_fixture.mapping_positive GROUP BY legacy_tenant_value HAVING count(DISTINCT tenant_uuid) <> 1) AS defects" \ + "SELECT count(*) FROM (SELECT legacy_tenant_value FROM omn15422_fixture.mapping_red GROUP BY legacy_tenant_value HAVING count(DISTINCT tenant_uuid) <> 1) AS defects" \ + ambiguous_mapping + +assert_pair \ + checksum_conflict \ + "SELECT count(*) FROM (SELECT migration_id FROM omn15422_fixture.checksum_positive GROUP BY migration_id HAVING count(DISTINCT checksum) <> 1) AS defects" \ + "SELECT count(*) FROM (SELECT migration_id FROM omn15422_fixture.checksum_red GROUP BY migration_id HAVING count(DISTINCT checksum) <> 1) AS defects" \ + checksum_conflict + +assert_pair \ + owner_drift \ + "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_roles r ON r.oid=c.relowner WHERE n.nspname='omn15422_fixture' AND c.relname='owner_positive' AND r.rolname <> 'role_omnidash'" \ + "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_roles r ON r.oid=c.relowner WHERE n.nspname='omn15422_fixture' AND c.relname='owner_red' AND r.rolname <> 'role_omnidash'" \ + owner_mismatch + +assert_pair \ + unsafe_rls_policy \ + "SELECT (CASE WHEN c.relrowsecurity AND c.relforcerowsecurity THEN 0 ELSE 1 END) + (SELECT count(*) FROM information_schema.columns WHERE table_schema='omn15422_fixture' AND table_name='tenant_usage_safe' AND column_name='tenant_id' AND (data_type <> 'uuid' OR is_nullable <> 'NO')) FROM pg_class c WHERE c.oid='omn15422_fixture.tenant_usage_safe'::regclass" \ + "SELECT (CASE WHEN c.relrowsecurity AND c.relforcerowsecurity THEN 0 ELSE 1 END) + (SELECT count(*) FROM information_schema.columns WHERE table_schema='public' AND table_name='tenant_usage_legacy' AND column_name='tenant_id' AND (data_type <> 'uuid' OR is_nullable <> 'NO')) FROM pg_class c WHERE c.oid='public.tenant_usage_legacy'::regclass" \ + legacy_varchar_enable_only + +assert_pair \ + unsafe_view \ + "SELECT CASE WHEN coalesce(reloptions, ARRAY[]::text[]) @> ARRAY['security_invoker=true'] THEN 0 ELSE 1 END FROM pg_class WHERE oid='omn15422_fixture.tenant_usage_safe_view'::regclass" \ + "SELECT CASE WHEN coalesce(reloptions, ARRAY[]::text[]) @> ARRAY['security_invoker=true'] THEN 0 ELSE 1 END FROM pg_class WHERE oid='omn15422_fixture.tenant_usage_red_view'::regclass" \ + missing_security_invoker + +assert_pair \ + unsafe_function \ + "SELECT (CASE WHEN p.prosecdef THEN 1 ELSE 0 END) + (CASE WHEN EXISTS (SELECT 1 FROM aclexplode(coalesce(p.proacl, acldefault('f', p.proowner))) acl WHERE acl.grantee=0 AND acl.privilege_type='EXECUTE') THEN 1 ELSE 0 END) FROM pg_proc p WHERE p.oid='omn15422_fixture.tenant_usage_safe_count()'::regprocedure" \ + "SELECT (CASE WHEN p.prosecdef THEN 1 ELSE 0 END) + (CASE WHEN EXISTS (SELECT 1 FROM aclexplode(coalesce(p.proacl, acldefault('f', p.proowner))) acl WHERE acl.grantee=0 AND acl.privilege_type='EXECUTE') THEN 1 ELSE 0 END) FROM pg_proc p WHERE p.oid='omn15422_fixture.tenant_usage_red_count()'::regprocedure" \ + definer_or_public_execute + +assert_pair \ + transformation_collision \ + "SELECT count(*) FROM (SELECT target_key FROM omn15422_fixture.transform_positive GROUP BY target_key HAVING count(*) > 1) AS defects" \ + "SELECT count(*) FROM (SELECT target_key FROM omn15422_fixture.transform_red GROUP BY target_key HAVING count(*) > 1) AS defects" \ + duplicate_target_key + +signature() { + host="$1" + database="$2" + relation="$3" + sql_value "$host" "$database" "SELECT string_agg(column_name || ':' || data_type || ':' || is_nullable, ',' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema='public' AND table_name='$relation'" +} + +flat_control="$(signature "$LEGACY_HOST" omnibase_infra flat_node_parity_control)" +node_control="$(signature "$LEGACY_HOST" omnidash_analytics flat_node_parity_control)" +[ "$flat_control" = "$node_control" ] || fail "flat/node positive shape control diverged" +flat_red="$(signature "$LEGACY_HOST" omnibase_infra llm_cost_aggregates)" +node_red="$(signature "$LEGACY_HOST" omnidash_analytics llm_cost_aggregates)" +[ "$flat_red" != "$node_red" ] || fail "flat/node RED shape collision was not discriminated" +echo "fixture_case=flat_node_shape_collision positive=PASS red=DETECTED red_signature=column_signature_mismatch" + +ledger_positive="$(signature "$LEGACY_HOST" omnibase_infra schema_migrations)" +ledger_red="$(signature "$LEGACY_HOST" omnidash_analytics schema_migrations)" +ledger_version="$(signature "$LEGACY_HOST" omninode_cloud schema_migrations)" +node_ledger="$(signature "$LEGACY_HOST" omnidash_analytics node_schema_migrations)" +case "$ledger_positive" in + migration_id:*checksum:*source_set:*) ;; + *) fail "checksum-capable positive ledger shape missing: $ledger_positive" ;; +esac +case "$ledger_red" in + filename:*applied_at:*) ;; + *) fail "filename/applied_at legacy ledger shape missing: $ledger_red" ;; +esac +case "$ledger_version" in + version:*applied_at:*checksum:*) ;; + *) fail "version/nullable-checksum legacy ledger shape missing: $ledger_version" ;; +esac +case "$node_ledger" in + version:*applied_at:*checksum:*) ;; + *) fail "node ledger shape missing: $node_ledger" ;; +esac +echo "fixture_case=legacy_shape_collision positive=PASS red=SEE_REAL_RUNNER" + +dependency_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT (SELECT count(*) FROM pg_constraint WHERE conrelid='public.tenant_usage_legacy'::regclass AND contype='f') + (SELECT count(*) FROM pg_indexes WHERE schemaname='public' AND tablename='tenant_usage_legacy') + (SELECT count(*) FROM pg_views WHERE schemaname='public' AND viewname='tenant_usage_legacy_view') + (SELECT count(*) FROM pg_proc WHERE oid='public.tenant_usage_legacy_count()'::regprocedure)")" +[ "$dependency_count" -ge 4 ] || fail "dependent FK/index/view/function catalog is incomplete" +sentinel_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT count(*) FROM public.tenants_legacy WHERE tenant_id IN ('', 'omninode', '00000000-0000-0000-0000-000000000000')")" +[ "$sentinel_count" = "3" ] || fail "synthetic sentinel corpus is incomplete" +acl_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT (SELECT count(*) FROM information_schema.role_table_grants WHERE table_schema='public' AND table_name='tenant_usage_legacy' AND grantee='role_omnidash') + (SELECT count(*) FROM pg_default_acl d CROSS JOIN LATERAL aclexplode(d.defaclacl) acl JOIN pg_roles r ON r.oid=acl.grantee WHERE r.rolname='app_dashboard' AND acl.privilege_type='SELECT')")" +[ "$acl_count" -ge 2 ] || fail "legacy grants/default privileges are incomplete" +policy_count="$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT count(*) FROM pg_policies WHERE schemaname='public' AND tablename='tenant_usage_legacy' AND policyname='tenant_usage_legacy_policy'")" +[ "$policy_count" = "1" ] || fail "legacy RLS policy is missing" +echo "fixture_case=dependencies_acl_and_sentinels status=PASS dependency_count=$dependency_count acl_count=$acl_count policy_count=$policy_count sentinel_count=$sentinel_count" + +# Reproduce OMN-15335 with the real migration as the non-owner role. The same +# file then runs twice as postgres to prove the OMN-15376 shape reconciliation +# is idempotent independently of the earlier ledger wall. +owner_log="$(mktemp)" +if psql -X -h "$LEGACY_HOST" -p "$LEGACY_PORT" -U role_omnidash -d omnidash_analytics \ + -v ON_ERROR_STOP=1 \ + -f "$MIGRATIONS_DIR/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql" \ + >"$owner_log" 2>&1; then + fail "owner-drift RED migration unexpectedly succeeded" +fi +grep -F 'must be owner of table llm_cost_aggregates' "$owner_log" >/dev/null \ + || { sed -n '1,160p' "$owner_log"; fail "owner-drift failure signature moved"; } +echo "fixture_case=owner_drift_real_migration red=DETECTED red_signature=must_be_owner" + +for pass in 1 2; do + psql -X -q -h "$LEGACY_HOST" -p "$LEGACY_PORT" -U postgres -d omnidash_analytics \ + -v ON_ERROR_STOP=1 \ + -f "$MIGRATIONS_DIR/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql" + psql -X -q -h "$LEGACY_HOST" -p "$LEGACY_PORT" -U postgres -d omnidash_analytics \ + -v ON_ERROR_STOP=1 \ + -f "$MIGRATIONS_DIR/nodes/node_projection_baselines/0001_create_baselines_tables.sql" + echo "fixture_case=legacy_shape_reconciliation pass=$pass status=PASS" +done +echo "fixture_case=owner_drift_real_migration positive=PASS red=DETECTED passes=2" + +run_forward() { + host="$1" + port="$2" + database="$3" + log="$4" + POSTGRES_HOST="$host" \ + POSTGRES_PORT="$port" \ + POSTGRES_USER=postgres \ + POSTGRES_PASSWORD='' \ + POSTGRES_DB="$database" \ + NODE_POSTGRES_DB=omnidash_analytics \ + MIGRATIONS_DIR="$MIGRATIONS_DIR" \ + sh "$RUNNER" >"$log" 2>&1 +} + +create_fixture_database() { + host="$1" + port="$2" + database="$3" + exists="$(psql -X -qAt -h "$host" -p "$port" -U postgres -d postgres \ + -v ON_ERROR_STOP=1 -v database="$database" -f - <<'EOSQL' +SELECT count(*) FROM pg_database WHERE datname = :'database'; +EOSQL +)" + if [ "$exists" = "0" ]; then + psql -X -q -h "$host" -p "$port" -U postgres -d postgres \ + -v ON_ERROR_STOP=1 -c "CREATE DATABASE ${database}" + fi +} + +run_control_forward() { + host="$1" + port="$2" + service_database="$3" + application_database="$4" + cloud_database="$5" + log="$6" + POSTGRES_HOST="$host" \ + POSTGRES_PORT="$port" \ + POSTGRES_USER=postgres \ + POSTGRES_PASSWORD='' \ + POSTGRES_DB="$service_database" \ + NODE_POSTGRES_DB="$application_database" \ + OMNINODE_CLOUD_HISTORY_DB="$cloud_database" \ + MIGRATIONS_DIR="$CONTROL_MIGRATIONS_DIR" \ + sh "$RUNNER" >"$log" 2>&1 +} + +# Positive ledger controls use separate synthetic databases and the same real +# runner/bootstrap artifact. They cross the OMN-15423 preflight hold without +# weakening it: the control tree contains one fully classified migration and +# an empty committed block set. Both fresh and legacy histories run twice. +for database in omn15413_control_service omn15413_control_app omn15413_control_cloud; do + create_fixture_database "$FRESH_HOST" "$FRESH_PORT" "$database" +done +for database in omn15413_control_service omn15413_control_app omn15413_control_cloud; do + create_fixture_database "$LEGACY_HOST" "$LEGACY_PORT" "$database" +done + +psql -X -q -h "$LEGACY_HOST" -p "$LEGACY_PORT" -U postgres \ + -d omn15413_control_app -v ON_ERROR_STOP=1 <<'EOSQL' +CREATE TABLE public.schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL +); +INSERT INTO public.schema_migrations VALUES + ('0001_legacy_control.sql', TIMESTAMPTZ '2026-01-01 00:00:00+00'); +CREATE TABLE public.node_schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL, + checksum TEXT NOT NULL +); +INSERT INTO public.node_schema_migrations VALUES ( + 'node:node_example:0001_create_example.sql', + TIMESTAMPTZ '2026-01-02 00:00:00+00', + '1f605f28cc1f4a1a7500862be51c35d01431cacba2d34201150f3ae3deb6c923' +); +EOSQL +psql -X -q -h "$LEGACY_HOST" -p "$LEGACY_PORT" -U postgres \ + -d omn15413_control_cloud -v ON_ERROR_STOP=1 <<'EOSQL' +CREATE TABLE public.schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL, + checksum TEXT +); +INSERT INTO public.schema_migrations VALUES + ('20260101_cloud_control.sql', TIMESTAMPTZ '2026-01-03 00:00:00+00', NULL); +CREATE TABLE public.migrations_log ( + migration_name TEXT NOT NULL, + direction TEXT NOT NULL, + executed_at TIMESTAMPTZ NOT NULL, + UNIQUE (migration_name, direction) +); +INSERT INTO public.migrations_log VALUES + ('20260101_cloud_control', 'forward', TIMESTAMPTZ '2026-01-03 00:00:00+00'); +EOSQL + +legacy_control_node_oid="$(sql_value "$LEGACY_HOST" omn15413_control_app \ + "SELECT 'public.node_schema_migrations'::regclass::oid")" +legacy_control_cloud_source="$(sql_value "$LEGACY_HOST" omn15413_control_cloud \ + "SELECT version || '|' || coalesce(checksum, '') || '|' || applied_at::text FROM public.schema_migrations")" + +for pass in 1 2; do + fresh_control_log="$(mktemp)" + run_control_forward "$FRESH_HOST" "$FRESH_PORT" \ + omn15413_control_service omn15413_control_app omn15413_control_cloud \ + "$fresh_control_log" \ + || { sed -n '1,240p' "$fresh_control_log"; fail "fresh ledger control pass $pass failed"; } + grep -F 'Sentinel set. Migration gate will report HEALTHY.' "$fresh_control_log" >/dev/null \ + || fail "fresh ledger control pass $pass omitted terminal sentinel proof" + if [ "$pass" = "2" ]; then + grep -F 'Complete: 0 infra applied, 1 infra skipped; 0 node applied, 1 node skipped' "$fresh_control_log" >/dev/null \ + || { tail -n 80 "$fresh_control_log"; fail "fresh ledger control second pass was not idempotent"; } + fi + echo "fixture_case=application_ledger_fresh pass=$pass status=PASS" + + legacy_control_log="$(mktemp)" + run_control_forward "$LEGACY_HOST" "$LEGACY_PORT" \ + omn15413_control_service omn15413_control_app omn15413_control_cloud \ + "$legacy_control_log" \ + || { sed -n '1,240p' "$legacy_control_log"; fail "legacy ledger control pass $pass failed"; } + grep -F 'Sentinel set. Migration gate will report HEALTHY.' "$legacy_control_log" >/dev/null \ + || fail "legacy ledger control pass $pass omitted terminal sentinel proof" + echo "fixture_case=application_ledger_legacy pass=$pass status=PASS" +done + +[ "$(sql_value "$FRESH_HOST" omn15413_control_app "SELECT count(*) FROM platform_catalog.schema_migrations")" = "1" ] \ + || fail "fresh ledger control canonical row count drifted" +[ "$(sql_value "$LEGACY_HOST" omn15413_control_app "SELECT count(*) FROM platform_catalog.schema_migrations")" = "3" ] \ + || fail "legacy ledger control did not import all three source shapes" +[ "$(sql_value "$LEGACY_HOST" omn15413_control_app "SELECT 'platform_catalog.schema_migrations'::regclass::oid")" = "$legacy_control_node_oid" ] \ + || fail "selected node ledger was copied instead of moved in place" +[ "$(sql_value "$LEGACY_HOST" omn15413_control_app "SELECT count(*) FROM public.schema_migrations")" = "1" ] \ + || fail "filename-only source ledger was rewritten" +[ "$(sql_value "$LEGACY_HOST" omn15413_control_cloud "SELECT version || '|' || coalesce(checksum, '') || '|' || applied_at::text FROM public.schema_migrations")" = "$legacy_control_cloud_source" ] \ + || fail "cloud applied-set source ledger was rewritten" +echo "fixture_case=application_ledger_sources status=PASS selected_oid_preserved=true sources_immutable=true" + +for pass in 1 2; do + fresh_log="$(mktemp)" + if run_forward "$FRESH_HOST" "$FRESH_PORT" omnibase_infra "$fresh_log"; then + fail "fresh real migration pass $pass crossed an unresolved domain" + fi + grep -F "$LEDGER_BLOCKER" "$fresh_log" >/dev/null \ + || { sed -n '1,240p' "$fresh_log"; fail "fresh blocker signature moved"; } + echo "fixture_case=fresh_install status=BLOCKED pass=$pass blocker=OMN-15423 signature=unresolved_domain" +done + +[ "$(sql_value "$FRESH_HOST" omnidash_analytics "SELECT to_regclass('platform_catalog.schema_migrations') IS NULL")" = "t" ] \ + || fail "fresh preflight blocker mutated the canonical application ledger" + +# The fixture executes the real legacy-upgrade entry point twice. OMN-15413 now +# crosses the old filename-ledger parser boundary, but OMN-15423 still has no +# authoritative domain for delegation_judge_verdict_events. That known, +# unfenced ambiguity must stop in preflight before any ledger or DDL mutation. +for pass in 1 2; do + legacy_log="$(mktemp)" + if run_forward "$LEGACY_HOST" "$LEGACY_PORT" omnibase_infra "$legacy_log"; then + fail "legacy upgrade pass $pass crossed an unresolved domain" + fi + grep -F "$LEDGER_BLOCKER" "$legacy_log" >/dev/null \ + || { sed -n '1,240p' "$legacy_log"; fail "legacy upgrade blocker signature moved"; } + echo "fixture_case=legacy_upgrade status=BLOCKED pass=$pass blocker=OMN-15423 signature=unresolved_domain" +done + +[ "$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT to_regclass('public.node_schema_migrations') IS NOT NULL")" = "t" ] \ + || fail "legacy preflight blocker moved the selected ledger" +[ "$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT to_regclass('platform_catalog.schema_migrations') IS NULL")" = "t" ] \ + || fail "legacy preflight blocker created the canonical ledger" +[ "$(sql_value "$LEGACY_HOST" omnidash_analytics "SELECT count(*) FROM public.schema_migrations")" = "23" ] \ + || fail "legacy preflight blocker rewrote filename history" + +sh /opt/omn15422/cutover-proof/prove.sh + +echo "fixture_status=PASS_WITH_EXPECTED_BLOCKER blocker=OMN-15423" diff --git a/tests/fixtures/omn15547/node-service-registry-tenant-rls-unqualified.sql.captured b/tests/fixtures/omn15547/node-service-registry-tenant-rls-unqualified.sql.captured new file mode 100644 index 0000000000..49f5f38b10 --- /dev/null +++ b/tests/fixtures/omn15547/node-service-registry-tenant-rls-unqualified.sql.captured @@ -0,0 +1,55 @@ +-- OMN-14894 (tranche 1): tenant_id + row-level tenant isolation for +-- node_service_registry. +-- +-- Unlike delegation_events (0022) and savings_estimates (080), this table +-- has no tenant_id column in the migration tree yet — OMN-14894 names it as +-- one of the three tables already carrying RLS on the manually-migrated +-- cloud database, so this migration brings the source-controlled tree to +-- parity: column + index + ENABLE/FORCE RLS + tenant_isolation policy + +-- SELECT grant to app_dashboard (OMN-14899's non-owner, NOSUPERUSER, +-- NOBYPASSRLS read role). +-- +-- DEFAULT 'omninode' mirrors the interim single-tenant convention used by +-- 0022/080: existing rows and unstamped registration writes land under the +-- default tenant; a writer-supplied tenant_id overrides it. +-- +-- SEAM DECISION: tenant_id TEXT, policy compares TEXT (no ::uuid cast) — +-- consistent with every landed tenant_id column on this projection surface. +-- See node_projection_delegation/0023 for the full rationale. +-- +-- BLAST RADIUS: FORCE constrains the table owner. Compose-lane writers are +-- the postgres SUPERUSER (never subject to RLS), so unaffected. Any +-- non-superuser owner-writer must SET app.tenant_id before this applies to +-- its database. + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app_dashboard') THEN + RAISE EXCEPTION + 'app_dashboard role missing — apply omnibase_infra forward migration ' + '094_create_app_dashboard_role.sql (OMN-14899) before this RLS ' + 'migration.'; + END IF; +END; +$$; + +-- Schema resolution for the read role (role-only migration 094 carries no +-- grants by design; USAGE is granted here, alongside the policies). +GRANT USAGE ON SCHEMA public TO app_dashboard; + +ALTER TABLE node_service_registry + ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'omninode'; + +CREATE INDEX IF NOT EXISTS idx_node_service_registry_tenant_id + ON node_service_registry (tenant_id); + +ALTER TABLE node_service_registry ENABLE ROW LEVEL SECURITY; +ALTER TABLE node_service_registry FORCE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS tenant_isolation ON node_service_registry; +CREATE POLICY tenant_isolation ON node_service_registry + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)); + +GRANT SELECT ON node_service_registry TO app_dashboard; diff --git a/tests/fixtures/omn15547/omnimarket-compare-dev-454c429f.gh-api.json.captured b/tests/fixtures/omn15547/omnimarket-compare-dev-454c429f.gh-api.json.captured new file mode 100644 index 0000000000..cb84c457a3 --- /dev/null +++ b/tests/fixtures/omn15547/omnimarket-compare-dev-454c429f.gh-api.json.captured @@ -0,0 +1 @@ +{"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/compare/dev...454c429f328e68e19300f33ffb8121f1bccc7f86","html_url":"https://github.com/OmniNode-ai/omnimarket/compare/dev...454c429f328e68e19300f33ffb8121f1bccc7f86","permalink_url":"https://github.com/OmniNode-ai/omnimarket/compare/OmniNode-ai:5dca587...OmniNode-ai:454c429","diff_url":"https://github.com/OmniNode-ai/omnimarket/compare/dev...454c429f328e68e19300f33ffb8121f1bccc7f86.diff","patch_url":"https://github.com/OmniNode-ai/omnimarket/compare/dev...454c429f328e68e19300f33ffb8121f1bccc7f86.patch","base_commit":{"sha":"5dca587009684aecab7c24bf1512cfc16380176b","node_id":"C_kwDOR6jjtdoAKDVkY2E1ODcwMDk2ODRhZWNhYjdjMjRiZjE1MTJjZmMxNjM4MDE3NmI","commit":{"author":{"name":"Jonah Gray","email":"jonah@omninode.ai","date":"2026-07-31T00:13:24Z"},"committer":{"name":"GitHub","email":"noreply@github.com","date":"2026-07-31T00:13:24Z"},"message":"fix(OMN-15469): make wiring the sole generation terminal producer (#1984)","tree":{"sha":"c16545132a07b8669ee8809093cd116abed6c7c7","url":"https://api.github.com/repos/OmniNode-ai/omnimarket/git/trees/c16545132a07b8669ee8809093cd116abed6c7c7"},"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/git/commits/5dca587009684aecab7c24bf1512cfc16380176b","comment_count":0,"verification":{"verified":true,"reason":"valid","signature":"-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJqa+ikCRC1aQ7uu5UhlAAACHkQADwlVFjFWcgsTaZoQkkINT9Q\nyCOLRS/6lagRZ3NX1p127bkcSWhipEWbtwg+WhDeSCDNF6898h8DcSb4xvO1koOz\n5/pLFJbuF7XYqsKw0QyB+Sy9I2YdM20GwM3oY7IgslOllEhZHIbTDNUk8JsAYHwM\nWtNQ1WAB2IMSc70sDo0z+sbmhaEqTdR6/mAtBYYuzGNtYT3wz5KMmMxcC8dc3GoV\nSF29pr5LGJPtVOMFGsoVnWq8OqI7BxUIx65qpsnJPCwuzMbeLghASNKUZ9tNDh73\nX+iUUGXDHmClhcCroBiEzyNRHM6gtaRLfZ6PiNKM78606kShMWqSpJZoqo0HTZ6y\nyYAfL5jx3QRCXyVCbhSoRNuPdKRIBpYI+Koks6hkbmHWZGWBdViZgtf0jMeLCAM+\n/qwyGeIaPD9kQAu2vx4c9iHJAuylaTqWoevaLsRUFcy4qQKxZzmP6HCR0l5HvMBs\nBkGJul0yD4E9ru60UZAsasE2rUAQU3pJ4MWitKsdxudxauhED+bwg0b9FDoozUvn\nvDP45Fy+viHziwqVLFyOB1NgNzxG/ZcnDmIZdVslSS2yzsBVO5+vncOqZXd7fGyj\nXZdLhCYR05tIptdwlli4DzjYv7K5v5X56zCoJ2zxqafaoHPHXR7IYS2aTdSkV00H\nSvt24sq6pv6XhkcyIiAM\n=Ji14\n-----END PGP SIGNATURE-----\n","payload":"tree c16545132a07b8669ee8809093cd116abed6c7c7\nparent 1c3139b60b8efaedd36bc9eadaf26f277c38dfbb\nauthor Jonah Gray 1785456804 -0400\ncommitter GitHub 1785456804 -0400\n\nfix(OMN-15469): make wiring the sole generation terminal producer (#1984)","verified_at":"2026-07-31T00:13:24Z"}},"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/5dca587009684aecab7c24bf1512cfc16380176b","html_url":"https://github.com/OmniNode-ai/omnimarket/commit/5dca587009684aecab7c24bf1512cfc16380176b","comments_url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/5dca587009684aecab7c24bf1512cfc16380176b/comments","author":{"login":"jonahgabriel","id":1002253,"node_id":"MDQ6VXNlcjEwMDIyNTM=","avatar_url":"https://avatars.githubusercontent.com/u/1002253?v=4","gravatar_id":"","url":"https://api.github.com/users/jonahgabriel","html_url":"https://github.com/jonahgabriel","followers_url":"https://api.github.com/users/jonahgabriel/followers","following_url":"https://api.github.com/users/jonahgabriel/following{/other_user}","gists_url":"https://api.github.com/users/jonahgabriel/gists{/gist_id}","starred_url":"https://api.github.com/users/jonahgabriel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jonahgabriel/subscriptions","organizations_url":"https://api.github.com/users/jonahgabriel/orgs","repos_url":"https://api.github.com/users/jonahgabriel/repos","events_url":"https://api.github.com/users/jonahgabriel/events{/privacy}","received_events_url":"https://api.github.com/users/jonahgabriel/received_events","type":"User","user_view_type":"public","site_admin":false},"committer":{"login":"web-flow","id":19864447,"node_id":"MDQ6VXNlcjE5ODY0NDQ3","avatar_url":"https://avatars.githubusercontent.com/u/19864447?v=4","gravatar_id":"","url":"https://api.github.com/users/web-flow","html_url":"https://github.com/web-flow","followers_url":"https://api.github.com/users/web-flow/followers","following_url":"https://api.github.com/users/web-flow/following{/other_user}","gists_url":"https://api.github.com/users/web-flow/gists{/gist_id}","starred_url":"https://api.github.com/users/web-flow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/web-flow/subscriptions","organizations_url":"https://api.github.com/users/web-flow/orgs","repos_url":"https://api.github.com/users/web-flow/repos","events_url":"https://api.github.com/users/web-flow/events{/privacy}","received_events_url":"https://api.github.com/users/web-flow/received_events","type":"User","user_view_type":"public","site_admin":false},"parents":[{"sha":"1c3139b60b8efaedd36bc9eadaf26f277c38dfbb","url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/1c3139b60b8efaedd36bc9eadaf26f277c38dfbb","html_url":"https://github.com/OmniNode-ai/omnimarket/commit/1c3139b60b8efaedd36bc9eadaf26f277c38dfbb"}]},"merge_base_commit":{"sha":"454c429f328e68e19300f33ffb8121f1bccc7f86","node_id":"C_kwDOR6jjtdoAKDQ1NGM0MjlmMzI4ZTY4ZTE5MzAwZjMzZmZiODEyMWYxYmNjYzdmODY","commit":{"author":{"name":"Jonah Gray","email":"jonah@omninode.ai","date":"2026-07-30T18:29:59Z"},"committer":{"name":"GitHub","email":"noreply@github.com","date":"2026-07-30T18:29:59Z"},"message":"feat(OMN-15484): one authored merge-hold gate the whole fleet can call (#1976)\n\nOMN-15483 built a required check that refuses a PR carrying a hold marker, so\nthe merge sweep cannot land it inside the adversarial-verification window. It\nshipped in omnimarket ONLY -- while every incident in that ticket own table\nhappened in onex_change_control (OCC#5588, #5586, #5530/#5531, #5584) or\nomnibase_infra (#2560). The mechanism did not cover the repos where the bug\noccurs.\n\nThis adds the ONE authored surface the fan-out needs, so adopting repos get the\ngate without vendoring a second vocabulary (AC1 rejects vendored-copy-plus-sync-\ntest outright -- that is how round 1 ended up with two divergent\n_DO_NOT_MERGE_RE definitions inside a single repo, neither a superset of the\nother).\n\n .github/workflows/merge-hold-gate-reusable.yml\n workflow_call surface. Checks out the caller, checks out omnimarket at the\n caller pinned ref, and renders four verdicts in order: the caller declares\n no second vocabulary (AC1); the check own context name is not itself a\n hold token (AC5); the gate demonstrably fires on a held title and clears\n on a clean one (AC3); then the real verdict for the caller PR.\n Unconditional, hosted runner, stdlib only, no uv sync (AC4).\n\n scripts/ci/check_single_hold_vocabulary.py\n The AC1 falsifier, run against the CALLING repo. Declares no tokens of its\n own -- it derives them from hold_marker.py at run time by reducing the\n canonical pattern alternation to literal skeletons, so a respelled copy\n (`do[ _-]*NOT[ _-]*merge`) is caught where a byte-compare would miss it.\n A single short token (draft/WIP/DNM) is deliberately NOT an offender: a\n false positive that wedges an adopting repo teaches people to disable the\n check, which is worse than a missing gate.\n\n scripts/ci/check_hold_gate_selftest.py\n A green hold check proves the job ran, not that the gate can say no. This\n drives the real CLI over a held vector (must exit 1) and a clean one (must\n exit 0), after validating both vectors against the canonical matcher so a\n vocabulary change cannot leave the probe silently vacuous.\n\n ci.yml: omnimarket becomes the first CALLER, locally, alongside the existing\n pr-hold-check. A reusable surface the defining repo does not call is\n unproven, and the local `uses:` resolves at the PR head so it exercises\n this diff. pr-hold-check is untouched and stays the registered enforcement\n so a defect in the new path cannot take working coverage down with it;\n collapsing the two is a follow-up.\n\nProofs by execution (all on the gate host, not asserted):\n - planted respelled vendored copy -> RED naming donotmerge/workinprogress/wip;\n empty-but-canonically-named copy -> RED; innocent `^\\[draft\\]` regex ->\n GREEN; clean tree -> GREEN.\n - mutating HOLD_MARKER_RE so the probe stops matching turns the self-test RED\n with \"went vacuous\" rather than passing while testing nothing.\n - context name \"Verification Hold Gate / evaluate\" -> RED naming the token;\n \"merge-hold-gate / evaluate\" -> clean.\n - 37 new tests, existing hold-gate suites 101 pass unchanged.\n\nRefs: OMN-15484 (this), OMN-15483 (the gate), OMN-14741 F-17 (the vocabulary).","tree":{"sha":"eec2c51666ed37cca5fe7665eeefd78e04cfb910","url":"https://api.github.com/repos/OmniNode-ai/omnimarket/git/trees/eec2c51666ed37cca5fe7665eeefd78e04cfb910"},"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/git/commits/454c429f328e68e19300f33ffb8121f1bccc7f86","comment_count":0,"verification":{"verified":true,"reason":"valid","signature":"-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJqa5gnCRC1aQ7uu5UhlAAAdlsQADk+y2BKF2IENACo7xfrpuOP\nwfbc5idSPLwlF5kJdrvEzLda1Ei59W0k2D5S8ExCUTddJYxdciurSxTech9EUouE\nCib+KmtFdlftgfGsZiYcdakmJQtjcDxXvTUZyk1Zot8V6yIQHvkON30qd5oE5s6H\nbj7zCFroNXwvi/2HZWWzfI6CfGfWOjDfYKBKPD323TtWqI/ET8uqsExKfnJ8qx2K\nsue8HQl51nZ6E6sjTjX+9MmH8IAxolC3q1TZLeihWEd57KpmfuKM1DXT2xePizpv\nj7CLbHT7nC+0EWC6hDj5A6XKvc4D4zCPy1L69vRfrBd2tUuwzB7WC6hAKf4wU3H8\nbCFiOPULTmATjco6i23csPxM+FlqXKdgfhEtbURIf5RAYo709cDvNJesBfU3Eo+h\nSaCSwtZDFyo7vpPHYuDbGPoRo+8q1oHbKSdlAVgPO2igFUd7zgIA1qHorSNoP5rz\nk4xyPsoqX8lsqiy9M8GGbbJDh/wmZMls2TbcdQGOAYnyDaPqkOi2mJzdtlA84XX4\n+CboScxHjS+HsrjNAOIbivugCjVZLvrU1rDWLlL+rlY1Ic6ObIQNnSTZczk9tBTk\n04cEJloAl36osn9UO1Hrgpg+sSRmEOv/KZZVVUWLI7TFVTnWTS/d0CydfGi03dKN\nlWIhsKmxy5+HddmkcIv0\n=wlCu\n-----END PGP SIGNATURE-----\n","payload":"tree eec2c51666ed37cca5fe7665eeefd78e04cfb910\nparent 3947ee54b83e1965a887e0b6f8be5007c04cfa73\nauthor Jonah Gray 1785436199 -0400\ncommitter GitHub 1785436199 -0400\n\nfeat(OMN-15484): one authored merge-hold gate the whole fleet can call (#1976)\n\nOMN-15483 built a required check that refuses a PR carrying a hold marker, so\nthe merge sweep cannot land it inside the adversarial-verification window. It\nshipped in omnimarket ONLY -- while every incident in that ticket own table\nhappened in onex_change_control (OCC#5588, #5586, #5530/#5531, #5584) or\nomnibase_infra (#2560). The mechanism did not cover the repos where the bug\noccurs.\n\nThis adds the ONE authored surface the fan-out needs, so adopting repos get the\ngate without vendoring a second vocabulary (AC1 rejects vendored-copy-plus-sync-\ntest outright -- that is how round 1 ended up with two divergent\n_DO_NOT_MERGE_RE definitions inside a single repo, neither a superset of the\nother).\n\n .github/workflows/merge-hold-gate-reusable.yml\n workflow_call surface. Checks out the caller, checks out omnimarket at the\n caller pinned ref, and renders four verdicts in order: the caller declares\n no second vocabulary (AC1); the check own context name is not itself a\n hold token (AC5); the gate demonstrably fires on a held title and clears\n on a clean one (AC3); then the real verdict for the caller PR.\n Unconditional, hosted runner, stdlib only, no uv sync (AC4).\n\n scripts/ci/check_single_hold_vocabulary.py\n The AC1 falsifier, run against the CALLING repo. Declares no tokens of its\n own -- it derives them from hold_marker.py at run time by reducing the\n canonical pattern alternation to literal skeletons, so a respelled copy\n (`do[ _-]*NOT[ _-]*merge`) is caught where a byte-compare would miss it.\n A single short token (draft/WIP/DNM) is deliberately NOT an offender: a\n false positive that wedges an adopting repo teaches people to disable the\n check, which is worse than a missing gate.\n\n scripts/ci/check_hold_gate_selftest.py\n A green hold check proves the job ran, not that the gate can say no. This\n drives the real CLI over a held vector (must exit 1) and a clean one (must\n exit 0), after validating both vectors against the canonical matcher so a\n vocabulary change cannot leave the probe silently vacuous.\n\n ci.yml: omnimarket becomes the first CALLER, locally, alongside the existing\n pr-hold-check. A reusable surface the defining repo does not call is\n unproven, and the local `uses:` resolves at the PR head so it exercises\n this diff. pr-hold-check is untouched and stays the registered enforcement\n so a defect in the new path cannot take working coverage down with it;\n collapsing the two is a follow-up.\n\nProofs by execution (all on the gate host, not asserted):\n - planted respelled vendored copy -> RED naming donotmerge/workinprogress/wip;\n empty-but-canonically-named copy -> RED; innocent `^\\[draft\\]` regex ->\n GREEN; clean tree -> GREEN.\n - mutating HOLD_MARKER_RE so the probe stops matching turns the self-test RED\n with \"went vacuous\" rather than passing while testing nothing.\n - context name \"Verification Hold Gate / evaluate\" -> RED naming the token;\n \"merge-hold-gate / evaluate\" -> clean.\n - 37 new tests, existing hold-gate suites 101 pass unchanged.\n\nRefs: OMN-15484 (this), OMN-15483 (the gate), OMN-14741 F-17 (the vocabulary).","verified_at":"2026-07-30T18:29:59Z"}},"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/454c429f328e68e19300f33ffb8121f1bccc7f86","html_url":"https://github.com/OmniNode-ai/omnimarket/commit/454c429f328e68e19300f33ffb8121f1bccc7f86","comments_url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/454c429f328e68e19300f33ffb8121f1bccc7f86/comments","author":{"login":"jonahgabriel","id":1002253,"node_id":"MDQ6VXNlcjEwMDIyNTM=","avatar_url":"https://avatars.githubusercontent.com/u/1002253?v=4","gravatar_id":"","url":"https://api.github.com/users/jonahgabriel","html_url":"https://github.com/jonahgabriel","followers_url":"https://api.github.com/users/jonahgabriel/followers","following_url":"https://api.github.com/users/jonahgabriel/following{/other_user}","gists_url":"https://api.github.com/users/jonahgabriel/gists{/gist_id}","starred_url":"https://api.github.com/users/jonahgabriel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jonahgabriel/subscriptions","organizations_url":"https://api.github.com/users/jonahgabriel/orgs","repos_url":"https://api.github.com/users/jonahgabriel/repos","events_url":"https://api.github.com/users/jonahgabriel/events{/privacy}","received_events_url":"https://api.github.com/users/jonahgabriel/received_events","type":"User","user_view_type":"public","site_admin":false},"committer":{"login":"web-flow","id":19864447,"node_id":"MDQ6VXNlcjE5ODY0NDQ3","avatar_url":"https://avatars.githubusercontent.com/u/19864447?v=4","gravatar_id":"","url":"https://api.github.com/users/web-flow","html_url":"https://github.com/web-flow","followers_url":"https://api.github.com/users/web-flow/followers","following_url":"https://api.github.com/users/web-flow/following{/other_user}","gists_url":"https://api.github.com/users/web-flow/gists{/gist_id}","starred_url":"https://api.github.com/users/web-flow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/web-flow/subscriptions","organizations_url":"https://api.github.com/users/web-flow/orgs","repos_url":"https://api.github.com/users/web-flow/repos","events_url":"https://api.github.com/users/web-flow/events{/privacy}","received_events_url":"https://api.github.com/users/web-flow/received_events","type":"User","user_view_type":"public","site_admin":false},"parents":[{"sha":"3947ee54b83e1965a887e0b6f8be5007c04cfa73","url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/3947ee54b83e1965a887e0b6f8be5007c04cfa73","html_url":"https://github.com/OmniNode-ai/omnimarket/commit/3947ee54b83e1965a887e0b6f8be5007c04cfa73"}]},"status":"behind","ahead_by":0,"behind_by":7,"total_commits":0,"commits":[],"files":[]} \ No newline at end of file diff --git a/tests/fixtures/omn15547/omnimarket-compare-dev-879d6fc6.gh-api.json.captured b/tests/fixtures/omn15547/omnimarket-compare-dev-879d6fc6.gh-api.json.captured new file mode 100644 index 0000000000..6a30284cfd --- /dev/null +++ b/tests/fixtures/omn15547/omnimarket-compare-dev-879d6fc6.gh-api.json.captured @@ -0,0 +1 @@ +{"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/compare/dev...879d6fc6825f876458c6d45ed670c8715de8ac95","html_url":"https://github.com/OmniNode-ai/omnimarket/compare/dev...879d6fc6825f876458c6d45ed670c8715de8ac95","permalink_url":"https://github.com/OmniNode-ai/omnimarket/compare/OmniNode-ai:5dca587...OmniNode-ai:879d6fc","diff_url":"https://github.com/OmniNode-ai/omnimarket/compare/dev...879d6fc6825f876458c6d45ed670c8715de8ac95.diff","patch_url":"https://github.com/OmniNode-ai/omnimarket/compare/dev...879d6fc6825f876458c6d45ed670c8715de8ac95.patch","base_commit":{"sha":"5dca587009684aecab7c24bf1512cfc16380176b","node_id":"C_kwDOR6jjtdoAKDVkY2E1ODcwMDk2ODRhZWNhYjdjMjRiZjE1MTJjZmMxNjM4MDE3NmI","commit":{"author":{"name":"Jonah Gray","email":"jonah@omninode.ai","date":"2026-07-31T00:13:24Z"},"committer":{"name":"GitHub","email":"noreply@github.com","date":"2026-07-31T00:13:24Z"},"message":"fix(OMN-15469): make wiring the sole generation terminal producer (#1984)","tree":{"sha":"c16545132a07b8669ee8809093cd116abed6c7c7","url":"https://api.github.com/repos/OmniNode-ai/omnimarket/git/trees/c16545132a07b8669ee8809093cd116abed6c7c7"},"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/git/commits/5dca587009684aecab7c24bf1512cfc16380176b","comment_count":0,"verification":{"verified":true,"reason":"valid","signature":"-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJqa+ikCRC1aQ7uu5UhlAAACHkQADwlVFjFWcgsTaZoQkkINT9Q\nyCOLRS/6lagRZ3NX1p127bkcSWhipEWbtwg+WhDeSCDNF6898h8DcSb4xvO1koOz\n5/pLFJbuF7XYqsKw0QyB+Sy9I2YdM20GwM3oY7IgslOllEhZHIbTDNUk8JsAYHwM\nWtNQ1WAB2IMSc70sDo0z+sbmhaEqTdR6/mAtBYYuzGNtYT3wz5KMmMxcC8dc3GoV\nSF29pr5LGJPtVOMFGsoVnWq8OqI7BxUIx65qpsnJPCwuzMbeLghASNKUZ9tNDh73\nX+iUUGXDHmClhcCroBiEzyNRHM6gtaRLfZ6PiNKM78606kShMWqSpJZoqo0HTZ6y\nyYAfL5jx3QRCXyVCbhSoRNuPdKRIBpYI+Koks6hkbmHWZGWBdViZgtf0jMeLCAM+\n/qwyGeIaPD9kQAu2vx4c9iHJAuylaTqWoevaLsRUFcy4qQKxZzmP6HCR0l5HvMBs\nBkGJul0yD4E9ru60UZAsasE2rUAQU3pJ4MWitKsdxudxauhED+bwg0b9FDoozUvn\nvDP45Fy+viHziwqVLFyOB1NgNzxG/ZcnDmIZdVslSS2yzsBVO5+vncOqZXd7fGyj\nXZdLhCYR05tIptdwlli4DzjYv7K5v5X56zCoJ2zxqafaoHPHXR7IYS2aTdSkV00H\nSvt24sq6pv6XhkcyIiAM\n=Ji14\n-----END PGP SIGNATURE-----\n","payload":"tree c16545132a07b8669ee8809093cd116abed6c7c7\nparent 1c3139b60b8efaedd36bc9eadaf26f277c38dfbb\nauthor Jonah Gray 1785456804 -0400\ncommitter GitHub 1785456804 -0400\n\nfix(OMN-15469): make wiring the sole generation terminal producer (#1984)","verified_at":"2026-07-31T00:13:24Z"}},"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/5dca587009684aecab7c24bf1512cfc16380176b","html_url":"https://github.com/OmniNode-ai/omnimarket/commit/5dca587009684aecab7c24bf1512cfc16380176b","comments_url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/5dca587009684aecab7c24bf1512cfc16380176b/comments","author":{"login":"jonahgabriel","id":1002253,"node_id":"MDQ6VXNlcjEwMDIyNTM=","avatar_url":"https://avatars.githubusercontent.com/u/1002253?v=4","gravatar_id":"","url":"https://api.github.com/users/jonahgabriel","html_url":"https://github.com/jonahgabriel","followers_url":"https://api.github.com/users/jonahgabriel/followers","following_url":"https://api.github.com/users/jonahgabriel/following{/other_user}","gists_url":"https://api.github.com/users/jonahgabriel/gists{/gist_id}","starred_url":"https://api.github.com/users/jonahgabriel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jonahgabriel/subscriptions","organizations_url":"https://api.github.com/users/jonahgabriel/orgs","repos_url":"https://api.github.com/users/jonahgabriel/repos","events_url":"https://api.github.com/users/jonahgabriel/events{/privacy}","received_events_url":"https://api.github.com/users/jonahgabriel/received_events","type":"User","user_view_type":"public","site_admin":false},"committer":{"login":"web-flow","id":19864447,"node_id":"MDQ6VXNlcjE5ODY0NDQ3","avatar_url":"https://avatars.githubusercontent.com/u/19864447?v=4","gravatar_id":"","url":"https://api.github.com/users/web-flow","html_url":"https://github.com/web-flow","followers_url":"https://api.github.com/users/web-flow/followers","following_url":"https://api.github.com/users/web-flow/following{/other_user}","gists_url":"https://api.github.com/users/web-flow/gists{/gist_id}","starred_url":"https://api.github.com/users/web-flow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/web-flow/subscriptions","organizations_url":"https://api.github.com/users/web-flow/orgs","repos_url":"https://api.github.com/users/web-flow/repos","events_url":"https://api.github.com/users/web-flow/events{/privacy}","received_events_url":"https://api.github.com/users/web-flow/received_events","type":"User","user_view_type":"public","site_admin":false},"parents":[{"sha":"1c3139b60b8efaedd36bc9eadaf26f277c38dfbb","url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/1c3139b60b8efaedd36bc9eadaf26f277c38dfbb","html_url":"https://github.com/OmniNode-ai/omnimarket/commit/1c3139b60b8efaedd36bc9eadaf26f277c38dfbb"}]},"merge_base_commit":{"sha":"5a2e8bf59a0d4f692cc7a4838d432a1bb989941a","node_id":"C_kwDOR6jjtdoAKDVhMmU4YmY1OWEwZDRmNjkyY2M3YTQ4MzhkNDMyYTFiYjk4OTk0MWE","commit":{"author":{"name":"Jonah Gray","email":"jonah@omninode.ai","date":"2026-07-30T14:34:51Z"},"committer":{"name":"GitHub","email":"noreply@github.com","date":"2026-07-30T14:34:51Z"},"message":"fix(OMN-15483): bind every merge consumer with a required CI hold check (#1973)\n\n* fix(OMN-15483): bind every merge consumer with a required CI hold check\n\nRound 1 taught the merge NODE to honor the hold marker, which bound one\nconsumer. It left unbound the consumer that performed every merge in the\nticket's incident table: the foreground Codex controller, which is a session\ndriving `gh pr merge` and contains no omnimarket code, so no amount of node\ncode can bind it. Chasing consumers does not converge either -- a human at a\nterminal, auto-merge, or the next node all start unbound.\n\nEnforcement therefore moves to the surface every consumer already respects:\nrequired status checks. A held PR fails the new unconditional `Merge Hold Gate\n(OMN-15483)` job, which is registered in the fail-closed CI Summary strict set,\nso a held PR can never be required-green for anyone.\n\n- scripts/ci/check_pr_hold_marker.py declares NO regex: it loads the canonical\n merge_control/hold_marker.py by path and calls evaluate_merge_hold(), so the\n CI verdict and the node verdict are one function over one vocabulary. Loading\n by path skips the package __init__, so the gate needs no uv sync and cannot be\n cascade-skipped by the dependency lane.\n- the job is unconditional (no needs/if): on a56e3819 a failing occ-preflight\n cascade-skipped Tests, typecheck, Contract Compliance and the whole E2E lane.\n- the criterion-1 single-vocabulary scan now walks scripts/ as well as src/.\n- the gate's display name is asserted not to be a hold token: the first name\n chosen was one, so a fan-out PR naming the gate would have been held by the\n gate it was installing.\n\nRED/GREEN by execution: a held title exits 1 naming the matched token; this\nPR's own live title and labels exit 0. Mutating the canonical vocabulary flips\nthe gate's verdict and turns 17 tests red across the CI-gate and node suites at\nonce -- single-source proven, not asserted. Vendoring a second regex into the\ngate turns the criterion-1 falsifier red with the offender named. Without the\nstrict registration, a skipped or absent hold job yields CI Summary SUCCESS --\nthe registration is the mechanism, not the job's existence.\n\n* fix(OMN-15483): resolve the hold gate GitHub base URL from the endpoint authority\n\nThe URL Authority Gate was red on this PR: the live-PR-state fetch fell back\nto a literal api.github.com host. omnimarket already declares exactly one\nauthority for external base URLs (src/omnimarket/configs/service_endpoints.yaml,\nOMN-12806) and github_api.py plus every GitHub-calling node handler resolves\ngithub.rest_url from it. The gate now does the same rather than carrying a\nsecond copy of that value.\n\nThe typed accessor (omnimarket.config.service_endpoints) cannot be imported\nhere: it is PyYAML-backed and this job runs a bare python3 with no uv sync, a\nproperty that exists so an unrelated dependency failure cannot skip the hold\ngate. PyYAML is absent from the hosted runner image system Python. So the gate\nreads the same authority file with a strict scalar reader over the exact\ngithub: rest_url: path, and a test asserts that reader and the typed accessor\nreturn an identical string.\n\nFail-closed, no default: a missing or malformed authority raises rather than\nfalling back to a host, and the gate reports FAIL. The authority is only read\nwhen a live fetch is warranted, so the payload-only path is unaffected.\n\nProofs by execution:\n - validator scan of this file, before 1 violation / after 0\n - reintroducing the literal (even annotated) turns two tests red\n - repointing the authority repoints the request URL\n - live fetch against real GitHub resolves and returns source=live\n\nRefs: OMN-15483\n\n* fix(OMN-15483): strip inline comments in hold gate URL authority\n\n* fix(OMN-15483): quote-aware authority scalar read, and make the URL falsifier the real gate\n\nTwo defects in the previous commit, both found by review rather than by my own\ntests, both now pinned.\n\n1. The scalar read was value.strip().strip(quotes), so an inline comment after a\n quoted value survived it: the returned host kept a stray quote and the comment\n text, and still passed the scheme-prefix check — a corrupted value returned\n where the docstring promised an error. _scalar_value now ends a quoted scalar\n at its matching close quote (a hash INSIDE quotes stays part of the value),\n allows only whitespace or a comment after it, refuses trailing junk and an\n unterminated quote, and for a bare scalar ends at the first whitespace-\n preceded hash, matching YAML comment rules. Seven vectors, both directions.\n\n2. The structural falsifier exempted whole docstrings, so it was more lenient\n than the gate it claimed to predict — it stayed green while the real URL\n Authority Gate went red on a URL inside a docstring BODY. It now imports\n omnibase_core validator_url_authority.scan_source and asserts zero violations,\n so the test IS the gate rather than an approximation of it. Same argument this\n PR makes everywhere else: do not re-implement the thing you must agree with.\n Proven by mutation — putting a quoted host back into the docstring body turns\n the test red naming rule/line/snippet, removing it turns it green.\n\nGates on the gate host, sha256-verified per file before running: full suite\n15517 passed / 86 skipped, ruff clean, mypy --strict clean on 2981 files,\npre-commit 68/68 rc=0. The 1 failed + 1 error are the same two Kafka integration\ntests, re-proven pre-existing at unmodified base aea0c33d (no broker on that\nhost).\n\nRefs: OMN-15483\n\n---------\n\nCo-authored-by: Jonah Gray ","tree":{"sha":"15a8904531134522a2b4a488b40640ebb09651dd","url":"https://api.github.com/repos/OmniNode-ai/omnimarket/git/trees/15a8904531134522a2b4a488b40640ebb09651dd"},"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/git/commits/5a2e8bf59a0d4f692cc7a4838d432a1bb989941a","comment_count":0,"verification":{"verified":true,"reason":"valid","signature":"-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJqa2ELCRC1aQ7uu5UhlAAAZ7EQAGCFNo5NRQRmFgWPNgWrXEdA\nrccyUCATdNJ1th+6U+eJW9nyjq6RAkHJDDYjS/xPLTpW9VL6WIQwo4qoLjLZHwvE\nGlSLWctGKtW3kM8asI7RrBnsIsBTb+r6ikyLITD6aY9+dCiXPA0rySVzu35kKe0H\nM9EmTTJLPPXAtdz93Grk6aYtBOhVPS3x/HfDOAicnv8ckQEsAgF6tOGne29THFhi\nwpuAAiQl8HDjmWGA52soZjSqN46169gMCP25RXiaXLW4talfW6sJLfvv4+OHl6vT\nNl79tJYYhY9lMg3xVgp+YLGY4BRtf+aq347SY5XBPy+FxUNsDC1qY7WnQnyVN/cH\nkkomj4/2JiVKFisfPj8z65pQNS1r/Xj4xtw8ob+NArfjn8m/qPw6MqFcG9vPUkhl\nt0btuw1NgtC6dFtLEsQ6iPkO0pnaeHoNbiVfeuVZ7Yr8nW3DyRGXFP4jf906RaRS\nlLFaVeaO2UGphEVw1xxXS/2sMtqZLu0GBr1ZSHeKV3VrL+f8STACq/5VN/iyVxH/\nTOvAbXDtBO/8oyhW39XCEKJ0sq0hoHr5+gj0hdwDFHRmvWd0DF0to6zuS+P0AHc7\nqn2gXPTdf1wPtWrV22a+hXGtkQ6MMWobVx0DTXAEGslfppI1evXeny/jRctvmb6N\nGLbA2Vx2lMZYAz0JAXn4\n=Lw6f\n-----END PGP SIGNATURE-----\n","payload":"tree 15a8904531134522a2b4a488b40640ebb09651dd\nparent e500668c8598325459f1f18f8c4f4e29d976d935\nauthor Jonah Gray 1785422091 -0400\ncommitter GitHub 1785422091 -0400\n\nfix(OMN-15483): bind every merge consumer with a required CI hold check (#1973)\n\n* fix(OMN-15483): bind every merge consumer with a required CI hold check\n\nRound 1 taught the merge NODE to honor the hold marker, which bound one\nconsumer. It left unbound the consumer that performed every merge in the\nticket's incident table: the foreground Codex controller, which is a session\ndriving `gh pr merge` and contains no omnimarket code, so no amount of node\ncode can bind it. Chasing consumers does not converge either -- a human at a\nterminal, auto-merge, or the next node all start unbound.\n\nEnforcement therefore moves to the surface every consumer already respects:\nrequired status checks. A held PR fails the new unconditional `Merge Hold Gate\n(OMN-15483)` job, which is registered in the fail-closed CI Summary strict set,\nso a held PR can never be required-green for anyone.\n\n- scripts/ci/check_pr_hold_marker.py declares NO regex: it loads the canonical\n merge_control/hold_marker.py by path and calls evaluate_merge_hold(), so the\n CI verdict and the node verdict are one function over one vocabulary. Loading\n by path skips the package __init__, so the gate needs no uv sync and cannot be\n cascade-skipped by the dependency lane.\n- the job is unconditional (no needs/if): on a56e3819 a failing occ-preflight\n cascade-skipped Tests, typecheck, Contract Compliance and the whole E2E lane.\n- the criterion-1 single-vocabulary scan now walks scripts/ as well as src/.\n- the gate's display name is asserted not to be a hold token: the first name\n chosen was one, so a fan-out PR naming the gate would have been held by the\n gate it was installing.\n\nRED/GREEN by execution: a held title exits 1 naming the matched token; this\nPR's own live title and labels exit 0. Mutating the canonical vocabulary flips\nthe gate's verdict and turns 17 tests red across the CI-gate and node suites at\nonce -- single-source proven, not asserted. Vendoring a second regex into the\ngate turns the criterion-1 falsifier red with the offender named. Without the\nstrict registration, a skipped or absent hold job yields CI Summary SUCCESS --\nthe registration is the mechanism, not the job's existence.\n\n* fix(OMN-15483): resolve the hold gate GitHub base URL from the endpoint authority\n\nThe URL Authority Gate was red on this PR: the live-PR-state fetch fell back\nto a literal api.github.com host. omnimarket already declares exactly one\nauthority for external base URLs (src/omnimarket/configs/service_endpoints.yaml,\nOMN-12806) and github_api.py plus every GitHub-calling node handler resolves\ngithub.rest_url from it. The gate now does the same rather than carrying a\nsecond copy of that value.\n\nThe typed accessor (omnimarket.config.service_endpoints) cannot be imported\nhere: it is PyYAML-backed and this job runs a bare python3 with no uv sync, a\nproperty that exists so an unrelated dependency failure cannot skip the hold\ngate. PyYAML is absent from the hosted runner image system Python. So the gate\nreads the same authority file with a strict scalar reader over the exact\ngithub: rest_url: path, and a test asserts that reader and the typed accessor\nreturn an identical string.\n\nFail-closed, no default: a missing or malformed authority raises rather than\nfalling back to a host, and the gate reports FAIL. The authority is only read\nwhen a live fetch is warranted, so the payload-only path is unaffected.\n\nProofs by execution:\n - validator scan of this file, before 1 violation / after 0\n - reintroducing the literal (even annotated) turns two tests red\n - repointing the authority repoints the request URL\n - live fetch against real GitHub resolves and returns source=live\n\nRefs: OMN-15483\n\n* fix(OMN-15483): strip inline comments in hold gate URL authority\n\n* fix(OMN-15483): quote-aware authority scalar read, and make the URL falsifier the real gate\n\nTwo defects in the previous commit, both found by review rather than by my own\ntests, both now pinned.\n\n1. The scalar read was value.strip().strip(quotes), so an inline comment after a\n quoted value survived it: the returned host kept a stray quote and the comment\n text, and still passed the scheme-prefix check — a corrupted value returned\n where the docstring promised an error. _scalar_value now ends a quoted scalar\n at its matching close quote (a hash INSIDE quotes stays part of the value),\n allows only whitespace or a comment after it, refuses trailing junk and an\n unterminated quote, and for a bare scalar ends at the first whitespace-\n preceded hash, matching YAML comment rules. Seven vectors, both directions.\n\n2. The structural falsifier exempted whole docstrings, so it was more lenient\n than the gate it claimed to predict — it stayed green while the real URL\n Authority Gate went red on a URL inside a docstring BODY. It now imports\n omnibase_core validator_url_authority.scan_source and asserts zero violations,\n so the test IS the gate rather than an approximation of it. Same argument this\n PR makes everywhere else: do not re-implement the thing you must agree with.\n Proven by mutation — putting a quoted host back into the docstring body turns\n the test red naming rule/line/snippet, removing it turns it green.\n\nGates on the gate host, sha256-verified per file before running: full suite\n15517 passed / 86 skipped, ruff clean, mypy --strict clean on 2981 files,\npre-commit 68/68 rc=0. The 1 failed + 1 error are the same two Kafka integration\ntests, re-proven pre-existing at unmodified base aea0c33d (no broker on that\nhost).\n\nRefs: OMN-15483\n\n---------\n\nCo-authored-by: Jonah Gray ","verified_at":"2026-07-30T14:34:51Z"}},"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/5a2e8bf59a0d4f692cc7a4838d432a1bb989941a","html_url":"https://github.com/OmniNode-ai/omnimarket/commit/5a2e8bf59a0d4f692cc7a4838d432a1bb989941a","comments_url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/5a2e8bf59a0d4f692cc7a4838d432a1bb989941a/comments","author":{"login":"jonahgabriel","id":1002253,"node_id":"MDQ6VXNlcjEwMDIyNTM=","avatar_url":"https://avatars.githubusercontent.com/u/1002253?v=4","gravatar_id":"","url":"https://api.github.com/users/jonahgabriel","html_url":"https://github.com/jonahgabriel","followers_url":"https://api.github.com/users/jonahgabriel/followers","following_url":"https://api.github.com/users/jonahgabriel/following{/other_user}","gists_url":"https://api.github.com/users/jonahgabriel/gists{/gist_id}","starred_url":"https://api.github.com/users/jonahgabriel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jonahgabriel/subscriptions","organizations_url":"https://api.github.com/users/jonahgabriel/orgs","repos_url":"https://api.github.com/users/jonahgabriel/repos","events_url":"https://api.github.com/users/jonahgabriel/events{/privacy}","received_events_url":"https://api.github.com/users/jonahgabriel/received_events","type":"User","user_view_type":"public","site_admin":false},"committer":{"login":"web-flow","id":19864447,"node_id":"MDQ6VXNlcjE5ODY0NDQ3","avatar_url":"https://avatars.githubusercontent.com/u/19864447?v=4","gravatar_id":"","url":"https://api.github.com/users/web-flow","html_url":"https://github.com/web-flow","followers_url":"https://api.github.com/users/web-flow/followers","following_url":"https://api.github.com/users/web-flow/following{/other_user}","gists_url":"https://api.github.com/users/web-flow/gists{/gist_id}","starred_url":"https://api.github.com/users/web-flow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/web-flow/subscriptions","organizations_url":"https://api.github.com/users/web-flow/orgs","repos_url":"https://api.github.com/users/web-flow/repos","events_url":"https://api.github.com/users/web-flow/events{/privacy}","received_events_url":"https://api.github.com/users/web-flow/received_events","type":"User","user_view_type":"public","site_admin":false},"parents":[{"sha":"e500668c8598325459f1f18f8c4f4e29d976d935","url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/e500668c8598325459f1f18f8c4f4e29d976d935","html_url":"https://github.com/OmniNode-ai/omnimarket/commit/e500668c8598325459f1f18f8c4f4e29d976d935"}]},"status":"diverged","ahead_by":1,"behind_by":12,"total_commits":1,"commits":[{"sha":"879d6fc6825f876458c6d45ed670c8715de8ac95","node_id":"C_kwDOR6jjtdoAKDg3OWQ2ZmM2ODI1Zjg3NjQ1OGM2ZDQ1ZWQ2NzBjODcxNWRlOGFjOTU","commit":{"author":{"name":"Jonah Gray","email":"jonah@omninode.ai","date":"2026-07-30T17:02:40Z"},"committer":{"name":"Jonah Gray","email":"jonah@omninode.ai","date":"2026-07-30T17:02:40Z"},"message":"feat(OMN-15484): one authored merge-hold gate the whole fleet can call\n\nOMN-15483 built a required check that refuses a PR carrying a hold marker, so\nthe merge sweep cannot land it inside the adversarial-verification window. It\nshipped in omnimarket ONLY -- while every incident in that ticket own table\nhappened in onex_change_control (OCC#5588, #5586, #5530/#5531, #5584) or\nomnibase_infra (#2560). The mechanism did not cover the repos where the bug\noccurs.\n\nThis adds the ONE authored surface the fan-out needs, so adopting repos get the\ngate without vendoring a second vocabulary (AC1 rejects vendored-copy-plus-sync-\ntest outright -- that is how round 1 ended up with two divergent\n_DO_NOT_MERGE_RE definitions inside a single repo, neither a superset of the\nother).\n\n .github/workflows/merge-hold-gate-reusable.yml\n workflow_call surface. Checks out the caller, checks out omnimarket at the\n caller pinned ref, and renders four verdicts in order: the caller declares\n no second vocabulary (AC1); the check own context name is not itself a\n hold token (AC5); the gate demonstrably fires on a held title and clears\n on a clean one (AC3); then the real verdict for the caller PR.\n Unconditional, hosted runner, stdlib only, no uv sync (AC4).\n\n scripts/ci/check_single_hold_vocabulary.py\n The AC1 falsifier, run against the CALLING repo. Declares no tokens of its\n own -- it derives them from hold_marker.py at run time by reducing the\n canonical pattern alternation to literal skeletons, so a respelled copy\n (`do[ _-]*NOT[ _-]*merge`) is caught where a byte-compare would miss it.\n A single short token (draft/WIP/DNM) is deliberately NOT an offender: a\n false positive that wedges an adopting repo teaches people to disable the\n check, which is worse than a missing gate.\n\n scripts/ci/check_hold_gate_selftest.py\n A green hold check proves the job ran, not that the gate can say no. This\n drives the real CLI over a held vector (must exit 1) and a clean one (must\n exit 0), after validating both vectors against the canonical matcher so a\n vocabulary change cannot leave the probe silently vacuous.\n\n ci.yml: omnimarket becomes the first CALLER, locally, alongside the existing\n pr-hold-check. A reusable surface the defining repo does not call is\n unproven, and the local `uses:` resolves at the PR head so it exercises\n this diff. pr-hold-check is untouched and stays the registered enforcement\n so a defect in the new path cannot take working coverage down with it;\n collapsing the two is a follow-up.\n\nProofs by execution (all on the gate host, not asserted):\n - planted respelled vendored copy -> RED naming donotmerge/workinprogress/wip;\n empty-but-canonically-named copy -> RED; innocent `^\\[draft\\]` regex ->\n GREEN; clean tree -> GREEN.\n - mutating HOLD_MARKER_RE so the probe stops matching turns the self-test RED\n with \"went vacuous\" rather than passing while testing nothing.\n - context name \"Verification Hold Gate / evaluate\" -> RED naming the token;\n \"merge-hold-gate / evaluate\" -> clean.\n - 37 new tests, existing hold-gate suites 101 pass unchanged.\n\nRefs: OMN-15484 (this), OMN-15483 (the gate), OMN-14741 F-17 (the vocabulary).","tree":{"sha":"bef206e313736d0dc05df06869d1e9276c371352","url":"https://api.github.com/repos/OmniNode-ai/omnimarket/git/trees/bef206e313736d0dc05df06869d1e9276c371352"},"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/git/commits/879d6fc6825f876458c6d45ed670c8715de8ac95","comment_count":0,"verification":{"verified":false,"reason":"unsigned","signature":null,"payload":null,"verified_at":null}},"url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/879d6fc6825f876458c6d45ed670c8715de8ac95","html_url":"https://github.com/OmniNode-ai/omnimarket/commit/879d6fc6825f876458c6d45ed670c8715de8ac95","comments_url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/879d6fc6825f876458c6d45ed670c8715de8ac95/comments","author":{"login":"jonahgabriel","id":1002253,"node_id":"MDQ6VXNlcjEwMDIyNTM=","avatar_url":"https://avatars.githubusercontent.com/u/1002253?v=4","gravatar_id":"","url":"https://api.github.com/users/jonahgabriel","html_url":"https://github.com/jonahgabriel","followers_url":"https://api.github.com/users/jonahgabriel/followers","following_url":"https://api.github.com/users/jonahgabriel/following{/other_user}","gists_url":"https://api.github.com/users/jonahgabriel/gists{/gist_id}","starred_url":"https://api.github.com/users/jonahgabriel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jonahgabriel/subscriptions","organizations_url":"https://api.github.com/users/jonahgabriel/orgs","repos_url":"https://api.github.com/users/jonahgabriel/repos","events_url":"https://api.github.com/users/jonahgabriel/events{/privacy}","received_events_url":"https://api.github.com/users/jonahgabriel/received_events","type":"User","user_view_type":"public","site_admin":false},"committer":{"login":"jonahgabriel","id":1002253,"node_id":"MDQ6VXNlcjEwMDIyNTM=","avatar_url":"https://avatars.githubusercontent.com/u/1002253?v=4","gravatar_id":"","url":"https://api.github.com/users/jonahgabriel","html_url":"https://github.com/jonahgabriel","followers_url":"https://api.github.com/users/jonahgabriel/followers","following_url":"https://api.github.com/users/jonahgabriel/following{/other_user}","gists_url":"https://api.github.com/users/jonahgabriel/gists{/gist_id}","starred_url":"https://api.github.com/users/jonahgabriel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jonahgabriel/subscriptions","organizations_url":"https://api.github.com/users/jonahgabriel/orgs","repos_url":"https://api.github.com/users/jonahgabriel/repos","events_url":"https://api.github.com/users/jonahgabriel/events{/privacy}","received_events_url":"https://api.github.com/users/jonahgabriel/received_events","type":"User","user_view_type":"public","site_admin":false},"parents":[{"sha":"5a2e8bf59a0d4f692cc7a4838d432a1bb989941a","url":"https://api.github.com/repos/OmniNode-ai/omnimarket/commits/5a2e8bf59a0d4f692cc7a4838d432a1bb989941a","html_url":"https://github.com/OmniNode-ai/omnimarket/commit/5a2e8bf59a0d4f692cc7a4838d432a1bb989941a"}]}],"files":[{"sha":"f42ec1f0e6d1bb9e77e07747307f924062bc38c4","filename":".github/workflows/ci.yml","status":"modified","additions":35,"deletions":0,"changes":35,"blob_url":"https://github.com/OmniNode-ai/omnimarket/blob/879d6fc6825f876458c6d45ed670c8715de8ac95/.github%2Fworkflows%2Fci.yml","raw_url":"https://github.com/OmniNode-ai/omnimarket/raw/879d6fc6825f876458c6d45ed670c8715de8ac95/.github%2Fworkflows%2Fci.yml","contents_url":"https://api.github.com/repos/OmniNode-ai/omnimarket/contents/.github%2Fworkflows%2Fci.yml?ref=879d6fc6825f876458c6d45ed670c8715de8ac95","patch":"@@ -2005,6 +2005,41 @@ jobs:\n - name: Refuse a PR that is held against landing\n run: python3 scripts/ci/check_pr_hold_marker.py\n \n+ # ============================================================================\n+ # OMN-15484 -- omnimarket is the FIRST CALLER of the shared fan-out surface.\n+ # ============================================================================\n+ # `merge-hold-gate-reusable.yml` is the one authored surface that\n+ # onex_change_control and omnibase_infra call to get this gate without\n+ # vendoring a second vocabulary (AC1). A reusable workflow that the repo\n+ # defining it does not itself call is an unproven surface: the adopting repos\n+ # would be the first to execute it, in a PR whose author cannot iterate on it.\n+ # So omnimarket calls it too, locally (`uses: ./...`), which resolves at the\n+ # PR head and therefore exercises the exact code under review.\n+ #\n+ # This runs ALONGSIDE `pr-hold-check` above, deliberately, and only for this\n+ # transition. `pr-hold-check` is the live-proven implementation registered in\n+ # STRICT_GATE_JOBS and is omnimarket's actual enforcement; this job is the\n+ # fan-out surface proving itself against an independent control. Collapsing\n+ # the two -- deleting `pr-hold-check` and registering this context instead --\n+ # is a follow-up, kept out of this PR so that a defect in the new path cannot\n+ # take omnimarket's working coverage down with it.\n+ #\n+ # NOT registered in STRICT_GATE_JOBS while it is a proof job. It is still not\n+ # toothless here: omnimarket's CI Summary poller runs a default-deny sweep\n+ # over every present+completed job, so a FAILURE in this job already fails the\n+ # required context. What strict registration would add is `skipped` -> FAILURE\n+ # and `absent` -> PENDING, and those belong with the collapse.\n+ merge-hold-gate:\n+ permissions:\n+ contents: read\n+ pull-requests: read\n+ uses: ./.github/workflows/merge-hold-gate-reusable.yml\n+ with:\n+ # The PR's own code, not `dev`: this call exists to prove the surface the\n+ # PR is adding. Adopting repos pin `dev` instead -- see the input's docs.\n+ vocabulary_ref: ${{ github.sha }}\n+ context_name: merge-hold-gate / evaluate\n+\n ci-summary:\n name: CI Summary\n # OMN-14127 fan-out: REQUIRED branch-protection context, implemented as a"},{"sha":"99ff7d5550574c60cf99c1aa55aecce5b4af87d0","filename":".github/workflows/merge-hold-gate-reusable.yml","status":"added","additions":147,"deletions":0,"changes":147,"blob_url":"https://github.com/OmniNode-ai/omnimarket/blob/879d6fc6825f876458c6d45ed670c8715de8ac95/.github%2Fworkflows%2Fmerge-hold-gate-reusable.yml","raw_url":"https://github.com/OmniNode-ai/omnimarket/raw/879d6fc6825f876458c6d45ed670c8715de8ac95/.github%2Fworkflows%2Fmerge-hold-gate-reusable.yml","contents_url":"https://api.github.com/repos/OmniNode-ai/omnimarket/contents/.github%2Fworkflows%2Fmerge-hold-gate-reusable.yml?ref=879d6fc6825f876458c6d45ed670c8715de8ac95","patch":"@@ -0,0 +1,147 @@\n+# ============================================================================\n+# Merge Hold Gate — the ONE authored surface, called by every adopting repo\n+# (OMN-15484, fan-out of OMN-15483)\n+# ============================================================================\n+#\n+# OMN-15483 landed the hold gate in omnimarket only. Every incident in that\n+# ticket's own table happened somewhere else: onex_change_control (OCC#5588,\n+# #5586, #5530/#5531, #5584) and omnibase_infra (#2560). The mechanism did not\n+# cover the repos where the bug occurs.\n+#\n+# The obvious fan-out — copy hold_marker.py + check_pr_hold_marker.py into each\n+# repo — is explicitly rejected by OMN-15484 AC1, because it rebuilds at fleet\n+# scale the exact bug round 1 found INSIDE omnimarket: two divergent\n+# `_DO_NOT_MERGE_RE` definitions, neither a superset of the other, so the same\n+# PR was suppressed by one consumer and authored by the other. Nine copies plus\n+# a sync test is that bug with more copies and more places to forget.\n+#\n+# So this is a reusable workflow: one authored surface, N callers. An adopting\n+# repo adds ~10 lines of `uses:` and declares NO vocabulary, NO regex and NO\n+# gate logic of its own. The vocabulary is read at run time out of omnimarket's\n+# src/omnimarket/merge_control/hold_marker.py — the same module the omnimarket\n+# node path calls — so there is nothing to drift.\n+#\n+# WHAT THIS JOB PROVES, in every calling repo, on every run\n+# ---------------------------------------------------------\n+# A green hold check proves only that the job ran and that THIS PR is not held.\n+# It does not prove the gate can say no, and every interesting failure mode is\n+# silent under that observation. So the job renders four verdicts, in order:\n+#\n+# 1. AC1 — the CALLING repo declares no second hold vocabulary.\n+# 2. AC5 — the check's own context name is not itself a hold token (the first\n+# draft of the omnimarket gate was named \"Verification Hold Gate\", and\n+# `verification hold` is in the vocabulary it enforces: a PR titled after\n+# the gate would have been held by the gate it was installing).\n+# 3. AC3 — a held title really does produce exit 1 HERE, and a clean title\n+# really does produce exit 0. Both directions: exit-1-always is as broken\n+# as exit-0-always.\n+# 4. The real verdict for the caller's PR.\n+#\n+# UNCONDITIONAL BY DESIGN (AC4). The calling job must carry no `needs:` and no\n+# `if:`. On omnimarket a56e3819 a failing `occ-preflight` cascade-skipped Tests,\n+# typecheck, Contract Compliance and the whole E2E lane; a hold gate an\n+# unrelated upstream can skip is not a gate. This job is hosted-runner,\n+# stdlib-only, no `uv sync`, and reaches a verdict in seconds, so it has no\n+# reason to depend on anything.\n+#\n+# REGISTRATION IS THE MECHANISM, NOT EXISTENCE. Adding this job without adding\n+# it to the caller's `CI Summary` strict set is worse than not adding it —\n+# proven by execution against the real evaluator on omnimarket#1973:\n+#\n+# hold job result | with strict registration | without registration\n+# --------------- | ------------------------ | --------------------\n+# failure | CI Summary FAILURE | CI Summary FAILURE\n+# skipped | CI Summary FAILURE | CI Summary SUCCESS <- bypass\n+# cancelled | CI Summary FAILURE | CI Summary FAILURE\n+# absent | CI Summary PENDING | CI Summary SUCCESS <- bypass\n+#\n+# An unregistered job looks like enforcement and enforces nothing. Each caller\n+# registers it in whatever shape its own `CI Summary` producer uses (a name in\n+# `STRICT_GATE_JOBS` for the poller repos; a `needs:` entry plus an explicit\n+# success-only check for the needs-based aggregators) and ships a test that\n+# drives that producer.\n+# ============================================================================\n+name: Merge Hold Gate (reusable)\n+\n+on:\n+ workflow_call:\n+ inputs:\n+ vocabulary_ref:\n+ description: >-\n+ omnimarket ref to resolve the canonical hold vocabulary from. Defaults\n+ to `dev`, which is deliberate for a VOCABULARY: adding a hold token\n+ must take effect fleet-wide immediately, and a per-caller pin would\n+ mean nine repos enforcing nine different vintages of the same rule —\n+ the divergence this fan-out exists to prevent. Same posture as\n+ omniclaude's deploy-gate-reusable, which checks itself out at `main`.\n+ required: false\n+ type: string\n+ default: dev\n+ scan_roots:\n+ description: >-\n+ Space-separated directories in the CALLING repo to scan for a second\n+ hold vocabulary (AC1). Missing directories are skipped, so the default\n+ is safe for repos with no `src/`.\n+ required: false\n+ type: string\n+ default: src scripts\n+ context_name:\n+ description: >-\n+ The check-run context this call produces, as the caller expects to\n+ register it (e.g. `merge-hold-gate / evaluate`). Validated against the\n+ canonical vocabulary for AC5, and asserted equal to the caller's real\n+ job wiring by the caller's own wiring test — the two halves together\n+ are what stop the registered name and the produced name drifting.\n+ required: true\n+ type: string\n+\n+permissions: {}\n+\n+jobs:\n+ evaluate:\n+ runs-on: ubuntu-latest\n+ timeout-minutes: 5\n+ permissions:\n+ contents: read\n+ pull-requests: read\n+ env:\n+ # Everything the gate reads, injected as ENV rather than interpolated into\n+ # a script body. `github.event.pull_request.title` is attacker-controlled\n+ # on a fork PR; as an env value it is data, never shell.\n+ GH_TOKEN: ${{ github.token }}\n+ GH_REPO: ${{ github.repository }}\n+ PR_NUMBER: ${{ github.event.pull_request.number }}\n+ GITHUB_EVENT_NAME: ${{ github.event_name }}\n+ PR_TITLE: ${{ github.event.pull_request.title }}\n+ PR_LABELS_JSON: ${{ toJSON(github.event.pull_request.labels) }}\n+ HOLD_GATE_CONTEXT_NAME: ${{ inputs.context_name }}\n+ CALLER_SCAN_ROOTS: ${{ inputs.scan_roots }}\n+ steps:\n+ - name: Checkout the calling repository\n+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4\n+ with:\n+ path: caller\n+ persist-credentials: false\n+\n+ - name: Checkout the canonical hold vocabulary (omnimarket)\n+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4\n+ with:\n+ repository: OmniNode-ai/omnimarket\n+ ref: ${{ inputs.vocabulary_ref }}\n+ path: vocabulary\n+ persist-credentials: false\n+\n+ - name: AC1 - this repository declares no second hold vocabulary\n+ working-directory: vocabulary\n+ run: |\n+ python3 -m scripts.ci.check_single_hold_vocabulary \\\n+ --tree \"${GITHUB_WORKSPACE}/caller\" \\\n+ --roots \"${CALLER_SCAN_ROOTS}\"\n+\n+ - name: AC5 + AC3 - the gate is not self-holding, and it demonstrably fires\n+ working-directory: vocabulary\n+ run: python3 -m scripts.ci.check_hold_gate_selftest\n+\n+ - name: Refuse a PR that is held against landing\n+ working-directory: vocabulary\n+ run: python3 scripts/ci/check_pr_hold_marker.py"},{"sha":"220c4a3ab8661657627b1df2cf914ccf6598846a","filename":"scripts/ci/check_hold_gate_selftest.py","status":"added","additions":316,"deletions":0,"changes":316,"blob_url":"https://github.com/OmniNode-ai/omnimarket/blob/879d6fc6825f876458c6d45ed670c8715de8ac95/scripts%2Fci%2Fcheck_hold_gate_selftest.py","raw_url":"https://github.com/OmniNode-ai/omnimarket/raw/879d6fc6825f876458c6d45ed670c8715de8ac95/scripts%2Fci%2Fcheck_hold_gate_selftest.py","contents_url":"https://api.github.com/repos/OmniNode-ai/omnimarket/contents/scripts%2Fci%2Fcheck_hold_gate_selftest.py?ref=879d6fc6825f876458c6d45ed670c8715de8ac95","patch":"@@ -0,0 +1,316 @@\n+#!/usr/bin/env python3\n+# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc.\n+# SPDX-License-Identifier: MIT\n+\"\"\"Prove the merge-hold gate FIRES, in the adopting repo's own CI (OMN-15484).\n+\n+Why a live self-proof and not a test\n+------------------------------------\n+OMN-15484 AC3 asks, for each adopting repo, for \"a held-title vector producing\n+exit 1 in that repo's CI (a workflow-dispatch or scratch PR is acceptable; an\n+assertion is not)\". A green ``Merge Hold Gate`` check-run proves only that the\n+job ran and that *this* PR is not held. It does not prove the gate can say no.\n+Every failure mode that matters is silent under that observation:\n+\n+* the canonical checkout resolved to a ref where the vocabulary has moved,\n+* the gate script exited 0 on an internal error a caller swallowed,\n+* the workflow passes the surfaces in a shape the evaluator reads as \"not\n+ observed\" — which on a *clear* PR still exits 0 via the payload path,\n+* a future edit narrows the vocabulary to nothing.\n+\n+In every one of those the adopting repo shows a green hold gate and enforces\n+nothing. That is precisely the \"looks like enforcement and enforces nothing\"\n+outcome the ticket calls worse than no gate at all.\n+\n+So this script runs BEFORE the real evaluation, on every execution in every\n+adopting repo, and drives the real ``check_pr_hold_marker.py`` CLI as a\n+subprocess over two vectors:\n+\n+1. a **held** title -> the CLI must exit 1,\n+2. a **clear** title -> the CLI must exit 0.\n+\n+Both directions are required. Exit-1-always is as broken as exit-0-always, and\n+only the pair distinguishes a working matcher from a stuck one.\n+\n+The probe vectors are validated against the canonical vocabulary first\n+-----------------------------------------------------------------------\n+A hardcoded \"held\" string would rot the moment the vocabulary changes, and it\n+would rot *silently* in the safe-looking direction: the vector stops being a\n+hold token, the gate correctly exits 0, and the self-test starts asserting\n+nothing. So the script loads the canonical module and asserts — via the\n+canonical ``match_hold_token`` — that its held vector really is a token and its\n+clear vector really is not, **before** running either through the CLI. If the\n+vocabulary ever stops matching the probe, this fails loudly instead of going\n+quietly vacuous.\n+\n+The vectors are exercised through the event-payload path with no token in the\n+environment, so the self-test performs no network I/O and cannot be answered by\n+the live PR under evaluation.\n+\n+Exit codes: ``0`` the gate demonstrably fires and demonstrably clears, ``1``\n+anything else.\n+\n+Related: OMN-15484 AC3, OMN-15483 (the gate this proves).\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import argparse\n+import os\n+import subprocess\n+import sys\n+from collections.abc import Sequence\n+from pathlib import Path\n+\n+from scripts.ci.check_pr_hold_marker import (\n+ CANONICAL_HOLD_MODULE,\n+ CanonicalVocabularyUnavailableError,\n+ load_canonical_hold_module,\n+)\n+\n+EXIT_OK = 0\n+EXIT_BROKEN = 1\n+\n+GATE_SCRIPT = Path(__file__).resolve().parent / \"check_pr_hold_marker.py\"\n+\n+# The probe vectors. Neither is trusted: both are checked against the canonical\n+# matcher before use (see the module docstring), so a vocabulary change that\n+# invalidates either one fails this script rather than hollowing it out.\n+#\n+# The held vector is deliberately the shape the OMN-15483 incident table records\n+# a human actually using on a PR title.\n+HELD_VECTOR = \"[OMN-15484 self-test] DO NOT MERGE — hold gate liveness probe\"\n+CLEAR_VECTOR = \"feat(OMN-15484): fan out the merge hold gate to this repository\"\n+\n+_SUBPROCESS_TIMEOUT_SECONDS = 60\n+\n+\n+class SelfTestBrokenError(RuntimeError):\n+ \"\"\"The gate did not behave as a gate. Never downgraded to a warning.\"\"\"\n+\n+\n+def _run_gate(title: str, *, gate_script: Path, repo_root: Path) -> tuple[int, str]:\n+ \"\"\"Run the real gate CLI over ``title`` on the payload path.\n+\n+ Args:\n+ title: The PR title to feed the gate.\n+ gate_script: Path to ``check_pr_hold_marker.py``.\n+ repo_root: Working directory (the canonical checkout root).\n+\n+ Returns:\n+ ``(exit_code, combined_output)``.\n+ \"\"\"\n+ env = {\n+ # Only what the gate reads. No GH_TOKEN/GITHUB_TOKEN: the live-fetch\n+ # branch must not be reachable, or the probe could be answered by the\n+ # real PR instead of by the vector.\n+ \"GITHUB_EVENT_NAME\": \"pull_request\",\n+ \"PR_TITLE\": title,\n+ \"PR_LABELS_JSON\": \"[]\",\n+ \"PATH\": os.environ.get(\"PATH\", \"\"),\n+ \"PYTHONPATH\": str(repo_root),\n+ \"PYTHONDONTWRITEBYTECODE\": \"1\",\n+ }\n+ completed = subprocess.run(\n+ [sys.executable, str(gate_script)],\n+ env=env,\n+ cwd=str(repo_root),\n+ capture_output=True,\n+ text=True,\n+ timeout=_SUBPROCESS_TIMEOUT_SECONDS,\n+ check=False,\n+ )\n+ return completed.returncode, (completed.stdout + completed.stderr).strip()\n+\n+\n+def check_context_name(canonical: object, context_name: str | None) -> str | None:\n+ \"\"\"AC5: the check's own context name must not be a hold token.\n+\n+ The first draft of the omnimarket gate was named ``Verification Hold Gate``,\n+ and ``verification hold`` is a token in the vocabulary the gate enforces.\n+ Nothing failed at the time — a job name only reaches the *title* surface\n+ when a human writes it there — but the fan-out is exactly that case: a PR\n+ titled \"fan out to onex_change_control\" would have been refused\n+ by the gate it was installing, which reads as the gate being broken rather\n+ than as the gate working.\n+\n+ Checking the CONTEXT name (`` / ``) rather than only\n+ the inner job name matters because the caller supplies half of it, and the\n+ caller is a different repo from the one holding this test.\n+\n+ Args:\n+ canonical: The loaded canonical vocabulary module.\n+ context_name: The context name to validate, or ``None`` to skip.\n+\n+ Returns:\n+ A failure report, or ``None`` when the name is acceptable.\n+ \"\"\"\n+ if not context_name:\n+ return None\n+ token = canonical.match_hold_token(context_name) # type: ignore[attr-defined]\n+ if token is None:\n+ return None\n+ return (\n+ \"FAIL — this gate's own check context is itself a hold token.\\n\"\n+ f\" context : {context_name!r}\\n\"\n+ f\" token : {token!r}\\n\"\n+ \"\\n\"\n+ \"Any PR whose title mentions this check would be held by the check \"\n+ \"itself — including the PR that installs it in the next repository. \"\n+ \"Rename the job to something the vocabulary does not match.\"\n+ )\n+\n+\n+def run(\n+ *,\n+ gate_script: Path = GATE_SCRIPT,\n+ module_path: Path = CANONICAL_HOLD_MODULE,\n+ held_vector: str = HELD_VECTOR,\n+ clear_vector: str = CLEAR_VECTOR,\n+ context_name: str | None = None,\n+) -> tuple[int, str]:\n+ \"\"\"Prove the gate fires on a held vector and clears on a clean one.\n+\n+ Args:\n+ gate_script: The gate CLI to drive.\n+ module_path: The canonical vocabulary module.\n+ held_vector: A title that MUST be held.\n+ clear_vector: A title that MUST NOT be held.\n+ context_name: The check context name to validate for AC5, if supplied.\n+\n+ Returns:\n+ ``(exit_code, report)``.\n+ \"\"\"\n+ try:\n+ canonical = load_canonical_hold_module(module_path)\n+ except CanonicalVocabularyUnavailableError as exc:\n+ return EXIT_BROKEN, f\"FAIL (fail-closed): {exc}\"\n+\n+ if not gate_script.is_file():\n+ return EXIT_BROKEN, (\n+ f\"FAIL (fail-closed): the gate CLI is not at {gate_script} — this \"\n+ \"repository's hold check cannot run at all\"\n+ )\n+\n+ name_failure = check_context_name(canonical, context_name)\n+ if name_failure is not None:\n+ return EXIT_BROKEN, name_failure\n+\n+ # (1) The vectors must still mean what they are named. Checked FIRST so a\n+ # vocabulary change can never leave this script asserting nothing.\n+ held_token = canonical.match_hold_token(held_vector)\n+ if held_token is None:\n+ return EXIT_BROKEN, (\n+ \"FAIL — the self-test's HELD vector is no longer a hold token \"\n+ f\"({held_vector!r}). The vocabulary changed and this probe went \"\n+ \"vacuous: it would have 'passed' by testing nothing. Update the \"\n+ \"vector in the same commit as the vocabulary.\"\n+ )\n+ if canonical.match_hold_token(clear_vector) is not None:\n+ return EXIT_BROKEN, (\n+ \"FAIL — the self-test's CLEAR vector now matches the vocabulary \"\n+ f\"({clear_vector!r}), so the negative direction proves nothing. \"\n+ \"Either the vocabulary over-matches or the vector needs replacing.\"\n+ )\n+\n+ repo_root = gate_script.resolve().parents[2]\n+\n+ # (2) Held vector -> the gate must refuse.\n+ held_code, held_output = _run_gate(\n+ held_vector, gate_script=gate_script, repo_root=repo_root\n+ )\n+ if held_code == 0:\n+ return EXIT_BROKEN, (\n+ \"FAIL — the hold gate did NOT fire on a held title.\\n\"\n+ f\" vector : {held_vector!r}\\n\"\n+ f\" token in it : {held_token!r}\\n\"\n+ f\" exit code : {held_code} (expected 1)\\n\"\n+ f\" gate output : {held_output}\\n\"\n+ \"\\n\"\n+ \"A gate that cannot say no is not enforcement. This repository's \"\n+ \"Merge Hold Gate would be green on every PR while holding nothing.\"\n+ )\n+\n+ # (3) Clear vector -> the gate must pass. Guards the opposite failure: a\n+ # gate stuck at exit 1 would block every PR in the adopting repo.\n+ clear_code, clear_output = _run_gate(\n+ clear_vector, gate_script=gate_script, repo_root=repo_root\n+ )\n+ if clear_code != 0:\n+ return EXIT_BROKEN, (\n+ \"FAIL — the hold gate refused an UNHELD title.\\n\"\n+ f\" vector : {clear_vector!r}\\n\"\n+ f\" exit code : {clear_code} (expected 0)\\n\"\n+ f\" gate output: {clear_output}\\n\"\n+ \"\\n\"\n+ \"Stuck-closed is a fleet outage, not a safe default: every PR in \"\n+ \"this repository would be unmergeable.\"\n+ )\n+\n+ return EXIT_OK, (\n+ \"PASS — the hold gate demonstrably fires and demonstrably clears.\\n\"\n+ f\" held {held_vector!r} -> exit {held_code} \"\n+ f\"(matched {held_token!r})\\n\"\n+ f\" clear {clear_vector!r} -> exit {clear_code}\\n\"\n+ f\" vocabulary: {module_path}\\n\"\n+ f\" gate CLI : {gate_script}\"\n+ )\n+\n+\n+def main(argv: Sequence[str] | None = None) -> int:\n+ \"\"\"CLI entrypoint.\n+\n+ Args:\n+ argv: Command-line arguments (defaults to ``sys.argv[1:]``).\n+\n+ Returns:\n+ Process exit code: 0 proven, 1 broken.\n+ \"\"\"\n+ parser = argparse.ArgumentParser(\n+ description=(\n+ \"Prove the merge-hold gate fires on a held title and clears on an \"\n+ \"unheld one, in the calling repository's own CI (OMN-15484 AC3).\"\n+ )\n+ )\n+ parser.add_argument(\n+ \"--gate-script\",\n+ type=Path,\n+ default=GATE_SCRIPT,\n+ help=\"Path to check_pr_hold_marker.py (default: the sibling script).\",\n+ )\n+ parser.add_argument(\n+ \"--module-path\",\n+ type=Path,\n+ default=CANONICAL_HOLD_MODULE,\n+ help=\"Path to the canonical hold_marker.py (default: the in-repo module).\",\n+ )\n+ parser.add_argument(\n+ \"--context-name\",\n+ default=None,\n+ help=(\n+ \"Check context name to validate for AC5. Defaults to the \"\n+ \"HOLD_GATE_CONTEXT_NAME environment variable, which is how the \"\n+ \"reusable workflow passes it (env, never interpolated into a \"\n+ \"script body).\"\n+ ),\n+ )\n+ args = parser.parse_args(argv)\n+\n+ code, report = run(\n+ gate_script=args.gate_script,\n+ module_path=args.module_path,\n+ context_name=args.context_name or os.environ.get(\"HOLD_GATE_CONTEXT_NAME\"),\n+ )\n+ if code == EXIT_OK:\n+ print(report)\n+ print(\"::notice::Merge Hold Gate self-test: the gate fires and clears.\")\n+ else:\n+ print(report, file=sys.stderr)\n+ print(\n+ f\"::error::Merge Hold Gate self-test: {report.splitlines()[0]}\",\n+ file=sys.stderr,\n+ )\n+ return code\n+\n+\n+if __name__ == \"__main__\":\n+ raise SystemExit(main())"},{"sha":"d315b2a45364dd20711a88d9f299a2ca3b2d176a","filename":"scripts/ci/check_single_hold_vocabulary.py","status":"added","additions":613,"deletions":0,"changes":613,"blob_url":"https://github.com/OmniNode-ai/omnimarket/blob/879d6fc6825f876458c6d45ed670c8715de8ac95/scripts%2Fci%2Fcheck_single_hold_vocabulary.py","raw_url":"https://github.com/OmniNode-ai/omnimarket/raw/879d6fc6825f876458c6d45ed670c8715de8ac95/scripts%2Fci%2Fcheck_single_hold_vocabulary.py","contents_url":"https://api.github.com/repos/OmniNode-ai/omnimarket/contents/scripts%2Fci%2Fcheck_single_hold_vocabulary.py?ref=879d6fc6825f876458c6d45ed670c8715de8ac95","patch":"@@ -0,0 +1,613 @@\n+#!/usr/bin/env python3\n+# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc.\n+# SPDX-License-Identifier: MIT\n+\"\"\"Cross-repo criterion-1 falsifier: exactly one hold vocabulary (OMN-15484).\n+\n+What this is for\n+----------------\n+OMN-15483 shipped the merge-hold gate in ``omnimarket`` only. OMN-15484 fans it\n+out to the repos where every incident in that ticket's table actually happened\n+(``onex_change_control``, ``omnibase_infra``). The obvious way to fan out — copy\n+``hold_marker.py`` into each repo — rebuilds, at fleet scale, the exact bug\n+OMN-15483 round 1 found *inside* omnimarket: two divergent ``_DO_NOT_MERGE_RE``\n+definitions (``occ_companion_emitter.py:163`` matched\n+``DO NOT MERGE``/``WORK IN PROGRESS``/``[WIP]``; ``handler_occ_companion_compute.py:110``\n+matched ``do not merge``/``DNM``/``WIP``/``[draft``), neither a superset of the\n+other, so the same PR was suppressed by one consumer and authored by the other.\n+\n+OMN-15484 AC1 therefore rejects \"vendored copy plus a sync test\" outright and\n+asks for a scan that FAILS when a second hold vocabulary is declared in an\n+adopting repo, **run as a check on each adopting repo**. This script is that\n+scan. The adopting repo does not host it: the reusable workflow\n+``.github/workflows/merge-hold-gate-reusable.yml`` checks the adopting repo out\n+and runs *this* file against it, so there is one authored copy with N callers.\n+\n+How \"a second vocabulary\" is detected without declaring one\n+-----------------------------------------------------------\n+A detector carrying its own token list would itself be a second vocabulary — the\n+thing it exists to forbid. So every token this script matches on is **derived at\n+runtime from the canonical module**:\n+\n+1. **Literal skeletons.** ``HOLD_MARKER_RE.pattern`` is split on its top-level\n+ ``|`` alternation and each alternative is reduced to its letter/digit\n+ skeleton (``do[\\\\s_-]?not[\\\\s_-]?merge`` -> ``donotmerge``, ``\\\\bWIP\\\\b`` ->\n+ ``wip``, ``\\\\[\\\\s*draft`` -> ``draft``). Every ``re.compile(...)`` literal in\n+ the scanned tree is reduced the same way; a candidate whose skeleton contains\n+ a canonical skeleton is a re-declaration however differently it is spelled.\n+ This survives separator/metacharacter noise, which a plain string compare\n+ does not: a vendored copy that swaps ``[\\\\s_-]?`` for ``[ _-]*`` still\n+ reduces to ``donotmerge``.\n+2. **Identifier names.** The canonical module's own exported regex names\n+ (``HOLD_MARKER_RE``, ``DO_NOT_MERGE_RE``) are reduced to name fragments, and\n+ any assignment target in the scanned tree that carries one of those fragments\n+ and binds a ``re.compile`` is an offender even if its pattern is empty today.\n+ This is the rule the in-repo omnimarket falsifier\n+ (``tests/test_merge_hold_marker_omn15483.py::test_only_one_module_declares_a_hold_regex``)\n+ already applies; keeping both rules means neither an empty-but-named copy nor\n+ an anonymous-but-equivalent copy slips through.\n+\n+Both rules read the canonical module. Delete or break it and this script FAILS\n+(``CanonicalVocabularyUnavailableError``) rather than reporting \"no offenders\" —\n+an unloadable vocabulary must never look like a clean scan.\n+\n+Stdlib only, by the same constraint as the gate itself: the reusable workflow\n+runs a bare ``python3`` with no ``uv sync``, so a dependency failure can never\n+skip this check.\n+\n+Exit codes: ``0`` exactly one vocabulary, ``1`` a second one was found or the\n+canonical vocabulary could not be loaded.\n+\n+Related: OMN-15484 (this fan-out), OMN-15483 (the gate), OMN-14741 F-17.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import argparse\n+import ast\n+import re\n+import sys\n+from collections.abc import Sequence\n+from pathlib import Path\n+\n+from scripts.ci.check_pr_hold_marker import (\n+ CANONICAL_HOLD_MODULE,\n+ CanonicalVocabularyUnavailableError,\n+ load_canonical_hold_module,\n+)\n+\n+EXIT_OK = 0\n+EXIT_OFFENDER = 1\n+\n+# Directory names never worth scanning: build/VCS noise, and vendored trees that\n+# are not the adopting repo's own source.\n+_SKIP_DIRS: frozenset[str] = frozenset(\n+ {\n+ \".git\",\n+ \".venv\",\n+ \"venv\",\n+ \"node_modules\",\n+ \"__pycache__\",\n+ \".mypy_cache\",\n+ \".pytest_cache\",\n+ \".ruff_cache\",\n+ \"site-packages\",\n+ \".tox\",\n+ \"build\",\n+ \"dist\",\n+ }\n+)\n+\n+_NON_WORD = re.compile(r\"[^0-9a-z]+\")\n+\n+# A single SHORT canonical token is not evidence of a re-declaration. ``wip``,\n+# ``dnm`` and ``draft`` are ordinary words that appear in perfectly innocent\n+# regexes (a draft-PR parser, a WIP-status enum), and flagging those would wedge\n+# an adopting repo on a false positive — the one outcome worse than a missing\n+# gate, because it teaches people to disable the check. A genuine vendored\n+# vocabulary is recognisable instead by reproducing the DISTINCTIVE multi-word\n+# tokens (``donotmerge``, ``workinprogress``, ``verificationhold``), or by\n+# carrying several tokens at once. So a candidate is an offender when it\n+# contains a long canonical token, OR at least two distinct canonical tokens.\n+_DISTINCTIVE_TOKEN_LENGTH = 8\n+_MULTI_TOKEN_THRESHOLD = 2\n+\n+\n+def literal_skeleton(text: str) -> str:\n+ \"\"\"Reduce a regex fragment to its lowercase letter/digit skeleton.\n+\n+ Regex metacharacters, escapes, separators and character classes all vanish,\n+ so two spellings of the same token collapse to the same string:\n+\n+ >>> literal_skeleton(r\"do[\\\\s_-]?not[\\\\s_-]?merge\")\n+ 'donotmerge'\n+ >>> literal_skeleton(r\"do[ _-]*NOT[ _-]*merge\")\n+ 'donotmerge'\n+ >>> literal_skeleton(r\"\\\\bWIP\\\\b\")\n+ 'wip'\n+\n+ Args:\n+ text: A regex fragment (or any string).\n+\n+ Returns:\n+ The skeleton: lowercase, alphanumerics only.\n+ \"\"\"\n+ # Drop escape backslashes first so ``\\b`` does not leave a stray ``b``.\n+ without_escapes = re.sub(r\"\\\\[A-Za-z]\", \" \", text)\n+ without_escapes = without_escapes.replace(\"\\\\\", \" \")\n+ return _NON_WORD.sub(\"\", without_escapes.lower())\n+\n+\n+def split_alternation(pattern: str) -> list[str]:\n+ \"\"\"Split a regex on its TOP-LEVEL ``|`` alternation.\n+\n+ ``|`` inside a character class or a group belongs to that construct, not to\n+ the outer alternation, so a naive ``pattern.split(\"|\")`` would shred a\n+ grouped vocabulary into meaningless fragments.\n+\n+ Args:\n+ pattern: The regex source.\n+\n+ Returns:\n+ The top-level alternatives, in order.\n+ \"\"\"\n+ parts: list[str] = []\n+ current: list[str] = []\n+ depth = 0\n+ in_class = False\n+ escaped = False\n+ for char in pattern:\n+ if escaped:\n+ current.append(char)\n+ escaped = False\n+ continue\n+ if char == \"\\\\\":\n+ current.append(char)\n+ escaped = True\n+ continue\n+ if in_class:\n+ current.append(char)\n+ if char == \"]\":\n+ in_class = False\n+ continue\n+ if char == \"[\":\n+ in_class = True\n+ current.append(char)\n+ continue\n+ if char == \"(\":\n+ depth += 1\n+ current.append(char)\n+ continue\n+ if char == \")\":\n+ depth = max(0, depth - 1)\n+ current.append(char)\n+ continue\n+ if char == \"|\" and depth == 0:\n+ parts.append(\"\".join(current))\n+ current = []\n+ continue\n+ current.append(char)\n+ parts.append(\"\".join(current))\n+ return parts\n+\n+\n+def canonical_skeletons(pattern: str) -> tuple[str, ...]:\n+ \"\"\"The canonical vocabulary's alternatives, as skeletons.\n+\n+ Args:\n+ pattern: ``HOLD_MARKER_RE.pattern`` from the canonical module.\n+\n+ Returns:\n+ Non-empty skeletons, deduplicated, longest first so an offender is\n+ reported against the most specific token it re-declares.\n+\n+ Raises:\n+ CanonicalVocabularyUnavailableError: If the canonical pattern reduces to\n+ nothing at all — a vocabulary with no literal content cannot be used\n+ to detect a copy of itself, and silently scanning for nothing would\n+ be a vacuous green.\n+ \"\"\"\n+ seen: dict[str, None] = {}\n+ for alternative in split_alternation(pattern):\n+ skeleton = literal_skeleton(alternative)\n+ if skeleton:\n+ seen[skeleton] = None\n+ if not seen:\n+ raise CanonicalVocabularyUnavailableError(\n+ \"the canonical hold vocabulary has no literal content to scan for \"\n+ f\"(pattern={pattern!r}) — refusing to report a clean scan\"\n+ )\n+ return tuple(sorted(seen, key=len, reverse=True))\n+\n+\n+def canonical_name_fragments(module_names: Sequence[str]) -> tuple[str, ...]:\n+ \"\"\"Identifier fragments derived from the canonical module's regex exports.\n+\n+ ``HOLD_MARKER_RE`` -> ``HOLD_MARKER``; ``DO_NOT_MERGE_RE`` -> ``DO_NOT_MERGE``.\n+\n+ Args:\n+ module_names: ``dir(canonical_module)`` or its ``__all__``.\n+\n+ Returns:\n+ Uppercase fragments, longest first.\n+ \"\"\"\n+ fragments: dict[str, None] = {}\n+ for name in module_names:\n+ if name.endswith(\"_RE\") and name.isupper():\n+ fragments[name[: -len(\"_RE\")]] = None\n+ return tuple(sorted(fragments, key=len, reverse=True))\n+\n+\n+def redeclared_tokens(\n+ candidate_skeleton: str,\n+ skeletons: Sequence[str],\n+) -> tuple[str, ...]:\n+ \"\"\"Which canonical tokens a candidate pattern re-declares, if any.\n+\n+ See :data:`_DISTINCTIVE_TOKEN_LENGTH` for why a single short token is not\n+ enough. Returns an empty tuple for the innocent cases so the caller can\n+ treat \"any hits\" as the offender predicate.\n+\n+ Args:\n+ candidate_skeleton: :func:`literal_skeleton` of a found ``re.compile``.\n+ skeletons: The canonical token skeletons.\n+\n+ Returns:\n+ The matched canonical tokens when the candidate qualifies as a\n+ re-declaration, else ``()``.\n+ \"\"\"\n+ if not candidate_skeleton:\n+ return ()\n+ matched = tuple(s for s in skeletons if s and s in candidate_skeleton)\n+ if not matched:\n+ return ()\n+ if any(len(s) >= _DISTINCTIVE_TOKEN_LENGTH for s in matched):\n+ return matched\n+ if len(set(matched)) >= _MULTI_TOKEN_THRESHOLD:\n+ return matched\n+ return ()\n+\n+\n+def _iter_python_files(root: Path) -> list[Path]:\n+ \"\"\"Every ``*.py`` under ``root``, skipping build/VCS/vendor noise.\"\"\"\n+ found: list[Path] = []\n+ for path in sorted(root.rglob(\"*.py\")):\n+ if any(part in _SKIP_DIRS for part in path.parts):\n+ continue\n+ found.append(path)\n+ return found\n+\n+\n+def scan_tree(\n+ *,\n+ tree_root: Path,\n+ scan_roots: Sequence[str],\n+ skeletons: Sequence[str],\n+ name_fragments: Sequence[str],\n+ canonical_module_path: Path,\n+) -> tuple[list[str], list[str]]:\n+ \"\"\"Find every second declaration of the hold vocabulary under ``tree_root``.\n+\n+ Args:\n+ tree_root: Root of the repository being scanned (the *adopting* repo\n+ when called from the reusable workflow).\n+ scan_roots: Relative directories to walk. Missing ones are skipped —\n+ not every repo has ``src/``.\n+ skeletons: Canonical literal skeletons (see :func:`canonical_skeletons`).\n+ name_fragments: Canonical identifier fragments.\n+ canonical_module_path: The one file allowed to declare the vocabulary.\n+ Exempted so scanning omnimarket itself does not flag the original.\n+\n+ Returns:\n+ ``(offenders, unscannable)`` — re-declarations found, and files that\n+ could not be cleared. Both empty means the tree is provably clean.\n+ \"\"\"\n+ offenders: list[str] = []\n+ unscannable: list[str] = []\n+ canonical_resolved = canonical_module_path.resolve()\n+ # The exemption must ALSO be relative, not just absolute. When omnimarket\n+ # calls this gate on itself, the caller checkout and the vocabulary checkout\n+ # are two different directories holding the same file, so an absolute-path\n+ # exemption alone would flag omnimarket's own canonical module as a second\n+ # vocabulary — the gate refusing the very repo that defines it.\n+ canonical_relative = _canonical_relative_path(canonical_resolved)\n+\n+ for relative in scan_roots:\n+ root = tree_root / relative\n+ if not root.is_dir():\n+ continue\n+ for path in _iter_python_files(root):\n+ resolved = path.resolve()\n+ if resolved == canonical_resolved:\n+ continue\n+ if (\n+ canonical_relative is not None\n+ and _relative_or_none(resolved, tree_root) == canonical_relative\n+ ):\n+ continue\n+ display = _relative_or_none(resolved, tree_root) or path\n+ try:\n+ source = path.read_text(encoding=\"utf-8\")\n+ except (OSError, UnicodeDecodeError) as exc:\n+ # Unreadable source is not evidence of cleanliness. Recorded as\n+ # UNSCANNABLE, not as an offender: the two have different causes\n+ # and different fixes, and conflating them makes the failure\n+ # message accuse the repo of a vocabulary violation it does not\n+ # have. Both still fail the check.\n+ unscannable.append(f\"{display}: unreadable ({exc})\")\n+ continue\n+ try:\n+ parsed = ast.parse(source, filename=str(path))\n+ except SyntaxError as exc:\n+ # Same posture. In practice the overwhelmingly likely cause is\n+ # an interpreter older than the source (PEP 695 generics parse\n+ # only on 3.12+), which is why the report prints the running\n+ # interpreter alongside — a version mismatch must be diagnosable\n+ # at a glance instead of looking like a fleet-wide violation.\n+ unscannable.append(f\"{display}: unparseable ({exc})\")\n+ continue\n+ offenders.extend(\n+ _offenders_in_module(\n+ parsed=parsed,\n+ path=path,\n+ tree_root=tree_root,\n+ skeletons=skeletons,\n+ name_fragments=name_fragments,\n+ )\n+ )\n+ return offenders, unscannable\n+\n+\n+def _canonical_relative_path(canonical_resolved: Path) -> Path | None:\n+ \"\"\"The canonical module's path relative to ITS OWN repo root.\n+\n+ ``...//src/omnimarket/merge_control/hold_marker.py`` ->\n+ ``src/omnimarket/merge_control/hold_marker.py``. Derived from the file's own\n+ location (``scripts/ci/.py`` puts the root at ``parents[2]``) rather\n+ than hardcoded, so moving the module inside omnimarket cannot leave a stale\n+ exemption behind.\n+\n+ Args:\n+ canonical_resolved: Absolute path to the canonical module.\n+\n+ Returns:\n+ The relative path, or ``None`` when it does not sit under the expected\n+ root (in which case only the absolute exemption applies).\n+ \"\"\"\n+ own_root = Path(__file__).resolve().parents[2]\n+ return _relative_or_none(canonical_resolved, own_root)\n+\n+\n+def _relative_or_none(path: Path, root: Path) -> Path | None:\n+ \"\"\"``path`` relative to ``root``, or ``None`` when it is not underneath.\"\"\"\n+ try:\n+ return path.resolve().relative_to(root.resolve())\n+ except ValueError:\n+ return None\n+\n+\n+def _offenders_in_module(\n+ *,\n+ parsed: ast.Module,\n+ path: Path,\n+ tree_root: Path,\n+ skeletons: Sequence[str],\n+ name_fragments: Sequence[str],\n+) -> list[str]:\n+ \"\"\"Offenders inside a single parsed module (see :func:`scan_tree`).\"\"\"\n+ offenders: list[str] = []\n+ try:\n+ display = path.relative_to(tree_root)\n+ except ValueError: # pragma: no cover - defensive\n+ display = path\n+\n+ for node in ast.walk(parsed):\n+ if not isinstance(node, ast.Call):\n+ continue\n+ func = node.func\n+ is_re_compile = (\n+ isinstance(func, ast.Attribute)\n+ and func.attr == \"compile\"\n+ and isinstance(func.value, ast.Name)\n+ and func.value.id == \"re\"\n+ )\n+ if not is_re_compile:\n+ continue\n+\n+ pattern_text = _joined_string_arg(node)\n+ candidate = literal_skeleton(pattern_text) if pattern_text else \"\"\n+ hits = redeclared_tokens(candidate, skeletons)\n+ if hits:\n+ listed = \", \".join(repr(h) for h in hits)\n+ offenders.append(\n+ f\"{display}:{node.lineno}: re.compile(...) re-declares the \"\n+ f\"canonical hold vocabulary ({listed})\"\n+ )\n+ continue\n+\n+ # Rule 2: named like the canonical export, bound to a re.compile.\n+ for target_name in _assignment_targets(parsed, node):\n+ normalized = target_name.lstrip(\"_\").upper()\n+ named = next((f for f in name_fragments if f and f in normalized), None)\n+ if named is not None:\n+ offenders.append(\n+ f\"{display}:{node.lineno}: {target_name} binds a re.compile \"\n+ f\"named after the canonical vocabulary ({named!r})\"\n+ )\n+ break\n+ return offenders\n+\n+\n+def _joined_string_arg(call: ast.Call) -> str:\n+ \"\"\"Concatenate the string parts of a ``re.compile`` first argument.\n+\n+ Handles the common shapes: a plain literal, implicit adjacent-literal\n+ concatenation (already folded by the parser), and explicit ``a + b``.\n+ Anything dynamic yields ``\"\"``, which falls through to the name rule.\n+ \"\"\"\n+ if not call.args:\n+ return \"\"\n+ return _string_of(call.args[0])\n+\n+\n+def _string_of(node: ast.expr) -> str:\n+ \"\"\"Best-effort static string value of ``node`` (empty when not static).\"\"\"\n+ if isinstance(node, ast.Constant) and isinstance(node.value, str):\n+ return node.value\n+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):\n+ return _string_of(node.left) + _string_of(node.right)\n+ if isinstance(node, ast.JoinedStr):\n+ return \"\".join(\n+ _string_of(value)\n+ for value in node.values\n+ if isinstance(value, ast.Constant | ast.BinOp | ast.JoinedStr)\n+ )\n+ return \"\"\n+\n+\n+def _assignment_targets(parsed: ast.Module, call: ast.Call) -> list[str]:\n+ \"\"\"Names assigned from ``call``, if it is the RHS of an assignment.\"\"\"\n+ names: list[str] = []\n+ for node in ast.walk(parsed):\n+ if isinstance(node, ast.Assign) and node.value is call:\n+ names.extend(t.id for t in node.targets if isinstance(t, ast.Name))\n+ elif (\n+ isinstance(node, ast.AnnAssign)\n+ and node.value is call\n+ and isinstance(node.target, ast.Name)\n+ ):\n+ names.append(node.target.id)\n+ return names\n+\n+\n+def run(\n+ *,\n+ tree_root: Path,\n+ scan_roots: Sequence[str],\n+ canonical_module_path: Path = CANONICAL_HOLD_MODULE,\n+) -> tuple[int, str]:\n+ \"\"\"Scan ``tree_root`` and render the verdict.\n+\n+ Args:\n+ tree_root: Repository root to scan.\n+ scan_roots: Relative directories inside it to walk.\n+ canonical_module_path: The canonical vocabulary module.\n+\n+ Returns:\n+ ``(exit_code, report)``.\n+ \"\"\"\n+ try:\n+ canonical = load_canonical_hold_module(canonical_module_path)\n+ skeletons = canonical_skeletons(canonical.HOLD_MARKER_RE.pattern)\n+ except CanonicalVocabularyUnavailableError as exc:\n+ return EXIT_OFFENDER, f\"FAIL (fail-closed): {exc}\"\n+\n+ fragments = canonical_name_fragments(\n+ getattr(canonical, \"__all__\", None) or dir(canonical)\n+ )\n+\n+ offenders, unscannable = scan_tree(\n+ tree_root=tree_root,\n+ scan_roots=scan_roots,\n+ skeletons=skeletons,\n+ name_fragments=fragments,\n+ canonical_module_path=canonical_module_path,\n+ )\n+\n+ scanned = \", \".join(scan_roots) or \"(nothing)\"\n+ interpreter = f\"python {sys.version.split()[0]} at {sys.executable}\"\n+\n+ if offenders:\n+ listed = \"\\n\".join(f\" - {o}\" for o in offenders)\n+ return EXIT_OFFENDER, (\n+ \"FAIL — a SECOND merge-hold vocabulary is declared in this \"\n+ f\"repository (OMN-15484 AC1 falsifier):\\n{listed}\\n\"\n+ \"\\n\"\n+ \"The hold vocabulary is declared exactly once, fleet-wide, in \"\n+ \"omnimarket's src/omnimarket/merge_control/hold_marker.py, and this \"\n+ \"repository's gate reads THAT module through the shared reusable \"\n+ \"workflow. A local copy is how OMN-15483 got two divergent \"\n+ \"definitions inside one repo; at fleet scale it is the same bug \"\n+ \"with more copies. Delete the copy and call the shared gate.\\n\"\n+ f\"\\n scanned: {tree_root} [{scanned}]\"\n+ )\n+\n+ if unscannable:\n+ listed = \"\\n\".join(f\" - {u}\" for u in unscannable)\n+ return EXIT_OFFENDER, (\n+ \"FAIL (fail-closed) — the scan could not CLEAR every file, so it \"\n+ \"cannot report that exactly one vocabulary exists. This is NOT a \"\n+ f\"vocabulary violation:\\n{listed}\\n\"\n+ \"\\n\"\n+ \"Most likely cause: the interpreter is older than the source it is \"\n+ \"parsing (PEP 695 generics, `type` statements and match patterns \"\n+ \"all parse only on new enough Pythons). Check the interpreter \"\n+ \"before touching any of the listed files.\\n\"\n+ f\"\\n interpreter: {interpreter}\"\n+ f\"\\n scanned: {tree_root} [{scanned}]\"\n+ )\n+\n+ return EXIT_OK, (\n+ \"PASS — exactly one merge-hold vocabulary.\\n\"\n+ f\" scanned: {tree_root} [{scanned}]\\n\"\n+ f\" canonical tokens derived from: {canonical_module_path}\\n\"\n+ f\" token skeletons: {', '.join(skeletons)}\\n\"\n+ f\" interpreter: {interpreter}\"\n+ )\n+\n+\n+def main(argv: Sequence[str] | None = None) -> int:\n+ \"\"\"CLI entrypoint.\n+\n+ Args:\n+ argv: Command-line arguments (defaults to ``sys.argv[1:]``).\n+\n+ Returns:\n+ Process exit code: 0 clean, 1 a second vocabulary or an unloadable one.\n+ \"\"\"\n+ parser = argparse.ArgumentParser(\n+ description=(\n+ \"Fail when a repository declares a SECOND merge-hold vocabulary \"\n+ \"(OMN-15484 AC1). Tokens are derived from the canonical module at \"\n+ \"runtime; this scanner declares none of its own.\"\n+ )\n+ )\n+ parser.add_argument(\n+ \"--tree\",\n+ type=Path,\n+ required=True,\n+ help=\"Repository root to scan (the adopting repo's checkout).\",\n+ )\n+ parser.add_argument(\n+ \"--roots\",\n+ default=\"src scripts\",\n+ help=(\n+ \"Space-separated directories inside --tree to walk. Missing \"\n+ \"directories are skipped. Default: 'src scripts'.\"\n+ ),\n+ )\n+ parser.add_argument(\n+ \"--module-path\",\n+ type=Path,\n+ default=CANONICAL_HOLD_MODULE,\n+ help=\"Path to the canonical hold_marker.py (default: the in-repo module).\",\n+ )\n+ args = parser.parse_args(argv)\n+\n+ code, report = run(\n+ tree_root=args.tree,\n+ scan_roots=[r for r in str(args.roots).split() if r],\n+ canonical_module_path=args.module_path,\n+ )\n+ if code == EXIT_OK:\n+ print(report)\n+ else:\n+ print(report, file=sys.stderr)\n+ print(\n+ f\"::error::Single Hold Vocabulary: {report.splitlines()[0]}\",\n+ file=sys.stderr,\n+ )\n+ return code\n+\n+\n+if __name__ == \"__main__\":\n+ raise SystemExit(main())"},{"sha":"4ad64721aad008f5ae6e4f11afb3fceafaa15d87","filename":"tests/unit/scripts/ci/test_merge_hold_gate_fanout_omn15484.py","status":"added","additions":519,"deletions":0,"changes":519,"blob_url":"https://github.com/OmniNode-ai/omnimarket/blob/879d6fc6825f876458c6d45ed670c8715de8ac95/tests%2Funit%2Fscripts%2Fci%2Ftest_merge_hold_gate_fanout_omn15484.py","raw_url":"https://github.com/OmniNode-ai/omnimarket/raw/879d6fc6825f876458c6d45ed670c8715de8ac95/tests%2Funit%2Fscripts%2Fci%2Ftest_merge_hold_gate_fanout_omn15484.py","contents_url":"https://api.github.com/repos/OmniNode-ai/omnimarket/contents/tests%2Funit%2Fscripts%2Fci%2Ftest_merge_hold_gate_fanout_omn15484.py?ref=879d6fc6825f876458c6d45ed670c8715de8ac95","patch":"@@ -0,0 +1,519 @@\n+# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc.\n+# SPDX-License-Identifier: MIT\n+\"\"\"The fan-out surface: one vocabulary, proven to fire, in every repo (OMN-15484).\n+\n+OMN-15483 shipped the hold gate in omnimarket only, while every incident in its\n+table happened in ``onex_change_control`` or ``omnibase_infra``. This module\n+covers the surface that closes that gap without vendoring: the reusable workflow\n+``.github/workflows/merge-hold-gate-reusable.yml`` plus the two scripts it runs\n+in the *calling* repo's CI.\n+\n+What is deliberately NOT asserted here: that the adopting repos are wired. That\n+lives in each adopting repo, driven against ITS OWN ``CI Summary`` producer,\n+because a strict slot in omnimarket's poller proves nothing about OCC's\n+needs-based aggregator. Cross-repo, the seam is the reusable workflow's input\n+contract, and each caller's wiring test asserts its half of it.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import ast\n+import pathlib\n+import subprocess\n+import sys\n+import textwrap\n+from typing import Any\n+\n+import pytest\n+import yaml\n+\n+from scripts.ci.check_hold_gate_selftest import (\n+ CLEAR_VECTOR,\n+ HELD_VECTOR,\n+ check_context_name,\n+)\n+from scripts.ci.check_hold_gate_selftest import run as selftest_run\n+from scripts.ci.check_pr_hold_marker import (\n+ CANONICAL_HOLD_MODULE,\n+ load_canonical_hold_module,\n+)\n+from scripts.ci.check_single_hold_vocabulary import (\n+ EXIT_OFFENDER,\n+ EXIT_OK,\n+ canonical_name_fragments,\n+ canonical_skeletons,\n+ literal_skeleton,\n+ redeclared_tokens,\n+ split_alternation,\n+)\n+from scripts.ci.check_single_hold_vocabulary import run as scan_run\n+\n+pytestmark = pytest.mark.unit\n+\n+_REPO_ROOT = pathlib.Path(__file__).resolve().parents[4]\n+_REUSABLE_WORKFLOW = (\n+ _REPO_ROOT / \".github\" / \"workflows\" / \"merge-hold-gate-reusable.yml\"\n+)\n+_CI_WORKFLOW = _REPO_ROOT / \".github\" / \"workflows\" / \"ci.yml\"\n+_CALLER_JOB_KEY = \"merge-hold-gate\"\n+_SCANNER = _REPO_ROOT / \"scripts\" / \"ci\" / \"check_single_hold_vocabulary.py\"\n+\n+\n+def _write_module(root: pathlib.Path, relative: str, body: str) -> pathlib.Path:\n+ \"\"\"Write a python module into a synthetic repo tree.\"\"\"\n+ path = root / relative\n+ path.parent.mkdir(parents=True, exist_ok=True)\n+ path.write_text(textwrap.dedent(body).lstrip(), encoding=\"utf-8\")\n+ return path\n+\n+\n+# ---------------------------------------------------------------------------\n+# AC1 — one vocabulary across repo boundaries\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestSingleVocabularyScanner:\n+ \"\"\"The AC1 falsifier, which runs against the CALLING repo's checkout.\"\"\"\n+\n+ def test_a_clean_tree_passes(self, tmp_path: pathlib.Path) -> None:\n+ _write_module(\n+ tmp_path,\n+ \"src/pkg/ordinary.py\",\n+ \"\"\"\n+ import re\n+ SEMVER_RE = re.compile(r\"^v?\\\\d+\\\\.\\\\d+\\\\.\\\\d+$\")\n+ \"\"\",\n+ )\n+ code, report = scan_run(tree_root=tmp_path, scan_roots=[\"src\", \"scripts\"])\n+ assert code == EXIT_OK, report\n+\n+ def test_a_respelled_vendored_copy_is_caught(self, tmp_path: pathlib.Path) -> None:\n+ \"\"\"The failure mode a byte-compare or a sync test misses.\n+\n+ A vendored copy never arrives as an exact duplicate — it arrives\n+ respelled, which is precisely how OMN-15483 round 1 ended up with two\n+ divergent definitions inside ONE repo. Detection is by literal skeleton,\n+ so separator and metacharacter noise does not launder it.\n+ \"\"\"\n+ _write_module(\n+ tmp_path,\n+ \"src/pkg/vendored.py\",\n+ \"\"\"\n+ import re\n+ _LOCAL = re.compile(r\"do[ _-]*NOT[ _-]*merge|work[ ]*in[ ]*progress\")\n+ \"\"\",\n+ )\n+ code, report = scan_run(tree_root=tmp_path, scan_roots=[\"src\"])\n+ assert code == EXIT_OFFENDER\n+ assert \"donotmerge\" in report\n+ assert \"vendored.py\" in report\n+\n+ def test_an_empty_but_canonically_named_copy_is_caught(\n+ self, tmp_path: pathlib.Path\n+ ) -> None:\n+ \"\"\"Name rule: a placeholder today is a divergent vocabulary tomorrow.\"\"\"\n+ _write_module(\n+ tmp_path,\n+ \"src/pkg/named.py\",\n+ \"\"\"\n+ import re\n+ _DO_NOT_MERGE_RE = re.compile(r\"\")\n+ \"\"\",\n+ )\n+ code, report = scan_run(tree_root=tmp_path, scan_roots=[\"src\"])\n+ assert code == EXIT_OFFENDER\n+ assert \"DO_NOT_MERGE\" in report\n+\n+ @pytest.mark.parametrize(\n+ \"pattern\",\n+ [\n+ r\"^\\[draft\\]\\s*\", # a draft-PR title parser\n+ r\"\\bWIP\\b\", # a status-label matcher\n+ r\"\\bDNM\\b\", # an abbreviation elsewhere in a codebase\n+ ],\n+ )\n+ def test_a_single_short_token_is_not_a_false_positive(\n+ self, tmp_path: pathlib.Path, pattern: str\n+ ) -> None:\n+ \"\"\"False positives are worse than a missing gate here.\n+\n+ ``draft``/``WIP``/``DNM`` are ordinary words. Flagging an innocent regex\n+ would wedge an adopting repo on a check it cannot satisfy, which teaches\n+ people to disable the check — the failure mode that ends enforcement\n+ everywhere. A real re-declaration is recognised by the distinctive\n+ multi-word tokens or by carrying several at once.\n+ \"\"\"\n+ _write_module(\n+ tmp_path,\n+ \"src/pkg/innocent.py\",\n+ f\"\"\"\n+ import re\n+ SOMETHING_RE = re.compile(r\"{pattern}\")\n+ \"\"\",\n+ )\n+ code, report = scan_run(tree_root=tmp_path, scan_roots=[\"src\"])\n+ assert code == EXIT_OK, report\n+\n+ def test_two_short_tokens_together_are_a_redeclaration(\n+ self, tmp_path: pathlib.Path\n+ ) -> None:\n+ \"\"\"One short token is a coincidence; two is a vocabulary.\"\"\"\n+ _write_module(\n+ tmp_path,\n+ \"src/pkg/sneaky.py\",\n+ \"\"\"\n+ import re\n+ _P = re.compile(r\"\\\\bWIP\\\\b|\\\\bDNM\\\\b\")\n+ \"\"\",\n+ )\n+ code, _ = scan_run(tree_root=tmp_path, scan_roots=[\"src\"])\n+ assert code == EXIT_OFFENDER\n+\n+ def test_the_canonical_module_itself_is_exempt_by_relative_path(\n+ self, tmp_path: pathlib.Path\n+ ) -> None:\n+ \"\"\"omnimarket calling the gate on itself must not flag its own source.\n+\n+ The caller checkout and the vocabulary checkout are two directories\n+ holding the same file, so an absolute-path exemption alone would have\n+ the gate refuse the very repo that defines the vocabulary. Caught here\n+ rather than in a red CI run on the fan-out PR.\n+ \"\"\"\n+ canonical_relative = \"src/omnimarket/merge_control/hold_marker.py\"\n+ _write_module(\n+ tmp_path,\n+ canonical_relative,\n+ CANONICAL_HOLD_MODULE.read_text(encoding=\"utf-8\"),\n+ )\n+ code, report = scan_run(tree_root=tmp_path, scan_roots=[\"src\"])\n+ assert code == EXIT_OK, report\n+\n+ def test_an_unparseable_file_fails_but_is_not_called_a_violation(\n+ self, tmp_path: pathlib.Path\n+ ) -> None:\n+ \"\"\"Fail-closed, and diagnosable.\n+\n+ A file the scanner cannot parse is not a file it cleared, so the check\n+ must fail. But the overwhelmingly likely cause is an interpreter older\n+ than the source, and reporting that as \"a second hold vocabulary is\n+ declared\" sends the reader hunting for a copy that does not exist.\n+ \"\"\"\n+ _write_module(tmp_path, \"src/pkg/broken.py\", \"def f( :\\n\")\n+ code, report = scan_run(tree_root=tmp_path, scan_roots=[\"src\"])\n+ assert code == EXIT_OFFENDER\n+ assert \"could not CLEAR\" in report\n+ assert \"NOT a vocabulary violation\" in report\n+ assert \"interpreter\" in report\n+\n+ def test_scanner_declares_no_vocabulary_of_its_own(self) -> None:\n+ \"\"\"The detector must not become the second copy it exists to forbid.\n+\n+ Every token it matches on is derived from the canonical module at run\n+ time. If this file ever grows its own token list, the fleet has two\n+ vocabularies again — one enforcing, one detecting, free to diverge.\n+\n+ Scoped to executable string literals on purpose. Prose — comments and\n+ docstrings — must be free to name the tokens, because explaining what\n+ the detector looks for is how the next reader understands it; the first\n+ cut of this test scanned raw file text and failed on its own\n+ explanatory comment. A hardcoded token list would be *data*, and data\n+ is what this asserts against.\n+ \"\"\"\n+ canonical = load_canonical_hold_module()\n+ tree = ast.parse(_SCANNER.read_text(encoding=\"utf-8\"), filename=str(_SCANNER))\n+\n+ docstrings = {\n+ id(node.body[0].value)\n+ for node in ast.walk(tree)\n+ if isinstance(\n+ node, ast.Module | ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef\n+ )\n+ and node.body\n+ and isinstance(node.body[0], ast.Expr)\n+ and isinstance(node.body[0].value, ast.Constant)\n+ and isinstance(node.body[0].value.value, str)\n+ }\n+ literals = [\n+ node.value\n+ for node in ast.walk(tree)\n+ if isinstance(node, ast.Constant)\n+ and isinstance(node.value, str)\n+ and id(node) not in docstrings\n+ ]\n+\n+ for skeleton in canonical_skeletons(canonical.HOLD_MARKER_RE.pattern):\n+ if len(skeleton) < 8:\n+ continue\n+ for literal in literals:\n+ assert skeleton not in literal_skeleton(literal), (\n+ f\"the AC1 scanner hardcodes the canonical token {skeleton!r} \"\n+ f\"in the string literal {literal!r} — it must derive tokens \"\n+ \"from hold_marker.py, not carry them\"\n+ )\n+\n+ def test_tokens_are_derived_from_the_canonical_pattern(self) -> None:\n+ \"\"\"Change the vocabulary and the scanner's targets change with it.\"\"\"\n+ derived = canonical_skeletons(r\"alpha[\\s_-]?beta|\\bGAMMA\\b\")\n+ assert set(derived) == {\"alphabeta\", \"gamma\"}\n+\n+ def test_top_level_alternation_split_respects_groups_and_classes(self) -> None:\n+ \"\"\"``|`` inside a class or group belongs to that construct.\"\"\"\n+ assert split_alternation(r\"a[b|c]d|(e|f)g|h\") == [r\"a[b|c]d\", r\"(e|f)g\", \"h\"]\n+\n+ def test_name_fragments_come_from_the_module_exports(self) -> None:\n+ assert set(canonical_name_fragments([\"HOLD_MARKER_RE\", \"DO_NOT_MERGE_RE\"])) == {\n+ \"HOLD_MARKER\",\n+ \"DO_NOT_MERGE\",\n+ }\n+\n+ def test_redeclaration_predicate_is_explicit(self) -> None:\n+ skeletons = (\"verificationhold\", \"workinprogress\", \"donotmerge\", \"wip\")\n+ assert redeclared_tokens(\"donotmerge\", skeletons) == (\"donotmerge\",)\n+ assert redeclared_tokens(\"wip\", skeletons) == ()\n+ assert redeclared_tokens(\"\", skeletons) == ()\n+\n+ def test_cli_exits_nonzero_on_an_offender(self, tmp_path: pathlib.Path) -> None:\n+ \"\"\"The CLI the workflow actually invokes, not just the function.\"\"\"\n+ _write_module(\n+ tmp_path,\n+ \"src/pkg/vendored.py\",\n+ \"\"\"\n+ import re\n+ _P = re.compile(r\"do not merge|work in progress\")\n+ \"\"\",\n+ )\n+ completed = subprocess.run(\n+ [\n+ sys.executable,\n+ \"-m\",\n+ \"scripts.ci.check_single_hold_vocabulary\",\n+ \"--tree\",\n+ str(tmp_path),\n+ \"--roots\",\n+ \"src\",\n+ ],\n+ cwd=str(_REPO_ROOT),\n+ capture_output=True,\n+ text=True,\n+ check=False,\n+ )\n+ assert completed.returncode == EXIT_OFFENDER, completed.stdout\n+ assert \"::error::\" in completed.stderr\n+\n+\n+# ---------------------------------------------------------------------------\n+# AC3 — the gate demonstrably fires, in the adopting repo's own CI\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestSelfProof:\n+ \"\"\"A green check proves the job ran, not that the gate can say no.\"\"\"\n+\n+ def test_the_selftest_passes_against_the_shipped_gate(self) -> None:\n+ code, report = selftest_run()\n+ assert code == 0, report\n+ assert \"fires and demonstrably clears\" in report\n+\n+ def test_the_probe_vectors_are_validated_before_use(self) -> None:\n+ \"\"\"A vector that stopped being a hold token must fail, not pass quietly.\n+\n+ This is the difference between a self-test and a decoration. If the\n+ vocabulary narrows and nobody updates the vector, the gate correctly\n+ exits 0 on it and a naive self-test records success while asserting\n+ nothing at all.\n+ \"\"\"\n+ code, report = selftest_run(held_vector=\"an ordinary title with no marker\")\n+ assert code == 1\n+ assert \"went vacuous\" in report\n+\n+ def test_an_over_matching_vocabulary_is_caught_by_the_clear_vector(self) -> None:\n+ \"\"\"Stuck-closed is a repo outage, not a safe default.\"\"\"\n+ code, report = selftest_run(clear_vector=HELD_VECTOR)\n+ assert code == 1\n+ assert \"negative direction proves nothing\" in report\n+\n+ def test_a_missing_gate_cli_fails_closed(self, tmp_path: pathlib.Path) -> None:\n+ code, report = selftest_run(gate_script=tmp_path / \"absent.py\")\n+ assert code == 1\n+ assert \"fail-closed\" in report\n+\n+ def test_a_missing_vocabulary_fails_closed(self, tmp_path: pathlib.Path) -> None:\n+ code, report = selftest_run(module_path=tmp_path / \"absent.py\")\n+ assert code == 1\n+ assert \"fail-closed\" in report\n+\n+ def test_the_clear_vector_is_a_realistic_fanout_title(self) -> None:\n+ \"\"\"AC5, from the other end: the PR that installs the gate is not held.\"\"\"\n+ canonical = load_canonical_hold_module()\n+ assert canonical.match_hold_token(CLEAR_VECTOR) is None\n+\n+\n+# ---------------------------------------------------------------------------\n+# AC5 — the gate does not hold its own fan-out\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestContextNameGuard:\n+ def test_a_self_holding_context_name_is_refused(self) -> None:\n+ \"\"\"The exact mistake made once already, now mechanised across repos.\n+\n+ The first draft of the omnimarket gate was named \"Verification Hold\n+ Gate\", and ``verification hold`` is a token in the vocabulary it\n+ enforces. In the fan-out the caller supplies half the context name from\n+ a different repository, so the check has to travel with the workflow.\n+ \"\"\"\n+ canonical = load_canonical_hold_module()\n+ failure = check_context_name(canonical, \"Verification Hold Gate / evaluate\")\n+ assert failure is not None\n+ assert \"Verification Hold\" in failure\n+\n+ def test_the_shipped_context_name_is_clean(self) -> None:\n+ canonical = load_canonical_hold_module()\n+ assert check_context_name(canonical, \"merge-hold-gate / evaluate\") is None\n+\n+ def test_no_context_name_supplied_is_not_an_error(self) -> None:\n+ canonical = load_canonical_hold_module()\n+ assert check_context_name(canonical, None) is None\n+\n+\n+# ---------------------------------------------------------------------------\n+# AC4 / AC6 — the workflow contract the adopting repos bind to\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestReusableWorkflowContract:\n+ \"\"\"Anything an adopting repo depends on is pinned here.\n+\n+ These are the fields OCC and omnibase_infra write into their ``ci.yml``.\n+ Changing one without changing the callers is a silent cross-repo break —\n+ the seam-mismatch class that is PAIR_INCOMPATIBLE even when both repos'\n+ CI is green.\n+ \"\"\"\n+\n+ @staticmethod\n+ def _workflow() -> dict[str, Any]:\n+ return dict(yaml.safe_load(_REUSABLE_WORKFLOW.read_text(encoding=\"utf-8\")))\n+\n+ @staticmethod\n+ def _job() -> dict[str, Any]:\n+ return dict(TestReusableWorkflowContract._workflow()[\"jobs\"][\"evaluate\"])\n+\n+ def test_it_is_callable(self) -> None:\n+ # `on:` is parsed by PyYAML 1.1 rules as the boolean True.\n+ triggers = self._workflow()[True]\n+ assert \"workflow_call\" in triggers\n+\n+ def test_the_inner_job_id_is_the_registered_context_suffix(self) -> None:\n+ \"\"\"Callers register `` / evaluate``; renaming breaks them all.\n+\n+ The check-run context of a reusable call is\n+ `` / ``, so this id is a cross-repo published\n+ name, not an internal detail.\n+ \"\"\"\n+ jobs = self._workflow()[\"jobs\"]\n+ assert list(jobs) == [\"evaluate\"], (\n+ \"the reusable workflow must expose exactly one job named `evaluate` \"\n+ \"— every caller's CI Summary registration is keyed on it\"\n+ )\n+ assert \"name\" not in jobs[\"evaluate\"], (\n+ \"a `name:` on the inner job would override `evaluate` in the \"\n+ \"composed context and unwire every caller's strict registration\"\n+ )\n+\n+ def test_the_input_contract_is_stable(self) -> None:\n+ inputs = self._workflow()[True][\"workflow_call\"][\"inputs\"]\n+ assert set(inputs) == {\"vocabulary_ref\", \"scan_roots\", \"context_name\"}\n+ assert inputs[\"context_name\"][\"required\"] is True, (\n+ \"context_name carries the AC5 check; making it optional would let a \"\n+ \"caller silently skip the self-holding-name guard\"\n+ )\n+ assert inputs[\"vocabulary_ref\"][\"default\"] == \"dev\"\n+\n+ def test_the_job_runs_on_a_github_hosted_runner(self) -> None:\n+ \"\"\"No self-hosted/LAN dependency: the gate must render a verdict always.\"\"\"\n+ assert self._job()[\"runs-on\"] == \"ubuntu-latest\"\n+\n+ def test_the_job_is_unconditional(self) -> None:\n+ job = self._job()\n+ assert \"needs\" not in job\n+ assert \"if\" not in job\n+\n+ def test_all_four_verdicts_are_wired(self) -> None:\n+ \"\"\"AC1 scan, AC5+AC3 self-proof, and the real evaluation all run.\"\"\"\n+ runs = \" \".join(\n+ str(step.get(\"run\", \"\")) for step in self._job()[\"steps\"] if \"run\" in step\n+ )\n+ assert \"check_single_hold_vocabulary\" in runs\n+ assert \"check_hold_gate_selftest\" in runs\n+ assert \"check_pr_hold_marker.py\" in runs\n+\n+ def test_pr_surfaces_are_passed_as_env_not_interpolated_into_a_script(\n+ self,\n+ ) -> None:\n+ \"\"\"A PR title is attacker-controlled on a fork; as env it is data.\"\"\"\n+ env = self._job()[\"env\"]\n+ assert \"github.event.pull_request.title\" in str(env[\"PR_TITLE\"])\n+ assert \"pull_request.labels\" in str(env[\"PR_LABELS_JSON\"])\n+ assert \"github.event_name\" in str(env[\"GITHUB_EVENT_NAME\"])\n+ assert \"github.repository\" in str(env[\"GH_REPO\"])\n+ runs = \" \".join(\n+ str(step.get(\"run\", \"\")) for step in self._job()[\"steps\"] if \"run\" in step\n+ )\n+ assert \"github.event\" not in runs, (\n+ \"no step may interpolate event data into a shell body — pass it \"\n+ \"through `env:` so it can never be evaluated as script\"\n+ )\n+\n+ def test_the_vocabulary_checkout_targets_omnimarket(self) -> None:\n+ steps = self._job()[\"steps\"]\n+ checkout = next(\n+ step\n+ for step in steps\n+ if str(step.get(\"uses\", \"\")).startswith(\"actions/checkout\")\n+ and step.get(\"with\", {}).get(\"repository\")\n+ )\n+ assert checkout[\"with\"][\"repository\"] == \"OmniNode-ai/omnimarket\"\n+ assert checkout[\"with\"][\"path\"] == \"vocabulary\"\n+ assert checkout[\"with\"][\"persist-credentials\"] is False\n+\n+\n+class TestOmnimarketIsItsOwnFirstCaller:\n+ \"\"\"A reusable surface the defining repo does not call is unproven.\"\"\"\n+\n+ @staticmethod\n+ def _caller_job() -> dict[str, Any]:\n+ workflow = yaml.safe_load(_CI_WORKFLOW.read_text(encoding=\"utf-8\"))\n+ jobs = workflow[\"jobs\"]\n+ assert _CALLER_JOB_KEY in jobs, f\"{_CALLER_JOB_KEY} is not a job in ci.yml\"\n+ return dict(jobs[_CALLER_JOB_KEY])\n+\n+ def test_ci_calls_the_reusable_workflow_locally(self) -> None:\n+ \"\"\"Local `uses:` resolves at the PR head — it proves THIS diff.\"\"\"\n+ assert (\n+ self._caller_job()[\"uses\"]\n+ == \"./.github/workflows/merge-hold-gate-reusable.yml\"\n+ )\n+\n+ def test_the_caller_is_unconditional(self) -> None:\n+ job = self._caller_job()\n+ assert \"needs\" not in job\n+ assert \"if\" not in job\n+\n+ def test_the_declared_context_matches_the_job_key_and_inner_job(self) -> None:\n+ \"\"\"The AC5 seam: what the caller declares must be what CI produces.\n+\n+ ``context_name`` is a string the caller writes by hand, and the guard it\n+ feeds is only as good as that string being true. This asserts it equals\n+ the context GitHub will actually mint for this call.\n+ \"\"\"\n+ declared = self._caller_job()[\"with\"][\"context_name\"]\n+ assert declared == f\"{_CALLER_JOB_KEY} / evaluate\"\n+\n+ def test_the_declared_context_is_not_itself_a_hold_token(self) -> None:\n+ canonical = load_canonical_hold_module()\n+ declared = self._caller_job()[\"with\"][\"context_name\"]\n+ assert check_context_name(canonical, declared) is None\n+\n+ def test_the_proof_call_pins_the_pr_under_review(self) -> None:\n+ \"\"\"Not `dev`: the point is to execute the surface this PR adds.\"\"\"\n+ assert \"github.sha\" in str(self._caller_job()[\"with\"][\"vocabulary_ref\"])"}]} \ No newline at end of file diff --git a/tests/fixtures/omn15547/onex-dev-topology-zero-table-grants.yaml.captured b/tests/fixtures/omn15547/onex-dev-topology-zero-table-grants.yaml.captured new file mode 100644 index 0000000000..98ee699114 --- /dev/null +++ b/tests/fixtures/omn15547/onex-dev-topology-zero-table-grants.yaml.captured @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +active_preset: application-cloud +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: DATABASE_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: omninode_internal + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + tenant: + domain: TENANT + owner: owner_onex_tenant +presets: + application-cloud: + - postgres + - onex_api + - omnidash + - omninode_runtime +schema_version: "2.0" +services: + omnidash: + mode: CLOUD + omninode_runtime: + mode: CLOUD + onex_api: + mode: CLOUD + postgres: + mode: CLOUD diff --git a/tests/fixtures/omn15547/test_omninode_system_slack_report.handtyped-fixture.py.captured b/tests/fixtures/omn15547/test_omninode_system_slack_report.handtyped-fixture.py.captured new file mode 100644 index 0000000000..7061c96331 --- /dev/null +++ b/tests/fixtures/omn15547/test_omninode_system_slack_report.handtyped-fixture.py.captured @@ -0,0 +1,1054 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""OMN-15509 -- the .201 system-health Slack reporter must see every lane's runtime. + +WHAT IS UNDER TEST + ``deploy/maintenance/omninode-system-slack-report.sh`` -- the real bash + artifact that root runs on ``.201`` via + ``/etc/cron.d/omninode-system-slack-report``. Every test here drives that + file itself, not a Python re-implementation of it: a surrogate would prove + nothing about the thing that actually alarms (memory + ``feedback_test_the_artifact_that_runs``). + +THE RED-BEFORE IS REAL, NOT ASSERTED + ``tests/fixtures/omn15509/omninode-system-slack-report.as-deployed-20260730.sh`` + is a byte-for-byte capture of the version that was live on ``.201`` during + the 2026-07-30T16:19-16:45Z outage. ``test_as_deployed_reports_green_on_the + _replayed_outage`` drives THAT file against the replayed outage state and + asserts it reports the dev runtime nowhere and every runtime endpoint as + HTTP 200 -- the false green. The paired test drives the fixed file against + the identical state and asserts CRITICAL naming the dev runtime. + +HERMETICITY + ``docker``/``curl``/``df``/``hostname``/``sha256sum``/``flock`` are replaced + by stubs on PATH. Both scripts pin ``PATH=`` at the top, so the harness + injects one identical line after that assignment in BOTH scripts. The + transformation is symmetric by construction (same helper, same regex), so a + difference in outcome can only come from the scripts' own logic. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[3] +FIXED_SCRIPT = REPO_ROOT / "deploy" / "maintenance" / "omninode-system-slack-report.sh" +AS_DEPLOYED_SCRIPT = ( + REPO_ROOT + / "tests" + / "fixtures" + / "omn15509" + / "omninode-system-slack-report.as-deployed-20260730.sh.captured" +) +# sha256 of the copy read off .201:/data/maintenance/bin/omninode-system-slack-report.sh +# at 2026-07-30T17:0xZ. Asserted below: if anyone edits the fixture, the +# "byte-for-byte capture" claim stops being true and the RED-before proof +# stops meaning anything, so the edit must fail loudly rather than pass. +# The `.captured` suffix keeps the SPDX-header hook off the file for the same +# reason -- a stamped header would no longer be the artifact that ran. +AS_DEPLOYED_SHA256 = "5fe6e5a61d6074922142006f5fc905e146bc4a1dcc18dbe4e0da99ddaec209da" +CRON_UNIT = ( + REPO_ROOT / "deploy" / "maintenance" / "cron.d" / "omninode-system-slack-report" +) +RUNTIME_POLICY_ENV = REPO_ROOT / "docker" / "runtime-policy.env" + +# Lane -> the runtime-policy.env key that carries its MAIN runtime port. +# Dropping a row here is the regression this module exists to catch, so the map +# is asserted against the rendered policy rather than trusted. +LANE_PORT_KEYS = { + "dev": "DEV_RUNTIME_MAIN_PORT", + "stability-test": "STABILITY_TEST_RUNTIME_MAIN_PORT", + "prod": "PROD_RUNTIME_MAIN_PORT", +} + +pytestmark = pytest.mark.skipif( + shutil.which("bash") is None or shutil.which("jq") is None, + reason="bash + jq are required to drive the real reporter artifact", +) + + +def test_as_deployed_fixture_is_the_unmodified_201_capture() -> None: + """Provenance guard for the RED-before artifact.""" + digest = hashlib.sha256(AS_DEPLOYED_SCRIPT.read_bytes()).hexdigest() + assert digest == AS_DEPLOYED_SHA256, ( + "the as-deployed fixture no longer matches the copy captured from " + ".201:/data/maintenance/bin/omninode-system-slack-report.sh on 2026-07-30; " + "the RED-before proof is only meaningful against the unmodified artifact" + ) + + +def _policy_port(key: str) -> str: + """Read one key out of the rendered runtime policy, same idiom as the script.""" + value = "" + for line in RUNTIME_POLICY_ENV.read_text().splitlines(): + if line.startswith(f"{key}="): + value = line.split("=", 1)[1].strip().strip("\"'") + assert value, f"{key} missing from {RUNTIME_POLICY_ENV}" + return value + + +def _write(path: Path, body: str, *, executable: bool = False) -> None: + path.write_text(body) + if executable: + path.chmod(0o755) + + +def _make_stub_bin( + tmp_path: Path, + *, + http: dict[str, tuple[int, str]], + docker_state: dict[str, Any], +) -> Path: + """Build a stub bin dir. ``http`` maps port -> (status_code, body).""" + bin_dir = tmp_path / "stubbin" + bin_dir.mkdir() + spec = tmp_path / "spec.json" + spec.write_text( + json.dumps( + {"http": {str(k): v for k, v in http.items()}, "docker": docker_state} + ) + ) + + # curl: honours -o and -w '%{http_code}'. A port absent from the spec + # is a connection failure (empty output, non-zero exit) exactly like a real + # refused connection, so "endpoint not reachable" is never silently a pass. + _write( + bin_dir / "curl", + f"""#!/usr/bin/env bash +SPEC={spec} +out=""; url="" +args=("$@") +for ((i=0; i<${{#args[@]}}; i++)); do + case "${{args[$i]}}" in + -o) out="${{args[$((i+1))]}}" ;; + http://*|https://*) url="${{args[$i]}}" ;; + esac +done +port=$(sed -E 's|.*:([0-9]+)/.*|\\1|' <<<"$url") +read -r code body < <(python3 - "$SPEC" "$port" <<'PY' +import json,sys +spec=json.load(open(sys.argv[1])) +e=spec["http"].get(sys.argv[2]) +print("NONE","" ) if e is None else print(e[0], json.dumps(e[1])) +PY +) +if [[ "$code" == "NONE" ]]; then + [[ -n "$out" ]] && : >"$out" + printf '%s' '' + exit 7 +fi +decoded=$(python3 -c 'import json,sys; print(json.loads(sys.argv[1]))' "$body") +[[ -n "$out" ]] && printf '%s' "$decoded" >"$out" +printf '%s' "$code" +exit 0 +""", + executable=True, + ) + + _write( + bin_dir / "docker", + f"""#!/usr/bin/env bash +SPEC={spec} +python3 - "$SPEC" "$@" <<'PY' +import json,sys +spec=json.load(open(sys.argv[1]))["docker"] +a=sys.argv[2:] +def ps_all(): + return [(c["name"], c["status"]) for c in spec["containers"]] +def running(): + return [c for c in spec["containers"] if c["status"].startswith("Up")] +if a[0]=="ps" and "-a" in a: + for n,s in ps_all(): print(f"{{n}}\\t{{s}}") +elif a[0]=="ps" and "health=starting" in " ".join(a): + for c in running(): + if "health: starting" in c["status"]: print(c["name"]) +elif a[0]=="ps": + for c in running(): print(c["name"]) +elif a[0]=="inspect": + name=a[-1] + c=next((c for c in spec["containers"] if c["name"]==name), None) + if c is None: sys.exit(1) + fmt=a[a.index("-f")+1] + print(c["started_at"] if "StartedAt" in fmt else c.get("start_period_ns",0)) +elif a[0]=="volume": + for v in spec.get("dangling", []): print(v) +PY +""", + executable=True, + ) + + _write( + bin_dir / "df", + """#!/usr/bin/env bash +echo "Filesystem 1G-blocks Used Avail Use% Mounted" +echo "target 1832G 75G 1664G 5%" +""", + executable=True, + ) + _write( + bin_dir / "hostname", "#!/usr/bin/env bash\necho omninode-pc\n", executable=True + ) + _write(bin_dir / "flock", "#!/usr/bin/env bash\nexit 0\n", executable=True) + if shutil.which("sha256sum") is None: + _write( + bin_dir / "sha256sum", + "#!/usr/bin/env bash\nshasum -a 256\n", + executable=True, + ) + return bin_dir + + +def _stage(script: Path, tmp_path: Path, bin_dir: Path) -> Path: + """Copy ``script`` and redirect it at the sandbox. + + Two textual transformations, applied by the SAME code to the fixed script + and to the as-deployed fixture so neither is advantaged: + + 1. inject the stub bin after the pinned ``PATH=`` assignment; + 2. repoint the ``/data/maintenance`` state/log/lock paths at ``tmp_path``. + + (2) is only needed because the as-deployed version hardcodes those paths + with no env override -- the fixed version reads them from the environment, + so for it the rewrite is equivalent to the env vars ``_run`` already sets. + Neither transformation touches probe selection, status classification, or + message formatting, which is all these tests assert on. + """ + staged = tmp_path / f"staged-{script.name}" + lines = script.read_text().splitlines(keepends=True) + for index, line in enumerate(lines): + if line.startswith("PATH="): + lines.insert(index + 1, f'PATH="{bin_dir}:$PATH"\n') + break + else: # pragma: no cover - both artifacts pin PATH; a miss is a real defect + raise AssertionError(f"no PATH assignment found in {script}") + patched = "".join(lines) + sandbox = tmp_path / "sandbox" + for var, sub in ( + ("STATE_DIR", sandbox / "state"), + ("LOG_DIR", sandbox / "logs"), + ("LOCK_FILE", sandbox / "lock"), + ("ENV_FILE", sandbox / "absent.env"), + ): + patched = re.sub( + rf"^{var}=.*$", f"{var}={sub}", patched, count=1, flags=re.MULTILINE + ) + staged.write_text(patched) + staged.chmod(0o755) + return staged + + +def _run( + script: Path, + tmp_path: Path, + bin_dir: Path, + *, + extra_env: dict[str, str] | None = None, +) -> str: + env = dict(os.environ) + env.update( + { + "OMNINODE_ALERT_ENV_FILE": str(tmp_path / "absent.env"), + "OMNINODE_ALERT_STATE_DIR": str(tmp_path / "state"), + "OMNINODE_ALERT_LOG_DIR": str(tmp_path / "logs"), + "OMNINODE_ALERT_LOCK_FILE": str(tmp_path / "lock"), + "OMNINODE_INFRA_REPO_ROOT": str(REPO_ROOT), + "OMNINODE_RUNTIME_POLICY_ENV": str(RUNTIME_POLICY_ENV), + "SLACK_BOT_TOKEN": "test-token", + "SLACK_CHANNEL_ID": "C-TEST", + } + ) + if extra_env: + env.update(extra_env) + staged = _stage(script, tmp_path, bin_dir) + proc = subprocess.run( + ["bash", str(staged), "--mode", "dry-run"], + capture_output=True, + text=True, + env=env, + timeout=120, + check=False, + ) + out = proc.stdout + if not out.strip(): + # The as-deployed script redirects stdout into its log file even in + # dry-run; read the report back from there. + logs = sorted((tmp_path / "sandbox" / "logs").glob("*.log")) + if logs: + out = logs[-1].read_text() + assert out.strip(), ( + f"no report produced: rc={proc.returncode} stderr={proc.stderr[-2000:]}" + ) + return out + + +# -------------------------------------------------------------------------- +# The 2026-07-30T16:19-16:45Z outage state, replayed verbatim. +# dev :8085 -> 503, healthy=false, is_running=false, no handlers +# dev :8086 -> connection refused (container never started) +# stability 18085-> 200 healthy +# prod 28085 -> 200 healthy +# everything else 200, all infra containers healthy +# -------------------------------------------------------------------------- +DEV_503_BODY = json.dumps( + { + "status": "unhealthy", + "healthy": False, + "is_running": False, + "registered_handlers": [], + "config_prefetch_status": "pending", + } +) +# A REALISTIC healthy runtime body (OMN-15525). +# +# The original fixture here was `{"status":"healthy","healthy":true, +# "version":"0.38.4"}` -- 63 bytes. Every real `/health` body on .201 is 2644 +# bytes, and `check_runtime_lane` truncated the body to 180 bytes before handing +# it to jq. Under the short fixture the truncation never bit, so the suite was +# green while the deployed artifact reported CRITICAL for all three lanes +# against a fully healthy fleet. The fixture must cross the excerpt boundary or +# it cannot see that class of defect at all. +# +# Shape mirrors the live body: top-level status/version/details, with the +# health-bearing booleans inside `details` and enough sibling keys to push the +# payload well past any display-truncation limit. +HEALTHY_BODY = json.dumps( + { + "status": "healthy", + "version": "0.38.4", + "details": { + "healthy": True, + "degraded": False, + "startup_in_progress": False, + "is_running": True, + "is_draining": False, + "pending_message_count": 0, + "max_concurrent_handlers": 32, + "handler_pool_size": 8, + "in_flight_tasks": 0, + "event_bus_healthy": True, + "event_bus": "kafka", + "runtime_attached": True, + "runtime_health": "healthy", + "local_ingress": True, + "no_handlers_registered": False, + "config_prefetch_status": "complete", + "batch_response_enabled": True, + "batch_response_pending": 0, + "failed_handlers": [], + "skipped_handlers": [], + "registered_handlers": [ + "handler_node_registration_effect", + "handler_session_state_reducer", + ], + "handler_pools": {"default": {"size": 8, "busy": 0}}, + "components": { + "postgres": "healthy", + "redpanda": "healthy", + "valkey": "healthy", + }, + "handlers": {"registered": 2, "failed": 0, "skipped": 0}, + }, + } +) +# Guard the guard: if someone shrinks this fixture the truncation defect becomes +# invisible again, so assert the property the fixture exists to provide. +assert len(HEALTHY_BODY) > 180, ( + "HEALTHY_BODY must exceed the reporter's display-excerpt limit or the " + "OMN-15525 truncation regression cannot be observed" +) + + +def _outage_http( + dev_port: str, stability_port: str, prod_port: str +) -> dict[str, tuple[int, str]]: + return { + dev_port: (503, DEV_503_BODY), + stability_port: (200, HEALTHY_BODY), + prod_port: (200, HEALTHY_BODY), + "13002": (200, json.dumps({"status": "ok"})), + "8099": (200, json.dumps({"state": "idle"})), + "3003": (200, "ok"), + } + + +def _outage_docker() -> dict[str, Any]: + return { + "containers": [ + { + "name": "omninode-runtime", + "status": "Up 26 minutes (health: starting)", + "started_at": "2026-07-30T16:19:00Z", + "start_period_ns": 120 * 10**9, + }, + { + "name": "omnibase-infra-redpanda", + "status": "Up 40 minutes (healthy)", + "started_at": "2026-07-30T16:05:00Z", + }, + { + "name": "omnibase-infra-postgres", + "status": "Up 40 minutes (healthy)", + "started_at": "2026-07-30T16:05:00Z", + }, + { + "name": "omnibase-infra-valkey", + "status": "Up 40 minutes (healthy)", + "started_at": "2026-07-30T16:05:00Z", + }, + ], + "dangling": [], + } + + +@pytest.fixture +def lane_ports() -> dict[str, str]: + return {lane: _policy_port(key) for lane, key in LANE_PORT_KEYS.items()} + + +# -------------------------------------------------------------------------- +# AC 6 -- RED-before / GREEN-after against the artifact that actually runs. +# -------------------------------------------------------------------------- + + +def test_as_deployed_reports_green_on_the_replayed_outage( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + """RED-before: the live 2026-07-30 script never looked at the dev runtime.""" + bin_dir = _make_stub_bin( + tmp_path, + http=_outage_http( + lane_ports["dev"], lane_ports["stability-test"], lane_ports["prod"] + ), + docker_state=_outage_docker(), + ) + report = _run(AS_DEPLOYED_SCRIPT, tmp_path, bin_dir) + + endpoints = report.split("*Runtime endpoints*", 1)[1].split("*Active issues*", 1)[0] + # Endpoint labels are backtick-delimited, so match the whole label rather + # than the bare port: "8085" is a substring of "runtime-18085". + probed_labels = re.findall(r"- `([^`]+)`: HTTP", endpoints) + assert probed_labels, endpoints + assert not [ + label + for label in probed_labels + if re.search(rf"(? None: + """GREEN-after: identical replayed state, fixed artifact, RED naming dev.""" + bin_dir = _make_stub_bin( + tmp_path, + http=_outage_http( + lane_ports["dev"], lane_ports["stability-test"], lane_ports["prod"] + ), + docker_state=_outage_docker(), + ) + report = _run(FIXED_SCRIPT, tmp_path, bin_dir) + + assert f"runtime-dev-{lane_ports['dev']}`: HTTP 503 (CRITICAL)" in report + assert ( + f"runtime-stability-test-{lane_ports['stability-test']}`: HTTP 200 (OK)" + in report + ) + assert f"runtime-prod-{lane_ports['prod']}`: HTTP 200 (OK)" in report + assert re.search(r"Issues: \*[1-9]\d* critical\*", report), report + + +# -------------------------------------------------------------------------- +# AC 7 -- the omission cannot silently reappear. +# -------------------------------------------------------------------------- + + +def test_every_lane_main_runtime_port_is_in_the_probe_set( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + """Drop a lane from RUNTIME_LANE_SPECS and this fails.""" + http = dict.fromkeys(lane_ports.values(), (200, HEALTHY_BODY)) + http.update( + { + "13002": (200, '{"status":"ok"}'), + "8099": (200, '{"state":"idle"}'), + "3003": (200, "ok"), + } + ) + bin_dir = _make_stub_bin( + tmp_path, http=http, docker_state={"containers": [], "dangling": []} + ) + report = _run(FIXED_SCRIPT, tmp_path, bin_dir) + + endpoints = report.split("*Runtime endpoints*", 1)[1].split("*Active issues*", 1)[0] + for lane, port in lane_ports.items(): + assert f"runtime-{lane}-{port}" in endpoints, ( + f"lane {lane} (:{port}) is not probed -- a lane's MAIN runtime health " + f"endpoint was dropped from the alert (OMN-15509 AC2/AC7)" + ) + + +def test_lane_specs_are_sourced_from_the_rendered_runtime_policy() -> None: + """The map is config-driven; hardcoding a port per call site regresses AC2.""" + body = FIXED_SCRIPT.read_text() + for lane, key in LANE_PORT_KEYS.items(): + assert f"{lane}|{key}" in body, f"lane {lane} not declared against {key}" + assert "policy_env_value" in body + + +def test_lane_specs_carry_no_hardcoded_fallback_ports() -> None: + """OMN-15525: the spec table must not smuggle literal ports back in. + + The OMN-15509 revision declared ``dev|DEV_RUNTIME_MAIN_PORT|8085`` and + substituted that literal whenever the policy lookup came back empty, so a + renamed key or an unrendered policy file silently probed a guessed port. + """ + source = FIXED_SCRIPT.read_text() + specs = re.search(r"RUNTIME_LANE_SPECS=\((.*?)\n\)", source, re.DOTALL) + assert specs, "RUNTIME_LANE_SPECS table not found" + for raw in specs.group(1).strip().splitlines(): + entry = raw.strip().strip('"') + if not entry: + continue + fields = entry.split("|") + assert len(fields) == 2, ( + f"lane spec {entry!r} carries more than lane|key -- a third field is " + "the hardcoded fallback port OMN-15525 removed" + ) + assert not re.fullmatch(r"\d+", fields[1]), entry + + +def test_cron_unit_points_at_the_versioned_script_name() -> None: + unit = CRON_UNIT.read_text() + assert "omninode-system-slack-report.sh" in unit + assert "--mode alert" in unit and "--mode digest" in unit + + +# -------------------------------------------------------------------------- +# AC 3 -- 200 with a non-healthy body is RED. +# -------------------------------------------------------------------------- + + +def test_http_200_with_unhealthy_body_is_critical( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + http = dict.fromkeys(lane_ports.values(), (200, HEALTHY_BODY)) + # 200 but the body says otherwise -- the exact case substring matching missed. + http[lane_ports["dev"]] = ( + 200, + json.dumps({"status": "degraded", "healthy": False}), + ) + http.update( + { + "13002": (200, '{"status":"ok"}'), + "8099": (200, '{"state":"idle"}'), + "3003": (200, "ok"), + } + ) + bin_dir = _make_stub_bin( + tmp_path, http=http, docker_state={"containers": [], "dangling": []} + ) + report = _run(FIXED_SCRIPT, tmp_path, bin_dir) + assert f"runtime-dev-{lane_ports['dev']}`: HTTP 200 (CRITICAL)" in report + + +def test_http_200_with_unresolvable_body_fails_closed( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + http = dict.fromkeys(lane_ports.values(), (200, HEALTHY_BODY)) + http[lane_ports["dev"]] = (200, "OK") # not JSON, no resolvable status + http.update( + { + "13002": (200, '{"status":"ok"}'), + "8099": (200, '{"state":"idle"}'), + "3003": (200, "ok"), + } + ) + bin_dir = _make_stub_bin( + tmp_path, http=http, docker_state={"containers": [], "dangling": []} + ) + report = _run(FIXED_SCRIPT, tmp_path, bin_dir) + assert f"runtime-dev-{lane_ports['dev']}`: HTTP 200 (CRITICAL)" in report + + +# -------------------------------------------------------------------------- +# AC 5 -- an endpoint that cannot be probed is RED, never omitted. +# -------------------------------------------------------------------------- + + +def test_unreachable_runtime_endpoint_is_critical_not_skipped( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + http = dict.fromkeys(lane_ports.values(), (200, HEALTHY_BODY)) + del http[lane_ports["dev"]] # connection refused + http.update( + { + "13002": (200, '{"status":"ok"}'), + "8099": (200, '{"state":"idle"}'), + "3003": (200, "ok"), + } + ) + bin_dir = _make_stub_bin( + tmp_path, http=http, docker_state={"containers": [], "dangling": []} + ) + report = _run(FIXED_SCRIPT, tmp_path, bin_dir) + assert f"runtime-dev-{lane_ports['dev']}`: HTTP 000 (CRITICAL)" in report + + +# -------------------------------------------------------------------------- +# AC 4 -- `health: starting` past start_period alarms; AC 5 -- Exit(0) does not. +# -------------------------------------------------------------------------- + + +def _all_green_http(lane_ports: dict[str, str]) -> dict[str, tuple[int, str]]: + http = dict.fromkeys(lane_ports.values(), (200, HEALTHY_BODY)) + http.update( + { + "13002": (200, '{"status":"ok"}'), + "8099": (200, '{"state":"idle"}'), + "3003": (200, "ok"), + } + ) + return http + + +def test_container_starting_past_start_period_is_critical( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + docker_state = { + "containers": [ + { + "name": "omninode-runtime", + "status": "Up 26 minutes (health: starting)", + "started_at": "2026-07-30T16:19:00Z", + "start_period_ns": 120 * 10**9, + } + ], + "dangling": [], + } + bin_dir = _make_stub_bin( + tmp_path, http=_all_green_http(lane_ports), docker_state=docker_state + ) + report = _run(FIXED_SCRIPT, tmp_path, bin_dir) + assert "starting_past_start_period=omninode-runtime" in report + assert "container_issues" in report + assert re.search(r"Issues: \*[1-9]\d* critical\*", report), report + + +def test_container_still_inside_start_period_does_not_alarm( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + """A genuinely-still-booting container is not a page.""" + from datetime import UTC, datetime + + docker_state = { + "containers": [ + { + "name": "omninode-runtime", + "status": "Up 3 seconds (health: starting)", + "started_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + "start_period_ns": 600 * 10**9, + } + ], + "dangling": [], + } + bin_dir = _make_stub_bin( + tmp_path, http=_all_green_http(lane_ports), docker_state=docker_state + ) + report = _run(FIXED_SCRIPT, tmp_path, bin_dir) + assert "starting_past_start_period=none" in report + assert "Issues: *0 critical*" in report + + +def test_expected_exit_zero_oneshot_does_not_alarm( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + docker_state = { + "containers": [ + { + "name": "omnibase-infra-migration", + "status": "Exited (0) 4 minutes ago", + "started_at": "2026-07-30T16:05:00Z", + } + ], + "dangling": [], + } + bin_dir = _make_stub_bin( + tmp_path, http=_all_green_http(lane_ports), docker_state=docker_state + ) + report = _run(FIXED_SCRIPT, tmp_path, bin_dir) + assert "exited_nonzero=none" in report + assert "Issues: *0 critical*" in report + + +def test_nonzero_exit_alarms(tmp_path: Path, lane_ports: dict[str, str]) -> None: + docker_state = { + "containers": [ + { + "name": "omninode-runtime-effects", + "status": "Exited (1) 2 minutes ago", + "started_at": "2026-07-30T16:05:00Z", + } + ], + "dangling": [], + } + bin_dir = _make_stub_bin( + tmp_path, http=_all_green_http(lane_ports), docker_state=docker_state + ) + report = _run(FIXED_SCRIPT, tmp_path, bin_dir) + assert "exited_nonzero=omninode-runtime-effects" in report + assert re.search(r"Issues: \*[1-9]\d* critical\*", report), report + + +# -------------------------------------------------------------------------- +# Prod stays read-only. +# -------------------------------------------------------------------------- + + +def test_prod_lane_is_probed_with_a_plain_get_only() -> None: + body = FIXED_SCRIPT.read_text() + assert re.search(r"curl .*-X GET .*/health", body), ( + "runtime probe must be an explicit GET" + ) + for verb in ("-X POST", "-X PUT", "-X DELETE", "-X PATCH"): + assert verb not in body, f"reporter must never issue {verb}" + + +# -------------------------------------------------------------------------- +# OMN-15525 -- the two false-green/false-RED defects found by DEPLOYING the +# OMN-15509 fix to .201 (installed 18:12Z, rolled back 18:13:29Z after it +# reported CRITICAL on all three demonstrably healthy lanes). +# -------------------------------------------------------------------------- + + +def test_healthy_lane_with_a_realistic_body_is_not_critical( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + """A 200 + healthy body larger than the display excerpt must read OK. + + RED-before: `check_runtime_lane` truncated the body to 180 bytes and then + parsed THAT with jq. A real .201 body is 2644 bytes, so jq always failed + ("Unfinished string at EOF"), the verdict was always "unresolvable", and + fail-closed reported CRITICAL for every lane on a healthy fleet. + """ + assert len(HEALTHY_BODY) > 180, "fixture must cross the excerpt boundary" + bin_dir = _make_stub_bin( + tmp_path, + http=_all_green_http(lane_ports), + docker_state={"containers": [], "dangling": []}, + ) + report = _run(FIXED_SCRIPT, tmp_path, bin_dir) + + for lane, port in lane_ports.items(): + assert f"runtime-{lane}-{port}`: HTTP 200 (OK)" in report, ( + f"lane {lane} went non-OK on a healthy 200 whose body merely exceeds " + f"the display excerpt -- the body is being parsed after truncation " + f"(OMN-15525):\n{report}" + ) + assert "could not be resolved from body" not in report, report + assert "Issues: *0 critical*, *0 warning*" in report, report + + +def test_verdict_is_independent_of_the_display_excerpt_size( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + """Shrinking the reported excerpt must not change any lane's status. + + This is the invariant the defect violated: display truncation is cosmetic, + so driving it to an absurdly small value must leave every verdict intact. + """ + bin_dir = _make_stub_bin( + tmp_path, + http=_all_green_http(lane_ports), + docker_state={"containers": [], "dangling": []}, + ) + baseline = _run(FIXED_SCRIPT, tmp_path, bin_dir) + + tiny_path = tmp_path / "tiny" + tiny_path.mkdir() + tiny_bin = _make_stub_bin( + tiny_path, + http=_all_green_http(lane_ports), + docker_state={"containers": [], "dangling": []}, + ) + tiny = _run( + FIXED_SCRIPT, + tiny_path, + tiny_bin, + extra_env={"OMNINODE_ALERT_BODY_EXCERPT_BYTES": "12"}, + ) + + def _statuses(report: str) -> list[tuple[str, str]]: + section = report.split("*Runtime endpoints*", 1)[1].split("*Active issues*", 1)[ + 0 + ] + return re.findall(r"- `([^`]+)`: HTTP \d+ \((\w+)\)", section) + + assert _statuses(baseline) == _statuses(tiny), ( + f"verdicts moved when only the display excerpt changed:\n" + f"{_statuses(baseline)}\nvs\n{_statuses(tiny)}" + ) + # Guard against a vacuous pass: "all CRITICAL == all CRITICAL" also + # satisfies the equality above, and that is precisely the broken state. + # The fleet here is healthy, so every verdict must be OK. + assert _statuses(baseline), baseline + assert all(status == "OK" for _, status in _statuses(baseline)), ( + f"stability check is vacuous -- the healthy baseline is not all OK:\n{baseline}" + ) + + +def test_missing_runtime_policy_file_is_critical_not_a_hardcoded_port( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + """Rule 8: an absent policy file must alarm, not fall back to 8085/18085/28085. + + RED-before: `policy_env_value` returned success on a missing file and + `lane_main_port` substituted the literal port, so the reporter kept probing + guessed ports and reported green with no indication the lane map had + stopped resolving. + """ + bin_dir = _make_stub_bin( + tmp_path, + http=_all_green_http(lane_ports), + docker_state={"containers": [], "dangling": []}, + ) + report = _run( + FIXED_SCRIPT, + tmp_path, + bin_dir, + extra_env={"OMNINODE_RUNTIME_POLICY_ENV": str(tmp_path / "no-such-policy.env")}, + ) + + for lane in LANE_PORT_KEYS: + assert f"runtime-{lane}-unresolved" in report, ( + f"lane {lane} did not report an unresolvable port with the policy " + f"file absent -- it fell back to a hardcoded port (OMN-15525):\n{report}" + ) + for lane, port in lane_ports.items(): + assert f"runtime-{lane}-{port}" not in report, ( + f"lane {lane} probed literal :{port} with no policy file present" + ) + assert re.search(r"Issues: \*[1-9]\d* critical\*", report), report + + +def test_renamed_policy_key_is_critical( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + """A policy file that exists but no longer carries the key must alarm.""" + partial = tmp_path / "partial-policy.env" + partial.write_text( + f"DEV_RUNTIME_MAIN_PORT_RENAMED={lane_ports['dev']}\n" + f"STABILITY_TEST_RUNTIME_MAIN_PORT={lane_ports['stability-test']}\n" + f"PROD_RUNTIME_MAIN_PORT={lane_ports['prod']}\n" + ) + bin_dir = _make_stub_bin( + tmp_path, + http=_all_green_http(lane_ports), + docker_state={"containers": [], "dangling": []}, + ) + report = _run( + FIXED_SCRIPT, + tmp_path, + bin_dir, + extra_env={"OMNINODE_RUNTIME_POLICY_ENV": str(partial)}, + ) + + assert "runtime-dev-unresolved" in report, report + assert f"runtime-dev-{lane_ports['dev']}" not in report, report + # The lanes whose keys still resolve are unaffected. + assert ( + f"runtime-stability-test-{lane_ports['stability-test']}`: HTTP 200 (OK)" + in report + ), report + assert re.search(r"Issues: \*[1-9]\d* critical\*", report), report + + +# -------------------------------------------------------------------------- +# OMN-15525 -- `--mode alert` is the path that actually pages, and it had NO +# behavioural coverage at all. Both prior revisions rendered a CRITICAL lane +# into the digest TEXT while computing an EMPTY `$issues`, so the alert branch +# took "clean" and posted nothing. Fixing the probe (OMN-15509) and the +# truncation is worthless if the alert still cannot fire. +# -------------------------------------------------------------------------- + +_SLACK_CURL = """#!/usr/bin/env bash +# Slack-aware curl: records chat.postMessage payloads, delegates everything +# else to the real stub so endpoint probing is unchanged. +is_slack=0 +payload="" +args=("$@") +for ((i=0; i<${#args[@]}; i++)); do + case "${args[$i]}" in + https://slack.com/*) is_slack=1 ;; + -d) payload="${args[$((i+1))]}" ;; + esac +done +if (( is_slack )); then + # One file per post: the payload is multi-line JSON (jq -n pretty-prints), so + # appending to a shared file would not round-trip. + mkdir -p "__POSTS__" + printf '%s' "$payload" > "__POSTS__/$(date +%s%N)-$$.json" + printf '%s' '{"ok":true}' + exit 0 +fi +exec "__REALCURL__" "$@" +""" + + +def _run_alert( + script: Path, + tmp_path: Path, + bin_dir: Path, + *, + extra_env: dict[str, str] | None = None, +) -> tuple[str, list[str]]: + """Drive ``script`` in ``--mode alert``; return (log text, Slack post texts).""" + posts = tmp_path / "slack-posts" + real_curl = bin_dir / "curl-http" + (bin_dir / "curl").rename(real_curl) + _write( + bin_dir / "curl", + _SLACK_CURL.replace("__POSTS__", str(posts)).replace( + "__REALCURL__", str(real_curl) + ), + executable=True, + ) + + env = dict(os.environ) + env.update( + { + "OMNINODE_ALERT_ENV_FILE": str(tmp_path / "absent.env"), + "OMNINODE_INFRA_REPO_ROOT": str(REPO_ROOT), + "OMNINODE_RUNTIME_POLICY_ENV": str(RUNTIME_POLICY_ENV), + "SLACK_BOT_TOKEN": "test-token", + "SLACK_CHANNEL_ID": "C-TEST", + } + ) + if extra_env: + env.update(extra_env) + staged = _stage(script, tmp_path, bin_dir) + proc = subprocess.run( + ["bash", str(staged), "--mode", "alert"], + capture_output=True, + text=True, + env=env, + timeout=120, + check=False, + ) + assert proc.returncode == 0, ( + f"alert run failed rc={proc.returncode} stderr={proc.stderr[-2000:]}" + ) + logs = sorted((tmp_path / "sandbox" / "logs").glob("*.log")) + log_text = logs[-1].read_text() if logs else proc.stdout + post_texts: list[str] = [] + if posts.is_dir(): + for payload in sorted(posts.glob("*.json")): + post_texts.append(json.loads(payload.read_text()).get("text", "")) + return log_text, post_texts + + +def test_alert_mode_pages_when_a_runtime_lane_is_down( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + """A 503 dev lane must produce a Slack alert naming that lane. + + RED-before: `$issues` was selected with `$2=="CRITICAL"`, but endpoint rows + carry their status in `$1`. With only endpoints failing, `$issues` was + empty, the alert branch wrote "clean" to the state file, and nothing was + ever posted -- while the digest text simultaneously listed the lane as + CRITICAL. The reporter could see the dead runtime and still not page. + """ + bin_dir = _make_stub_bin( + tmp_path, + http=_outage_http( + lane_ports["dev"], lane_ports["stability-test"], lane_ports["prod"] + ), + docker_state={"containers": [], "dangling": []}, + ) + log_text, posts = _run_alert(FIXED_SCRIPT, tmp_path, bin_dir) + + assert posts, ( + "no Slack post was attempted while the dev runtime lane was 503 -- the " + f"alert branch took the 'clean' path (OMN-15525).\nlog:\n{log_text}" + ) + joined = "\n".join(posts) + assert f"runtime-dev-{lane_ports['dev']}" in joined, ( + f"alert fired but never named the dead dev lane:\n{joined}" + ) + assert re.search(r"Issues: \*[1-9]\d* critical\*", joined), joined + + state = (tmp_path / "sandbox" / "state" / "omninode-system-alert.hash").read_text() + assert state.strip() != "clean", ( + "alert run recorded the fleet as clean while a lane was 503" + ) + + +def test_alert_mode_stays_quiet_on_a_healthy_fleet( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + """Control for the test above: an all-green fleet must post nothing. + + Without this, `test_alert_mode_pages_when_a_runtime_lane_is_down` could pass + against a script that posts unconditionally. + """ + bin_dir = _make_stub_bin( + tmp_path, + http=_all_green_http(lane_ports), + docker_state={"containers": [], "dangling": []}, + ) + log_text, posts = _run_alert(FIXED_SCRIPT, tmp_path, bin_dir) + + assert not posts, ( + f"alert posted against a fully healthy fleet:\n{posts}\n{log_text}" + ) + state = (tmp_path / "sandbox" / "state" / "omninode-system-alert.hash").read_text() + assert state.strip() == "clean", f"healthy fleet not recorded clean: {state!r}" + + +def test_endpoint_failures_are_counted_in_the_header( + tmp_path: Path, lane_ports: dict[str, str] +) -> None: + """The header count must agree with the *Active issues* list. + + The two were computed by different awk programs over different columns, so + the digest could say `0 critical` directly above three CRITICAL lanes -- + observed verbatim on .201 against the merged OMN-15509 revision. + """ + bin_dir = _make_stub_bin( + tmp_path, + http=_outage_http( + lane_ports["dev"], lane_ports["stability-test"], lane_ports["prod"] + ), + docker_state={"containers": [], "dangling": []}, + ) + report = _run(FIXED_SCRIPT, tmp_path, bin_dir) + + header = re.search(r"Issues: \*(\d+) critical\*, \*(\d+) warning\*", report) + assert header, report + listed_critical = len( + re.findall(r"^- CRITICAL ", report.split("*Active issues*", 1)[1], re.MULTILINE) + ) + assert int(header.group(1)) == listed_critical, ( + f"header claims {header.group(1)} critical but {listed_critical} are " + f"listed under *Active issues*:\n{report}" + ) + assert listed_critical > 0, report diff --git a/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json b/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json new file mode 100644 index 0000000000..c060b5851a --- /dev/null +++ b/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json @@ -0,0 +1,174 @@ +{ + "workflow_runs": [ + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145773, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T19:17:26Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574146135, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T19:18:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145723, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T19:18:18Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145227, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T19:17:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145181, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T19:17:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145170, + "name": "Main target guard", + "status": "completed", + "updated_at": "2026-07-30T19:17:14Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145349, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T19:17:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145140, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T19:17:46Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:49Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058841, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T19:16:07Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:49Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574059012, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T19:23:08Z" + }, + { + "conclusion": "failure", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058377, + "name": ".github/workflows/ci.yml", + "status": "completed", + "updated_at": "2026-07-30T19:15:48Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058150, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T19:15:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057942, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057936, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T19:16:04Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057920, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:05Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057886, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057926, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T19:16:07Z" + } + ] +} diff --git a/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json b/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json new file mode 100644 index 0000000000..f7f8c02e97 --- /dev/null +++ b/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json @@ -0,0 +1,6 @@ +{ + "contexts": [ + "CI Summary" + ], + "strict": false +} diff --git a/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json b/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json new file mode 100644 index 0000000000..08b0d9a227 --- /dev/null +++ b/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json @@ -0,0 +1,370 @@ +{ + "check_runs": [ + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T19:24:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T19:23:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T19:23:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T19:22:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T19:22:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T19:22:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T19:18:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T19:18:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T19:17:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T19:17:20Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T19:17:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T19:17:25Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T19:17:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T19:17:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T19:17:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T19:17:04Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T19:17:08Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T19:15:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-30T19:16:05Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T19:16:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T19:15:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T19:16:12Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T19:15:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CodeQL", + "started_at": "2026-07-30T19:16:20Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T19:16:11Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T19:16:08Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T19:16:09Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T19:16:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T19:16:04Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T19:16:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T19:16:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T19:16:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T19:16:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T19:15:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T19:16:05Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T19:16:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T19:16:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T19:15:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T19:16:12Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T19:16:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T19:16:13Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T19:15:55Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T19:16:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T19:16:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T19:15:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T19:15:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T19:16:17Z", + "status": "completed" + } + ] +} diff --git a/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json b/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json new file mode 100644 index 0000000000..0c428a268a --- /dev/null +++ b/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json @@ -0,0 +1,10 @@ +{ + "state": "success", + "statuses": [ + { + "context": "CodeRabbit", + "created_at": "2026-07-30T19:15:58Z", + "state": "success" + } + ] +} diff --git a/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json b/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json new file mode 100644 index 0000000000..7086402a1a --- /dev/null +++ b/tests/fixtures/omn15550/absent/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json @@ -0,0 +1,12 @@ +[ + { + "base": { + "ref": "dev" + }, + "draft": false, + "head": { + "sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b" + }, + "number": 2575 + } +] diff --git a/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json b/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json new file mode 100644 index 0000000000..ca954ad092 --- /dev/null +++ b/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json @@ -0,0 +1,1004 @@ +{ + "workflow_runs": [ + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768690, + "name": "Auto-Tag on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:47:28Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768467, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:47:48Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768504, + "name": "TODO Audit on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:47:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768458, + "name": "Runtime Rebuild Trigger", + "status": "completed", + "updated_at": "2026-07-30T21:48:32Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768457, + "name": "PR Merged Event Producer", + "status": "completed", + "updated_at": "2026-07-30T21:48:29Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727782, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T21:32:45Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727222, + "name": "Hostile Reviewer", + "status": "completed", + "updated_at": "2026-07-30T21:33:36Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727021, + "name": "product-readiness-shadow", + "status": "completed", + "updated_at": "2026-07-30T21:33:31Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727328, + "name": "CI", + "status": "completed", + "updated_at": "2026-07-30T21:40:00Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727411, + "name": "OCC Companion Effect Publisher", + "status": "completed", + "updated_at": "2026-07-30T21:31:44Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727000, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:31:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727249, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T21:31:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727234, + "name": "Imperative Contract Guard", + "status": "completed", + "updated_at": "2026-07-30T21:32:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727313, + "name": "docs-validate", + "status": "completed", + "updated_at": "2026-07-30T21:32:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727335, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T21:31:57Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727047, + "name": "No bare compose teardown (OMN-13886)", + "status": "completed", + "updated_at": "2026-07-30T21:32:00Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727320, + "name": "Deploy Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:01Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727282, + "name": "Required-Check Skip-Vector Guard", + "status": "completed", + "updated_at": "2026-07-30T21:31:56Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727278, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:38Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726730, + "name": "Omni* Ecosystem Standards Compliance", + "status": "completed", + "updated_at": "2026-07-30T21:33:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726962, + "name": "Security Scan", + "status": "completed", + "updated_at": "2026-07-30T21:38:18Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726938, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T21:31:41Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726907, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:42Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726936, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:40Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726803, + "name": "Dep Provenance Gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:44Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726770, + "name": "Dispatcher Route Coverage", + "status": "completed", + "updated_at": "2026-07-30T21:31:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726784, + "name": "Shell Hygiene Gate (OMN-14761)", + "status": "completed", + "updated_at": "2026-07-30T21:31:46Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726946, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T21:31:48Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726967, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T21:31:27Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726747, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:38Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726930, + "name": "skill-node-mapping-sync", + "status": "completed", + "updated_at": "2026-07-30T21:32:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726900, + "name": "receipt-honesty", + "status": "completed", + "updated_at": "2026-07-30T21:32:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726899, + "name": "Integration Test Check", + "status": "completed", + "updated_at": "2026-07-30T21:32:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726856, + "name": "fresh-deploy-fitness", + "status": "completed", + "updated_at": "2026-07-30T21:32:10Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726866, + "name": "node-migration-sync", + "status": "completed", + "updated_at": "2026-07-30T21:32:03Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726943, + "name": "Check Architecture Handshake", + "status": "completed", + "updated_at": "2026-07-30T21:31:57Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726893, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:29Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726986, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:25Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726966, + "name": "Contract Validation", + "status": "completed", + "updated_at": "2026-07-30T21:32:20Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704580, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:31:18Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704517, + "name": "Runtime Rebuild Trigger", + "status": "completed", + "updated_at": "2026-07-30T21:31:06Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704315, + "name": "TODO Audit on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:31:05Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704769, + "name": "Auto-Tag on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:31:06Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704541, + "name": "PR Merged Event Producer", + "status": "completed", + "updated_at": "2026-07-30T21:31:14Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:32Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583529379, + "name": "docs-validate", + "status": "completed", + "updated_at": "2026-07-30T21:29:04Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:32Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528727, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T21:30:03Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527959, + "name": "OCC Companion Effect Publisher", + "status": "completed", + "updated_at": "2026-07-30T21:28:45Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528511, + "name": "Deploy Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:04Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527536, + "name": "Imperative Contract Guard", + "status": "completed", + "updated_at": "2026-07-30T21:29:27Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528008, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:32Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527982, + "name": "Hostile Reviewer", + "status": "completed", + "updated_at": "2026-07-30T21:30:53Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528083, + "name": "Required-Check Skip-Vector Guard", + "status": "completed", + "updated_at": "2026-07-30T21:28:59Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527480, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T21:28:55Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527641, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T21:28:55Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526350, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T21:28:31Z" + }, + { + "conclusion": "failure", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527197, + "name": ".github/workflows/ci.yml", + "status": "completed", + "updated_at": "2026-07-30T21:28:30Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526328, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T21:28:36Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526975, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T21:28:45Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526372, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T21:28:41Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526428, + "name": "Check Architecture Handshake", + "status": "completed", + "updated_at": "2026-07-30T21:28:44Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526501, + "name": "Dep Provenance Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:02Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526267, + "name": "node-migration-sync", + "status": "completed", + "updated_at": "2026-07-30T21:29:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526653, + "name": "receipt-honesty", + "status": "completed", + "updated_at": "2026-07-30T21:29:22Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526321, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:26Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526326, + "name": "Contract Validation", + "status": "completed", + "updated_at": "2026-07-30T21:29:01Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526616, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526405, + "name": "fresh-deploy-fitness", + "status": "completed", + "updated_at": "2026-07-30T21:29:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526568, + "name": "skill-node-mapping-sync", + "status": "completed", + "updated_at": "2026-07-30T21:29:35Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526360, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:31Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526420, + "name": "product-readiness-shadow", + "status": "completed", + "updated_at": "2026-07-30T21:30:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526998, + "name": "Omni* Ecosystem Standards Compliance", + "status": "completed", + "updated_at": "2026-07-30T21:31:17Z" + }, + { + "conclusion": "cancelled", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526601, + "name": "Security Scan", + "status": "completed", + "updated_at": "2026-07-30T21:31:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526986, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T21:28:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526269, + "name": "Shell Hygiene Gate (OMN-14761)", + "status": "completed", + "updated_at": "2026-07-30T21:28:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527092, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:28:53Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526270, + "name": "Dispatcher Route Coverage", + "status": "completed", + "updated_at": "2026-07-30T21:28:53Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526377, + "name": "Integration Test Check", + "status": "completed", + "updated_at": "2026-07-30T21:28:55Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526987, + "name": "No bare compose teardown (OMN-13886)", + "status": "completed", + "updated_at": "2026-07-30T21:28:56Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515034, + "name": "Runtime Rebuild Trigger", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515192, + "name": "Auto-Tag on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515354, + "name": "TODO Audit on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515225, + "name": "PR Merged Event Producer", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515230, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:28:39Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145773, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T19:17:26Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574146135, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T19:18:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145723, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T19:18:18Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145227, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T19:17:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145181, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T19:17:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145170, + "name": "Main target guard", + "status": "completed", + "updated_at": "2026-07-30T19:17:14Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145349, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T19:17:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145140, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T19:17:46Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:49Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058841, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T19:16:07Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:49Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574059012, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T19:23:08Z" + }, + { + "conclusion": "failure", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058377, + "name": ".github/workflows/ci.yml", + "status": "completed", + "updated_at": "2026-07-30T19:15:48Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058150, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T19:15:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057942, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057936, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T19:16:04Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057920, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:05Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057886, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057926, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T19:16:07Z" + } + ] +} diff --git a/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json b/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json new file mode 100644 index 0000000000..f7f8c02e97 --- /dev/null +++ b/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json @@ -0,0 +1,6 @@ +{ + "contexts": [ + "CI Summary" + ], + "strict": false +} diff --git a/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json b/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json new file mode 100644 index 0000000000..629eea69ff --- /dev/null +++ b/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json @@ -0,0 +1,1300 @@ +{ + "check_runs": [ + { + "conclusion": "skipped", + "name": "auto-tag", + "started_at": "2026-07-30T21:47:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "TODO Audit", + "started_at": "2026-07-30T21:47:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:47:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Trigger node_redeploy Start", + "started_at": "2026-07-30T21:47:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Merged Event", + "started_at": "2026-07-30T21:47:28Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Runtime Boot Smoke (compose)", + "started_at": "2026-07-30T21:39:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Tests Gate", + "started_at": "2026-07-30T21:39:41Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Test-Failure Ratchet Gate", + "started_at": "2026-07-30T21:39:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 1/1)", + "started_at": "2026-07-30T21:37:52Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Detect Changes", + "started_at": "2026-07-30T21:36:59Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T21:33:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T21:33:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T21:33:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Silent-Skip Guard (OMN-14172)", + "started_at": "2026-07-30T21:32:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Migration Integration Test", + "started_at": "2026-07-30T21:32:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T21:32:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Version Pin Compliance", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "zone-filter / Zone Filter (docs-only check)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Drift Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Compliance Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "OCC Born-Path Trigger Coverage (OMN-14987)", + "started_at": "2026-07-30T21:32:36Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Kafka Boundary Compat (OMN-3256)", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Kafka Schema Handshake (OMN-3411)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Naming Lint", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Compliance", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Path Pre-Flight", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Compose Required-Env Coverage (OMN-5439)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Migration Freeze Check", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Arch Invariants (OMN-3343)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Demo Loop Gate", + "started_at": "2026-07-30T21:32:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "AI-Slop Pattern Check (strict, PR diff)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Cross-Repo Migration Conflicts", + "started_at": "2026-07-30T21:32:30Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Writer-Migration Coupling Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Sync Gate (Wave C) [OMN-8915]", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Plugin ENV->Service Completeness (OMN-4313)", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Effect-Assertion Gate (RT-5)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Enum Drift Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "ONEX Validators", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Infra Node Handler Ownership", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Fingerprint Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Lint", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T21:32:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CodeQL", + "started_at": "2026-07-30T21:31:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T21:32:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T21:31:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T21:31:50Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T21:31:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T21:31:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "merge-hold-gate / evaluate", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Deploy Agent Tests (OMN-15378) / deploy-agent-tests", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "no-noncanonical-lifecycle-classes", + "started_at": "2026-07-30T21:31:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runtime Profiles / validate", + "started_at": "2026-07-30T21:31:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Summary", + "started_at": "2026-07-30T21:31:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T21:31:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Enforce validator-requirements.yaml (OMN-13291)", + "started_at": "2026-07-30T21:31:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "OCC Companion Merged Gate (OMN-15214)", + "started_at": "2026-07-30T21:31:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T21:31:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T21:31:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T21:31:40Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T21:31:26Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T21:31:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T21:31:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T21:31:40Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T21:31:46Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T21:31:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T21:31:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T21:31:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T21:31:40Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T21:31:54Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T21:31:36Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T21:31:36Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T21:31:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T21:31:42Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T21:31:37Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T21:31:31Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T21:31:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T21:31:31Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T21:31:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "TODO Audit", + "started_at": "2026-07-30T21:31:11Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "auto-tag", + "started_at": "2026-07-30T21:31:06Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Trigger node_redeploy Start", + "started_at": "2026-07-30T21:31:05Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Publish PR Merged Event", + "started_at": "2026-07-30T21:31:05Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:31:09Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T21:31:13Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T21:30:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T21:30:42Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T21:29:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T21:29:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T21:28:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T21:29:00Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T21:28:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-30T21:28:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T21:28:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T21:28:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T21:28:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T21:28:45Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T21:28:52Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T21:28:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T21:28:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:28:46Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T21:28:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T21:29:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T21:29:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T21:29:11Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T21:28:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T21:28:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T21:28:44Z", + "status": "completed" + }, + { + "conclusion": "cancelled", + "name": "CodeQL", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T21:28:58Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T21:28:30Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T21:28:58Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T21:28:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T21:28:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T21:28:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T21:28:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T21:28:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T21:28:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T21:28:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T21:28:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "TODO Audit", + "started_at": "2026-07-30T21:28:20Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Publish PR Merged Event", + "started_at": "2026-07-30T21:28:20Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "auto-tag", + "started_at": "2026-07-30T21:28:20Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:28:29Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Trigger node_redeploy Start", + "started_at": "2026-07-30T21:28:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T19:24:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T19:23:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T19:23:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T19:22:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T19:22:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T19:22:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T19:18:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T19:18:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T19:17:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T19:17:20Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T19:17:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T19:17:25Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T19:17:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T19:17:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T19:17:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T19:17:04Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T19:17:08Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T19:15:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-30T19:16:05Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T19:16:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T19:15:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T19:16:12Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T19:15:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CodeQL", + "started_at": "2026-07-30T19:16:20Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T19:16:11Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T19:16:08Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T19:16:09Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T19:16:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T19:16:04Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T19:16:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T19:16:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T19:16:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T19:16:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T19:15:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T19:16:05Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T19:16:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T19:16:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T19:15:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T19:16:12Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T19:16:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T19:16:13Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T19:15:55Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T19:16:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T19:16:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T19:15:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T19:15:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T19:16:17Z", + "status": "completed" + } + ] +} diff --git a/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json b/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json new file mode 100644 index 0000000000..0c428a268a --- /dev/null +++ b/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json @@ -0,0 +1,10 @@ +{ + "state": "success", + "statuses": [ + { + "context": "CodeRabbit", + "created_at": "2026-07-30T19:15:58Z", + "state": "success" + } + ] +} diff --git a/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json b/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json new file mode 100644 index 0000000000..7086402a1a --- /dev/null +++ b/tests/fixtures/omn15550/healed/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json @@ -0,0 +1,12 @@ +[ + { + "base": { + "ref": "dev" + }, + "draft": false, + "head": { + "sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b" + }, + "number": 2575 + } +] diff --git a/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_39c888104009e884a845c9ca971fd0314e836e55_per_page_100.json b/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_39c888104009e884a845c9ca971fd0314e836e55_per_page_100.json new file mode 100644 index 0000000000..518ab4968c --- /dev/null +++ b/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_39c888104009e884a845c9ca971fd0314e836e55_per_page_100.json @@ -0,0 +1,484 @@ +{ + "workflow_runs": [ + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035737, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T23:17:40Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035755, + "name": "Main target guard", + "status": "completed", + "updated_at": "2026-07-30T23:17:42Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035745, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T23:17:33Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035979, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T23:17:34Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035789, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T23:18:07Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035771, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T23:18:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590036005, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T23:18:23Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590036376, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T23:18:40Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:16:59Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590014922, + "name": "OCC Companion Effect Publisher", + "status": "completed", + "updated_at": "2026-07-30T23:17:13Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:16:59Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590014991, + "name": "Required-Check Skip-Vector Guard", + "status": "completed", + "updated_at": "2026-07-30T23:17:13Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:16:59Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590014525, + "name": "product-readiness-shadow", + "status": "completed", + "updated_at": "2026-07-30T23:18:58Z" + }, + { + "conclusion": null, + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542895, + "name": "Deploy Gate", + "status": "in_progress", + "updated_at": "2026-07-30T23:56:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542724, + "name": "Contract Validation", + "status": "completed", + "updated_at": "2026-07-30T23:08:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542793, + "name": "Docker Build", + "status": "completed", + "updated_at": "2026-07-30T23:13:29Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542744, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T23:08:19Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542760, + "name": "Integration Test Check", + "status": "completed", + "updated_at": "2026-07-30T23:08:32Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542701, + "name": "Check Architecture Handshake", + "status": "completed", + "updated_at": "2026-07-30T23:08:33Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542735, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T23:08:26Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589543012, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T23:08:26Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542878, + "name": "OCC Companion Effect Publisher", + "status": "completed", + "updated_at": "2026-07-30T23:08:45Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542990, + "name": "Required-Check Skip-Vector Guard", + "status": "completed", + "updated_at": "2026-07-30T23:08:39Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542694, + "name": "receipt-honesty", + "status": "completed", + "updated_at": "2026-07-30T23:08:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542797, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T23:08:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542740, + "name": "Shell Hygiene Gate (OMN-14761)", + "status": "completed", + "updated_at": "2026-07-30T23:08:38Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542874, + "name": "docs-validate", + "status": "completed", + "updated_at": "2026-07-30T23:08:42Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542754, + "name": "Dispatcher Route Coverage", + "status": "completed", + "updated_at": "2026-07-30T23:08:43Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542885, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T23:08:35Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542734, + "name": "No bare compose teardown (OMN-13886)", + "status": "completed", + "updated_at": "2026-07-30T23:08:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542705, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T23:08:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542985, + "name": "Imperative Contract Guard", + "status": "completed", + "updated_at": "2026-07-30T23:08:55Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542725, + "name": "node-migration-sync", + "status": "completed", + "updated_at": "2026-07-30T23:08:59Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542953, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T23:26:08Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542695, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T23:25:58Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589543315, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T23:26:20Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542914, + "name": "Hostile Reviewer", + "status": "completed", + "updated_at": "2026-07-30T23:27:10Z" + }, + { + "conclusion": null, + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589543003, + "name": "CI", + "status": "in_progress", + "updated_at": "2026-07-31T00:06:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542676, + "name": "Omni* Ecosystem Standards Compliance", + "status": "completed", + "updated_at": "2026-07-30T23:09:51Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542683, + "name": "product-readiness-shadow", + "status": "completed", + "updated_at": "2026-07-30T23:10:36Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542661, + "name": "Security Scan", + "status": "completed", + "updated_at": "2026-07-30T23:13:17Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542665, + "name": "Main target guard", + "status": "completed", + "updated_at": "2026-07-30T23:08:25Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542573, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T23:08:30Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542668, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T23:08:33Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542647, + "name": "Dep Provenance Gate", + "status": "completed", + "updated_at": "2026-07-30T23:08:44Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542598, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T23:08:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542611, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T23:08:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542681, + "name": "fresh-deploy-fitness", + "status": "completed", + "updated_at": "2026-07-30T23:08:59Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542585, + "name": "skill-node-mapping-sync", + "status": "completed", + "updated_at": "2026-07-30T23:09:01Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542571, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T23:09:07Z" + } + ] +} diff --git a/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json b/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json new file mode 100644 index 0000000000..f7f8c02e97 --- /dev/null +++ b/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json @@ -0,0 +1,6 @@ +{ + "contexts": [ + "CI Summary" + ], + "strict": false +} diff --git a/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_check_runs_per_page_100.json b/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_check_runs_per_page_100.json new file mode 100644 index 0000000000..574adc0a18 --- /dev/null +++ b/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_check_runs_per_page_100.json @@ -0,0 +1,760 @@ +{ + "check_runs": [ + { + "conclusion": "success", + "name": "Runtime Profiles / validate", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Deploy Agent Tests (OMN-15378) / deploy-agent-tests", + "started_at": "2026-07-30T23:08:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "merge-hold-gate / evaluate", + "started_at": "2026-07-30T23:08:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T23:25:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Enforce validator-requirements.yaml (OMN-13291)", + "started_at": "2026-07-30T23:08:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Test-Failure Ratchet Gate", + "started_at": "2026-07-30T23:34:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Tests Gate", + "started_at": "2026-07-30T23:34:01Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 14/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 13/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 15/15)", + "started_at": "2026-07-30T23:29:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 12/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 11/15)", + "started_at": "2026-07-30T23:29:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 9/15)", + "started_at": "2026-07-30T23:29:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 4/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 10/15)", + "started_at": "2026-07-30T23:29:52Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 8/15)", + "started_at": "2026-07-30T23:29:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 5/15)", + "started_at": "2026-07-30T23:29:55Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 7/15)", + "started_at": "2026-07-30T23:29:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 6/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 1/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 3/15)", + "started_at": "2026-07-30T23:29:54Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Silent-Skip Guard (OMN-14172)", + "started_at": "2026-07-30T23:28:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 2/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Migration Integration Test", + "started_at": "2026-07-30T23:28:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Detect Changes", + "started_at": "2026-07-30T23:29:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Cross-Repo Migration Conflicts", + "started_at": "2026-07-30T23:26:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Migration Freeze Check", + "started_at": "2026-07-30T23:28:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "OCC Born-Path Trigger Coverage (OMN-14987)", + "started_at": "2026-07-30T23:27:18Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Demo Loop Gate", + "started_at": "2026-07-30T23:26:26Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Path Pre-Flight", + "started_at": "2026-07-30T23:28:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Compliance Check", + "started_at": "2026-07-30T23:27:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Plugin ENV->Service Completeness (OMN-4313)", + "started_at": "2026-07-30T23:26:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Enum Drift Check", + "started_at": "2026-07-30T23:26:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Writer-Migration Coupling Check", + "started_at": "2026-07-30T23:27:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Naming Lint", + "started_at": "2026-07-30T23:28:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Sync Gate (Wave C) [OMN-8915]", + "started_at": "2026-07-30T23:26:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Effect-Assertion Gate (RT-5)", + "started_at": "2026-07-30T23:27:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Arch Invariants (OMN-3343)", + "started_at": "2026-07-30T23:28:11Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Version Pin Compliance", + "started_at": "2026-07-30T23:27:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Lint", + "started_at": "2026-07-30T23:26:18Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Compose Required-Env Coverage (OMN-5439)", + "started_at": "2026-07-30T23:26:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Compliance", + "started_at": "2026-07-30T23:28:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Fingerprint Check", + "started_at": "2026-07-30T23:26:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Kafka Schema Handshake (OMN-3411)", + "started_at": "2026-07-30T23:26:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Drift Check", + "started_at": "2026-07-30T23:28:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "AI-Slop Pattern Check (strict, PR diff)", + "started_at": "2026-07-30T23:27:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "ONEX Validators", + "started_at": "2026-07-30T23:26:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Infra Node Handler Ownership", + "started_at": "2026-07-30T23:26:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Kafka Boundary Compat (OMN-3256)", + "started_at": "2026-07-30T23:28:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "zone-filter / Zone Filter (docs-only check)", + "started_at": "2026-07-30T23:26:37Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "OCC Companion Merged Gate (OMN-15214)", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "no-noncanonical-lifecycle-classes", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T23:27:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T23:26:13Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T23:26:04Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T23:25:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T23:25:18Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T23:25:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T23:25:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T23:18:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T23:18:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T23:17:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T23:17:31Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T23:17:31Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T23:17:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T23:17:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T23:17:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T23:17:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T23:17:28Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T23:17:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T23:17:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T23:17:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T23:17:09Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T23:17:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T23:17:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Build Summary", + "started_at": "2026-07-30T23:13:19Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Docker Integration Tests", + "started_at": "2026-07-30T23:12:44Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Security Scan (Trivy)", + "started_at": "2026-07-30T23:12:44Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Image Size Analysis", + "started_at": "2026-07-30T23:12:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Attest Source Hash (OMN-9139) / Verify source hash attestation", + "started_at": "2026-07-30T23:12:46Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T23:10:26Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T23:09:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Build Runtime Image", + "started_at": "2026-07-30T23:08:47Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T23:08:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T23:08:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T23:08:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T23:08:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T23:08:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T23:08:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T23:08:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Validate Dockerfile Plugin Pins", + "started_at": "2026-07-30T23:08:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T23:08:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T23:08:30Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T23:08:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T23:08:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T23:08:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CodeQL", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T23:08:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + } + ] +} diff --git a/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_status_per_page_100.json b/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_status_per_page_100.json new file mode 100644 index 0000000000..b5486d2824 --- /dev/null +++ b/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_status_per_page_100.json @@ -0,0 +1,10 @@ +{ + "state": "success", + "statuses": [ + { + "context": "CodeRabbit", + "created_at": "2026-07-30T23:17:06Z", + "state": "success" + } + ] +} diff --git a/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json b/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json new file mode 100644 index 0000000000..4405567c3a --- /dev/null +++ b/tests/fixtures/omn15550/late_start/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json @@ -0,0 +1,12 @@ +[ + { + "base": { + "ref": "dev" + }, + "draft": false, + "head": { + "sha": "39c888104009e884a845c9ca971fd0314e836e55" + }, + "number": 2581 + } +] diff --git a/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json new file mode 100644 index 0000000000..ca954ad092 --- /dev/null +++ b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json @@ -0,0 +1,1004 @@ +{ + "workflow_runs": [ + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768690, + "name": "Auto-Tag on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:47:28Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768467, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:47:48Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768504, + "name": "TODO Audit on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:47:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768458, + "name": "Runtime Rebuild Trigger", + "status": "completed", + "updated_at": "2026-07-30T21:48:32Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768457, + "name": "PR Merged Event Producer", + "status": "completed", + "updated_at": "2026-07-30T21:48:29Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727782, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T21:32:45Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727222, + "name": "Hostile Reviewer", + "status": "completed", + "updated_at": "2026-07-30T21:33:36Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727021, + "name": "product-readiness-shadow", + "status": "completed", + "updated_at": "2026-07-30T21:33:31Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727328, + "name": "CI", + "status": "completed", + "updated_at": "2026-07-30T21:40:00Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727411, + "name": "OCC Companion Effect Publisher", + "status": "completed", + "updated_at": "2026-07-30T21:31:44Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727000, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:31:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727249, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T21:31:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727234, + "name": "Imperative Contract Guard", + "status": "completed", + "updated_at": "2026-07-30T21:32:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727313, + "name": "docs-validate", + "status": "completed", + "updated_at": "2026-07-30T21:32:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727335, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T21:31:57Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727047, + "name": "No bare compose teardown (OMN-13886)", + "status": "completed", + "updated_at": "2026-07-30T21:32:00Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727320, + "name": "Deploy Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:01Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727282, + "name": "Required-Check Skip-Vector Guard", + "status": "completed", + "updated_at": "2026-07-30T21:31:56Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727278, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:38Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726730, + "name": "Omni* Ecosystem Standards Compliance", + "status": "completed", + "updated_at": "2026-07-30T21:33:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726962, + "name": "Security Scan", + "status": "completed", + "updated_at": "2026-07-30T21:38:18Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726938, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T21:31:41Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726907, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:42Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726936, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:40Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726803, + "name": "Dep Provenance Gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:44Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726770, + "name": "Dispatcher Route Coverage", + "status": "completed", + "updated_at": "2026-07-30T21:31:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726784, + "name": "Shell Hygiene Gate (OMN-14761)", + "status": "completed", + "updated_at": "2026-07-30T21:31:46Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726946, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T21:31:48Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726967, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T21:31:27Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726747, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:38Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726930, + "name": "skill-node-mapping-sync", + "status": "completed", + "updated_at": "2026-07-30T21:32:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726900, + "name": "receipt-honesty", + "status": "completed", + "updated_at": "2026-07-30T21:32:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726899, + "name": "Integration Test Check", + "status": "completed", + "updated_at": "2026-07-30T21:32:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726856, + "name": "fresh-deploy-fitness", + "status": "completed", + "updated_at": "2026-07-30T21:32:10Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726866, + "name": "node-migration-sync", + "status": "completed", + "updated_at": "2026-07-30T21:32:03Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726943, + "name": "Check Architecture Handshake", + "status": "completed", + "updated_at": "2026-07-30T21:31:57Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726893, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:29Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726986, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:25Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726966, + "name": "Contract Validation", + "status": "completed", + "updated_at": "2026-07-30T21:32:20Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704580, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:31:18Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704517, + "name": "Runtime Rebuild Trigger", + "status": "completed", + "updated_at": "2026-07-30T21:31:06Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704315, + "name": "TODO Audit on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:31:05Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704769, + "name": "Auto-Tag on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:31:06Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704541, + "name": "PR Merged Event Producer", + "status": "completed", + "updated_at": "2026-07-30T21:31:14Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:32Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583529379, + "name": "docs-validate", + "status": "completed", + "updated_at": "2026-07-30T21:29:04Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:32Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528727, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T21:30:03Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527959, + "name": "OCC Companion Effect Publisher", + "status": "completed", + "updated_at": "2026-07-30T21:28:45Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528511, + "name": "Deploy Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:04Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527536, + "name": "Imperative Contract Guard", + "status": "completed", + "updated_at": "2026-07-30T21:29:27Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528008, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:32Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527982, + "name": "Hostile Reviewer", + "status": "completed", + "updated_at": "2026-07-30T21:30:53Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528083, + "name": "Required-Check Skip-Vector Guard", + "status": "completed", + "updated_at": "2026-07-30T21:28:59Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527480, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T21:28:55Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527641, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T21:28:55Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526350, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T21:28:31Z" + }, + { + "conclusion": "failure", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527197, + "name": ".github/workflows/ci.yml", + "status": "completed", + "updated_at": "2026-07-30T21:28:30Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526328, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T21:28:36Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526975, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T21:28:45Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526372, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T21:28:41Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526428, + "name": "Check Architecture Handshake", + "status": "completed", + "updated_at": "2026-07-30T21:28:44Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526501, + "name": "Dep Provenance Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:02Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526267, + "name": "node-migration-sync", + "status": "completed", + "updated_at": "2026-07-30T21:29:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526653, + "name": "receipt-honesty", + "status": "completed", + "updated_at": "2026-07-30T21:29:22Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526321, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:26Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526326, + "name": "Contract Validation", + "status": "completed", + "updated_at": "2026-07-30T21:29:01Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526616, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526405, + "name": "fresh-deploy-fitness", + "status": "completed", + "updated_at": "2026-07-30T21:29:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526568, + "name": "skill-node-mapping-sync", + "status": "completed", + "updated_at": "2026-07-30T21:29:35Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526360, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:31Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526420, + "name": "product-readiness-shadow", + "status": "completed", + "updated_at": "2026-07-30T21:30:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526998, + "name": "Omni* Ecosystem Standards Compliance", + "status": "completed", + "updated_at": "2026-07-30T21:31:17Z" + }, + { + "conclusion": "cancelled", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526601, + "name": "Security Scan", + "status": "completed", + "updated_at": "2026-07-30T21:31:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526986, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T21:28:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526269, + "name": "Shell Hygiene Gate (OMN-14761)", + "status": "completed", + "updated_at": "2026-07-30T21:28:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527092, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:28:53Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526270, + "name": "Dispatcher Route Coverage", + "status": "completed", + "updated_at": "2026-07-30T21:28:53Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526377, + "name": "Integration Test Check", + "status": "completed", + "updated_at": "2026-07-30T21:28:55Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526987, + "name": "No bare compose teardown (OMN-13886)", + "status": "completed", + "updated_at": "2026-07-30T21:28:56Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515034, + "name": "Runtime Rebuild Trigger", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515192, + "name": "Auto-Tag on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515354, + "name": "TODO Audit on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515225, + "name": "PR Merged Event Producer", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515230, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:28:39Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145773, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T19:17:26Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574146135, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T19:18:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145723, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T19:18:18Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145227, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T19:17:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145181, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T19:17:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145170, + "name": "Main target guard", + "status": "completed", + "updated_at": "2026-07-30T19:17:14Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145349, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T19:17:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145140, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T19:17:46Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:49Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058841, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T19:16:07Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:49Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574059012, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T19:23:08Z" + }, + { + "conclusion": "failure", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058377, + "name": ".github/workflows/ci.yml", + "status": "completed", + "updated_at": "2026-07-30T19:15:48Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058150, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T19:15:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057942, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057936, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T19:16:04Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057920, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:05Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057886, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057926, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T19:16:07Z" + } + ] +} diff --git a/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json new file mode 100644 index 0000000000..f7f8c02e97 --- /dev/null +++ b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json @@ -0,0 +1,6 @@ +{ + "contexts": [ + "CI Summary" + ], + "strict": false +} diff --git a/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json new file mode 100644 index 0000000000..da795e79f8 --- /dev/null +++ b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json @@ -0,0 +1,605 @@ +{ + "__next__": "/repos/OmniNode-ai/omnibase_infra/commits/2173c0c53a46624be1efd4cff91cd07f3f8df13b/check-runs?per_page=100&page=2", + "check_runs": [ + { + "conclusion": "skipped", + "name": "auto-tag", + "started_at": "2026-07-30T21:47:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "TODO Audit", + "started_at": "2026-07-30T21:47:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:47:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Trigger node_redeploy Start", + "started_at": "2026-07-30T21:47:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Merged Event", + "started_at": "2026-07-30T21:47:28Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Runtime Boot Smoke (compose)", + "started_at": "2026-07-30T21:39:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Tests Gate", + "started_at": "2026-07-30T21:39:41Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Test-Failure Ratchet Gate", + "started_at": "2026-07-30T21:39:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 1/1)", + "started_at": "2026-07-30T21:37:52Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Detect Changes", + "started_at": "2026-07-30T21:36:59Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T21:33:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T21:33:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T21:33:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Silent-Skip Guard (OMN-14172)", + "started_at": "2026-07-30T21:32:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Migration Integration Test", + "started_at": "2026-07-30T21:32:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T21:32:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Version Pin Compliance", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "zone-filter / Zone Filter (docs-only check)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Drift Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Compliance Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "OCC Born-Path Trigger Coverage (OMN-14987)", + "started_at": "2026-07-30T21:32:36Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Kafka Boundary Compat (OMN-3256)", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Kafka Schema Handshake (OMN-3411)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Naming Lint", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Compliance", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Path Pre-Flight", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Compose Required-Env Coverage (OMN-5439)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Migration Freeze Check", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Arch Invariants (OMN-3343)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Demo Loop Gate", + "started_at": "2026-07-30T21:32:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "AI-Slop Pattern Check (strict, PR diff)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Cross-Repo Migration Conflicts", + "started_at": "2026-07-30T21:32:30Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Writer-Migration Coupling Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Sync Gate (Wave C) [OMN-8915]", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Plugin ENV->Service Completeness (OMN-4313)", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Effect-Assertion Gate (RT-5)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Enum Drift Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "ONEX Validators", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Infra Node Handler Ownership", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Fingerprint Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Lint", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T21:32:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CodeQL", + "started_at": "2026-07-30T21:31:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T21:32:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T21:31:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T21:31:50Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T21:31:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T21:31:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "merge-hold-gate / evaluate", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Deploy Agent Tests (OMN-15378) / deploy-agent-tests", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "no-noncanonical-lifecycle-classes", + "started_at": "2026-07-30T21:31:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runtime Profiles / validate", + "started_at": "2026-07-30T21:31:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T21:31:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Enforce validator-requirements.yaml (OMN-13291)", + "started_at": "2026-07-30T21:31:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "OCC Companion Merged Gate (OMN-15214)", + "started_at": "2026-07-30T21:31:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T21:31:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T21:31:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T21:31:40Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T21:31:26Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T21:31:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T21:31:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T21:31:40Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T21:31:46Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T21:31:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T21:31:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T21:31:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T21:31:40Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T21:31:54Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T21:31:36Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T21:31:36Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T21:31:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T21:31:42Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T21:31:37Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T21:31:31Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T21:31:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T21:31:31Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T21:31:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "TODO Audit", + "started_at": "2026-07-30T21:31:11Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "auto-tag", + "started_at": "2026-07-30T21:31:06Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Trigger node_redeploy Start", + "started_at": "2026-07-30T21:31:05Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Publish PR Merged Event", + "started_at": "2026-07-30T21:31:05Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:31:09Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T21:31:13Z", + "status": "completed" + } + ] +} diff --git a/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100_page_2.json b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100_page_2.json new file mode 100644 index 0000000000..0ecc8b77d6 --- /dev/null +++ b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100_page_2.json @@ -0,0 +1,700 @@ +{ + "check_runs": [ + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T21:30:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T21:30:42Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T21:29:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T21:29:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T21:28:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T21:29:00Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T21:28:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-30T21:28:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T21:28:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T21:28:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T21:28:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T21:28:45Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T21:28:52Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T21:28:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T21:28:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:28:46Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T21:28:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T21:29:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T21:29:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T21:29:11Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T21:28:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T21:28:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T21:28:44Z", + "status": "completed" + }, + { + "conclusion": "cancelled", + "name": "CodeQL", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T21:28:58Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T21:28:30Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T21:28:58Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T21:28:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T21:28:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T21:28:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T21:28:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T21:28:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T21:28:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T21:28:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T21:28:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "TODO Audit", + "started_at": "2026-07-30T21:28:20Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Publish PR Merged Event", + "started_at": "2026-07-30T21:28:20Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "auto-tag", + "started_at": "2026-07-30T21:28:20Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:28:29Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Trigger node_redeploy Start", + "started_at": "2026-07-30T21:28:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T19:24:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T19:23:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T19:23:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T19:22:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T19:22:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T19:22:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T19:18:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T19:18:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T19:17:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T19:17:20Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T19:17:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T19:17:25Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T19:17:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T19:17:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T19:17:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T19:17:04Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T19:17:08Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T19:15:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-30T19:16:05Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T19:16:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T19:15:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T19:16:12Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T19:15:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CodeQL", + "started_at": "2026-07-30T19:16:20Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T19:16:11Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T19:16:08Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T19:16:09Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T19:16:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T19:16:04Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T19:16:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T19:16:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T19:16:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T19:16:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T19:15:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T19:16:05Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T19:16:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T19:16:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T19:15:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T19:16:12Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T19:16:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T19:16:13Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T19:15:55Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T19:16:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T19:16:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T19:15:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T19:15:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T19:16:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Summary", + "started_at": "2026-07-30T21:31:53Z", + "status": "completed" + } + ] +} diff --git a/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json new file mode 100644 index 0000000000..0c428a268a --- /dev/null +++ b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json @@ -0,0 +1,10 @@ +{ + "state": "success", + "statuses": [ + { + "context": "CodeRabbit", + "created_at": "2026-07-30T19:15:58Z", + "state": "success" + } + ] +} diff --git a/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json new file mode 100644 index 0000000000..7086402a1a --- /dev/null +++ b/tests/fixtures/omn15550/paginated/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json @@ -0,0 +1,12 @@ +[ + { + "base": { + "ref": "dev" + }, + "draft": false, + "head": { + "sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b" + }, + "number": 2575 + } +] diff --git a/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_39c888104009e884a845c9ca971fd0314e836e55_per_page_100.json b/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_39c888104009e884a845c9ca971fd0314e836e55_per_page_100.json new file mode 100644 index 0000000000..d91537faa3 --- /dev/null +++ b/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_39c888104009e884a845c9ca971fd0314e836e55_per_page_100.json @@ -0,0 +1,564 @@ +{ + "workflow_runs": [ + { + "conclusion": "success", + "created_at": "2026-07-30T23:55:04Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30591979006, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T23:55:18Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:55:04Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30591979240, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T23:55:12Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:55:04Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30591979328, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T23:56:07Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:55:04Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30591979615, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T23:56:01Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:55:03Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30591978966, + "name": "Main target guard", + "status": "completed", + "updated_at": "2026-07-30T23:55:13Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:55:03Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30591978972, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T23:55:10Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:55:03Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30591978962, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T23:55:43Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:55:03Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30591978979, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T23:55:48Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035737, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T23:17:40Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035755, + "name": "Main target guard", + "status": "completed", + "updated_at": "2026-07-30T23:17:42Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035745, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T23:17:33Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035979, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T23:17:34Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035789, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T23:18:07Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590035771, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T23:18:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590036005, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T23:18:23Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:17:25Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590036376, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T23:18:40Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:16:59Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590014922, + "name": "OCC Companion Effect Publisher", + "status": "completed", + "updated_at": "2026-07-30T23:17:13Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:16:59Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590014991, + "name": "Required-Check Skip-Vector Guard", + "status": "completed", + "updated_at": "2026-07-30T23:17:13Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:16:59Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30590014525, + "name": "product-readiness-shadow", + "status": "completed", + "updated_at": "2026-07-30T23:18:58Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542895, + "name": "Deploy Gate", + "status": "completed", + "updated_at": "2026-07-30T23:56:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542724, + "name": "Contract Validation", + "status": "completed", + "updated_at": "2026-07-30T23:08:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542793, + "name": "Docker Build", + "status": "completed", + "updated_at": "2026-07-30T23:13:29Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542744, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T23:08:19Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542760, + "name": "Integration Test Check", + "status": "completed", + "updated_at": "2026-07-30T23:08:32Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542701, + "name": "Check Architecture Handshake", + "status": "completed", + "updated_at": "2026-07-30T23:08:33Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542735, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T23:08:26Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589543012, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T23:08:26Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542878, + "name": "OCC Companion Effect Publisher", + "status": "completed", + "updated_at": "2026-07-30T23:08:45Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542990, + "name": "Required-Check Skip-Vector Guard", + "status": "completed", + "updated_at": "2026-07-30T23:08:39Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542694, + "name": "receipt-honesty", + "status": "completed", + "updated_at": "2026-07-30T23:08:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542797, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T23:08:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542740, + "name": "Shell Hygiene Gate (OMN-14761)", + "status": "completed", + "updated_at": "2026-07-30T23:08:38Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542874, + "name": "docs-validate", + "status": "completed", + "updated_at": "2026-07-30T23:08:42Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542754, + "name": "Dispatcher Route Coverage", + "status": "completed", + "updated_at": "2026-07-30T23:08:43Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542885, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T23:08:35Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542734, + "name": "No bare compose teardown (OMN-13886)", + "status": "completed", + "updated_at": "2026-07-30T23:08:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542705, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T23:08:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542985, + "name": "Imperative Contract Guard", + "status": "completed", + "updated_at": "2026-07-30T23:08:55Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542725, + "name": "node-migration-sync", + "status": "completed", + "updated_at": "2026-07-30T23:08:59Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542953, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T23:26:08Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542695, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T23:25:58Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589543315, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T23:26:20Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542914, + "name": "Hostile Reviewer", + "status": "completed", + "updated_at": "2026-07-30T23:27:10Z" + }, + { + "conclusion": null, + "created_at": "2026-07-30T23:08:19Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589543003, + "name": "CI", + "status": "in_progress", + "updated_at": "2026-07-31T00:06:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542676, + "name": "Omni* Ecosystem Standards Compliance", + "status": "completed", + "updated_at": "2026-07-30T23:09:51Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542683, + "name": "product-readiness-shadow", + "status": "completed", + "updated_at": "2026-07-30T23:10:36Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542661, + "name": "Security Scan", + "status": "completed", + "updated_at": "2026-07-30T23:13:17Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542665, + "name": "Main target guard", + "status": "completed", + "updated_at": "2026-07-30T23:08:25Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542573, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T23:08:30Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542668, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T23:08:33Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542647, + "name": "Dep Provenance Gate", + "status": "completed", + "updated_at": "2026-07-30T23:08:44Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542598, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T23:08:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542611, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T23:08:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542681, + "name": "fresh-deploy-fitness", + "status": "completed", + "updated_at": "2026-07-30T23:08:59Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542585, + "name": "skill-node-mapping-sync", + "status": "completed", + "updated_at": "2026-07-30T23:09:01Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T23:08:18Z", + "head_branch": "jonah/omn-15520-cost-propagation", + "head_sha": "39c888104009e884a845c9ca971fd0314e836e55", + "id": 30589542571, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T23:09:07Z" + } + ] +} diff --git a/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json b/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json new file mode 100644 index 0000000000..f7f8c02e97 --- /dev/null +++ b/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json @@ -0,0 +1,6 @@ +{ + "contexts": [ + "CI Summary" + ], + "strict": false +} diff --git a/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_check_runs_per_page_100.json b/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_check_runs_per_page_100.json new file mode 100644 index 0000000000..e8f3644f00 --- /dev/null +++ b/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_check_runs_per_page_100.json @@ -0,0 +1,832 @@ +{ + "check_runs": [ + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T23:55:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runtime Profiles / validate", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Deploy Agent Tests (OMN-15378) / deploy-agent-tests", + "started_at": "2026-07-30T23:08:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "merge-hold-gate / evaluate", + "started_at": "2026-07-30T23:08:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T23:25:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Enforce validator-requirements.yaml (OMN-13291)", + "started_at": "2026-07-30T23:08:24Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Runtime Boot Smoke (compose)", + "started_at": "2026-07-30T23:55:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Test-Failure Ratchet Gate", + "started_at": "2026-07-30T23:34:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Tests Gate", + "started_at": "2026-07-30T23:34:01Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 14/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 13/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 15/15)", + "started_at": "2026-07-30T23:29:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 12/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 11/15)", + "started_at": "2026-07-30T23:29:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 9/15)", + "started_at": "2026-07-30T23:29:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 4/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 10/15)", + "started_at": "2026-07-30T23:29:52Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 8/15)", + "started_at": "2026-07-30T23:29:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 5/15)", + "started_at": "2026-07-30T23:29:55Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 7/15)", + "started_at": "2026-07-30T23:29:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 6/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 1/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 3/15)", + "started_at": "2026-07-30T23:29:54Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Silent-Skip Guard (OMN-14172)", + "started_at": "2026-07-30T23:28:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 2/15)", + "started_at": "2026-07-30T23:29:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Migration Integration Test", + "started_at": "2026-07-30T23:28:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Detect Changes", + "started_at": "2026-07-30T23:29:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Cross-Repo Migration Conflicts", + "started_at": "2026-07-30T23:26:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Migration Freeze Check", + "started_at": "2026-07-30T23:28:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "OCC Born-Path Trigger Coverage (OMN-14987)", + "started_at": "2026-07-30T23:27:18Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Demo Loop Gate", + "started_at": "2026-07-30T23:26:26Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Path Pre-Flight", + "started_at": "2026-07-30T23:28:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Compliance Check", + "started_at": "2026-07-30T23:27:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Plugin ENV->Service Completeness (OMN-4313)", + "started_at": "2026-07-30T23:26:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Enum Drift Check", + "started_at": "2026-07-30T23:26:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Writer-Migration Coupling Check", + "started_at": "2026-07-30T23:27:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Naming Lint", + "started_at": "2026-07-30T23:28:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Sync Gate (Wave C) [OMN-8915]", + "started_at": "2026-07-30T23:26:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Effect-Assertion Gate (RT-5)", + "started_at": "2026-07-30T23:27:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Arch Invariants (OMN-3343)", + "started_at": "2026-07-30T23:28:11Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Version Pin Compliance", + "started_at": "2026-07-30T23:27:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Lint", + "started_at": "2026-07-30T23:26:18Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Compose Required-Env Coverage (OMN-5439)", + "started_at": "2026-07-30T23:26:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Compliance", + "started_at": "2026-07-30T23:28:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Fingerprint Check", + "started_at": "2026-07-30T23:26:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Kafka Schema Handshake (OMN-3411)", + "started_at": "2026-07-30T23:26:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Drift Check", + "started_at": "2026-07-30T23:28:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "AI-Slop Pattern Check (strict, PR diff)", + "started_at": "2026-07-30T23:27:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "ONEX Validators", + "started_at": "2026-07-30T23:26:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Infra Node Handler Ownership", + "started_at": "2026-07-30T23:26:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Kafka Boundary Compat (OMN-3256)", + "started_at": "2026-07-30T23:28:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "zone-filter / Zone Filter (docs-only check)", + "started_at": "2026-07-30T23:26:37Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "OCC Companion Merged Gate (OMN-15214)", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "no-noncanonical-lifecycle-classes", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": null, + "name": "CI Summary", + "started_at": "2026-07-30T23:55:52Z", + "status": "in_progress" + }, + { + "conclusion": "success", + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-30T23:55:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T23:55:08Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T23:55:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T23:55:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T23:55:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T23:55:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T23:55:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T23:55:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T23:55:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T23:27:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T23:26:13Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T23:26:04Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T23:25:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T23:25:18Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T23:25:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T23:25:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T23:18:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T23:18:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T23:17:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T23:17:31Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T23:17:31Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T23:17:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T23:17:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T23:17:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T23:17:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T23:17:28Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T23:17:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T23:17:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T23:17:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T23:17:09Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T23:17:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T23:17:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Build Summary", + "started_at": "2026-07-30T23:13:19Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Docker Integration Tests", + "started_at": "2026-07-30T23:12:44Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Security Scan (Trivy)", + "started_at": "2026-07-30T23:12:44Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Image Size Analysis", + "started_at": "2026-07-30T23:12:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Attest Source Hash (OMN-9139) / Verify source hash attestation", + "started_at": "2026-07-30T23:12:46Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T23:10:26Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T23:09:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Build Runtime Image", + "started_at": "2026-07-30T23:08:47Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T23:08:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T23:08:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T23:08:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T23:08:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T23:08:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T23:08:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T23:08:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Validate Dockerfile Plugin Pins", + "started_at": "2026-07-30T23:08:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T23:08:23Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T23:08:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T23:08:30Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T23:08:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T23:08:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T23:08:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CodeQL", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T23:08:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T23:08:22Z", + "status": "completed" + } + ] +} diff --git a/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_status_per_page_100.json b/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_status_per_page_100.json new file mode 100644 index 0000000000..b5486d2824 --- /dev/null +++ b/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_commits_39c888104009e884a845c9ca971fd0314e836e55_status_per_page_100.json @@ -0,0 +1,10 @@ +{ + "state": "success", + "statuses": [ + { + "context": "CodeRabbit", + "created_at": "2026-07-30T23:17:06Z", + "state": "success" + } + ] +} diff --git a/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json b/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json new file mode 100644 index 0000000000..4405567c3a --- /dev/null +++ b/tests/fixtures/omn15550/pending/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json @@ -0,0 +1,12 @@ +[ + { + "base": { + "ref": "dev" + }, + "draft": false, + "head": { + "sha": "39c888104009e884a845c9ca971fd0314e836e55" + }, + "number": 2581 + } +] diff --git a/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json b/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json new file mode 100644 index 0000000000..ca954ad092 --- /dev/null +++ b/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_actions_runs_head_sha_2173c0c53a46624be1efd4cff91cd07f3f8df13b_per_page_100.json @@ -0,0 +1,1004 @@ +{ + "workflow_runs": [ + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768690, + "name": "Auto-Tag on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:47:28Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768467, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:47:48Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768504, + "name": "TODO Audit on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:47:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768458, + "name": "Runtime Rebuild Trigger", + "status": "completed", + "updated_at": "2026-07-30T21:48:32Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:47:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30584768457, + "name": "PR Merged Event Producer", + "status": "completed", + "updated_at": "2026-07-30T21:48:29Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727782, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T21:32:45Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727222, + "name": "Hostile Reviewer", + "status": "completed", + "updated_at": "2026-07-30T21:33:36Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727021, + "name": "product-readiness-shadow", + "status": "completed", + "updated_at": "2026-07-30T21:33:31Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727328, + "name": "CI", + "status": "completed", + "updated_at": "2026-07-30T21:40:00Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727411, + "name": "OCC Companion Effect Publisher", + "status": "completed", + "updated_at": "2026-07-30T21:31:44Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727000, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:31:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727249, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T21:31:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727234, + "name": "Imperative Contract Guard", + "status": "completed", + "updated_at": "2026-07-30T21:32:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727313, + "name": "docs-validate", + "status": "completed", + "updated_at": "2026-07-30T21:32:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727335, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T21:31:57Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727047, + "name": "No bare compose teardown (OMN-13886)", + "status": "completed", + "updated_at": "2026-07-30T21:32:00Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727320, + "name": "Deploy Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:01Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727282, + "name": "Required-Check Skip-Vector Guard", + "status": "completed", + "updated_at": "2026-07-30T21:31:56Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:26Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583727278, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:38Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726730, + "name": "Omni* Ecosystem Standards Compliance", + "status": "completed", + "updated_at": "2026-07-30T21:33:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726962, + "name": "Security Scan", + "status": "completed", + "updated_at": "2026-07-30T21:38:18Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726938, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T21:31:41Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726907, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:42Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726936, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:40Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726803, + "name": "Dep Provenance Gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:44Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726770, + "name": "Dispatcher Route Coverage", + "status": "completed", + "updated_at": "2026-07-30T21:31:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726784, + "name": "Shell Hygiene Gate (OMN-14761)", + "status": "completed", + "updated_at": "2026-07-30T21:31:46Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726946, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T21:31:48Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726967, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T21:31:27Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726747, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T21:31:38Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726930, + "name": "skill-node-mapping-sync", + "status": "completed", + "updated_at": "2026-07-30T21:32:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726900, + "name": "receipt-honesty", + "status": "completed", + "updated_at": "2026-07-30T21:32:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726899, + "name": "Integration Test Check", + "status": "completed", + "updated_at": "2026-07-30T21:32:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726856, + "name": "fresh-deploy-fitness", + "status": "completed", + "updated_at": "2026-07-30T21:32:10Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726866, + "name": "node-migration-sync", + "status": "completed", + "updated_at": "2026-07-30T21:32:03Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726943, + "name": "Check Architecture Handshake", + "status": "completed", + "updated_at": "2026-07-30T21:31:57Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726893, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:29Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726986, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T21:32:25Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:25Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583726966, + "name": "Contract Validation", + "status": "completed", + "updated_at": "2026-07-30T21:32:20Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704580, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:31:18Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704517, + "name": "Runtime Rebuild Trigger", + "status": "completed", + "updated_at": "2026-07-30T21:31:06Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704315, + "name": "TODO Audit on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:31:05Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704769, + "name": "Auto-Tag on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:31:06Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:31:05Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583704541, + "name": "PR Merged Event Producer", + "status": "completed", + "updated_at": "2026-07-30T21:31:14Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:32Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583529379, + "name": "docs-validate", + "status": "completed", + "updated_at": "2026-07-30T21:29:04Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:32Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528727, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T21:30:03Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527959, + "name": "OCC Companion Effect Publisher", + "status": "completed", + "updated_at": "2026-07-30T21:28:45Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528511, + "name": "Deploy Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:04Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527536, + "name": "Imperative Contract Guard", + "status": "completed", + "updated_at": "2026-07-30T21:29:27Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528008, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:32Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527982, + "name": "Hostile Reviewer", + "status": "completed", + "updated_at": "2026-07-30T21:30:53Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583528083, + "name": "Required-Check Skip-Vector Guard", + "status": "completed", + "updated_at": "2026-07-30T21:28:59Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527480, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T21:28:55Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:31Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527641, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T21:28:55Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526350, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T21:28:31Z" + }, + { + "conclusion": "failure", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527197, + "name": ".github/workflows/ci.yml", + "status": "completed", + "updated_at": "2026-07-30T21:28:30Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526328, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T21:28:36Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526975, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T21:28:45Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526372, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T21:28:41Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526428, + "name": "Check Architecture Handshake", + "status": "completed", + "updated_at": "2026-07-30T21:28:44Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526501, + "name": "Dep Provenance Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:02Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526267, + "name": "node-migration-sync", + "status": "completed", + "updated_at": "2026-07-30T21:29:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526653, + "name": "receipt-honesty", + "status": "completed", + "updated_at": "2026-07-30T21:29:22Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526321, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:26Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526326, + "name": "Contract Validation", + "status": "completed", + "updated_at": "2026-07-30T21:29:01Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526616, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526405, + "name": "fresh-deploy-fitness", + "status": "completed", + "updated_at": "2026-07-30T21:29:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526568, + "name": "skill-node-mapping-sync", + "status": "completed", + "updated_at": "2026-07-30T21:29:35Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526360, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T21:29:31Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526420, + "name": "product-readiness-shadow", + "status": "completed", + "updated_at": "2026-07-30T21:30:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526998, + "name": "Omni* Ecosystem Standards Compliance", + "status": "completed", + "updated_at": "2026-07-30T21:31:17Z" + }, + { + "conclusion": "cancelled", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526601, + "name": "Security Scan", + "status": "completed", + "updated_at": "2026-07-30T21:31:37Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526986, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T21:28:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526269, + "name": "Shell Hygiene Gate (OMN-14761)", + "status": "completed", + "updated_at": "2026-07-30T21:28:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527092, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:28:53Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526270, + "name": "Dispatcher Route Coverage", + "status": "completed", + "updated_at": "2026-07-30T21:28:53Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526377, + "name": "Integration Test Check", + "status": "completed", + "updated_at": "2026-07-30T21:28:55Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583526987, + "name": "No bare compose teardown (OMN-13886)", + "status": "completed", + "updated_at": "2026-07-30T21:28:56Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515034, + "name": "Runtime Rebuild Trigger", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515192, + "name": "Auto-Tag on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515354, + "name": "TODO Audit on Merge", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515225, + "name": "PR Merged Event Producer", + "status": "completed", + "updated_at": "2026-07-30T21:28:20Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T21:28:19Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583515230, + "name": "Artifact Reconciliation Webhook", + "status": "completed", + "updated_at": "2026-07-30T21:28:39Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145773, + "name": "PR Title Check", + "status": "completed", + "updated_at": "2026-07-30T19:17:26Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574146135, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T19:18:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:17:00Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145723, + "name": "Receipt Gate", + "status": "completed", + "updated_at": "2026-07-30T19:18:18Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145227, + "name": "Duplication Sweep", + "status": "completed", + "updated_at": "2026-07-30T19:17:15Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145181, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T19:17:11Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145170, + "name": "Main target guard", + "status": "completed", + "updated_at": "2026-07-30T19:17:14Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145349, + "name": "URL Authority Gate", + "status": "completed", + "updated_at": "2026-07-30T19:17:52Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:16:59Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574145140, + "name": "Canonical Inference Gate", + "status": "completed", + "updated_at": "2026-07-30T19:17:46Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:49Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058841, + "name": "CR Thread Gate (caller)", + "status": "completed", + "updated_at": "2026-07-30T19:16:07Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:49Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574059012, + "name": "Reject skip-gate bypass tokens", + "status": "completed", + "updated_at": "2026-07-30T19:23:08Z" + }, + { + "conclusion": "failure", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058377, + "name": ".github/workflows/ci.yml", + "status": "completed", + "updated_at": "2026-07-30T19:15:48Z" + }, + { + "conclusion": "skipped", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574058150, + "name": "OmniGate", + "status": "completed", + "updated_at": "2026-07-30T19:15:49Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057942, + "name": "check-env-reads-gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:09Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057936, + "name": "Non-dev base guard", + "status": "completed", + "updated_at": "2026-07-30T19:16:04Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057920, + "name": "Stale TODO Gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:05Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057886, + "name": "Precommit Parity Gate", + "status": "completed", + "updated_at": "2026-07-30T19:16:06Z" + }, + { + "conclusion": "success", + "created_at": "2026-07-30T19:15:48Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30574057926, + "name": "Runner Disk Preflight", + "status": "completed", + "updated_at": "2026-07-30T19:16:07Z" + } + ] +} diff --git a/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json b/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json new file mode 100644 index 0000000000..5384c58b73 --- /dev/null +++ b/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_branches_dev_protection_required_status_checks.json @@ -0,0 +1,6 @@ +{ + "contexts": [ + "CodeRabbit" + ], + "strict": false +} diff --git a/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json b/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json new file mode 100644 index 0000000000..629eea69ff --- /dev/null +++ b/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_check_runs_per_page_100.json @@ -0,0 +1,1300 @@ +{ + "check_runs": [ + { + "conclusion": "skipped", + "name": "auto-tag", + "started_at": "2026-07-30T21:47:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "TODO Audit", + "started_at": "2026-07-30T21:47:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:47:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Trigger node_redeploy Start", + "started_at": "2026-07-30T21:47:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Merged Event", + "started_at": "2026-07-30T21:47:28Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Runtime Boot Smoke (compose)", + "started_at": "2026-07-30T21:39:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Tests Gate", + "started_at": "2026-07-30T21:39:41Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Test-Failure Ratchet Gate", + "started_at": "2026-07-30T21:39:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Tests (Split 1/1)", + "started_at": "2026-07-30T21:37:52Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Detect Changes", + "started_at": "2026-07-30T21:36:59Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T21:33:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T21:33:24Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T21:33:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Silent-Skip Guard (OMN-14172)", + "started_at": "2026-07-30T21:32:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Migration Integration Test", + "started_at": "2026-07-30T21:32:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T21:32:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Version Pin Compliance", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "zone-filter / Zone Filter (docs-only check)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Drift Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Compliance Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "OCC Born-Path Trigger Coverage (OMN-14987)", + "started_at": "2026-07-30T21:32:36Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Kafka Boundary Compat (OMN-3256)", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Kafka Schema Handshake (OMN-3411)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Naming Lint", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Compliance", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Path Pre-Flight", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Compose Required-Env Coverage (OMN-5439)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Migration Freeze Check", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Arch Invariants (OMN-3343)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Demo Loop Gate", + "started_at": "2026-07-30T21:32:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "AI-Slop Pattern Check (strict, PR diff)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Cross-Repo Migration Conflicts", + "started_at": "2026-07-30T21:32:30Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Writer-Migration Coupling Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Contract Sync Gate (Wave C) [OMN-8915]", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Plugin ENV->Service Completeness (OMN-4313)", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Effect-Assertion Gate (RT-5)", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Topic Enum Drift Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "ONEX Validators", + "started_at": "2026-07-30T21:32:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Infra Node Handler Ownership", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Fingerprint Check", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Lint", + "started_at": "2026-07-30T21:32:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T21:32:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CodeQL", + "started_at": "2026-07-30T21:31:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T21:32:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T21:31:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T21:31:50Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T21:31:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T21:31:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "merge-hold-gate / evaluate", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Deploy Agent Tests (OMN-15378) / deploy-agent-tests", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "no-noncanonical-lifecycle-classes", + "started_at": "2026-07-30T21:31:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runtime Profiles / validate", + "started_at": "2026-07-30T21:31:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Summary", + "started_at": "2026-07-30T21:31:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T21:31:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Enforce validator-requirements.yaml (OMN-13291)", + "started_at": "2026-07-30T21:31:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "OCC Companion Merged Gate (OMN-15214)", + "started_at": "2026-07-30T21:31:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T21:31:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T21:31:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T21:31:40Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T21:31:26Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T21:31:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T21:31:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T21:31:40Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T21:31:46Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T21:31:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T21:31:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T21:31:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:31:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T21:31:40Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T21:31:54Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T21:31:36Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T21:31:36Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T21:31:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T21:31:42Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T21:31:37Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T21:31:31Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T21:31:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T21:31:31Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T21:31:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T21:31:28Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T21:31:29Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "TODO Audit", + "started_at": "2026-07-30T21:31:11Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "auto-tag", + "started_at": "2026-07-30T21:31:06Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Trigger node_redeploy Start", + "started_at": "2026-07-30T21:31:05Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Publish PR Merged Event", + "started_at": "2026-07-30T21:31:05Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:31:09Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T21:31:13Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T21:30:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T21:30:42Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T21:29:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T21:29:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T21:28:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T21:29:00Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T21:28:32Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-30T21:28:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T21:28:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T21:28:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T21:28:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T21:28:45Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T21:28:52Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T21:28:48Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T21:28:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:28:46Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T21:28:50Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T21:29:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T21:29:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T21:29:11Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T21:28:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T21:28:43Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T21:28:44Z", + "status": "completed" + }, + { + "conclusion": "cancelled", + "name": "CodeQL", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T21:28:58Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T21:28:30Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T21:28:58Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T21:28:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T21:28:38Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T21:28:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T21:28:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T21:28:35Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T21:28:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T21:28:47Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T21:28:39Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T21:28:34Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T21:28:33Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "TODO Audit", + "started_at": "2026-07-30T21:28:20Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Publish PR Merged Event", + "started_at": "2026-07-30T21:28:20Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "auto-tag", + "started_at": "2026-07-30T21:28:20Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T21:28:29Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "Trigger node_redeploy Start", + "started_at": "2026-07-30T21:28:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Review Gate", + "started_at": "2026-07-30T19:24:44Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T19:23:02Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Hostile Reviewer (adversarial gate)", + "started_at": "2026-07-30T19:23:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T19:22:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T19:22:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-preflight / eligibility", + "started_at": "2026-07-30T19:22:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / scan / reject-skip-gate-token", + "started_at": "2026-07-30T19:18:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "reason-graph", + "started_at": "2026-07-30T19:18:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Omni Standards Gate", + "started_at": "2026-07-30T19:17:56Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call-reject-skip-token / occ-preflight / eligibility", + "started_at": "2026-07-30T19:17:20Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T19:17:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "verify / verify", + "started_at": "2026-07-30T19:17:25Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T19:17:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T19:17:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T19:17:10Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T19:17:04Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T19:17:08Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "gate / CodeRabbit Thread Check", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "occ-companion-effect-manual-replay", + "started_at": "2026-07-30T19:15:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "imperative-contract-guard / Imperative Contract Guard", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "call / validate-docs", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deploy-gate / deploy-gate", + "started_at": "2026-07-30T19:16:05Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pr-title / check-title", + "started_at": "2026-07-30T19:16:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "occ-companion-effect / Publish occ-companion-effect command", + "started_at": "2026-07-30T19:15:51Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "required-check-skip-guard / check-skip-vectors", + "started_at": "2026-07-30T19:16:12Z", + "status": "completed" + }, + { + "conclusion": "skipped", + "name": "verify", + "started_at": "2026-07-30T19:15:49Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "typecheck (shadow)", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CodeQL", + "started_at": "2026-07-30T19:16:20Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "lint (shadow)", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "tests+coverage (shadow)", + "started_at": "2026-07-30T19:16:11Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "main-target-guard", + "started_at": "2026-07-30T19:16:08Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "contract-validation", + "started_at": "2026-07-30T19:16:09Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "non-dev-base-guard", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "CI Naming Convention", + "started_at": "2026-07-30T19:16:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "receipt-honesty", + "started_at": "2026-07-30T19:16:04Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Handler Contract Compliance", + "started_at": "2026-07-30T19:16:16Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Coverage", + "started_at": "2026-07-30T19:16:27Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Runner Disk Preflight", + "started_at": "2026-07-30T19:16:03Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "check-env-reads-gate", + "started_at": "2026-07-30T19:16:00Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Type Safety Validation", + "started_at": "2026-07-30T19:16:17Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Integration Test Removal Gate", + "started_at": "2026-07-30T19:16:21Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "terminal-cost-completeness", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "context-field-presence", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "PEP 604 Type Union Check (UP007)", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Stale TODO Gate", + "started_at": "2026-07-30T19:15:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "pinned-wheel-skew", + "started_at": "2026-07-30T19:16:22Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "No bare compose teardown", + "started_at": "2026-07-30T19:16:05Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "deployed-migration-tree-sync-logic", + "started_at": "2026-07-30T19:16:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "skill-node-mapping-sync", + "started_at": "2026-07-30T19:16:19Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "sibling-lock-pins", + "started_at": "2026-07-30T19:15:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish PR Webhook Event", + "started_at": "2026-07-30T19:16:12Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "node-migration-sync", + "started_at": "2026-07-30T19:16:07Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "dispatcher-route-coverage", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "release-identity", + "started_at": "2026-07-30T19:16:15Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Canonical Inference Gate", + "started_at": "2026-07-30T19:16:13Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Check architecture handshake", + "started_at": "2026-07-30T19:15:55Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Dep Provenance Gate", + "started_at": "2026-07-30T19:16:14Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Duplication Sweep", + "started_at": "2026-07-30T19:16:06Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "Precommit Parity Gate", + "started_at": "2026-07-30T19:15:53Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "URL Authority Gate", + "started_at": "2026-07-30T19:15:57Z", + "status": "completed" + }, + { + "conclusion": "success", + "name": "shell-hygiene", + "started_at": "2026-07-30T19:16:17Z", + "status": "completed" + } + ] +} diff --git a/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json b/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json new file mode 100644 index 0000000000..0c428a268a --- /dev/null +++ b/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_commits_2173c0c53a46624be1efd4cff91cd07f3f8df13b_status_per_page_100.json @@ -0,0 +1,10 @@ +{ + "state": "success", + "statuses": [ + { + "context": "CodeRabbit", + "created_at": "2026-07-30T19:15:58Z", + "state": "success" + } + ] +} diff --git a/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json b/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json new file mode 100644 index 0000000000..7086402a1a --- /dev/null +++ b/tests/fixtures/omn15550/status_surface/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json @@ -0,0 +1,12 @@ +[ + { + "base": { + "ref": "dev" + }, + "draft": false, + "head": { + "sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b" + }, + "number": 2575 + } +] diff --git a/tests/fixtures/omn15550/zerojob/repos_OmniNode_ai_omnibase_infra_actions_runs_30583527197_jobs_per_page_1.json b/tests/fixtures/omn15550/zerojob/repos_OmniNode_ai_omnibase_infra_actions_runs_30583527197_jobs_per_page_1.json new file mode 100644 index 0000000000..8615d88006 --- /dev/null +++ b/tests/fixtures/omn15550/zerojob/repos_OmniNode_ai_omnibase_infra_actions_runs_30583527197_jobs_per_page_1.json @@ -0,0 +1,4 @@ +{ + "jobs": [], + "total_count": 0 +} diff --git a/tests/fixtures/omn15550/zerojob/repos_OmniNode_ai_omnibase_infra_actions_runs_per_page_50.json b/tests/fixtures/omn15550/zerojob/repos_OmniNode_ai_omnibase_infra_actions_runs_per_page_50.json new file mode 100644 index 0000000000..9579ebc512 --- /dev/null +++ b/tests/fixtures/omn15550/zerojob/repos_OmniNode_ai_omnibase_infra_actions_runs_per_page_50.json @@ -0,0 +1,15 @@ +{ + "workflow_runs": [ + { + "conclusion": "failure", + "created_at": "2026-07-30T21:28:30Z", + "head_branch": "jonah/omn-15525-alert-body-truncation-and-drift-gate", + "head_sha": "2173c0c53a46624be1efd4cff91cd07f3f8df13b", + "id": 30583527197, + "name": ".github/workflows/ci.yml", + "path": ".github/workflows/ci.yml", + "status": "completed", + "updated_at": "2026-07-30T21:28:30Z" + } + ] +} diff --git a/tests/fixtures/omn15550/zerojob/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json b/tests/fixtures/omn15550/zerojob/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/tests/fixtures/omn15550/zerojob/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json @@ -0,0 +1 @@ +[] diff --git a/tests/fixtures/omn15550/zerojob_healthy/repos_OmniNode_ai_omnibase_infra_actions_runs_30586119120_jobs_per_page_1.json b/tests/fixtures/omn15550/zerojob_healthy/repos_OmniNode_ai_omnibase_infra_actions_runs_30586119120_jobs_per_page_1.json new file mode 100644 index 0000000000..4249490655 --- /dev/null +++ b/tests/fixtures/omn15550/zerojob_healthy/repos_OmniNode_ai_omnibase_infra_actions_runs_30586119120_jobs_per_page_1.json @@ -0,0 +1,4 @@ +{ + "jobs": [], + "total_count": 54 +} diff --git a/tests/fixtures/omn15550/zerojob_healthy/repos_OmniNode_ai_omnibase_infra_actions_runs_per_page_50.json b/tests/fixtures/omn15550/zerojob_healthy/repos_OmniNode_ai_omnibase_infra_actions_runs_per_page_50.json new file mode 100644 index 0000000000..34d0e70f14 --- /dev/null +++ b/tests/fixtures/omn15550/zerojob_healthy/repos_OmniNode_ai_omnibase_infra_actions_runs_per_page_50.json @@ -0,0 +1,15 @@ +{ + "workflow_runs": [ + { + "conclusion": "success", + "created_at": "2026-07-30T22:09:17Z", + "head_branch": "jonah/omn-15468-envelope-failure-terminal-emission", + "head_sha": "31ebe15df19d61def4e0a8247877fa1979cc29fe", + "id": 30586119120, + "name": "CI", + "path": ".github/workflows/ci.yml", + "status": "completed", + "updated_at": "2026-07-30T22:20:01Z" + } + ] +} diff --git a/tests/fixtures/omn15550/zerojob_healthy/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json b/tests/fixtures/omn15550/zerojob_healthy/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/tests/fixtures/omn15550/zerojob_healthy/repos_OmniNode_ai_omnibase_infra_pulls_state_open_per_page_100.json @@ -0,0 +1 @@ +[] diff --git a/tests/fixtures/omn15676/runtime-image-config-probe-403b6092.txt.captured b/tests/fixtures/omn15676/runtime-image-config-probe-403b6092.txt.captured new file mode 100644 index 0000000000..88868f7c9b --- /dev/null +++ b/tests/fixtures/omn15676/runtime-image-config-probe-403b6092.txt.captured @@ -0,0 +1,2 @@ +MISSING /app/config/runner_fleet.yaml +PRESENT /app/config/delegation/routing_tiers.yaml diff --git a/tests/fixtures/omn15701/onex-dev-topology-reverted-tenant-grants.yaml.captured b/tests/fixtures/omn15701/onex-dev-topology-reverted-tenant-grants.yaml.captured new file mode 100644 index 0000000000..cb46f4e300 --- /dev/null +++ b/tests/fixtures/omn15701/onex-dev-topology-reverted-tenant-grants.yaml.captured @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +active_preset: application-cloud +databases: + application: + bindings: + app_dashboard: + database_ref: application + dsn_env: DATABASE_URL + principal: app_dashboard + omninode_runtime_service: + database_ref: application + dsn_env: OMNINODE_INTERNAL_DB_URL + principal: omninode_runtime + onex_api: + database_ref: application + dsn_env: OMNINODE_CLOUD_DB_URL + principal: onex_api + tenant_projection: + database_ref: application + dsn_env: OMNIDASH_ANALYTICS_DB_URL + principal: tenant_projection_writer + checksum_ledger: canonical + checksum_ledgers: + canonical: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: platform_catalog + stream_column: migration_stream + version_column: version + migration_stream: omnibase_infra.application + owners: + owner_omninode_internal: + login: false + owner_onex_tenant: + login: false + owner_platform_catalog: + login: false + physical_name: omnidash_analytics + principals: + app_dashboard: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: platform_catalog + login: true + omninode_runtime: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: omninode_internal + - object_type: TABLE + objects: + - agent_routing_decisions + - baselines_breakdown + - baselines_comparisons + - baselines_quality_snapshots + - baselines_roi_snapshots + - baselines_snapshots + - baselines_trend + - capability_scores + - capsule_store + - context_roi_scores + - contract_registry + - cost_by_repo_snapshots + - dep_health_findings + - deployment_evidence_projection + - deployment_readiness_projection + - event_chain + - evidence_correlation_trace_projection + - evidence_dashboard_projection + - evidence_readiness_aggregate_projection + - gate_activity + - gate_metrics + - generation_events + - instruction_eval_aggregate_snapshots + - intent_classification_events + - live_events + - llm_call_metrics + - llm_cost_aggregates + - llm_delegation_daily_projection + - llm_routing_decisions + - mcp_tools + - merge_state_transitions + - nightly_loop_decisions + - nightly_loop_iterations + - node_service_registry + - overnight_session_phases + - overnight_sessions + - pattern_learning_artifacts + - pr_lifecycle_ledger_entries + - pr_merged_events + - receipt_gate_rows + - renderer_capability_projection + - sandbox_decisions + - session_outcomes + - session_replay_snapshots + - skill_execution_snapshots + - swarm_runs + - traces + - voice_sessions + privileges: [INSERT, SELECT, UPDATE] + schema: omninode_internal + login: true + onex_api: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: platform_catalog + login: true + tenant_projection_writer: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: tenant + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - delegation_budget_state + - delegation_events + - delegation_shadow_comparisons + - projection_delegation_inference_response_text + - savings_estimates + privileges: [INSERT, SELECT, UPDATE] + schema: public + login: true + schemas: + omninode_internal: + domain: OMNINODE_INTERNAL + owner: owner_omninode_internal + platform_catalog: + domain: PLATFORM_CATALOG + owner: owner_platform_catalog + public: + domain: TENANT + owner: owner_onex_tenant + tenant: + domain: TENANT + owner: owner_onex_tenant + omniintelligence: + bindings: + omninode_runtime_service: + database_ref: omniintelligence + dsn_env: OMNIINTELLIGENCE_DB_URL + principal: role_omniintelligence + checksum_ledger: service_owned + checksum_ledgers: + service_owned: + checksum_column: checksum + domain_column: domain + relation: schema_migrations + schema: public + stream_column: migration_stream + version_column: version + migration_stream: omniintelligence.service + owners: + owner_omniintelligence: + login: false + physical_name: omniintelligence + principals: + role_omniintelligence: + bypass_rls: false + grants: + - object_type: DATABASE + privileges: [CONNECT] + - object_type: SCHEMA + privileges: [USAGE] + schema: public + - object_type: TABLE + objects: + - dispatch_eval_results + privileges: [INSERT, SELECT, UPDATE] + schema: public + login: true + schemas: + public: + domain: OMNINODE_INTERNAL + owner: owner_omniintelligence +presets: + application-cloud: + - postgres + - onex_api + - omnidash + - omninode_runtime +schema_version: "2.0" +services: + omnidash: + mode: CLOUD + omninode_runtime: + mode: CLOUD + onex_api: + mode: CLOUD + postgres: + mode: CLOUD diff --git a/tests/fixtures/omn15717/001_create_review_bot_bypass_log.sql.captured b/tests/fixtures/omn15717/001_create_review_bot_bypass_log.sql.captured new file mode 100644 index 0000000000..31294af894 --- /dev/null +++ b/tests/fixtures/omn15717/001_create_review_bot_bypass_log.sql.captured @@ -0,0 +1,17 @@ +-- Migration: Create review_bot_bypass_log table — OMN-8497 +-- Target DB: omnidash_analytics (omnibase_infra postgres on .201:5436) +-- Node: node_pr_review_bot / HandlerEmergencyBypassParser + +CREATE TABLE IF NOT EXISTS review_bot_bypass_log ( + audit_id UUID PRIMARY KEY, + pr_url TEXT NOT NULL, + actor TEXT NOT NULL, + reason TEXT NOT NULL, + bypass_timestamp TIMESTAMPTZ NOT NULL, + kafka_event_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_rbl_actor ON review_bot_bypass_log (actor); +CREATE INDEX IF NOT EXISTS idx_rbl_pr_url ON review_bot_bypass_log (pr_url); +CREATE INDEX IF NOT EXISTS idx_rbl_bypass_timestamp ON review_bot_bypass_log (bypass_timestamp); diff --git a/tests/fixtures/omn15776/job-93294479341.json.captured b/tests/fixtures/omn15776/job-93294479341.json.captured new file mode 100644 index 0000000000..aaaf0317ae --- /dev/null +++ b/tests/fixtures/omn15776/job-93294479341.json.captured @@ -0,0 +1 @@ +{"id":93294479341,"run_id":31333147268,"workflow_name":"CI","head_branch":"codex/omn-14551-supersede-invalid-2670-receipt","run_url":"https://api.github.com/repos/OmniNode-ai/onex_change_control/actions/runs/31333147268","run_attempt":1,"node_id":"CR_kwDOQr55Hc8AAAAVuMjH7Q","head_sha":"266381339d530b5b5799769a887840563e2b9783","url":"https://api.github.com/repos/OmniNode-ai/onex_change_control/actions/jobs/93294479341","html_url":"https://github.com/OmniNode-ai/onex_change_control/actions/runs/31333147268/job/93294479341","status":"completed","conclusion":"failure","created_at":"2026-08-09T20:00:49Z","started_at":"2026-08-09T20:06:56Z","completed_at":"2026-08-09T20:16:56Z","name":"No Divergent Automation PRs (OMN-14778)","steps":[],"check_run_url":"https://api.github.com/repos/OmniNode-ai/onex_change_control/check-runs/93294479341","labels":["self-hosted","omnibase-ci"],"runner_id":33606,"runner_name":"omninode-runner-17","runner_group_id":3,"runner_group_name":"omnibase-ci"} \ No newline at end of file diff --git a/tests/fixtures/runtime_path_classifier.py b/tests/fixtures/runtime_path_classifier.py new file mode 100644 index 0000000000..e02a3d3c92 --- /dev/null +++ b/tests/fixtures/runtime_path_classifier.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Hermetic test double for omniclaude's canonical deploy-path classifier.""" + +from __future__ import annotations + + +def find_runtime_paths(changed_files: list[str]) -> list[str]: + """Return the runtime paths exercised by rebuild-trigger unit tests.""" + runtime_prefixes = ("src/omnimarket/", "src/omnibase_infra/nodes/") + return [ + path + for path in changed_files + if path.startswith(runtime_prefixes) or path.startswith("docker/docker-compose") + ] diff --git a/tests/fixtures/seams/ack_dlq/poisoned_command.json b/tests/fixtures/seams/ack_dlq/poisoned_command.json new file mode 100644 index 0000000000..710fdef126 --- /dev/null +++ b/tests/fixtures/seams/ack_dlq/poisoned_command.json @@ -0,0 +1,11 @@ +{ + "_comment": "OMN-14498 Lane C seam fixture. A poisoned command as it arrives at the auto-wired consume boundary: the ingress correlation_id is carried on the Kafka message HEADERS (ModelEventMessage.headers.correlation_id), while the message VALUE is truncated JSON that json.loads() cannot decode. This is the exists-but-wrong shape: the boundary cannot recover lineage from the body, so a pre-fix boundary mints a fresh uuid4() and the DLQ record -- and therefore every replay of it -- carries a VALID id with the WRONG lineage, producing an orphan terminal.", + "topic": "onex.cmd.platform.node-rebuild.v1", + "ingress_correlation_id": "3f9d2c17-8b4a-4e51-9a26-7c0d5e1b8f34", + "ingress_key": "node-rebuild-omn14498", + "ingress_partition": 3, + "ingress_offset": "417", + "poisoned_value": "{\"event_type\": \"onex.cmd.platform.node-rebuild.v1\", \"correlation_id\": \"3f9d2c17-8b4a-4e51-9a26-7c0d5e1b8f34\", \"payload\": {\"node_id\": \"node_rebuild\", \"truncated_here\":", + "expected_dlq_topic_category": "commands", + "expected_failure_type": "handler_exception" +} diff --git a/tests/fixtures/seams/core_release/0.46.8_expected_symbols.json b/tests/fixtures/seams/core_release/0.46.8_expected_symbols.json new file mode 100644 index 0000000000..3142eb4212 --- /dev/null +++ b/tests/fixtures/seams/core_release/0.46.8_expected_symbols.json @@ -0,0 +1,68 @@ +{ + "_comment": [ + "OMN-14628 seam fixture. Freezes the omnibase-core RELEASED (PyPI) 0.46.8", + "surface that omnibase_infra's delegation dispatch path consumes.", + "This exists so that a core pin move (git-rev -> published release, or a", + "release bump) fails LOUDLY on the exact symbols and field TYPES infra", + "relies on, rather than silently degrading at runtime.", + "Update procedure: bump core, re-run the seam test, and record the new", + "surface here only after confirming every consumer still type-checks." + ], + "core_version": "0.46.8", + "source": "pypi", + "modules": { + "omnibase_core.models.delegation.wire": [ + "EnumBudgetAction", + "EnumDelegationTerminalFailureCause", + "EnumQualityContractMode", + "EnumQualityGateCategory", + "EnumQualityScoreComparison", + "ModelBifrostDelegationConfig", + "ModelDelegationBackendConfig", + "ModelDelegationCompleted", + "ModelDelegationConfig", + "ModelDelegationEventEnvelope", + "ModelDelegationFailed", + "ModelDelegationRequest", + "ModelDelegationResult", + "ModelQualityGateInput", + "ModelQualityGateResult", + "ModelTaskDelegatedEvent" + ], + "omnibase_core.enums.ticket": [ + "EnumDodEvidenceExecutionScope" + ], + "omnibase_core.enums.enum_delegation_terminal_failure_cause": [ + "EnumDelegationTerminalFailureCause" + ], + "omnibase_core.models.ticket.model_contract_dod_item": [ + "ModelContractDodItem" + ] + }, + "typed_fields": [ + { + "module": "omnibase_core.models.ticket.model_contract_dod_item", + "model": "ModelContractDodItem", + "field": "execution_scope", + "expected_type": "EnumDodEvidenceExecutionScope", + "optional": false, + "why": [ + "Released 0.46.7 imports ModelContractDodItem successfully but has NO", + "execution_scope field at all. That is the exists-but-wrong case this", + "seam test must catch as a TYPE failure, not an ImportError." + ] + }, + { + "module": "omnibase_core.models.delegation.wire", + "model": "ModelDelegationResult", + "field": "terminal_failure_cause", + "expected_type": "EnumDelegationTerminalFailureCause", + "optional": true, + "why": [ + "Infra's delegation terminalization readback classifies failures off a", + "typed cause; an untyped str here silently re-opens stringly-typed", + "failure routing." + ] + } + ] +} diff --git a/tests/fixtures/seams/core_release/dispatch_kwargs_frozen.json b/tests/fixtures/seams/core_release/dispatch_kwargs_frozen.json new file mode 100644 index 0000000000..12a764ff69 --- /dev/null +++ b/tests/fixtures/seams/core_release/dispatch_kwargs_frozen.json @@ -0,0 +1,53 @@ +{ + "_comment": [ + "OMN-14628 seam fixture. Freezes the RuntimeDelegationDispatchPort.dispatch()", + "keyword surface that current OmniMarket callers rely on.", + "PR #2595 restored callable parity for system_prompt/temperature/response_format;", + "PR #2312 added tenant_id. Nothing mechanically held that surface afterwards, so", + "a rename, a reordering into positional, or a dropped default silently breaks the", + "cross-repo call site with no local test failure. This fixture is that mechanism.", + "Note: several of these fail closed with NotImplementedError when non-None. The", + "CALLABLE contract (name + kind + annotation + default) is what is frozen here;", + "the accept/reject behavior is asserted by the existing dispatch-port tests." + ], + "module": "omnibase_infra.runtime.service_delegation_dispatch_port", + "qualname": "RuntimeDelegationDispatchPort.dispatch", + "required_keyword_only_params": [ + { + "name": "tenant_id", + "annotation": "str | None", + "default": null, + "added_by": "OMN-14628 / PR #2312" + }, + { + "name": "system_prompt", + "annotation": "str | None", + "default": null, + "added_by": "OMN-14628 / PR #2595" + }, + { + "name": "temperature", + "annotation": "float | None", + "default": null, + "added_by": "OMN-14628 / PR #2595" + }, + { + "name": "response_format", + "annotation": "dict[str, object] | None", + "default": null, + "added_by": "OMN-14628 / PR #2595" + }, + { + "name": "backend_id", + "annotation": "str | None", + "default": null, + "added_by": "OMN-14628" + }, + { + "name": "response_contract", + "annotation": "dict[str, object] | None", + "default": null, + "added_by": "OMN-14628" + } + ] +} diff --git a/tests/fixtures/seams/dispatch_correlation/accepted_command.json b/tests/fixtures/seams/dispatch_correlation/accepted_command.json new file mode 100644 index 0000000000..1e2eede9c8 --- /dev/null +++ b/tests/fixtures/seams/dispatch_correlation/accepted_command.json @@ -0,0 +1,21 @@ +{ + "_comment": "Seam fixture for OMN-15474 / OMN-15546. One ingress-ACCEPTED delegation command, exactly as the gateway hands it to the runtime. `ingress_correlation_id` is the AUTHORITY: it is assigned at ingress and every command/event emitted downstream must carry it byte-identical. The dispatcher must never mint one, and must never emit the command more than once.", + "ticket": "OMN-15474", + "also_used_by": "OMN-15546", + "seam": "ingress accept -> auto-wiring subscription attach -> dispatch engine -> emitted command", + "topic": "onex.cmd.omnibase-infra.delegation-request.v1", + "ingress_correlation_id": "a4000001-0000-4000-8000-000000000001", + "expected_emitted_command_count": 1, + "envelope": { + "correlation_id": "a4000001-0000-4000-8000-000000000001", + "event_type": "delegation-request", + "source_tool": "onex-gateway-ingress", + "envelope_timestamp": "2026-07-30T04:49:47.000000Z", + "payload": { + "correlation_id": "a4000001-0000-4000-8000-000000000001", + "workflow_id": "5f90f524-52f7-4b32-a654-d5aa68e58a80", + "task_type": "test", + "prompt": "seam probe: one accepted command must execute exactly once" + } + } +} diff --git a/tests/fixtures/seams/quota_terminal/forced_429_terminal.json b/tests/fixtures/seams/quota_terminal/forced_429_terminal.json new file mode 100644 index 0000000000..9047340952 --- /dev/null +++ b/tests/fixtures/seams/quota_terminal/forced_429_terminal.json @@ -0,0 +1,156 @@ +{ + "ticket": "OMN-15503", + "scenario": "forced_429_provider_quota_exhaustion", + "delegation_class": "refactor", + "description": [ + "Deterministic forced-429 capture for ONE accepted delegation command.", + "Reproduces the two live defects recorded in the 2026-07-29 deep dive matrix", + "row `refactor | FAIL | Google Gemini HTTP 429 after two escalation attempts`:", + "(1) THREE terminal events are emitted for a single accepted command, and", + "(2) the LAST of them is an outer `delegate-skill-completed` declaring", + " status=completed / quality_gate_passed=true even though every inner", + " attempt in the ladder was refused with HTTP 429 RESOURCE_EXHAUSTED.", + "This is a frozen fixture, not a live opportunistic run (OMN-15503 AC3):", + "no provider is contacted and no runtime lane is touched when it replays." + ], + "correlation_id": "c4f2a1de-0000-4d15-9503-000000000429", + "session_id": "5b1d6e2c-1111-4a44-8f01-0000000000aa", + "tenant_id": "omninode", + "expected_projection": { + "durable_terminal_rows": 1, + "terminal_failure_cause": "provider_quota_exhausted", + "terminal_ok": false, + "attempt_history_min_length": 3 + }, + "terminal_events": [ + { + "_event_type": "onex.evt.omnimarket.delegate-skill-failed.v1", + "status": "failed", + "correlation_id": "c4f2a1de-0000-4d15-9503-000000000429", + "session_id": "5b1d6e2c-1111-4a44-8f01-0000000000aa", + "tenant_id": "omninode", + "task_type": "refactor", + "provider": "google", + "model_name": "gemini-2.5-flash", + "prompt_text": "Refactor the attempt-ladder reducer for readability.", + "response": "", + "quality_gate_passed": false, + "quality_score": 0.0, + "error_message": "Google Gemini HTTP 429: RESOURCE_EXHAUSTED (free-tier quota exceeded)", + "escalation_count": 0, + "emitted_at": "2026-07-29T18:04:11+00:00", + "metrics": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "latency_ms": 812 + }, + "attempts": [ + { + "tier": "cloud-free", + "backend_id": "google-gemini", + "model_id": "gemini-2.5-flash", + "quality_gate_passed": false, + "cost_usd": 0.0, + "failure_class": "provider_quota_exhausted", + "error_message": "HTTP 429 RESOURCE_EXHAUSTED: quota exceeded for generativelanguage.googleapis.com" + } + ] + }, + { + "_event_type": "onex.evt.omnimarket.delegate-skill-failed.v1", + "status": "failed", + "correlation_id": "c4f2a1de-0000-4d15-9503-000000000429", + "session_id": "5b1d6e2c-1111-4a44-8f01-0000000000aa", + "tenant_id": "omninode", + "task_type": "refactor", + "provider": "google", + "model_name": "gemini-2.5-pro", + "prompt_text": "Refactor the attempt-ladder reducer for readability.", + "response": "", + "quality_gate_passed": false, + "quality_score": 0.0, + "error_message": "Google Gemini HTTP 429: RESOURCE_EXHAUSTED (free-tier quota exceeded)", + "escalation_count": 1, + "emitted_at": "2026-07-29T18:04:19+00:00", + "metrics": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "latency_ms": 1544 + }, + "attempts": [ + { + "tier": "cloud-free", + "backend_id": "google-gemini", + "model_id": "gemini-2.5-flash", + "quality_gate_passed": false, + "cost_usd": 0.0, + "failure_class": "provider_quota_exhausted", + "error_message": "HTTP 429 RESOURCE_EXHAUSTED: quota exceeded for generativelanguage.googleapis.com" + }, + { + "tier": "cloud-free-escalated-1", + "backend_id": "google-gemini", + "model_id": "gemini-2.5-pro", + "quality_gate_passed": false, + "cost_usd": 0.0, + "failure_class": "provider_quota_exhausted", + "error_message": "HTTP 429 RESOURCE_EXHAUSTED: quota exceeded for generativelanguage.googleapis.com" + } + ] + }, + { + "_event_type": "onex.evt.omnimarket.delegate-skill-completed.v1", + "status": "completed", + "correlation_id": "c4f2a1de-0000-4d15-9503-000000000429", + "session_id": "5b1d6e2c-1111-4a44-8f01-0000000000aa", + "tenant_id": "omninode", + "task_type": "refactor", + "provider": "google", + "model_name": "gemini-2.5-pro", + "prompt_text": "Refactor the attempt-ladder reducer for readability.", + "response": "", + "quality_gate_passed": true, + "quality_score": 1.0, + "error_message": "", + "escalation_count": 2, + "emitted_at": "2026-07-29T18:04:27+00:00", + "metrics": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "latency_ms": 2301 + }, + "attempts": [ + { + "tier": "cloud-free", + "backend_id": "google-gemini", + "model_id": "gemini-2.5-flash", + "quality_gate_passed": false, + "cost_usd": 0.0, + "failure_class": "provider_quota_exhausted", + "error_message": "HTTP 429 RESOURCE_EXHAUSTED: quota exceeded for generativelanguage.googleapis.com" + }, + { + "tier": "cloud-free-escalated-1", + "backend_id": "google-gemini", + "model_id": "gemini-2.5-pro", + "quality_gate_passed": false, + "cost_usd": 0.0, + "failure_class": "provider_quota_exhausted", + "error_message": "HTTP 429 RESOURCE_EXHAUSTED: quota exceeded for generativelanguage.googleapis.com" + }, + { + "tier": "cloud-free-escalated-2", + "backend_id": "google-gemini", + "model_id": "gemini-2.5-pro", + "quality_gate_passed": false, + "cost_usd": 0.0, + "failure_class": "provider_quota_exhausted", + "error_message": "HTTP 429 RESOURCE_EXHAUSTED: quota exceeded for generativelanguage.googleapis.com" + } + ] + } + ] +} diff --git a/tests/helpers/application_db_topology.py b/tests/helpers/application_db_topology.py new file mode 100644 index 0000000000..000b7d155a --- /dev/null +++ b/tests/helpers/application_db_topology.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed application-database targets shared by projection wiring tests.""" + +from __future__ import annotations + +from functools import lru_cache +from typing import Literal + +from omnibase_core.enums.enum_database_grant_object_type import ( + EnumDatabaseGrantObjectType, +) +from omnibase_core.enums.enum_database_privilege import EnumDatabasePrivilege +from omnibase_core.models.contracts.subcontracts.model_db_table_declaration import ( + ModelDbTableDeclaration, +) +from omnibase_core.models.core.model_deployment_topology import ModelDeploymentTopology +from omnibase_core.models.core.model_deployment_topology_database_grant import ( + ModelDeploymentTopologyDatabaseGrant, +) +from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + ProjectionDatabaseTarget, + _resolve_projection_database_target, +) +from omnibase_infra.topology import load_topology_profile +from omnibase_infra.topology.physical_schema_mapping import ( + physical_grant_schema_for_table, +) + + +@lru_cache(maxsize=1) +def application_topology() -> ModelDeploymentTopology: + """Return the real shipped topology every runtime profile resolves. + + OMN-15656: this deliberately loads the platform instance rather than + ``tests/fixtures/application_relation_ownership/topology.yaml``. The fixture + topology carried hand-written TABLE grants that the shipped instances did + not, so wiring tests proved a topology that does not exist and a 43/43 + strict-wiring failure shipped to onex-dev undetected. + """ + return load_topology_profile("local") + + +ProjectionAccess = Literal["read", "write", "read_write"] + + +def _shipped_table_grant_exists( + topology: ModelDeploymentTopology, principal: str, schema: str, table: str +) -> bool: + """Return whether the shipped topology already grants ``table`` to ``principal``. + + Compares against the PHYSICAL grant schema + (``physical_grant_schema_for_table``), not the caller-supplied logical + schema: the shipped generator already applies the tenant/omninode_internal + physical bridge when it writes TABLE grants, so a table pending its + family's copy migration is checked-in with ``schema: public`` even though + its logical domain is ``tenant``/``omninode_internal``. Comparing against + the raw logical schema here would silently stop detecting an + already-shipped grant for every bridged table -- exactly the false + negative this helper exists to prevent. + """ + database = topology.databases["application"] + physical_schema = physical_grant_schema_for_table(schema, table) + return any( + grant.object_type is EnumDatabaseGrantObjectType.TABLE + and grant.schema == physical_schema + and table in grant.objects + for grant in database.principals[principal].grants + ) + + +def _topology_with_unshipped_grants( + topology: ModelDeploymentTopology, + tables: tuple[ModelDbTableDeclaration, ...], + *, + principal: str, + privileges: tuple[EnumDatabasePrivilege, ...], + reason: str, +) -> ModelDeploymentTopology: + """Grant relations the platform deliberately does not ship yet. + + This is the ONLY sanctioned grant synthesis left in the test tree + (OMN-15656). It exists for domains whose grants cannot be derived from node + contracts — today only ``PLATFORM_CATALOG``, which requires a + caller-supplied binding and which no ``db_io.db_tables`` block declares. + + It is fail-closed against the failure class this ticket fixed: synthesising + a grant the shipped topology is *supposed* to carry is refused outright, so + this helper can never be used to re-hide a missing platform grant the way + the old ``_with_projection_fixture_grants`` did. + """ + if not reason.strip(): + raise ValueError("unshipped grant synthesis requires an explicit reason") + database = topology.databases["application"] + for table in tables: + if _shipped_table_grant_exists(topology, principal, table.schema, table.name): + raise AssertionError( + f"{table.schema}.{table.name} is already granted to {principal!r} by " + "the shipped topology; use the real grant instead of synthesising one" + ) + principals = dict(database.principals) + target = principals[principal] + principals[principal] = target.model_copy( + update={ + "grants": ( + *target.grants, + *( + ModelDeploymentTopologyDatabaseGrant( + object_type=EnumDatabaseGrantObjectType.TABLE, + schema=table.schema, + objects=(table.name,), + privileges=privileges, + ) + for table in tables + ), + ) + } + ) + return topology.model_copy( + update={ + "databases": { + "application": database.model_copy(update={"principals": principals}) + } + } + ) + + +def projection_database_target( + *table_names: str, + schema: str = "tenant", + physical_database: str = "omnidash_analytics", + access: ProjectionAccess = "read_write", + catalog_read_binding: str | None = None, + catalog_write_binding: str | None = None, + unshipped_grant_principal: str | None = None, + unshipped_grant_reason: str = "", +) -> ProjectionDatabaseTarget: + names = table_names or ("projection_fixture",) + tables = tuple( + ModelDbTableDeclaration( + name=name, + database_ref="application", + schema=schema, + migration=f"tests/{name}.sql", + access=access, + role=f"{name}_projection", + ) + for name in names + ) + topology = application_topology() + if unshipped_grant_principal is not None: + topology = _topology_with_unshipped_grants( + topology, + tables, + principal=unshipped_grant_principal, + privileges=( + (EnumDatabasePrivilege.SELECT,) + if access == "read" + else ( + EnumDatabasePrivilege.SELECT, + EnumDatabasePrivilege.INSERT, + EnumDatabasePrivilege.UPDATE, + ) + ), + reason=unshipped_grant_reason, + ) + if physical_database != topology.databases["application"].physical_name: + database = topology.databases["application"].model_copy( + update={"physical_name": physical_database} + ) + topology = topology.model_copy(update={"databases": {"application": database}}) + return _resolve_projection_database_target( + tables, + topology, + catalog_read_binding=catalog_read_binding, + catalog_write_binding=catalog_write_binding, + ) + + +def projection_database_urls( + target: ProjectionDatabaseTarget, + default_url: str, + **binding_urls: str, +) -> dict[str, str]: + """Build an exact binding→DSN map for focused adapter tests.""" + return { + binding.binding_ref: binding_urls.get(binding.binding_ref, default_url) + for binding in target.bindings + } + + +__all__ = [ + "application_topology", + "projection_database_target", + "projection_database_urls", +] diff --git a/tests/helpers/projection_tenant_authority.py b/tests/helpers/projection_tenant_authority.py new file mode 100644 index 0000000000..f6b73d91de --- /dev/null +++ b/tests/helpers/projection_tenant_authority.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Real Ed25519 projection-authority fixtures.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +from omnibase_core.crypto.crypto_ed25519_signer import generate_keypair +from omnibase_core.models.envelope.model_message_envelope import ModelMessageEnvelope +from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope +from omnibase_infra.runtime.projection_tenant_authority import ( + VerifiedProjectionTenantAuthority, + verify_signed_projection_tenant_authority, +) + + +class InMemoryKeyProvider: + """Minimal test provider implementing the core key-provider protocol.""" + + def __init__(self, keys: dict[str, bytes] | None = None) -> None: + self._keys = dict(keys or {}) + + def get_public_key(self, runtime_id: str) -> bytes | None: + return self._keys.get(runtime_id) + + def register_key(self, runtime_id: str, public_key: bytes) -> None: + if len(public_key) != 32: + raise ValueError("Ed25519 public keys must be 32 bytes") + self._keys[runtime_id] = public_key + + def has_key(self, runtime_id: str) -> bool: + return runtime_id in self._keys + + def list_runtime_ids(self) -> list[str]: + return sorted(self._keys) + + +@dataclass(frozen=True) +class StaticTenantBindingResolver: + runtime_id: str + realm: str + bus_id: str + tenant_id: UUID + + def resolve_tenant_id( + self, + *, + runtime_id: str, + realm: str, + bus_id: str, + ) -> UUID | None: + if (runtime_id, realm, bus_id) != ( + self.runtime_id, + self.realm, + self.bus_id, + ): + return None + return self.tenant_id + + +@dataclass(frozen=True) +class SignedTenantAuthorityFixture: + envelope: ModelMessageEnvelope[ModelEventEnvelope[dict[str, object]]] + key_provider: InMemoryKeyProvider + binding_resolver: StaticTenantBindingResolver + + def verify(self) -> VerifiedProjectionTenantAuthority: + return verify_signed_projection_tenant_authority( + self.envelope, + self.key_provider, + self.binding_resolver, + ) + + +def signed_tenant_authority_fixture( + tenant_id: UUID, + *, + payload: dict[str, object] | None = None, + event_envelope: ModelEventEnvelope[dict[str, object]] | None = None, + runtime_id: str = "tenant-gateway-proof", + realm: str = "test", + bus_id: str = "proof-bus", +) -> SignedTenantAuthorityFixture: + keypair = generate_keypair() + typed_event = event_envelope or ModelEventEnvelope[dict[str, object]]( + payload=payload or {}, + correlation_id=uuid4(), + ) + if typed_event.correlation_id is None: + raise ValueError("projection authority fixture requires correlation_id") + envelope = ModelMessageEnvelope[ + ModelEventEnvelope[dict[str, object]] + ].create_signed( + realm=realm, + runtime_id=runtime_id, + bus_id=bus_id, + tenant_id=str(tenant_id), + payload=typed_event, + trace_id=typed_event.correlation_id, + private_key=keypair.private_key_bytes, + emitted_at=datetime.now(UTC), + ) + return SignedTenantAuthorityFixture( + envelope=envelope, + key_provider=InMemoryKeyProvider({runtime_id: keypair.public_key_bytes}), + binding_resolver=StaticTenantBindingResolver( + runtime_id=runtime_id, + realm=realm, + bus_id=bus_id, + tenant_id=tenant_id, + ), + ) + + +def verified_tenant_authority( + tenant_id: UUID, +) -> VerifiedProjectionTenantAuthority: + return signed_tenant_authority_fixture(tenant_id).verify() + + +def verified_tenant_dispatch( + tenant_id: UUID, +) -> tuple[VerifiedProjectionTenantAuthority, ModelEventEnvelope[dict[str, object]]]: + fixture = signed_tenant_authority_fixture(tenant_id) + return fixture.verify(), fixture.envelope.payload + + +__all__ = [ + "InMemoryKeyProvider", + "SignedTenantAuthorityFixture", + "StaticTenantBindingResolver", + "signed_tenant_authority_fixture", + "verified_tenant_authority", + "verified_tenant_dispatch", +] diff --git a/tests/helpers/util_kafka.py b/tests/helpers/util_kafka.py index d1eaa09837..b1a777230a 100644 --- a/tests/helpers/util_kafka.py +++ b/tests/helpers/util_kafka.py @@ -1344,15 +1344,44 @@ async def create_topic( self.created_topics.append(topic_name) # Wait for topic metadata to propagate - await wait_for_topic_metadata( + topic_ready = await wait_for_topic_metadata( admin, topic_name, expected_partitions=partitions ) + if not topic_ready: + correlation_id = uuid4() + context = ModelInfraErrorContext.with_correlation( + correlation_id=correlation_id, + transport_type=EnumInfraTransportType.KAFKA, + operation="create_topic", + target_name=topic_name, + ) + raise InfraUnavailableError( + f"Topic '{topic_name}' metadata did not propagate within " + f"timeout (expected {partitions} partition(s)); create_topic " + "cannot confirm the topic actually exists on the broker.", + context=context, + ) except TopicAlreadyExistsError: # Topic already exists - still wait for metadata - await wait_for_topic_metadata( + topic_ready = await wait_for_topic_metadata( admin, topic_name, timeout=5.0, expected_partitions=partitions ) + if not topic_ready: + correlation_id = uuid4() + context = ModelInfraErrorContext.with_correlation( + correlation_id=correlation_id, + transport_type=EnumInfraTransportType.KAFKA, + operation="create_topic", + target_name=topic_name, + ) + raise InfraUnavailableError( + f"Topic '{topic_name}' already existed but its metadata did " + f"not propagate within timeout (expected {partitions} " + "partition(s)); create_topic cannot confirm the topic " + "actually exists on the broker.", + context=context, + ) return topic_name diff --git a/tests/helpers/util_migration_shape.py b/tests/helpers/util_migration_shape.py new file mode 100644 index 0000000000..a43f81a7cf --- /dev/null +++ b/tests/helpers/util_migration_shape.py @@ -0,0 +1,392 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Shape helpers for the node-owned migration corpus (OMN-15376). + +Parses ``docker/migrations/forward/nodes//*.sql`` well enough to answer +two questions that the shape-drift gate and its execution proof both need: + +* which columns does a ``CREATE TABLE IF NOT EXISTS`` DECLARE, and +* which columns does the same file RECONCILE with a guarded + ``ALTER TABLE ... ADD COLUMN IF NOT EXISTS``. + +The operator fence is READ FROM the single-sourced manifest +(``docker/migrations/forward/fenced-node-migrations.yaml``, OMN-15349) rather +than restated here: a second hand-maintained copy of that list is exactly the +cross-repo drift OMN-15336 was filed about. Before OMN-15349 this read from +the runner script's own literal copy; the runner no longer carries one (it +parses the same manifest at runtime), so this helper now points at the +manifest directly instead. + +Ticket: OMN-15376 +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +FORWARD_DIR = REPO_ROOT / "docker" / "migrations" / "forward" +NODE_MIGRATIONS_DIR = FORWARD_DIR / "nodes" +FORWARD_RUNNER = REPO_ROOT / "scripts" / "run-forward-migrations.sh" +FENCE_MANIFEST = ( + REPO_ROOT / "docker" / "migrations" / "forward" / "fenced-node-migrations.yaml" +) + +_CREATE_TABLE_GUARDED = re.compile( + r"CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\s+([A-Za-z0-9_.\"]+)\s*\(", re.I +) +_ADD_COLUMN_GUARDED = re.compile( + r"ALTER\s+TABLE\s+([A-Za-z0-9_.\"]+)\s+ADD\s+COLUMN\s+IF\s+NOT\s+EXISTS\s+" + r'("?[A-Za-z_][A-Za-z0-9_]*"?)', + re.I, +) +_MANIFEST_ID_LINE = re.compile(r'^\s*-\s*id:\s*"([^"]*)"', re.MULTILINE) + +# A leading keyword that marks a table-level constraint rather than a column. +_CONSTRAINT_HEADS = frozenset( + {"CONSTRAINT", "PRIMARY", "UNIQUE", "CHECK", "FOREIGN", "EXCLUDE", "LIKE"} +) +_COLUMN_TERMINATORS = ( + "NOT NULL", + "NULL", + "PRIMARY KEY", + "UNIQUE", + "REFERENCES", + "CHECK", + "DEFAULT", + "GENERATED", + "COLLATE", + "CONSTRAINT", + "DEFERRABLE", +) + + +def mask_literals(sql: str) -> str: + """Blank out comments, dollar-quoted bodies and string literals in place. + + Byte offsets are preserved so the mask can be used for paren/comma scanning + while substrings are still taken from the ORIGINAL text. + """ + out = list(sql) + i, n = 0, len(sql) + while i < n: + if sql.startswith("--", i): + end = sql.find("\n", i) + end = n if end == -1 else end + elif sql.startswith("/*", i): + end = sql.find("*/", i + 2) + end = n if end == -1 else end + 2 + elif sql.startswith("$$", i): + end = sql.find("$$", i + 2) + end = n if end == -1 else end + 2 + elif sql[i] == "'": + end = _string_end(sql, i) + else: + i += 1 + continue + for k in range(i, end): + out[k] = " " + i = end + return "".join(out) + + +def _string_end(sql: str, start: int) -> int: + j = start + 1 + n = len(sql) + while j < n: + if sql[j] == "'": + if j + 1 < n and sql[j + 1] == "'": + j += 2 + continue + return j + 1 + j += 1 + return n + + +def _balanced(text: str, start: int) -> int: + """Index just past the balanced paren group opening at ``text[start]``.""" + depth = 0 + for i in range(start, len(text)): + if text[i] == "(": + depth += 1 + elif text[i] == ")": + depth -= 1 + if depth == 0: + return i + 1 + return len(text) + + +@dataclass(frozen=True) +class DeclaredColumn: + """One column of a ``CREATE TABLE``: identifier plus its type/default text.""" + + name: str + type_text: str + default_text: str + generated: bool + + def seed_ddl_fragment(self) -> str: + """Column definition for a drift seed: type + DEFAULT, no constraints.""" + fragment = f"{self.name} {self.type_text}" + if self.default_text and not self.generated: + fragment += f" DEFAULT {self.default_text}" + return fragment + + +@dataclass(frozen=True) +class GuardedTable: + """A ``CREATE TABLE IF NOT EXISTS`` and the columns it declares.""" + + qualified_name: str + bare_name: str + columns: tuple[DeclaredColumn, ...] + + +def _split_body_items(body: str) -> list[str]: + masked = mask_literals(body) + items, depth, start = [], 0, 0 + for i, ch in enumerate(masked): + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + elif ch == "," and depth == 0: + items.append(body[start:i]) + start = i + 1 + items.append(body[start:]) + cleaned = [] + for item in items: + stripped = re.sub(r"--[^\n]*", "", item) + stripped = re.sub(r"/\*.*?\*/", "", stripped, flags=re.S).strip() + if stripped: + cleaned.append(" ".join(stripped.split())) + return cleaned + + +def _parse_column(item: str) -> DeclaredColumn | None: + match = re.match(r'("?[A-Za-z_][A-Za-z0-9_]*"?)\s+(.*)$', item, re.S) + if match is None: + return None + rest = match.group(2) + masked = mask_literals(rest) + type_end = len(rest) + default_text = "" + generated = False + i = 0 + while i < len(masked): + if masked[i] == "(": + i = _balanced(masked, i) + continue + upper = masked[i:].upper() + hit = next( + ( + kw + for kw in _COLUMN_TERMINATORS + if upper.startswith(kw) + and (i == 0 or not (masked[i - 1].isalnum() or masked[i - 1] == "_")) + and ( + i + len(kw) >= len(masked) + or not (masked[i + len(kw)].isalnum() or masked[i + len(kw)] == "_") + ) + ), + None, + ) + if hit is None: + i += 1 + continue + type_end = min(type_end, i) + if hit == "DEFAULT": + start = i + len(hit) + # Whitespace is skipped against the ORIGINAL text: mask_literals() + # blanks string bodies to spaces, so skipping on the mask would walk + # straight past a literal default and yield "DEFAULT ::jsonb". + while start < len(rest) and rest[start] == " ": + start += 1 + end = _scan_default(masked, start) + default_text = rest[start:end].strip() + i = end + elif hit == "GENERATED": + generated = True + i += len(hit) + else: + i += len(hit) + return DeclaredColumn( + name=match.group(1), + type_text=rest[:type_end].strip(), + default_text=default_text, + generated=generated, + ) + + +def _scan_default(masked: str, start: int) -> int: + i = start + while i < len(masked): + if masked[i] == "(": + i = _balanced(masked, i) + continue + upper = masked[i:].upper() + if any( + upper.startswith(kw) + and not (masked[i - 1].isalnum() or masked[i - 1] == "_") + for kw in _COLUMN_TERMINATORS + ): + return i + i += 1 + return len(masked) + + +def guarded_create_tables(sql: str) -> list[GuardedTable]: + """Every ``CREATE TABLE IF NOT EXISTS`` in ``sql``, with declared columns.""" + masked = mask_literals(sql) + tables: list[GuardedTable] = [] + for match in _CREATE_TABLE_GUARDED.finditer(masked): + qualified = match.group(1) + open_idx = match.end() - 1 + close_idx = _balanced(masked, open_idx) - 1 + body = sql[open_idx + 1 : close_idx] + columns = [] + for item in _split_body_items(body): + head = re.match(r'"?([A-Za-z_][A-Za-z0-9_]*)', item) + if head is not None and head.group(1).upper() in _CONSTRAINT_HEADS: + continue + column = _parse_column(item) + if column is not None: + columns.append(column) + tables.append( + GuardedTable( + qualified_name=qualified, + bare_name=qualified.split(".")[-1].strip('"'), + columns=tuple(columns), + ) + ) + return tables + + +def reconciled_columns(sql: str, table: GuardedTable) -> set[str]: + """Columns of ``table`` covered by a guarded ADD COLUMN in the same file.""" + masked = mask_literals(sql) + covered: set[str] = set() + for match in _ADD_COLUMN_GUARDED.finditer(masked): + target = match.group(1).split(".")[-1].strip('"').lower() + if target != table.bare_name.lower(): + continue + covered.add(match.group(2).strip('"').lower()) + return covered + + +def fenced_migration_ids() -> frozenset[str]: + """The operator fence baseline, read from the single-sourced manifest + (OMN-15349) so this helper cannot drift from what the runners actually + load. Note this is the BASELINE fence, not either runner's post-release + effective fence — callers that need the effective (post-release) set for + a specific lane must apply that lane's release policy themselves. + """ + ids = tuple(_MANIFEST_ID_LINE.findall(FENCE_MANIFEST.read_text(encoding="utf-8"))) + if not ids: # pragma: no cover - structural guard + raise AssertionError( + f"no fenced ids parsed from {FENCE_MANIFEST} — the manifest is " + "missing, empty, or the fence seam moved; fix this reader, do not " + "restate the list here." + ) + return frozenset(ids) + + +def node_migration_files() -> list[tuple[str, Path]]: + """Every vendored node migration as ``(namespaced_id, path)``, runner order.""" + files: list[tuple[str, Path]] = [] + for node_dir in sorted(p for p in NODE_MIGRATIONS_DIR.iterdir() if p.is_dir()): + for sql_file in sorted(node_dir.glob("*.sql")): + files.append((f"node:{node_dir.name}:{sql_file.name}", sql_file)) + return files + + +def flat_migration_files() -> list[tuple[str, Path]]: + """Every top-level (non-node) forward migration as ``(namespaced_id, path)``. + + Deliberately excludes ``NODE_MIGRATIONS_DIR`` (``forward/nodes/``) -- that + corpus is `node_migration_files`'s. This is the OTHER producer OMN-15384 + is about: ``docker/migrations/forward/*.sql`` at the top level, applied + against the ``omnibase_infra`` database rather than a node's own DB. + """ + files: list[tuple[str, Path]] = [] + for sql_file in sorted(FORWARD_DIR.glob("*.sql")): + files.append((f"flat:{sql_file.name}", sql_file)) + return files + + +def normalized_column_type(type_text: str) -> str: + """Collapse insignificant whitespace so ``DECIMAL(5,4)`` == ``DECIMAL(5, 4)``. + + Comparison, not display: two type spellings that differ only in whitespace + inside the type/precision text are the same declared type. Genuine type + differences (``REAL`` vs ``DOUBLE PRECISION``, ``UUID`` vs ``TEXT``, + ``NUMERIC(14,6)`` vs ``NUMERIC(18,6)``) are NOT collapsed by this -- those + are real shape divergence, the thing OMN-15384's gate exists to catch. + """ + return re.sub(r"\s+", "", type_text).upper() + + +@dataclass(frozen=True) +class ShapeDiff: + """Column-level divergence between one table's flat and node declarations.""" + + only_flat: tuple[str, ...] + only_node: tuple[str, ...] + type_diff: tuple[tuple[str, str, str], ...] # (column, flat_type, node_type) + + def __bool__(self) -> bool: + """True iff there IS a divergence -- empty diff is falsy, like a set.""" + return bool(self.only_flat or self.only_node or self.type_diff) + + def describe(self, *, table: str) -> str: + parts = [f"{table}:"] + if self.only_flat: + parts.append(f" only in flat: {list(self.only_flat)}") + if self.only_node: + parts.append(f" only in node: {list(self.only_node)}") + for column, flat_type, node_type in self.type_diff: + parts.append(f" type diff {column}: flat={flat_type!r} node={node_type!r}") + return "\n".join(parts) + + +def diff_column_shapes( + flat_columns: dict[str, str], node_columns: dict[str, str] +) -> ShapeDiff: + """Pure comparison of two ``{column: normalized_type}`` maps for one table.""" + flat_names = set(flat_columns) + node_names = set(node_columns) + type_diff = tuple( + sorted( + (column, flat_columns[column], node_columns[column]) + for column in flat_names & node_names + if flat_columns[column] != node_columns[column] + ) + ) + return ShapeDiff( + only_flat=tuple(sorted(flat_names - node_names)), + only_node=tuple(sorted(node_names - flat_names)), + type_diff=type_diff, + ) + + +def table_column_shapes(files: list[tuple[str, Path]]) -> dict[str, dict[str, str]]: + """Map ``bare table name -> {column name: normalized type}`` across *files*. + + Later files win per column on a repeat ``CREATE TABLE IF NOT EXISTS`` for + the same table in the same corpus (e.g. ``baselines`` has both a + ``0001_create`` and a later ``0002_realign`` file) -- the same + last-applied-wins semantics the migrations themselves have at runtime. + """ + shapes: dict[str, dict[str, str]] = {} + for _migration_id, path in files: + sql = path.read_text(encoding="utf-8") + for table in guarded_create_tables(sql): + bucket = shapes.setdefault(table.bare_name.lower(), {}) + for column in table.columns: + bucket[column.name.strip('"').lower()] = normalized_column_type( + column.type_text + ) + return shapes diff --git a/tests/incident_replays/registry.yaml b/tests/incident_replays/registry.yaml new file mode 100644 index 0000000000..da52772b2e --- /dev/null +++ b/tests/incident_replays/registry.yaml @@ -0,0 +1,328 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# INCIDENT REPLAY REGISTRY (OMN-15547) +# +# Enforced by `scripts/ci/check_incident_replay_coverage.py`. Read that module's +# docstring for rules R1-R5 and why they are worded the way they are. +# +# The one-line version: a guard that has never been run against the real thing +# it exists to catch is decorative. On 2026-07-30 three separate guards were +# green while enforcing nothing, each because its tests fed it a synthetic input +# that could not exhibit the failure. +# +# HOW TO ADD A CASE (the honest bar) +# 1. Get the real bytes. Fetch them from the surface that actually failed -- +# `gh api` a run/PR, `git cat-file` the object, `curl` the live endpoint, +# `scp` the host file. Do NOT retype them from memory or from a report; a +# plausible-looking reconstruction is the defect, not the fix. +# 2. Commit them verbatim under `tests/fixtures//`. Use a +# `.captured` infix so header/format hooks leave the bytes alone -- a +# reformatted artifact is no longer the artifact that failed. +# 3. Record the sha256 and a `capture.source` locator someone else can +# re-fetch. Free-text provenance is rejected on purpose: a hand-typed +# fixture has no locator that resolves, and that is the only difference a +# machine can see between a capture and an invention. +# 4. Write the test that drives the REAL guard with those bytes and asserts +# the verdict the buggy guard got wrong. +version: 1 +scope: + # Guards whose enforcement is load-bearing enough that a missing replay case + # is a hard failure. APPEND-ONLY -- removing a line is removing a proof + # obligation. An entry whose file does not exist yet reports PENDING rather + # than failing, which is how a requirement is armed BEFORE the guard lands. + required_guards: + # The .201 system-health alert. Incident OMN-15525: a 63-byte synthetic + # HEALTHY_BODY hid a 180-byte pre-parse truncation, so a 654-line suite was + # green while the deployed artifact paged CRITICAL on a healthy fleet. + - deploy/maintenance/omninode-system-slack-report.sh + # Pre-registered, not yet present. Lands with OMN-15538 (omnibase_infra + # #2583), which replays 879d6fc6 -- the pin that wedged every open infra PR + # for ~2.5h on 2026-07-30 (OMN-15536). Listing it here now means the + # requirement arms itself the moment that PR merges, with no follow-up edit. + - scripts/ci/check_pin_reachability.py + # Wired enforcement guards that carry NO replay case yet: the honest + # outstanding debt, enumerated so it is countable. This list may only SHRINK. + # Generated 2026-07-31 from the wired-guard inventory + # (`check_incident_replay_coverage.wired_guards`) at dev fea3f3e6. + # + # Being on this list is not permission -- it is a debt record. Anything NOT on + # it and NOT covered fails DEFAULT-DENY, which is the rule that stops the + # detection shelf growing faster than the proof behind it. + debt_baseline: + - scripts/audit-branch-protection.sh + - scripts/audit-merge-method-on-main.sh + - scripts/audit-runner-routing.py + - scripts/audit_required_context_parity_cli.py + - scripts/batch_validate_node_contracts.py + - scripts/check-env-reads.sh + - scripts/check-pinned-wheels.py + - scripts/check-realm-no-clients.sh + - scripts/check-strict-gate-ordering.sh + - scripts/check_context_field_presence.py + - scripts/check_contract_topic_parity.py + - scripts/check_dep_provenance.py + - scripts/check_dispatcher_route_coverage.py + - scripts/check_dockerfile_pins.py + - scripts/check_event_registry_fingerprint.py + - scripts/check_evidence_provenance_probe.py + - scripts/check_lane_attestation.py + - scripts/check_lane_census_age.py + - scripts/check_lane_env_drift.py + - scripts/check_migration_freeze.sh + - scripts/check_migration_required.py + - scripts/check_no_cloud_bus_wrapper.sh + - scripts/check_prod_promotion_lineage.py + - scripts/check_release_identity.py + - scripts/check_required_env_vars.py + - scripts/check_runtime_image_identity.py + - scripts/check_schema_fingerprint.py + - scripts/check_seed_provenance.py + - scripts/check_shared_enum_ownership.py + - scripts/check_terminal_cost_completeness.py + - scripts/check_test_failure_ratchet.py + - scripts/check_topic_drift.py + - scripts/check_version_pins.py + - scripts/ci/build_runner_image.sh + - scripts/ci/canonical_handler_shape_baseline.py + - scripts/ci/check_deploy_scope_dod.py + - scripts/ci/check_integration_skips.py + - scripts/ci/check_occ_companion_merged.py + - scripts/ci/check_shell_hygiene.sh + - scripts/ci/ci_env_digest.py + - scripts/ci/ci_summary_gate.py + - scripts/ci/detect_test_paths.py + - scripts/ci/ensure_ci_env.sh + - scripts/ci/merge_queue_enqueue.py + - scripts/ci/occ_manual_replay_precheck.py + - scripts/ci/parity_red_on_base_baseline.py + - scripts/ci/publish_with_retry.py + - scripts/ci/reject-node-implementations.sh + - scripts/ci/run_duplication_sweep.py + - scripts/ci/runner_fleet_canary.sh + - scripts/ci/runner_image_identity.py + - scripts/ci/verify_pypi_pin_resolvability.py + - scripts/ci_check_kafka_no_localhost_default.py + - scripts/deploy-agent/deploy_agent/executor.py + - scripts/deploy-runtime.sh + - scripts/generate_claude_lane_block.py + - scripts/generate_topic_enums.py + - scripts/hooks/prepush_smart_tests.sh + - scripts/lint_compose_dangling_deps.py + - scripts/lint_pricing_manifest.py + - scripts/publish_pr_merged_event.py + - scripts/publish_pr_webhook_event.py + - scripts/publish_test_failure_baseline.py + - scripts/release_drift_monitor.py + - scripts/resolve_node_migration_source_ref.py + - scripts/run-migrations.py + - scripts/run_baselines_batch_compute.py + - scripts/runtime_build/check_sibling_lock_pins.py + - scripts/runtime_build/cut_release_train_tag.sh + - scripts/runtime_build/refresh_dev_lane.sh + - scripts/runtime_build/refresh_stability_lane.sh + - scripts/runtime_build/stage_workspace.sh + - scripts/sync-node-migrations.sh + - scripts/system_health_check.sh + - scripts/trigger_rebuild_on_merge.py + - scripts/update-plugin-pins.py + - scripts/update_version_matrix.py + - scripts/validate-kafka-schema-handshake.py + - scripts/validate-pr-contract-sync.sh + - scripts/validate.py + - scripts/validate_no_env_fallbacks.py + - scripts/validation/check_ai_slop.py + - scripts/validation/check_kafka_no_hardcoded_fallback.sh + - scripts/validation/check_no_infra_inmemory_import.sh + - scripts/validation/check_published_events_consistency.py + - scripts/validation/check_topic_literals.py + - scripts/validation/check_topic_suffix_exports.py + - scripts/validation/lint_topic_names.py + - scripts/validation/run_topic_lint.sh + - scripts/validation/validate_migration_freeze.py + - scripts/validation/validate_migration_sequence.py + - scripts/validation/validate_model_dump_json_mode.py + - scripts/validation/validate_naming.py + - scripts/validation/validate_precommit_fail_loud.py + - scripts/validation/validate_precommit_pin_parity.py + - scripts/validation/validate_test_removal_gate.py + - scripts/validation/validate_test_root_collection.py + - scripts/verify-merge-queue-no-bypass.sh +cases: + - id: omn15525-health-body-truncation + guard: deploy/maintenance/omninode-system-slack-report.sh + incident: OMN-15525 + regression_class: false_red + guard_verdict_on_artifact: accept + artifact: + fixture: tests/fixtures/omn15547/health-dev-8085.json.captured + sha256: 240178e33079f76b38f4995c39b7a90da68c37d4f08365c13752411a9da6050a + capture: + method: live-fetch + source: live-http:omni-201-ts:8085/health + captured_at: 2026-07-31T00:03:06Z + captured_by: lane replay (OMN-15547) + test: tests/unit/scripts/test_omninode_system_slack_report.py::test_healthy_lane_with_a_realistic_body_is_not_critical + discriminator: tests/unit/scripts/test_omninode_system_slack_report.py::test_unreachable_runtime_endpoint_is_critical_not_skipped + why: >- + The reporter truncated the /health body to 180 bytes before handing it to jq. Every real body on .201 is 2079-2644 bytes, so jq died on every lane ("Unfinished string at EOF"), the verdict fell through to `unresolvable`, and the alert paged CRITICAL for all three lanes against a fully healthy fleet. The suite could not see any of it because HEALTHY_BODY was 63 bytes -- under the cut. The OMN-15525 repair made the literal bigger but kept it hand-typed ("Shape mirrors the live body"); this case replaces it with the 2644 bytes the endpoint actually returned, so the fixture can exhibit failure modes nobody anticipated. Direction is false_red, so the `discriminator` is required: it proves the same reporter still says CRITICAL when an endpoint is genuinely unreachable, which is what stops an accept-only proof from passing a stuck-open guard. + - id: omn15547-handtyped-fixture-passed-as-proof + guard: scripts/ci/check_incident_replay_coverage.py + incident: OMN-15547 + regression_class: false_green + guard_verdict_on_artifact: reject + artifact: + fixture: tests/fixtures/omn15547/test_omninode_system_slack_report.handtyped-fixture.py.captured + sha256: 10cfdc48ef1a80dbc10f2c9cf1a84cfc20a68877111e15b5cbe1e56c428c5a0c + capture: + method: git-object + source: git-object:OmniNode-ai/omnibase_infra@0f05039434996594000db85cd8d3947523bfebcf:tests/unit/scripts/test_omninode_system_slack_report.py + captured_at: "2026-07-31T00:12:00Z" + captured_by: lane replay (OMN-15547) + test: tests/ci/test_incident_replay_coverage_omn15547.py::test_the_shipped_handtyped_fixture_fails_r1_and_r2 + why: >- + This lint is itself a new guard, so its own default-deny rule demands a real regression case rather than a baseline exemption -- and it fired on itself the first time it was wired, which is how this case came to exist. The artifact is the verbatim blob that shipped on dev at 0f050394 as the REPAIR for OMN-15525: a hand-typed HEALTHY_BODY literal defended by the comment "Shape mirrors the live body". It was accepted as proof, and it was not proof -- the key names, nesting and value shapes were guesses, so the fixture could only ever exhibit failure modes its author had already imagined. The case drives the real R1/R2 predicates over those bytes and requires BOTH to say no: R1 because the body is a literal inside the test rather than a committed capture, R2 because "Shape mirrors the live body" is the strongest provenance the file offers and it resolves to nothing. A world with this lint could not have shipped that blob; a world without it did, twice. + - id: omn15536-unreachable-pin-outage + guard: scripts/ci/check_pin_reachability.py + incident: OMN-15536 + regression_class: false_green + guard_verdict_on_artifact: reject + artifact: + fixture: tests/fixtures/omn15547/omnimarket-compare-dev-879d6fc6.gh-api.json.captured + sha256: 4e957324574cf01581701cc662adba35cb092670ee67db48f31118f17239f3b5 + capture: + method: live-fetch + source: gh-api:repos/OmniNode-ai/omnimarket/compare/dev...879d6fc6825f876458c6d45ed670c8715de8ac95 + captured_at: "2026-07-31T00:18:28Z" + captured_by: lane replay (OMN-15547) + test: tests/ci/test_incident_replay_coverage_omn15547.py::test_the_omn15536_outage_pin_is_unreachable_by_the_real_predicate + why: >- + omnibase_infra#2570 pinned the merge-hold reusable workflow at omnimarket@879d6fc6 -- the pre-squash head of omnimarket#1976, whose branch GitHub deleted when it squash-merged as 454c429f. The ref then resolved to nothing, so ci.yml failed to LOAD (0 jobs, conclusion=failure) and the sole required context CI Summary went ABSENT rather than red: all seven open infra PRs blocked for ~2.5h with no failing check to point at. Both repos already shipped pin validators and both PASSED the dead pin, because both only checked that it looked like 40 hex characters. The captured artifact is GitHub answering the question those validators never asked -- `status: diverged`, i.e. unreachable from the protected branch. OMN-15538 (#2583) built the predicate; this case pins it against the real response rather than a hardcoded string, and its `behind` sibling (omnimarket-compare-dev-454c429f, the squash that actually landed) is the control that stops a blanket-reject implementation looking correct. + - id: omn15355-acl-generator-source-lock-missing-roots + guard: scripts/generate_application_database_acl.py + incident: OMN-15355 + regression_class: false_green + guard_verdict_on_artifact: reject + artifact: + fixture: tests/fixtures/omn15547/application-acl-source-lock.yaml.captured + sha256: 79d329c20554dcd4508322dc9f0d73630464e20e2f84dd712e8584a7caea0a97 + capture: + method: git-object + source: git-object:OmniNode-ai/omnibase_infra@b65dd286f373c3363e14066e0bbf272ddbc0aec6:docker/application-acl-proof/source-lock.yaml + captured_at: "2026-07-31T05:55:00Z" + captured_by: Codex merge sweep (OMN-15418 repair) + test: tests/unit/scripts/test_prove_application_database_acl.py::test_generator_rejects_source_lock_without_repository_roots + why: >- + The ACL matrix generator is an enforcement guard only if it refuses to render from a source lock whose repository roots are not explicitly supplied. A missing-root run can otherwise produce proof-shaped output divorced from the immutable Git objects named by the lock. This fixture is the real source lock from the rebased #2548 head; the replay drives the actual generator entrypoint with no repository roots and requires the fail-closed missing-root verdict. + - id: omn15355-acl-proof-prechange-drift + guard: scripts/ci/prove_application_database_acl.py + incident: OMN-15355 + regression_class: false_green + guard_verdict_on_artifact: reject + artifact: + fixture: tests/fixtures/omn15547/application-acl-prechange-fixture.json.captured + sha256: a38988de6a5f7927491bb36c47e45202577b254db4b60731f0b1d24cfad235d5 + capture: + method: git-object + source: git-object:OmniNode-ai/omnibase_infra@b65dd286f373c3363e14066e0bbf272ddbc0aec6:docker/application-acl-proof/generated/prechange-fixture-acl.json + captured_at: "2026-07-31T05:55:00Z" + captured_by: Codex merge sweep (OMN-15418 repair) + test: tests/unit/scripts/test_prove_application_database_acl.py::test_prechange_capture_pins_durable_acl_drift_predicate + why: >- + The PostgreSQL ACL proof is only durable if its pre-change rollback artifact is exact. A proof that accepts drift in that artifact can pass while rollback no longer restores the real pre-change graph. This fixture is the real generated pre-change artifact from the rebased #2548 head; the replay loads it through the proof module's EXPECTED_PRECHANGE path and requires the same durable-prechange equality predicate to reject a one-row drift. + - id: omn15361-application-sql-unqualified-node-migration + guard: scripts/ci/check_application_database_sql.py + incident: OMN-15361 + regression_class: false_green + guard_verdict_on_artifact: reject + artifact: + fixture: tests/fixtures/omn15547/node-service-registry-tenant-rls-unqualified.sql.captured + sha256: 3e8a3c12dbefc4432262a1c2533667f4a5f342c39149915bd80b45241c182e98 + capture: + method: git-object + source: git-object:OmniNode-ai/omnibase_infra@5d845c16c348e5c1e337dfd56f7f452f5547235a:docker/migrations/forward/nodes/node_projection_registration/0002_node_service_registry_tenant_rls.sql + captured_at: "2026-07-31T06:49:00Z" + captured_by: Codex merge sweep (OMN-15361 repair) + test: tests/ci/test_application_database_sql_gate.py::test_omn15361_replay_rejects_real_unqualified_application_migration + why: >- + The changed-SQL gate is load-bearing only if it rejects deployable application migrations exactly as they appear in the migration tree, not only small SQL literals assembled inside unit tests. This fixture captures the real node_service_registry tenant-RLS migration bytes from the #2590 head: it targets application relations with unqualified names and no authoritative ownership manifest, the shape OMN-15361 exists to stop from silently extending the application authority surface. The replay stages those captured bytes as an actual changed .sql path in a Git repository and drives the real scripts/ci/check_application_database_sql.py validator through validate_changed_sql, requiring the schema-qualification rejection. + - id: omn15422-run-forward-legacy-ledger-fixture + guard: scripts/run-forward-migrations.sh + incident: OMN-15422 + regression_class: false_green + guard_verdict_on_artifact: reject + artifact: + fixture: tests/fixtures/omn15547/legacy-rds-fixture-prove.sh.captured + sha256: dbaa9701ceaa9fbdc3a7d011f82d6931f4714c6b6574b322b8d7e46b513f168c + capture: + method: git-object + source: git-object:OmniNode-ai/omnibase_infra@60f542619f2df5db9c173982f3a06bd1dc4134b8:docker/legacy-rds-fixture/prove.sh + captured_at: "2026-07-31T07:35:00Z" + captured_by: Codex merge sweep (OMN-15422 repair) + test: tests/ci/test_legacy_rds_fixture_contract.py::test_proof_runs_real_migrations_twice_and_pins_the_blocked_upgrade + why: >- + The forward migration runner became newly wired enforcement when the OMN-15422 rebuilt legacy-RDS proof workflow started invoking it against both fresh and legacy PostgreSQL 16 histories. The incident was a false-green upgrade proof: a runner could pass a fresh path while legacy ledger shapes, duplicate checksums, and the unresolved OMN-15423 node migration wall were never exercised together. This captured proof script is the committed artifact that drives the real runner twice through the synthetic legacy ledger corpus and requires the blocked-upgrade signature, so the runner cannot be counted as wired enforcement without the regression surface it exists to catch. + - id: omn15656-topology-zero-table-grants + guard: scripts/generate_application_database_table_grants.py + incident: OMN-15656 + regression_class: false_green + guard_verdict_on_artifact: reject + artifact: + fixture: tests/fixtures/omn15547/onex-dev-topology-zero-table-grants.yaml.captured + sha256: b34206e7512eca5eba1502e969fea52c641b4b5eda4eff538ba5c7a459e7b07d + capture: + method: git-object + source: git-object:OmniNode-ai/omnibase_infra@27630ec1be051faaef0a9f61a59615af11b28a39:src/omnibase_infra/topology/instances/onex-dev.yaml + captured_at: "2026-08-02T10:40:00Z" + captured_by: lane replay (OMN-15656) + test: tests/unit/topology/test_application_database_table_grants.py::test_replay_derivation_rejects_the_captured_topology_as_drifted + why: >- + The onex-dev runtime plane failed to boot on deploy run 30737415706 because every topology instance declared ZERO `object_type: TABLE` grants, so the OMN-15418 privilege validator refused all 43 contract-declared projections on all 7 profiles. This artifact is the exact instance the cluster consumed -- its sha256 b34206e7 is the same digest recorded in the rendered catalogs' `source.sha256` and in omninode_infra's k8s source-lock, so it is the deployed bytes rather than a reconstruction. Direction is false_green: nothing was red. CI passed, the ACL matrix had already serialized the finding as a BLOCKED blocker two days earlier, and the wiring tests were green because `_with_projection_fixture_grants` manufactured the missing grants at test time and loaded a fixture topology instead of the shipped one. The replay drives the real derivation over the captured bytes and requires the guard to reject them, and its sibling `test_replay_captured_topology_reproduces_the_deploy_failure_verbatim` pins the resolver's error string character-for-character against the pod log, so a future change that merely reshapes the message cannot quietly satisfy this case. + - id: omn15676-runner-fleet-missing-from-image + guard: scripts/ci/assert_image_config_paths.py + incident: OMN-15676 + regression_class: false_green + guard_verdict_on_artifact: reject + artifact: + fixture: tests/fixtures/omn15676/runtime-image-config-probe-403b6092.txt.captured + sha256: 9be8ff420b04bd0b96ef944ff4b67d639501f32810320fa96294d1f4b5d60a41 + capture: + method: container-probe + source: container-probe:ghcr.io/omninode-ai/omnibase-infra-runtime@sha256:403b60921fe0a624ab788d9444e710a2223bd8d3c39304e7e8d9809e7ef1fcdd:/app/config + captured_at: "2026-08-02T23:41:00Z" + captured_by: lane replay (OMN-15676) + test: tests/ci/test_assert_image_config_paths.py::test_replay_real_prefix_image_probe_is_rejected + why: >- + config/runner_fleet.yaml was tracked, valid, referenced by config code, and shipped in NO runtime image, because no Dockerfile COPY placed it. The runtime resolves it at /app/config/runner_fleet.yaml (default_runner_fleet_config_path is repo-root-relative, and the image sets PYTHONPATH=/app/src, so parents[4] is /app), HandlerRunnerFleetSnapshot.__init__ calls load_runner_fleet_config() which raises rather than falling back to lab defaults, and the handler is instantiated during auto-wiring -- so with ONEX_WIRING_STRICT_MODE=1 bound on onex-dev the absence is a boot failure, not a degraded mode. Direction is false_green in the strongest sense: there was no red anywhere. Every source-level check passed, because every source-level check was looking at a working tree that was correct. This is the THIRD instance of the class (grants fixture, then routing_tiers.yaml under OMN-15645, now this), which is why the repair ships the generalized in-image assertion and not just the missing COPY. The artifact is the verbatim stdout of the guard's own probe run inside the digest-pinned image ghcr.io/omninode-ai/omnibase-infra-runtime@sha256:403b6092 on omni-201-ts -- two lines, one MISSING and one PRESENT, so it is also the control: a blanket-reject implementation that failed every path would look correct against an all-missing capture, and cannot against this one. The locator grammar `container-probe` was added to this checker in the same commit for the same reason -- an incident whose failing bytes exist only inside an image had no re-fetchable locator, so this class could not be replayed at all. + - id: omn15776-broker-dispatch-wedge + guard: scripts/ci/runner_broker_dispatch_wedge_rerun.sh + incident: OMN-15776 + regression_class: false_green + guard_verdict_on_artifact: reject + artifact: + fixture: tests/fixtures/omn15776/job-93294479341.json.captured + sha256: 5f4305bd74e2d7ee2ceb9d09e9861d1f8d04cdc198150c64aa2befff8ff0c0bf + capture: + method: live-fetch + source: gh-api:repos/OmniNode-ai/onex_change_control/actions/jobs/93294479341 + captured_at: "2026-08-09T21:00:00Z" + captured_by: lane replay (OMN-15776) + test: tests/ci/test_runner_broker_dispatch_wedge_rerun.py::TestOmn15776IncidentReplay::test_real_wedge_job_is_rejected_ie_flagged_for_rerun + why: >- + GitHub's Actions broker dispatched job 93294479341 ("No Divergent Automation PRs (OMN-14778)", onex_change_control run 31333147268) to omninode-runner-17 within seconds of that same runner finishing its prior job, exactly while the runner's Runner.Listener was mid-reconnect on its broker long-poll after the retry/backoff storm every job completion triggers on that connection. The dispatch was never delivered to the runner's local message loop -- the runner's own _diag log has zero "Running job" entry for it, confirmed independently across 4 runners in the 2026-08-09 investigation -- so GitHub's server-side clock ran alone and orphaned the assignment at exactly 600s (started_at 20:06:56Z, completed_at 20:16:56Z), recording steps=[] because no Runner.Worker ever spawned. Direction is false_green: before this guard existed, that was the entire lifecycle -- a real, unrecoverable red that no local process (not the OMN-14564 heartbeat watchdog, which never saw a listener go silent or a Worker exist to guard) could observe or act on, so it sat there until a human happened to notice across several 2026-08-09 ledger passes and diagnosed it without being able to fix it ("not fixable via rerun or from this session's access level" -- see the mergesweep-0809-runnerfix ledger entry). The replay drives the real script against the verbatim captured job and requires it to REJECT the artifact: recognize the structural signature (runner assigned, conclusion=failure, steps=[], duration in the tight 595-605s band around the proven fixed ~10m0-1s server-side timeout) and reissue exactly that job -- proving the guard actually closes the gap on the real incident bytes, not a hand-typed approximation of them. + - id: omn15717-pr-review-bot-undeclared-migration + guard: scripts/validation/validate_application_migration_manifest.py + incident: OMN-15717 + regression_class: false_green + guard_verdict_on_artifact: reject + artifact: + fixture: tests/fixtures/omn15717/001_create_review_bot_bypass_log.sql.captured + sha256: 63e2646a7f8767fad9ec969b224982e00b83aacb665107ab5b103964d5616e00 + capture: + method: git-object + source: git-object:OmniNode-ai/omnimarket@cedd24311ed320d34cc5ab5f8f79f5b04e9abf25:src/omnimarket/nodes/node_pr_review_bot/migrations/001_create_review_bot_bypass_log.sql + captured_at: "2026-08-06T00:30:00Z" + captured_by: lane replay (OMN-15717, lab-runtimes) + test: tests/unit/scripts/validation/test_application_migration_manifest.py::TestOmn15717IncidentReplay::test_captured_undeclared_migration_is_rejected + why: >- + node_pr_review_bot's 001_create_review_bot_bypass_log.sql was vendored into docker/migrations/forward/nodes/ (commit history: added, applied to at least one live database, later removed as "stale" by scripts/sync-node-migrations.sh once omnimarket deleted the shell node in the OMN-13212 canonical rebuild) without ever gaining a row in docker/migrations/forward/_ledger/application-migrations.tsv. scripts/validation/validate_application_migration_manifest.py already implements the exact "declared set matches the vendored tree" check that would have caught this (its missing/extra set-difference assertion), but it was wired to NOTHING -- no CI job, no pre-commit hook, only its own test file called it directly. So the omission shipped silently -- false_green in the strongest sense, there was no red anywhere in CI. The gap surfaced only weeks later, at deploy time, when a workspace-mode refresh_stability_lane.sh run hit a live database carrying the legacy runner's applied-history row for this exact migration id and bootstrap.sql raised "unknown migration stream/domain: adopted node version node:node_pr_review_bot:001_create_review_bot_bypass_log.sql has no checked-in declaration" (bootstrap.sql:673, forensic log .201:/tmp/refresh_stability_lane_20260805.log). The fixture is the verbatim file content from the omnimarket commit that introduced it (content never changed across its lifetime -- confirmed via `git diff` between the add and delete commits, both empty); content-identical, unmodified bytes are what still sits in this repo's own vendored tree post-fix. The replay stages those captured bytes as a vendored node migration file with zero declaration rows (the exact pre-fix tree shape) and drives the real validate_manifests() function, requiring rejection. diff --git a/tests/integration/ci/test_dep_provenance_lineage_live_omn15604.py b/tests/integration/ci/test_dep_provenance_lineage_live_omn15604.py new file mode 100644 index 0000000000..7beaadd5fb --- /dev/null +++ b/tests/integration/ci/test_dep_provenance_lineage_live_omn15604.py @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Live half of the dep-provenance content-lineage gate (OMN-15604). + +Hermetic decision-logic tests (fake resolver, no network) live in +``tests/scripts/test_check_dep_provenance_lineage_omn15604.py``. This module +holds only the tests that hit the live GitHub REST API, under +``tests/integration`` for the same reason as the OMN-15538 pin-reachability +gate's live half: the pre-push selector always ignores this tree, so a +transient network failure here can never make a branch locally unpushable. + +What this proves, against the REAL commits (not synthetic values, per memory +``feedback_prove_red_against_exists_but_wrong``): + +1. **RED on the actual incident pin.** ``omnibase_core`` rev ``3d51b047`` (the + ``omnibase_infra@dev`` pyproject.toml:222 pin this ticket exists because + of) has a ``src/`` tree that genuinely differs from released tag + ``v0.46.8``'s ``src/`` tree. ``resolve_src_tree_sha`` must resolve both to + their real GitHub tree SHAs and ``find_lineage_violations`` must flag the + mismatch. +2. **GREEN on a pin that IS the release.** Pinning tag ``v0.46.8``'s own + resolved commit against declared version ``0.46.8`` must report no + violation -- content identical, by construction. +""" + +from __future__ import annotations + +import importlib.util +import os +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).resolve().parents[3] / "scripts" / "check_dep_provenance.py" +_IN_CI = bool(os.environ.get("CI")) + +# The exact live incident pin (OMN-15604 ticket evidence). +_PINNED_REV = "3d51b047a43ee412a7521502619d35c216dc7811" +_RELEASED_TAG_COMMIT = "105f7ce0a8f4b31f6f01fc94e9b43e75984f166a" # v0.46.8 + + +def _load_module(): + spec = importlib.util.spec_from_file_location("check_dep_provenance", _SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def mod(): + return _load_module() + + +def _require_resolved(sha: str | None, detail: str, label: str) -> str: + """Fail closed in CI, skip only on a developer machine, on a network miss.""" + if sha is not None: + return sha + message = f"{label}: could not resolve via the live GitHub API ({detail})" + if _IN_CI: + pytest.fail(f"{message} — an unresolvable pin is not a passing pin") + pytest.skip(message) + raise AssertionError("unreachable") # pragma: no cover + + +@pytest.mark.integration +def test_live_pinned_and_released_src_trees_are_independently_resolvable(mod) -> None: + """Sanity: both real refs resolve to real (and DIFFERENT) tree SHAs. + + This is the ground-truth measurement the ticket's own evidence cites + (`git rev-parse 3d51b047:src` = a74566fd..., `git rev-parse v0.46.8:src` + = 008efdba...) -- reproduced here via the REST path the CI gate actually + uses, not the local-clone path the ticket used to discover the defect. + """ + pinned_sha, pinned_detail = mod.resolve_src_tree_sha("omnibase_core", _PINNED_REV) + released_sha, released_detail = mod.resolve_src_tree_sha("omnibase_core", "v0.46.8") + pinned_sha = _require_resolved(pinned_sha, pinned_detail, "pinned rev 3d51b047") + released_sha = _require_resolved(released_sha, released_detail, "released v0.46.8") + + assert pinned_sha == "a74566fd92b0ca9bb86919df5e7f804cc4307793" + assert released_sha == "008efdba12b39cf04d90d17468523daa281fe4fd" + assert pinned_sha != released_sha + + +@pytest.mark.integration +def test_live_red_on_the_real_incident_pin(mod) -> None: + """RED-before proof: the exact live pin (3d51b047 + ==0.46.8), not a + synthetic value, drives find_lineage_violations end to end.""" + pyproject_text = ( + "[project]\n" + 'name = "omnibase-infra"\n' + 'version = "0.0.0"\n' + "dependencies = [\n" + ' "omnibase-core==0.46.8",\n' + "]\n" + "\n" + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414\n' + ) + + violations = mod.find_lineage_violations(pyproject_text) + undetermined = [v for v in violations if "UNDETERMINED lineage" in v] + if undetermined: + _require_resolved(None, undetermined[0], "live lineage check") + return # pragma: no cover - _require_resolved always fails/skips above + + assert len(violations) == 1, violations + assert "omnibase-core" in violations[0] + assert "differs from" in violations[0] + + +@pytest.mark.integration +def test_live_green_when_pin_is_the_release_commit(mod) -> None: + """GREEN-after proof: pinning the release tag's own commit reports clean.""" + pyproject_text = ( + "[project]\n" + 'name = "omnibase-infra"\n' + 'version = "0.0.0"\n' + "dependencies = [\n" + ' "omnibase-core==0.46.8",\n' + "]\n" + "\n" + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_RELEASED_TAG_COMMIT}" }}\n' + ) + + violations = mod.find_lineage_violations(pyproject_text) + undetermined = [v for v in violations if "UNDETERMINED lineage" in v] + if undetermined: + _require_resolved(None, undetermined[0], "live lineage check (green case)") + return # pragma: no cover - _require_resolved always fails/skips above + + assert violations == [] + + +@pytest.mark.integration +def test_live_this_repos_own_pyproject_is_lineage_clean() -> None: + """This repo's own tree (AC1): no forbidden git override present at all, + so the lineage check has nothing to compare -- zero live calls, exit 0.""" + mod = _load_module() + repo_root = Path(__file__).resolve().parents[3] + pyproject_path = repo_root / "pyproject.toml" + text = pyproject_path.read_text() + + violations = mod.find_lineage_violations(text) + assert violations == [], ( + "omnibase_infra's own pyproject.toml should carry no forbidden " + f"git-pinned override post-OMN-14628; got: {violations}" + ) + + +@pytest.mark.integration +def test_live_omn_15414_resolves_to_done(mod) -> None: + """AC3 ground truth: the exact ticket the live incident's escape token + cites (`# raw-override-ok: OMN-15414`) really is Done today, via the + real Linear API -- not an assumed/stale fact. Skipped (not failed) when + LINEAR_API_KEY is unavailable, matching the check's own graceful + degradation posture.""" + if not os.environ.get("LINEAR_API_KEY"): + pytest.skip("LINEAR_API_KEY not set in this environment") + + status_name, detail = mod.resolve_ticket_status("OMN-15414") + if status_name is None: + if _IN_CI: + pytest.fail(f"could not resolve OMN-15414 via live Linear API: {detail}") + pytest.skip(f"could not resolve OMN-15414 via live Linear API: {detail}") + assert status_name.strip().lower() in mod._TICKET_DONE_STATUSES, ( + f"expected OMN-15414 to be closed (Done at ticket-file time), got " + f"{status_name!r}" + ) + + +@pytest.mark.integration +def test_live_red_escape_token_end_to_end_against_the_real_incident_token( + mod, +) -> None: + """RED end-to-end: the EXACT live incident line (rev 3d51b047, token + `# raw-override-ok: OMN-15414`, no until=) fails + find_escape_token_violations, reproducing the ticket's cited proof + requirement ("RED case where a well-formed token cites a Done ticket") + against the real production code path. + + Deliberately does NOT skip on a missing LINEAR_API_KEY: LINEAR_API_KEY is + not provisioned as a repo or org secret anywhere in OmniNode-ai today, so + a skip-on-unset guard here would mean this proof never actually executes + in the live enforcing CI environment -- precisely the gap a remediation + round found ("Both live AC3 proofs also skip in CI"). The mandatory- + until= enforcement path (`find_escape_token_violations`) fires on this + exact token shape with zero network calls, so this test is unconditional. + """ + pyproject_text = ( + "[project]\n" + 'name = "omnibase-infra"\n' + 'version = "0.0.0"\n' + "dependencies = [\n" + ' "omnibase-core==0.46.8",\n' + "]\n" + "\n" + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414\n' + ) + + violations = mod.find_escape_token_violations(pyproject_text) + assert len(violations) == 1, violations + assert "OMN-15414" in violations[0] diff --git a/tests/integration/ci/test_pin_reachability_live_omn15538.py b/tests/integration/ci/test_pin_reachability_live_omn15538.py new file mode 100644 index 0000000000..e3fd654522 --- /dev/null +++ b/tests/integration/ci/test_pin_reachability_live_omn15538.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Live half of the protected-branch pin-reachability gate (OMN-15538). + +Static extraction + oracle tests are hermetic and live in +``tests/ci/test_pin_reachability_omn15538.py``. This module holds only the +tests that hit the live GitHub API, under ``tests/integration`` for the same +reasons as its OMN-14941 sibling: the pre-push selector always ignores that +tree, and a red here must not make a branch locally unpushable. + +Two things are proven live, and both matter: + +1. **Discrimination on the real incident vectors.** The two SHAs that caused + the 2026-07-30 incidents must resolve UNREACHABLE and their correct + counterparts REACHABLE. These are the actual pins, not synthetic values — + proving RED against *exists-but-wrong* rather than against absent (memory + ``feedback_prove_red_against_exists_but_wrong``). ``879d6fc6`` and + ``5a907b71`` are both real commits that GitHub still serves; an existence + probe passes on both, which is exactly why the oracle is compare-based. + +2. **This repo's own tree is clean.** Expected GREEN. There is no standing + authorization for a red here — if it fails, a pin in this repo is not + durable; re-pin it (OMN-15248 posture: a blessed failure is how a real + regression gets waved through). + +Note on ``5a907b71``'s expected lifetime: it is currently ``ahead`` of core +``dev`` (the live head of ``jonah/omn-15392-evidence-execution-scope``). When +OMN-15392 lands, it becomes ``diverged`` — still UNREACHABLE, so the assertion +below holds across that transition. That is the point: the verdict is stable +while the reason changes, because the pin was never durable in either state. +""" + +from __future__ import annotations + +import os + +import pytest + +from scripts.ci.check_pin_reachability import ( + Verdict, + _Resolver, + extract_pins, +) +from tests.ci.test_pin_reachability_omn15538 import ( + INCIDENT_A_DEAD_SHA, + INCIDENT_A_GOOD_SHA, + INCIDENT_B_DEAD_SHA, + INCIDENT_B_GOOD_SHA, + REPO_ROOT, + WORKFLOWS_DIR, +) + +_IN_CI = bool(os.environ.get("CI")) + + +def _resolve(repo: str, ref: str) -> tuple[Verdict, str]: + resolution = _Resolver(("dev", "main")).resolve(repo, ref) + return resolution.verdict, resolution.detail + + +def _require_determined(verdict: Verdict, detail: str, label: str) -> None: + """Fail closed in CI, skip only on a developer machine.""" + if verdict is not Verdict.UNDETERMINED: + return + message = f"{label}: could not resolve against the live GitHub API ({detail})" + if _IN_CI: + pytest.fail(f"{message} — an unresolvable pin is not a passing pin") + pytest.skip(message) + + +@pytest.mark.integration +def test_incident_a_workflow_pin_is_unreachable() -> None: + """The pin that wedged omnibase_infra CI for ~2.5h (OMN-15536). + + ``omnimarket@879d6fc6`` was the head of a PR branch GitHub deleted on + squash-merge. The shape-only validators in this repo passed on it. + """ + verdict, detail = _resolve("omnimarket", INCIDENT_A_DEAD_SHA) + _require_determined(verdict, detail, "incident A dead pin") + assert verdict is Verdict.UNREACHABLE, ( + f"expected UNREACHABLE for the OMN-15536 wedging pin, got {verdict} ({detail})" + ) + + +@pytest.mark.integration +def test_incident_a_correct_counterpart_is_reachable() -> None: + verdict, detail = _resolve("omnimarket", INCIDENT_A_GOOD_SHA) + _require_determined(verdict, detail, "incident A good pin") + assert verdict is Verdict.REACHABLE, ( + f"expected REACHABLE for the merged dev squash, got {verdict} ({detail})" + ) + + +@pytest.mark.integration +def test_incident_b_dependency_pin_is_unreachable() -> None: + """The dependency pin live on ``omnimarket@dev pyproject.toml``. + + ``omnibase_core@5a907b71`` is the head of an unlanded branch: ``ahead`` of + dev, ``diverged`` from main. It resolves today and dies on merge. The + pre-existing uv.lock reachability check (OMN-14449) calls this pin OK, + because ``git branch -r --contains`` accepts any remote branch including + the live feature branch. + """ + verdict, detail = _resolve("omnibase_core", INCIDENT_B_DEAD_SHA) + _require_determined(verdict, detail, "incident B dead pin") + assert verdict is Verdict.UNREACHABLE, ( + f"expected UNREACHABLE for the unlanded-branch-head dependency pin, " + f"got {verdict} ({detail})" + ) + + +@pytest.mark.integration +def test_incident_b_correct_counterpart_is_reachable() -> None: + verdict, detail = _resolve("omnibase_core", INCIDENT_B_GOOD_SHA) + _require_determined(verdict, detail, "incident B good pin") + assert verdict is Verdict.REACHABLE, ( + f"expected REACHABLE for the merged dev squash, got {verdict} ({detail})" + ) + + +@pytest.mark.integration +def test_protected_branch_union_is_load_bearing() -> None: + """A dev-only oracle would false-RED a correct pin in this very repo. + + ``onex_change_control@2dd26ade`` is ``diverged`` from OCC ``dev`` and + ``behind`` OCC ``main``; this repo pins it in both pyproject.toml and + uv.lock. Narrowing the oracle to ``dev`` alone turns a durable pin red, + and a gate that cries wolf gets bypassed. + """ + dev_only, dev_detail = ( + _Resolver(("dev",)) + .resolve("onex_change_control", "2dd26ade7caaa7131e532473ec9d8a207d0e77ab") + .verdict, + "dev-only oracle", + ) + union, union_detail = _resolve( + "onex_change_control", "2dd26ade7caaa7131e532473ec9d8a207d0e77ab" + ) + _require_determined(union, union_detail, "OCC main-reachable pin") + if dev_only is Verdict.UNDETERMINED: + pytest.skip(f"{dev_detail}: undetermined") + assert dev_only is Verdict.UNREACHABLE + assert union is Verdict.REACHABLE + + +@pytest.mark.integration +def test_this_repo_has_no_undurable_pins() -> None: + """Expected GREEN. A failure here means a pin in this repo is a time bomb.""" + pins = extract_pins( + [WORKFLOWS_DIR, REPO_ROOT / "pyproject.toml", REPO_ROOT / "uv.lock"] + ) + assert pins, "no pins extracted — the extractor is broken, not the tree clean" + + resolver = _Resolver(("dev", "main")) + bad: list[str] = [] + undetermined: list[str] = [] + for pin in sorted(pins): + resolution = resolver.resolve(pin.repo, pin.ref) + located = f"{pin.source}::{pin.locus} -> {pin.repo}@{pin.ref[:12]}" + if resolution.verdict is Verdict.UNREACHABLE: + bad.append(f"{located} ({resolution.detail})") + elif resolution.verdict is Verdict.UNDETERMINED: + undetermined.append(f"{located} ({resolution.detail})") + + assert not bad, "pins not reachable from dev/main:\n " + "\n ".join(bad) + if undetermined: + message = "unresolved pins:\n " + "\n ".join(undetermined) + if _IN_CI: + pytest.fail(f"{message}\n(an unresolvable pin is not a passing pin)") + pytest.skip(message) diff --git a/tests/integration/ci/test_workflow_uses_refs_resolve_live.py b/tests/integration/ci/test_workflow_uses_refs_resolve_live.py index 66db110640..488df79b0d 100644 --- a/tests/integration/ci/test_workflow_uses_refs_resolve_live.py +++ b/tests/integration/ci/test_workflow_uses_refs_resolve_live.py @@ -15,12 +15,25 @@ slow/chaos/kafka/performance markers), where it FAILS CLOSED: a definitive 404 fails, and an unverifiable pin (no network / rate-limited) also fails when ``CI`` is set; -* keeping the born-path sequencing honest: while the omniclaude OMN-14941 PR - (call-occ-companion-effect-reusable.yml) is unmerged, the @dev pin in - call-occ-companion-effect.yml is a genuine 404 and this test is - DELIBERATELY RED in CI on this repo — that red must not also make the - branch unpushable at the local pre-push hook, which is exactly what - happened when this test sat in ``tests/ci/``. +* a red here must not make the branch unpushable at the local pre-push hook, + which is exactly what happened while this test sat in ``tests/ci/``. + +**This test is expected GREEN. There is no standing authorization for a red +here — if it fails, a pin is broken; fix the pin.** (OMN-15248.) The file +previously carried a born-path pre-authorization: while the omniclaude +OMN-14941 reusable (``call-occ-companion-effect-reusable.yml``) was unmerged, +the ``@dev`` pin in ``call-occ-companion-effect.yml`` was a genuine 404 and +this test was documented as deliberately red. That condition EXPIRED — both +formerly-404 pins resolve on omniclaude ``dev`` (verified 2026-07-27 via +``gh api repos/OmniNode-ai/omniclaude/contents/?ref=dev``: +``call-occ-companion-effect-reusable.yml`` -> ``c540d981``, +``call-occ-autobind-reusable.yml`` -> ``5f8f64e4``). An expired +red-authorization is how a real regression gets waved through: a reader sees +a blessed failure and stops investigating. Never restate one here — if a pin +genuinely cannot resolve yet, the born-path ordering is the fix (land the +upstream ref first), not a docstring waiver. Mechanizing that rule (so the +next stale waiver self-retires instead of aging into a blind spot) is +OMN-15257; until it lands this paragraph is prose, not enforcement. """ from __future__ import annotations @@ -74,7 +87,21 @@ def test_every_cross_repo_uses_ref_resolves_live() -> None: pytest.fail( "cross-repo `uses:` pins that DO NOT resolve on the live remote " "(the OMN-14941/E1 silent-outage class — the workflow fails at " - "parse time and no job ever runs):\n" + "\n".join(lines) + "parse time and no job ever runs):\n" + + "\n".join(lines) + + "\n\nFIX THE PIN, DO NOT WAIVE THIS TEST. Every red here is a " + "live broken workflow reference. Take exactly one of:\n" + " 1. the target ref moved or was never merged -> re-pin the " + "`uses:` line to a ref that exists on the remote (a merged SHA, " + "or the branch the file actually lives on), or land the upstream " + "PR that creates it FIRST (born-path ordering);\n" + " 2. the path/filename changed upstream -> update the path in " + "the `uses:` line;\n" + " 3. the repo is private/unreachable to this token -> that is an " + "`undetermined`, not a 404; a 404 here means GitHub answered " + "definitively that the path does not exist at that ref.\n" + "Adding an xfail/skip or a 'deliberately red pending ' " + "docstring is NOT an accepted fix (OMN-15248)." ) if undetermined: @@ -83,8 +110,15 @@ def test_every_cross_repo_uses_ref_resolves_live() -> None: pytest.fail( "could not resolve these cross-repo `uses:` pins against the " "live GitHub API — failing CLOSED in CI (an unverifiable pin " - "is not a passing pin; thread GH_TOKEN into the test step or " - "fix runner egress):\n" + detail + "is not a passing pin):\n" + + detail + + "\n\nFIX THE RESOLUTION PATH, DO NOT WAIVE THIS TEST: " + "HTTP 401/403/404-on-a-private-repo -> thread a token with " + "read access to the target repo into the test step " + "(`GH_TOKEN`/`GITHUB_TOKEN` env, e.g. CROSS_REPO_PAT for " + "private cross-repo reads); HTTP 403/429 rate-limit -> the " + "run is unauthenticated, supply the token; transport errors " + "-> fix runner egress to api.github.com." ) pytest.skip( "GitHub API unreachable from this local machine; live resolution " diff --git a/tests/integration/db/test_generation_events_role_tolerance_omn15351.py b/tests/integration/db/test_generation_events_role_tolerance_omn15351.py new file mode 100644 index 0000000000..adfeda3286 --- /dev/null +++ b/tests/integration/db/test_generation_events_role_tolerance_omn15351.py @@ -0,0 +1,315 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Real-Postgres execution proof for node migration 0027 role tolerance (OMN-15351). + +``docker/migrations/forward/nodes/node_projection_delegation/0027_generation_events_tenant_rls.sql`` +opened with a fail-closed guard that raised on a missing ``role_omnidash``:: + + psql:/migrations/forward/nodes/node_projection_delegation/0027_generation_events_tenant_rls.sql:26: + ERROR: role_omnidash role missing — generation_events writer access cannot be granted + +``role_omnidash`` is ENVIRONMENT-provisioned (out-of-band on cloud RDS, or from +``ROLE_OMNIDASH_PASSWORD`` at first-startup init in +``docker/migrations/forward/000_create_multiple_databases.sh``); no forward +migration anywhere creates it. The ``.201`` dev-lane cluster carries +``pg_roles = {app_dashboard, postgres, role_omniweb}``, so that guard made EVERY +dev-lane deploy fatal at this file (OMN-15348 AC4 redeploy, workflow +``wf_55998f90``). + +This module executes the REAL vendored SQL file with the SAME psql invocation the +deploy-time runner uses (``scripts/run-forward-migrations.sh``: +``psql -v ON_ERROR_STOP=1 -f ``) against a REAL ephemeral +Postgres, in three role states: + +* ``role_omnidash`` absent -> exit 0, WARNING naming BOTH skipped grants, and + zero ``role_omnidash`` privileges anywhere in the resulting ACLs. +* ``role_omnidash`` present -> the exact pre-OMN-15351 grant set is applied + (schema USAGE + SELECT/INSERT/UPDATE on ``generation_events``). +* ``app_dashboard`` absent -> STILL FATAL. Forward migration 094 (OMN-14899) + creates ``app_dashboard`` in-repo, so its absence is a real ordering bug, not + an environment difference. Nothing but the ``role_omnidash`` guard was softened. + +Hermetic by construction: each test initdb's its own cluster into a temp +directory listening on a unix socket only, so role state (which is CLUSTER-wide, +not database-wide) cannot leak between tests and nothing can collide with a lane +database or a TCP port. Skips when Postgres binaries are unavailable. + +Run: uv run pytest tests/integration/db/test_generation_events_role_tolerance_omn15351.py -v + +Ticket: OMN-15351 +""" + +from __future__ import annotations + +import shutil +import subprocess +import tempfile +from collections.abc import Iterator +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.integration, pytest.mark.postgres, pytest.mark.slow] + +REPO_ROOT = Path(__file__).resolve().parents[3] +NODE_MIGRATIONS = ( + REPO_ROOT + / "docker" + / "migrations" + / "forward" + / "nodes" + / "node_projection_delegation" +) +BASE_TABLE_SQL = NODE_MIGRATIONS / "0008_generation_events.sql" +MIGRATION_SQL = NODE_MIGRATIONS / "0027_generation_events_tenant_rls.sql" + +OWNER_ROLE = "postgres" +DB = "omn15351" +# Non-owner/NOSUPERUSER/NOBYPASSRLS mirrors the live posture of both roles. +_ROLE_ATTRS = "NOSUPERUSER NOBYPASSRLS LOGIN PASSWORD 'omn15351_proof_only'" # pragma: allowlist secret + +# The two grants the migration must skip (and name) when role_omnidash is absent, +# and must apply verbatim when it is present. +SKIPPED_GRANT_SCHEMA = "GRANT USAGE ON SCHEMA public TO role_omnidash" +SKIPPED_GRANT_TABLE = ( + "GRANT SELECT, INSERT, UPDATE ON generation_events TO role_omnidash" +) +# Pre-OMN-15351 role_omnidash privileges on generation_events, proven by running +# the unmodified file on the same fixture (see the PR body's differential run). +EXPECTED_OMNIDASH_TABLE_PRIVILEGES = {"SELECT", "INSERT", "UPDATE"} + + +def _pg_bin(name: str) -> str | None: + """Resolve a Postgres binary from PATH or a brew keg-only prefix.""" + found = shutil.which(name) + if found: + return found + for prefix in sorted(Path("/opt/homebrew/opt").glob("postgresql@*"), reverse=True): + candidate = prefix / "bin" / name + if candidate.exists(): + return str(candidate) + return None + + +_INITDB = _pg_bin("initdb") +_PG_CTL = _pg_bin("pg_ctl") +_PSQL = _pg_bin("psql") + +if not _INITDB or not _PG_CTL or not _PSQL: # pragma: no cover - environment dependent + pytest.skip( + "initdb/pg_ctl/psql not available — cannot bring up an ephemeral Postgres", + allow_module_level=True, + ) + + +@pytest.fixture +def cluster() -> Iterator[str]: + """Bring up a fresh ephemeral, unix-socket-only Postgres cluster per test. + + Per-test (not per-module) because ``CREATE ROLE`` is cluster-wide: a shared + cluster would let one test's role leak into another's role-absent premise and + silently make it vacuous. + """ + root = Path(tempfile.mkdtemp(prefix="omn15351-pg-")) + data_dir = root / "data" + sock_dir = root / "sock" + sock_dir.mkdir() + + subprocess.run( + [str(_INITDB), "-D", str(data_dir), "-U", OWNER_ROLE, "-A", "trust"], + check=True, + capture_output=True, + ) + subprocess.run( + [ + str(_PG_CTL), + "-D", + str(data_dir), + "-l", + str(root / "postgres.log"), + "-o", + f"-k {sock_dir} -h '' -c listen_addresses=''", + "-w", + "start", + ], + check=True, + capture_output=True, + ) + try: + yield str(sock_dir) + finally: + subprocess.run( + [str(_PG_CTL), "-D", str(data_dir), "-m", "immediate", "stop"], + check=False, + capture_output=True, + ) + shutil.rmtree(root, ignore_errors=True) + + +def _psql(sock: str, database: str, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [str(_PSQL), "-X", "-q", "-h", sock, "-U", OWNER_ROLE, "-d", database, *args], + capture_output=True, + text=True, + check=False, # the exit code IS the assertion in the RED/fatal-guard cases + ) + + +def _query(sock: str, sql: str) -> str: + result = _psql(sock, DB, "-t", "-A", "-v", "ON_ERROR_STOP=1", "-c", sql) + assert result.returncode == 0, f"probe query failed: {result.stderr}" + return result.stdout.strip() + + +def _prepare(sock: str, *, app_dashboard: bool, role_omnidash: bool) -> None: + """Create the database, the requested roles, and the 0008 base table.""" + assert _psql(sock, "postgres", "-c", f"CREATE DATABASE {DB}").returncode == 0 + for role, wanted in ( + ("app_dashboard", app_dashboard), + ("role_omnidash", role_omnidash), + ): + if wanted: + created = _psql(sock, "postgres", "-c", f"CREATE ROLE {role} {_ROLE_ATTRS}") + assert created.returncode == 0, created.stderr + + base = _psql(sock, DB, "-v", "ON_ERROR_STOP=1", "-f", str(BASE_TABLE_SQL)) + assert base.returncode == 0, base.stderr + + # Anti-vacuity: assert the premise this test rests on actually holds. + roles = _query(sock, "SELECT rolname FROM pg_roles WHERE rolname !~ '^pg_'").split() + assert ("app_dashboard" in roles) is app_dashboard, roles + assert ("role_omnidash" in roles) is role_omnidash, roles + + +def _apply_migration(sock: str) -> subprocess.CompletedProcess[str]: + """Apply 0027 exactly as scripts/run-forward-migrations.sh does.""" + return _psql(sock, DB, "-v", "ON_ERROR_STOP=1", "-f", str(MIGRATION_SQL)) + + +def test_role_omnidash_absent_completes_and_names_every_skipped_grant( + cluster: str, +) -> None: + """Role absent: migration succeeds, warns by name, and grants nothing to it.""" + _prepare(cluster, app_dashboard=True, role_omnidash=False) + + result = _apply_migration(cluster) + output = result.stdout + result.stderr + + assert result.returncode == 0, output + assert "WARNING:" in output, output + assert "role_omnidash role missing" in output, output + # No silent skip: the warning must enumerate what it skipped. + assert SKIPPED_GRANT_SCHEMA in output, output + assert SKIPPED_GRANT_TABLE in output, output + + # Readback: zero role_omnidash privileges anywhere. + assert ( + _query( + cluster, + "SELECT count(*) FROM information_schema.role_table_grants " + "WHERE table_name = 'generation_events' AND grantee = 'role_omnidash'", + ) + == "0" + ) + assert ( + _query( + cluster, + "SELECT coalesce(nspacl::text, '') LIKE '%role_omnidash%' " + "FROM pg_namespace WHERE nspname = 'public'", + ) + == "f" + ) + + # Everything the migration does NOT gate on role_omnidash still applied. + assert ( + _query( + cluster, + "SELECT relrowsecurity FROM pg_class WHERE relname = 'generation_events'", + ) + == "t" + ) + assert ( + _query( + cluster, + "SELECT count(*) FROM pg_policies WHERE tablename = 'generation_events' " + "AND policyname = 'tenant_isolation'", + ) + == "1" + ) + assert ( + _query( + cluster, + "SELECT count(*) FROM information_schema.columns " + "WHERE table_name = 'generation_events' AND column_name = 'tenant_id'", + ) + == "1" + ) + assert ( + _query( + cluster, + "SELECT privilege_type FROM information_schema.role_table_grants " + "WHERE table_name = 'generation_events' AND grantee = 'app_dashboard'", + ) + == "SELECT" + ) + + +def test_role_omnidash_absent_migration_is_idempotent(cluster: str) -> None: + """Re-applying on a role-less lane stays green (the deploy retry path).""" + _prepare(cluster, app_dashboard=True, role_omnidash=False) + + assert _apply_migration(cluster).returncode == 0 + second = _apply_migration(cluster) + assert second.returncode == 0, second.stdout + second.stderr + assert "role_omnidash role missing" in second.stdout + second.stderr + + +def test_role_omnidash_present_applies_the_unchanged_grant_set(cluster: str) -> None: + """Role present: identical grants to the pre-OMN-15351 file, no warning.""" + _prepare(cluster, app_dashboard=True, role_omnidash=True) + + result = _apply_migration(cluster) + output = result.stdout + result.stderr + assert result.returncode == 0, output + assert "role_omnidash role missing" not in output, output + + table_privileges = set( + _query( + cluster, + "SELECT privilege_type FROM information_schema.role_table_grants " + "WHERE table_name = 'generation_events' AND grantee = 'role_omnidash'", + ).split() + ) + assert table_privileges == EXPECTED_OMNIDASH_TABLE_PRIVILEGES + + assert ( + _query( + cluster, + "SELECT has_schema_privilege('role_omnidash', 'public', 'USAGE')", + ) + == "t" + ) + # arw = INSERT/SELECT/UPDATE, and nothing more (no DELETE 'd', no owner bits). + assert "role_omnidash=arw/postgres" in _query( + cluster, + "SELECT relacl::text FROM pg_class WHERE relname = 'generation_events'", + ) + + +def test_app_dashboard_guard_is_still_fatal(cluster: str) -> None: + """Only the role_omnidash guard was relaxed; 094's role stays fail-closed.""" + _prepare(cluster, app_dashboard=False, role_omnidash=True) + + result = _apply_migration(cluster) + output = result.stdout + result.stderr + + assert result.returncode != 0, output + assert "app_dashboard role missing" in output, output + # Fail-closed means the table must NOT be left half-migrated with RLS on. + assert ( + _query( + cluster, + "SELECT relrowsecurity FROM pg_class WHERE relname = 'generation_events'", + ) + == "f" + ) diff --git a/tests/integration/db/test_omn14899_app_dashboard_reproof.py b/tests/integration/db/test_omn14899_app_dashboard_reproof.py new file mode 100644 index 0000000000..8ecac2f80a --- /dev/null +++ b/tests/integration/db/test_omn14899_app_dashboard_reproof.py @@ -0,0 +1,533 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +r"""OMN-14899 re-proof — tenant isolation, asserted under the REAL app role. + +WHY THIS EXISTS +--------------- +OMN-14899's earlier isolation matrix went green while the session was +effectively exempt from row-level security. Postgres bypasses RLS +unconditionally for SUPERUSER, for BYPASSRLS, and for a table's OWNER, so an +isolation suite run under any of those returns exactly the same PASS whether +the policies are correct, wrong, or absent. That is not a weak proof — it is a +proof of nothing, and it is indistinguishable from a real one by looking at the +result. + +Operator ruling 28 (2026-07-31) keeps OMN-14899 open on exactly this: merged is +not proven, and the acceptance criterion naming the *running application's* +connection was never satisfied. + +HOW THE VACUITY IS MADE STRUCTURALLY IMPOSSIBLE +----------------------------------------------- +Every isolation assertion in this module depends on the ``proven_role`` +fixture, and that fixture is an ADMISSION GATE, not a setup step. It refuses to +yield a connection until it has read, IN THIS SESSION, that: + +1. ``current_user`` and ``session_user`` are both the expected read role. Not a + role that can ``SET ROLE`` to it — ``session_user`` pins the authenticated + identity, so a superuser that did ``SET ROLE app_dashboard`` is rejected. + (``SET ROLE`` does not clear BYPASSRLS inherited from the session role.) +2. ``rolsuper`` and ``rolbypassrls`` are both false, read ``WHERE rolname = + current_user`` — keyed on the live session identity, never on the literal + role name, so the check cannot be satisfied by reading some *other* role's + catalog row. +3. The connecting role owns zero tables in the schema under test. An owner is + exempt from RLS with FORCE included. +4. The table under test actually carries ``relrowsecurity``, + ``relforcerowsecurity`` and a policy named ``tenant_isolation``. A "denied" + result against a table with no policy proves nothing about isolation. +5. Both tenants have rows the role could in principle see. A cross-tenant read + returning zero is only evidence of DENIAL if the data exists to be denied + (memory: prove RED against exists-but-wrong, not against empty). + +A failure in the gate is a hard ``pytest.fail``, never a skip: a proof that +quietly degrades into a weaker proof is the failure mode this module exists to +remove. + +HOW TO RUN IT +------------- +Read-only. It issues no ``INSERT``/``UPDATE``/``DELETE``/DDL and creates no +rows, so it is safe against a live lane — but it is NOT run automatically. It +skips unless a DSN is supplied, so CI never executes it: + + OMN14899_APP_DASHBOARD_DSN='postgresql://app_dashboard:...@host:5432/omnidash_analytics?sslmode=verify-full' \ + OMN14899_TENANT_A='' \ + OMN14899_TENANT_B='' \ + uv run pytest tests/integration/db/test_omn14899_app_dashboard_reproof.py -v + +The DSN must authenticate AS the read role. Pointing it at a superuser is not a +shortcut — the gate rejects it and names why. + +Never paste the DSN into a PR body, ticket, or commit. The credential is +deployment-owned (AWS Secrets Manager ``omninode/staging/rds/app-dashboard``). + +WHAT THIS DOES NOT PROVE +------------------------ +It proves the ROLE and the POLICIES hold under a real pooled connection. It +does NOT by itself prove the *deployed application* connects as this role — +that is OMN-15358 (workload wiring), and the honest way to close it is to point +this harness at the DSN the running workload actually resolves, then record +which manifest/secret that DSN came from alongside the result. +""" + +from __future__ import annotations + +import os +import warnings +from collections.abc import Iterator +from typing import Any + +import psycopg2 +import psycopg2.pool +import pytest + +# The role whose identity every assertion below is conditioned on. +EXPECTED_READ_ROLE = "app_dashboard" + +# The table OMN-14899's evidence trail uses. Pinned as a constant rather than +# read from an env var: an operator-supplied table name could silently point +# the proof at a table with no policy, and the gate's job is to make that +# class of substitution impossible rather than merely detectable. +TARGET_SCHEMA = "public" +TARGET_TABLE = "delegation_events" +TENANT_GUC = "app.tenant_id" + +_DSN_ENV = "OMN14899_APP_DASHBOARD_DSN" +_TENANT_A_ENV = "OMN14899_TENANT_A" +_TENANT_B_ENV = "OMN14899_TENANT_B" + + +def _qualified() -> str: + return f"{TARGET_SCHEMA}.{TARGET_TABLE}" + + +@pytest.fixture(scope="module") +def dsn() -> str: + value = os.environ.get(_DSN_ENV) + if not value: + pytest.skip( + f"{_DSN_ENV} not set — the OMN-14899 re-proof runs only against an " + "explicitly supplied live DSN, never implicitly in CI" + ) + return value + + +@pytest.fixture(scope="module") +def tenants() -> tuple[str, str]: + """Two distinct tenant ids, both required. + + Deliberately no defaults. A silent default here would make a cross-tenant + assertion pass against a tenant that does not exist, which is the same + vacuity in a different costume. + """ + tenant_a = os.environ.get(_TENANT_A_ENV) + tenant_b = os.environ.get(_TENANT_B_ENV) + missing = [ + name + for name, value in ((_TENANT_A_ENV, tenant_a), (_TENANT_B_ENV, tenant_b)) + if not value + ] + if missing: + pytest.fail( + f"{_DSN_ENV} is set but {', '.join(missing)} is not. The isolation " + "legs need two real, distinct tenants; running with one would " + "reduce the cross-tenant check to a tautology." + ) + assert tenant_a is not None and tenant_b is not None + if tenant_a == tenant_b: + pytest.fail( + f"{_TENANT_A_ENV} and {_TENANT_B_ENV} are the same value " + f"({tenant_a!r}) — a cross-tenant denial cannot be observed " + "between a tenant and itself" + ) + return tenant_a, tenant_b + + +def _scalar(cursor: Any, sql: str, params: tuple[Any, ...] = ()) -> Any: + cursor.execute(sql, params) + row = cursor.fetchone() + assert row is not None, f"query returned no row: {sql}" + return row[0] + + +def _assert_role_identity(cursor: Any) -> None: + """Gate step 1+2 — the connecting identity, read in this very session.""" + cursor.execute("SELECT current_user, session_user") + identity = cursor.fetchone() + assert identity is not None + current_user, session_user = identity + + if session_user != EXPECTED_READ_ROLE: + pytest.fail( + f"session_user is {session_user!r}, expected {EXPECTED_READ_ROLE!r}. " + "The DSN must AUTHENTICATE as the read role. A privileged session " + "that reached the role via SET ROLE is rejected on purpose: SET " + "ROLE does not drop BYPASSRLS held by the session role, so every " + "isolation result below would be vacuous." + ) + if current_user != EXPECTED_READ_ROLE: + pytest.fail( + f"current_user is {current_user!r} but session_user is " + f"{session_user!r} — the session has switched roles mid-flight; " + "refusing to certify isolation under an ambiguous identity" + ) + + # Keyed on current_user, NOT on the literal role name: reading + # `WHERE rolname = 'app_dashboard'` would report the flags of a role that + # might not be the one this session is running as. + cursor.execute( + "SELECT rolsuper, rolbypassrls FROM pg_roles WHERE rolname = current_user" + ) + flags = cursor.fetchone() + if flags is None: + pytest.fail( + "pg_roles has no row for current_user — cannot certify that this " + "session is not RLS-exempt" + ) + rolsuper, rolbypassrls = flags + if rolsuper or rolbypassrls: + pytest.fail( + f"the connected role {current_user!r} has rolsuper={rolsuper}, " + f"rolbypassrls={rolbypassrls}. Postgres bypasses row-level " + "security unconditionally for such a role, so every isolation " + "assertion below would pass regardless of whether the policies " + "are correct. This is the exact vacuity that made the earlier " + "OMN-14899 green worthless." + ) + + +def _assert_not_owner(cursor: Any) -> None: + """Gate step 3 — ownership is an RLS exemption FORCE does not close.""" + cursor.execute( + "SELECT tablename FROM pg_tables " + "WHERE schemaname = %s AND tableowner = current_user " + "ORDER BY tablename", + (TARGET_SCHEMA,), + ) + owned = [row[0] for row in cursor.fetchall()] + if owned: + pytest.fail( + f"the connected role owns table(s) {owned} in {TARGET_SCHEMA}. An " + "owner is exempt from row-level security (FORCE included), so any " + "isolation reading taken from them is a false clean." + ) + + +def _assert_table_is_actually_protected(cursor: Any) -> None: + """Gate step 4 — a denial against an unprotected table proves nothing.""" + cursor.execute( + "SELECT c.relrowsecurity, c.relforcerowsecurity, " + " EXISTS (SELECT 1 FROM pg_policy p " + " WHERE p.polrelid = c.oid AND p.polname = 'tenant_isolation') " + " FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " + " WHERE n.nspname = %s AND c.relname = %s AND c.relkind = 'r'", + (TARGET_SCHEMA, TARGET_TABLE), + ) + row = cursor.fetchone() + if row is None: + pytest.fail( + f"{_qualified()} is not visible to the connected role — cannot " + "certify isolation against a table this session cannot even " + "resolve in the catalog" + ) + enabled, forced, has_policy = row + if not (enabled and has_policy): + pytest.fail( + f"{_qualified()} has relrowsecurity={enabled}, " + f"tenant_isolation policy present={has_policy}. Zero-row results " + "from an unprotected table are just an empty table, not isolation." + ) + if not forced: + # Not fatal for a NON-owner (which the gate already established this + # role is), but recorded: FORCE is what protects the OWNER's own reads, + # and its absence changes what a future writer-side proof would mean. + warnings.warn( + f"{_qualified()} has relforcerowsecurity=False. The connected role " + "is a non-owner so its reads are still policed, but the owner's " + "are not — do not cite this run as evidence about writer-side " + "isolation.", + stacklevel=2, + ) + + +def _assert_both_tenants_have_rows(cursor: Any, tenants: tuple[str, str]) -> None: + """Gate step 5 — denial is only observable where data exists to deny.""" + for tenant in tenants: + cursor.execute(f"SET {TENANT_GUC} = %s", (tenant,)) + visible = _scalar(cursor, f"SELECT count(*) FROM {_qualified()}") # noqa: S608 - identifier is a module constant, tenants are bound params + if visible == 0: + pytest.fail( + f"tenant {tenant!r} has zero visible rows in {_qualified()}. " + "Every cross-tenant assertion below would then return zero for " + "the trivial reason that the data is absent, not because the " + "policy denied it. Point the harness at tenants that have rows." + ) + cursor.execute(f"RESET {TENANT_GUC}") + + +@pytest.fixture(scope="module") +def proven_role(dsn: str, tenants: tuple[str, str]) -> Iterator[Any]: + """A connection that has PROVEN it is the constrained role before yielding. + + The gate runs first and fails hard. No isolation assertion in this module + can execute against an unproven identity, because none of them can obtain a + connection except through here. + """ + try: + conn = psycopg2.connect(dsn) + except psycopg2.OperationalError as exc: + pytest.fail( + f"could not open a session with {_DSN_ENV}: {exc}\n" + "If this reads 'permission denied for database ... does not have " + "CONNECT privilege', that is OMN-15297 — apply forward migration " + "097_grant_app_dashboard_connect_omnidash_analytics.sql." + ) + + conn.autocommit = True + with conn.cursor() as cur: + _assert_role_identity(cur) + _assert_not_owner(cur) + _assert_table_is_actually_protected(cur) + _assert_both_tenants_have_rows(cur, tenants) + + try: + yield conn + finally: + conn.close() + + +# ============================================================================= +# Leg 0 — the gate itself, stated as a test so the evidence names it. +# ============================================================================= + + +@pytest.mark.integration +@pytest.mark.postgres +def test_leg0_connected_identity_is_the_constrained_read_role( + proven_role: Any, +) -> None: + """Records the identity the other legs are conditioned on. + + If this test is absent from a run's output, the other legs' results mean + nothing — that is the point of stating it separately rather than leaving it + implicit in a fixture. + """ + with proven_role.cursor() as cur: + cur.execute( + "SELECT current_user, session_user, " + " (SELECT rolsuper FROM pg_roles WHERE rolname = current_user), " + " (SELECT rolbypassrls FROM pg_roles WHERE rolname = current_user)" + ) + row = cur.fetchone() + + assert row is not None + current_user, session_user, rolsuper, rolbypassrls = row + print( + f"[OMN-14899] current_user={current_user} session_user={session_user} " + f"rolsuper={rolsuper} rolbypassrls={rolbypassrls}" + ) + assert current_user == EXPECTED_READ_ROLE + assert session_user == EXPECTED_READ_ROLE + assert rolsuper is False + assert rolbypassrls is False + + +# ============================================================================= +# Leg 1 — positive same-tenant access. +# ============================================================================= + + +@pytest.mark.integration +@pytest.mark.postgres +def test_leg1_same_tenant_rows_are_visible( + proven_role: Any, tenants: tuple[str, str] +) -> None: + """With the tenant context set, the role sees that tenant's rows — and ONLY + that tenant's rows. + + The second half matters as much as the first: a policy that lets everything + through also satisfies "sees its own rows". + """ + tenant_a, _ = tenants + with proven_role.cursor() as cur: + cur.execute(f"SET {TENANT_GUC} = %s", (tenant_a,)) + visible = _scalar(cur, f"SELECT count(*) FROM {_qualified()}") # noqa: S608 - identifier is a module constant, tenants are bound params + foreign = _scalar( + cur, + f"SELECT count(*) FROM {_qualified()} WHERE tenant_id <> %s", # noqa: S608 - identifier is a module constant, tenants are bound params + (tenant_a,), + ) + cur.execute(f"RESET {TENANT_GUC}") + + assert visible > 0, ( + f"tenant {tenant_a!r} sees zero rows with its own context set — the " + "policy is denying the tenant its own data" + ) + assert foreign == 0, ( + f"tenant {tenant_a!r} can see {foreign} row(s) belonging to other " + "tenants — the tenant_isolation policy is not constraining reads" + ) + + +# ============================================================================= +# Leg 2 — cross-tenant denial. +# ============================================================================= + + +@pytest.mark.integration +@pytest.mark.postgres +def test_leg2_other_tenants_rows_are_denied( + proven_role: Any, tenants: tuple[str, str] +) -> None: + """Tenant A's context must not reach tenant B's rows. + + The gate already proved tenant B HAS rows, so a zero here is denial rather + than emptiness. Both directions are checked: a policy broken in one + direction only is still broken. + """ + tenant_a, tenant_b = tenants + with proven_role.cursor() as cur: + cur.execute(f"SET {TENANT_GUC} = %s", (tenant_a,)) + b_from_a = _scalar( + cur, + f"SELECT count(*) FROM {_qualified()} WHERE tenant_id = %s", # noqa: S608 - identifier is a module constant, tenants are bound params + (tenant_b,), + ) + cur.execute(f"SET {TENANT_GUC} = %s", (tenant_b,)) + a_from_b = _scalar( + cur, + f"SELECT count(*) FROM {_qualified()} WHERE tenant_id = %s", # noqa: S608 - identifier is a module constant, tenants are bound params + (tenant_a,), + ) + cur.execute(f"RESET {TENANT_GUC}") + + assert b_from_a == 0, ( + f"context={tenant_a!r} can read {b_from_a} row(s) of tenant " + f"{tenant_b!r} — cross-tenant read" + ) + assert a_from_b == 0, ( + f"context={tenant_b!r} can read {a_from_b} row(s) of tenant " + f"{tenant_a!r} — cross-tenant read" + ) + + +# ============================================================================= +# Leg 3 — unset context denies everything. +# ============================================================================= + + +@pytest.mark.integration +@pytest.mark.postgres +def test_leg3_unset_tenant_context_returns_zero_rows(proven_role: Any) -> None: + """Fail-closed, with no default-tenant fallback. + + ``current_setting('app.tenant_id', true)`` is NULL when the GUC is unset, + the policy predicate is NULL, and nothing is visible. This is OMN-14899's + named negative proof, and it is the one that distinguishes a real policy + from a permissive one. + + Both RESET and an explicit empty string are exercised: an application that + "clears" the context by writing '' rather than resetting it must not + thereby match a row whose tenant_id is ''. + """ + with proven_role.cursor() as cur: + cur.execute(f"RESET {TENANT_GUC}") + after_reset = _scalar(cur, f"SELECT count(*) FROM {_qualified()}") # noqa: S608 - identifier is a module constant, tenants are bound params + raw = _scalar(cur, f"SELECT current_setting('{TENANT_GUC}', true)") + + cur.execute(f"SET {TENANT_GUC} = ''") + after_empty = _scalar(cur, f"SELECT count(*) FROM {_qualified()}") # noqa: S608 - identifier is a module constant, tenants are bound params + cur.execute(f"RESET {TENANT_GUC}") + + assert after_reset == 0, ( + f"{after_reset} row(s) visible with {TENANT_GUC} unset " + f"(current_setting returned {raw!r}) — the policy is not fail-closed " + "and an un-scoped connection reads across every tenant" + ) + assert after_empty == 0, ( + f"{after_empty} row(s) visible with {TENANT_GUC} set to the empty " + "string — an application clearing context by writing '' would read " + "another tenant's data" + ) + + +# ============================================================================= +# Leg 4 — pooled-connection reuse. +# ============================================================================= + + +@pytest.mark.integration +@pytest.mark.postgres +def test_leg4_pooled_connection_reuse_does_not_leak_tenant_context( + proven_role: Any, dsn: str, tenants: tuple[str, str] +) -> None: + """A recycled physical connection must not carry the previous checkout's + tenant context. + + ``app.tenant_id`` is a session GUC. A pool hands the SAME backend to the + next caller, so a session-level ``SET`` that is never reset survives the + checkout boundary — and the next request, possibly for a different tenant + or for no tenant at all, inherits it. That is a cross-tenant read produced + entirely by connection management, with correct policies and a correct + role. + + ``minconn=maxconn=1`` forces reuse, and ``pg_backend_pid()`` is compared + across checkouts so REUSE ITSELF IS PROVEN rather than assumed — without + that comparison a pool that quietly opened a second backend would make this + test pass while testing nothing. + + ``proven_role`` is a parameter (not merely ordering) so the identity gate is + a hard precondition of this leg too. + """ + assert proven_role is not None + tenant_a, tenant_b = tenants + + pool = psycopg2.pool.SimpleConnectionPool(1, 1, dsn) + try: + first = pool.getconn() + first.autocommit = True + with first.cursor() as cur: + first_pid = _scalar(cur, "SELECT pg_backend_pid()") + cur.execute(f"SET {TENANT_GUC} = %s", (tenant_a,)) + seen_a = _scalar(cur, f"SELECT count(*) FROM {_qualified()}") # noqa: S608 - identifier is a module constant, tenants are bound params + pool.putconn(first) + + assert seen_a > 0, "setup: tenant A must see its own rows on checkout 1" + + second = pool.getconn() + second.autocommit = True + with second.cursor() as cur: + second_pid = _scalar(cur, "SELECT pg_backend_pid()") + leaked_setting = _scalar( + cur, f"SELECT current_setting('{TENANT_GUC}', true)" + ) + unscoped_rows = _scalar(cur, f"SELECT count(*) FROM {_qualified()}") # noqa: S608 - identifier is a module constant, tenants are bound params + + cur.execute(f"SET {TENANT_GUC} = %s", (tenant_b,)) + b_rows = _scalar(cur, f"SELECT count(*) FROM {_qualified()}") # noqa: S608 - identifier is a module constant, tenants are bound params + a_rows_from_b = _scalar( + cur, + f"SELECT count(*) FROM {_qualified()} WHERE tenant_id = %s", # noqa: S608 - identifier is a module constant, tenants are bound params + (tenant_a,), + ) + cur.execute(f"RESET {TENANT_GUC}") + pool.putconn(second) + finally: + pool.closeall() + + assert first_pid == second_pid, ( + f"the pool handed out two different backends ({first_pid} then " + f"{second_pid}) — connection reuse did not occur, so this leg proved " + "nothing about pooled context leakage. Re-run against a pool that " + "actually recycles." + ) + assert unscoped_rows == 0, ( + f"a recycled connection returned {unscoped_rows} row(s) before the new " + f"caller set any tenant context (current_setting={leaked_setting!r}) — " + f"tenant {tenant_a!r}'s context leaked across the checkout boundary" + ) + assert b_rows > 0, "tenant B must see its own rows on the recycled connection" + assert a_rows_from_b == 0, ( + f"the recycled connection, scoped to tenant {tenant_b!r}, can read " + f"{a_rows_from_b} row(s) of tenant {tenant_a!r}" + ) diff --git a/tests/integration/db/test_projection_writer_tenant_rls_omn15301.py b/tests/integration/db/test_projection_writer_tenant_rls_omn15301.py new file mode 100644 index 0000000000..9071b7e26b --- /dev/null +++ b/tests/integration/db/test_projection_writer_tenant_rls_omn15301.py @@ -0,0 +1,533 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Real PostgreSQL proof for split domain projection adapters (OMN-15421).""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import patch +from uuid import UUID, uuid4 + +import pytest + +psycopg2 = pytest.importorskip("psycopg2", reason="psycopg2 required for RLS proof") +psycopg2_extras = pytest.importorskip( + "psycopg2.extras", reason="psycopg2 extras required for UUID proof" +) +psycopg2_extras.register_uuid() + +from omnibase_infra.errors.error_projection import ProjectionTenantContextError +from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _build_projection_db_adapter, +) +from tests.helpers.application_db_topology import ( + projection_database_target, + projection_database_urls, +) +from tests.helpers.projection_tenant_authority import verified_tenant_dispatch + +pytestmark = [pytest.mark.integration, pytest.mark.postgres, pytest.mark.slow] + +DATABASE = "omnidash_analytics" +TENANT_TABLE = "delegation_events" +INTERNAL_TABLE = "generation_events" +CATALOG_TABLE = "plan_tiers" +TENANT_ROLE = "tenant_projection_writer" +INTERNAL_ROLE = "omninode_runtime" +CATALOG_READER_ROLE = "app_dashboard" +ROLE_PASSWORD = "domain_adapter_proof_only" # pragma: allowlist secret +OWNER_ROLE = "postgres" +TENANT_A = UUID("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa") +TENANT_B = UUID("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb") + +_SCHEMA_SQL = """ +CREATE SCHEMA tenant; +CREATE SCHEMA omninode_internal; +CREATE SCHEMA platform_catalog; +CREATE TABLE tenant.delegation_events ( + correlation_id UUID PRIMARY KEY, + task_type TEXT NOT NULL, + tenant_id UUID NOT NULL +); +ALTER TABLE tenant.delegation_events ENABLE ROW LEVEL SECURITY; +ALTER TABLE tenant.delegation_events FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON tenant.delegation_events + FOR ALL + USING (tenant_id = current_setting('app.tenant_id', true)::uuid) + WITH CHECK (tenant_id = current_setting('app.tenant_id', true)::uuid); +CREATE TABLE omninode_internal.generation_events ( + correlation_id UUID PRIMARY KEY, + source_tenant_id UUID NULL, + status TEXT NOT NULL +); +CREATE TABLE platform_catalog.plan_tiers ( + tier_id TEXT PRIMARY KEY, + display_name TEXT NOT NULL +); +""" + + +def _pg_bin(name: str) -> str | None: + found = shutil.which(name) + if found: + return found + for prefix in sorted(Path("/opt/homebrew/opt").glob("postgresql@*"), reverse=True): + candidate = prefix / "bin" / name + if candidate.exists(): + return str(candidate) + return None + + +_INITDB = _pg_bin("initdb") +_PG_CTL = _pg_bin("pg_ctl") +if not _INITDB or not _PG_CTL: # pragma: no cover - environment dependent + pytest.skip( + "initdb/pg_ctl not available — cannot bring up ephemeral PostgreSQL", + allow_module_level=True, + ) + + +@pytest.fixture(scope="module") +def pg_socket_dir() -> Iterator[str]: + root = Path(tempfile.mkdtemp(prefix="omn15421-pg-")) + data_dir = root / "data" + socket_dir = root / "socket" + socket_dir.mkdir() + postgres_env = {**os.environ, "LANG": "C", "LC_ALL": "C", "LC_CTYPE": "C"} + subprocess.run( + [ + str(_INITDB), + "-D", + str(data_dir), + "-U", + OWNER_ROLE, + "--auth-local=trust", + "--auth-host=trust", + "-E", + "UTF8", + ], + check=True, + capture_output=True, + env=postgres_env, + ) + subprocess.run( + [ + str(_PG_CTL), + "-D", + str(data_dir), + "-l", + str(root / "postgres.log"), + "-o", + f"-k {socket_dir} -h ''", + "-w", + "start", + ], + check=True, + capture_output=True, + env=postgres_env, + ) + try: + yield str(socket_dir) + finally: + subprocess.run( + [str(_PG_CTL), "-D", str(data_dir), "-m", "immediate", "-w", "stop"], + check=False, + capture_output=True, + env=postgres_env, + ) + shutil.rmtree(root, ignore_errors=True) + + +def _dsn(socket_dir: str, user: str, password: str | None = None) -> str: + value = f"host={socket_dir} dbname={DATABASE} user={user}" + return f"{value} password={password}" if password else value + + +@pytest.fixture(scope="module") +def domain_dsns(pg_socket_dir: str) -> dict[str, str]: + bootstrap = psycopg2.connect( + f"host={pg_socket_dir} dbname=postgres user={OWNER_ROLE}" + ) + bootstrap.autocommit = True + with bootstrap.cursor() as cursor: + cursor.execute(f"CREATE DATABASE {DATABASE}") + bootstrap.close() + + owner_dsn = _dsn(pg_socket_dir, OWNER_ROLE) + conn = psycopg2.connect(owner_dsn) + conn.autocommit = True + with conn.cursor() as cursor: + cursor.execute(_SCHEMA_SQL) + for role in (TENANT_ROLE, INTERNAL_ROLE, CATALOG_READER_ROLE): + cursor.execute( + f"CREATE ROLE {role} LOGIN PASSWORD %s NOSUPERUSER NOBYPASSRLS", + (ROLE_PASSWORD,), + ) + cursor.execute(f"GRANT CONNECT ON DATABASE {DATABASE} TO {role}") + cursor.execute(f"GRANT USAGE ON SCHEMA tenant TO {TENANT_ROLE}") + cursor.execute( + f"GRANT SELECT, INSERT, UPDATE ON tenant.{TENANT_TABLE} TO {TENANT_ROLE}" + ) + cursor.execute(f"GRANT USAGE ON SCHEMA omninode_internal TO {INTERNAL_ROLE}") + cursor.execute( + "GRANT SELECT, INSERT, UPDATE ON " + f"omninode_internal.{INTERNAL_TABLE} TO {INTERNAL_ROLE}" + ) + cursor.execute( + f"GRANT USAGE ON SCHEMA platform_catalog TO {CATALOG_READER_ROLE}" + ) + cursor.execute( + f"GRANT SELECT ON platform_catalog.{CATALOG_TABLE} TO {CATALOG_READER_ROLE}" + ) + conn.close() + return { + "owner": owner_dsn, + "tenant": _dsn(pg_socket_dir, TENANT_ROLE, ROLE_PASSWORD), + "internal": _dsn(pg_socket_dir, INTERNAL_ROLE, ROLE_PASSWORD), + "catalog": _dsn(pg_socket_dir, CATALOG_READER_ROLE, ROLE_PASSWORD), + } + + +@pytest.fixture(autouse=True) +def _clean_tables(domain_dsns: dict[str, str]) -> Iterator[None]: + yield + conn = psycopg2.connect(domain_dsns["owner"]) + conn.autocommit = True + with conn.cursor() as cursor: + cursor.execute(f"TRUNCATE tenant.{TENANT_TABLE}") + cursor.execute(f"TRUNCATE omninode_internal.{INTERNAL_TABLE}") + cursor.execute(f"TRUNCATE platform_catalog.{CATALOG_TABLE}") + conn.close() + + +def _tenant_adapter(dsn: str, tenant_id: UUID | None) -> object: + target = projection_database_target(TENANT_TABLE, schema="tenant") + authority = None + event = None + if tenant_id is not None: + authority, event = verified_tenant_dispatch(tenant_id) + return _build_projection_db_adapter( + projection_database_urls(target, dsn), target, authority, event + ) + + +def _internal_adapter(dsn: str) -> object: + target = projection_database_target(INTERNAL_TABLE, schema="omninode_internal") + return _build_projection_db_adapter( + projection_database_urls(target, dsn), target, None, None + ) + + +def _catalog_reader_adapter(dsn: str) -> object: + target = projection_database_target( + CATALOG_TABLE, + schema="platform_catalog", + access="read", + catalog_read_binding="app_dashboard", + unshipped_grant_principal="app_dashboard", + unshipped_grant_reason=( + "PLATFORM_CATALOG grants are not derivable from node contracts: no " + "db_io.db_tables block declares a catalog relation, so the shipped " + "topology carries none (OMN-15355/OMN-15424 own that grant set). " + "This asserts catalog read isolation, not catalog coverage." + ), + ) + return _build_projection_db_adapter( + projection_database_urls(target, dsn), target, None, None + ) + + +def _tenant_rows(owner_dsn: str) -> list[tuple[UUID, UUID]]: + conn = psycopg2.connect(owner_dsn) + conn.autocommit = True + try: + with conn.cursor() as cursor: + cursor.execute( + "SELECT correlation_id, tenant_id FROM tenant.delegation_events " + "ORDER BY tenant_id" + ) + return list(cursor.fetchall()) + finally: + conn.close() + + +def test_verified_tenant_write_read_and_uuid_preservation( + domain_dsns: dict[str, str], +) -> None: + correlation_id = uuid4() + adapter = _tenant_adapter(domain_dsns["tenant"], TENANT_A) + try: + assert adapter.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": correlation_id, "task_type": "proof"}, + ) + found = adapter.query(TENANT_TABLE, {"correlation_id": correlation_id}) + assert found[0]["correlation_id"] == correlation_id + assert isinstance(found[0]["correlation_id"], UUID) + finally: + adapter.close() + assert _tenant_rows(domain_dsns["owner"]) == [(correlation_id, TENANT_A)] + + +def test_untrusted_row_tenant_cannot_choose_context( + domain_dsns: dict[str, str], +) -> None: + adapter = _tenant_adapter(domain_dsns["tenant"], TENANT_A) + with pytest.raises(ProjectionTenantContextError, match="does not match"): + adapter.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": uuid4(), "task_type": "proof", "tenant_id": TENANT_B}, + ) + assert _tenant_rows(domain_dsns["owner"]) == [] + + +def test_equal_tenant_string_is_replaced_with_uuid(domain_dsns: dict[str, str]) -> None: + adapter = _tenant_adapter(domain_dsns["tenant"], TENANT_A) + try: + assert adapter.upsert( + TENANT_TABLE, + "correlation_id", + { + "correlation_id": uuid4(), + "task_type": "proof", + "tenant_id": str(TENANT_A), + }, + ) + finally: + adapter.close() + + +def test_missing_authority_fails_before_connect(domain_dsns: dict[str, str]) -> None: + adapter = _tenant_adapter(domain_dsns["tenant"], None) + with patch("psycopg2.connect") as connect: + with pytest.raises(ProjectionTenantContextError, match="verified authority"): + adapter.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": uuid4(), "task_type": "proof"}, + ) + connect.assert_not_called() + + +def test_tenant_b_cannot_read_tenant_a(domain_dsns: dict[str, str]) -> None: + correlation_id = uuid4() + tenant_a = _tenant_adapter(domain_dsns["tenant"], TENANT_A) + tenant_b = _tenant_adapter(domain_dsns["tenant"], TENANT_B) + try: + tenant_a.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": correlation_id, "task_type": "proof"}, + ) + assert tenant_b.query(TENANT_TABLE, {"correlation_id": correlation_id}) == [] + finally: + tenant_a.close() + tenant_b.close() + + +def test_reused_tenant_connection_does_not_leak_context( + domain_dsns: dict[str, str], +) -> None: + shared_connection = psycopg2.connect(domain_dsns["tenant"]) + try: + with patch("psycopg2.connect", return_value=shared_connection): + for tenant_id in (TENANT_A, TENANT_B): + _tenant_adapter(domain_dsns["tenant"], tenant_id).upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": uuid4(), "task_type": "pooled"}, + ) + with shared_connection.cursor() as cursor: + cursor.execute("SELECT current_setting('app.tenant_id', true)") + assert cursor.fetchone()[0] in (None, "") + finally: + shared_connection.close() + assert [row[1] for row in _tenant_rows(domain_dsns["owner"])] == [ + TENANT_A, + TENANT_B, + ] + + +def test_real_rls_with_check_rejects_tenant_b_insert_and_update_under_a( + domain_dsns: dict[str, str], +) -> None: + conn = psycopg2.connect(domain_dsns["tenant"]) + try: + with conn.cursor() as cursor: + with pytest.raises(psycopg2.errors.InsufficientPrivilege): + cursor.execute( + "INSERT INTO tenant.delegation_events VALUES (%s, %s, %s)", + (uuid4(), "unset-context", TENANT_A), + ) + conn.rollback() + + with conn.cursor() as cursor: + cursor.execute("SET LOCAL app.tenant_id = %s", (str(TENANT_A),)) + with pytest.raises(psycopg2.errors.InsufficientPrivilege): + cursor.execute( + "INSERT INTO tenant.delegation_events VALUES (%s, %s, %s)", + (uuid4(), "wrong-insert", TENANT_B), + ) + conn.rollback() + + correlation_id = uuid4() + with conn.cursor() as cursor: + cursor.execute("SET LOCAL app.tenant_id = %s", (str(TENANT_A),)) + cursor.execute( + "INSERT INTO tenant.delegation_events VALUES (%s, %s, %s)", + (correlation_id, "valid-a", TENANT_A), + ) + conn.commit() + with conn.cursor() as cursor: + cursor.execute("SET LOCAL app.tenant_id = %s", (str(TENANT_A),)) + with pytest.raises(psycopg2.errors.InsufficientPrivilege): + cursor.execute( + "UPDATE tenant.delegation_events SET tenant_id = %s " + "WHERE correlation_id = %s", + (TENANT_B, correlation_id), + ) + conn.rollback() + finally: + conn.close() + + +def test_rollback_clears_guc_before_reusing_connection_for_tenant_b( + domain_dsns: dict[str, str], +) -> None: + shared_connection = psycopg2.connect(domain_dsns["tenant"]) + try: + with patch("psycopg2.connect", return_value=shared_connection): + tenant_a = _tenant_adapter(domain_dsns["tenant"], TENANT_A) + with pytest.raises(psycopg2.errors.NotNullViolation): + tenant_a.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": uuid4()}, + ) + with shared_connection.cursor() as cursor: + cursor.execute("SELECT current_setting('app.tenant_id', true)") + assert cursor.fetchone()[0] in (None, "") + + tenant_b = _tenant_adapter(domain_dsns["tenant"], TENANT_B) + tenant_b.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": uuid4(), "task_type": "after-rollback"}, + ) + with shared_connection.cursor() as cursor: + cursor.execute("SELECT current_setting('app.tenant_id', true)") + assert cursor.fetchone()[0] in (None, "") + finally: + shared_connection.close() + + assert _tenant_rows(domain_dsns["owner"])[0][1] == TENANT_B + + +def test_internal_write_read_has_no_tenant_guc(domain_dsns: dict[str, str]) -> None: + correlation_id = uuid4() + adapter = _internal_adapter(domain_dsns["internal"]) + try: + adapter.upsert( + INTERNAL_TABLE, + "correlation_id", + { + "correlation_id": correlation_id, + "source_tenant_id": TENANT_A, + "status": "complete", + }, + ) + assert ( + adapter.query(INTERNAL_TABLE, {"correlation_id": correlation_id})[0][ + "status" + ] + == "complete" + ) + conn = next(iter(adapter._connections.values())) + with conn.cursor() as cursor: + cursor.execute("SELECT current_setting('app.tenant_id', true)") + assert cursor.fetchone()[0] in (None, "") + finally: + adapter.close() + + +def test_catalog_reader_can_read_and_has_no_writer_operation( + domain_dsns: dict[str, str], +) -> None: + owner = psycopg2.connect(domain_dsns["owner"]) + owner.autocommit = True + with owner.cursor() as cursor: + cursor.execute( + "INSERT INTO platform_catalog.plan_tiers VALUES (%s, %s)", + ("beta", "Beta"), + ) + owner.close() + adapter = _catalog_reader_adapter(domain_dsns["catalog"]) + try: + assert adapter.query(CATALOG_TABLE)[0]["tier_id"] == "beta" + with pytest.raises(PermissionError, match="write refused"): + adapter.upsert( + CATALOG_TABLE, + "tier_id", + {"tier_id": "pro", "display_name": "Pro"}, + ) + finally: + adapter.close() + + +@pytest.mark.parametrize( + ("dsn_key", "sql"), + [ + ("tenant", "SELECT * FROM omninode_internal.generation_events"), + ("internal", "SELECT * FROM tenant.delegation_events"), + ("catalog", "INSERT INTO platform_catalog.plan_tiers VALUES ('x', 'X')"), + ], +) +def test_cross_domain_roles_are_denied( + domain_dsns: dict[str, str], dsn_key: str, sql: str +) -> None: + conn = psycopg2.connect(domain_dsns[dsn_key]) + conn.autocommit = True + try: + with pytest.raises(psycopg2.errors.InsufficientPrivilege): + with conn.cursor() as cursor: + cursor.execute(sql) + finally: + conn.close() + + +def test_miswired_dsn_fails_identity_attestation(domain_dsns: dict[str, str]) -> None: + target = projection_database_target(TENANT_TABLE, schema="tenant") + authority, event = verified_tenant_dispatch(TENANT_A) + adapter = _build_projection_db_adapter( + projection_database_urls(target, domain_dsns["internal"]), + target, + authority, + event, + ) + with pytest.raises(PermissionError, match="expected"): + adapter.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": uuid4(), "task_type": "proof"}, + ) + + +def test_environment_tenant_is_not_authority( + domain_dsns: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("ONEX_TENANT_ID", str(TENANT_A)) + monkeypatch.setenv("ENFORCE_TENANT_ISOLATION", "false") + adapter = _tenant_adapter(domain_dsns["tenant"], None) + with pytest.raises(ProjectionTenantContextError): + adapter.upsert( + TENANT_TABLE, + "correlation_id", + {"correlation_id": uuid4(), "task_type": "proof"}, + ) diff --git a/tests/integration/docker/conftest.py b/tests/integration/docker/conftest.py index a14198e1c4..027d43562f 100644 --- a/tests/integration/docker/conftest.py +++ b/tests/integration/docker/conftest.py @@ -36,11 +36,17 @@ - run_container(): Context manager for container lifecycle - wait_for_healthy(): Poll container health status - wait_for_log_message(): Wait for log output + - _run_subprocess_with_group_kill(): group-kill-safe subprocess runner, + shared with test_docker_integration.py via ``from .conftest import ...`` + (OMN-15567 -- see docstring on the function itself for why this matters + for the ``built_test_image`` fixture specifically). """ from __future__ import annotations +import contextlib import os +import signal import socket import subprocess import time @@ -288,6 +294,49 @@ def test_image_name() -> str: return f"omnibase-infra-test:{os.getpid()}" +def _run_subprocess_with_group_kill( + cmd: list[str], + *, + timeout: int, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + """Run a subprocess, killing its whole process group on timeout. + + ``subprocess.run(..., timeout=...)`` only terminates the immediate child on + a ``TimeoutExpired`` -- a killed ``docker build`` invocation (and any + grandchild ``buildkit``/``runc`` processes it spawned) is left orphaned on + the runner, still consuming CPU/network after pytest has moved on. Running + the child in its own session (``start_new_session=True``) lets us kill the + whole process group via ``os.killpg`` instead of just the direct child. + + OMN-15567: defined here (not in test_docker_integration.py) so the + ``built_test_image`` fixture below and the explicit build tests in + test_docker_integration.py share one group-kill implementation instead of + the fixture using a bare ``subprocess.run`` that can orphan build + processes. test_docker_integration.py imports this via + ``from .conftest import _run_subprocess_with_group_kill``. + """ + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + start_new_session=True, + ) + try: + stdout, stderr = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(proc.pid, signal.SIGKILL) + # Drain pipes so the killed child doesn't leave zombie fds behind. + stdout, stderr = proc.communicate() + raise + return subprocess.CompletedProcess( + cmd, proc.returncode, stdout=stdout, stderr=stderr + ) + + @pytest.fixture(scope="module") def built_test_image( docker_available: bool, @@ -299,6 +348,16 @@ def built_test_image( This fixture builds the Docker image once per test module and cleans up after all tests in the module complete. + OMN-15567: this is a module-scoped fixture, so its build only actually + runs for the first test in the module that requests it -- that test item + is charged the full build wall-clock under pytest-timeout (func_only + defaults to False, so the timer wraps setup+call+teardown). Every class + below that consumes this fixture carries a class-level + ``pytestmark = [pytest.mark.timeout(...)]`` sized for a cold build, so + the budget applies regardless of which test happens to be the first + consumer -- see the module docstring in test_docker_integration.py for + the full "why this can't regress via reordering" argument. + Args: docker_available: Whether Docker is available. project_root: Project root directory. @@ -326,14 +385,10 @@ def built_test_image( env = os.environ.copy() env["DOCKER_BUILDKIT"] = "1" - result = subprocess.run( + result = _run_subprocess_with_group_kill( build_cmd, - capture_output=True, - text=True, timeout=int(os.getenv("OMNI_DOCKER_BUILD_TIMEOUT_SECONDS", "1200")), env=env, - check=False, - shell=False, ) if result.returncode != 0: diff --git a/tests/integration/docker/test_docker_integration.py b/tests/integration/docker/test_docker_integration.py index fe4284850e..01367d6406 100644 --- a/tests/integration/docker/test_docker_integration.py +++ b/tests/integration/docker/test_docker_integration.py @@ -16,6 +16,39 @@ This test suite addresses PR #32 reviewer feedback requesting CI/CD integration tests for Docker infrastructure implementation. + +Cold-build timeout budget (OMN-15567) +-------------------------------------- +The nightly suite runs under a blanket ``pytest --timeout=300``. Two kinds of +test in this module actually invoke ``docker build`` and need a much larger +budget than that: + +1. The two explicit build tests in ``TestDockerBuild`` + (``test_build_succeeds_with_public_deps``, + ``test_build_uses_buildkit_cache_mounts``) carry their own function-level + ``@pytest.mark.timeout(BUILD_TIMEOUT + BUILD_TEST_TIMEOUT_MARGIN_SECONDS)``. +2. The module-scoped ``built_test_image`` fixture (defined in ``conftest.py``) + does the same build for every other class below that consumes it. Because + the fixture is module-scoped, only the *first* test in the module that + requests it actually pays the build cost -- and pytest-timeout charges + that cost to whichever test item happens to be first (``func_only`` + defaults to False, so the timer spans setup+call+teardown of that item, + not just its body). + + Rather than mark only "the currently-first" consumer -- which would + silently regress the instant a test is added, removed, or reordered ahead + of it -- every class below that declares a fixture parameter typed + ``built_test_image: str`` carries its own class-level + ``pytestmark = [pytest.mark.timeout(BUILD_TIMEOUT + BUILD_TEST_TIMEOUT_MARGIN_SECONDS)]``. + ``pytest.Item.get_closest_marker("timeout")`` resolves per test item by + walking function -> class -> module, so no matter which member of one of + these classes pytest picks as the actual first consumer, that item's + closest "timeout" marker is always at least the cold-build budget. A + function-level marker (as in ``TestDockerBuild``'s two explicit build + tests) still wins over the class-level one where both are present, since + it is closer. This makes the budget invariant to collection/execution + order within the file, and any new test added to one of these classes + inherits it automatically instead of needing its own opt-in marker. """ from __future__ import annotations @@ -33,6 +66,11 @@ import pytest import yaml +# OMN-15567: _run_subprocess_with_group_kill lives in conftest.py so the +# built_test_image fixture and these explicit build tests share one +# group-kill implementation -- see conftest.py's docstring on the function. +from .conftest import _run_subprocess_with_group_kill + # ============================================================================= # Test Markers and Constants # ============================================================================= @@ -52,6 +90,16 @@ HEALTH_CHECK_TIMEOUT = 90 SHUTDOWN_TIMEOUT = int(os.getenv("OMNI_DOCKER_SHUTDOWN_TIMEOUT_SECONDS", "120")) +# Headroom above BUILD_TIMEOUT for the pytest-level per-test timeout marker. +# OMN-15567: nightly-integration.yml invokes the whole suite with a blanket +# `pytest --timeout=300 --timeout-method=thread`, sized for the rest of the +# suite. Without a per-test `@pytest.mark.timeout(...)` override, that 300s +# CLI ceiling silently pre-empts BUILD_TIMEOUT (default 1200s) and kills a +# cold-cache `docker build` long before the subprocess-level timeout the test +# was actually designed around ever fires. pytest-timeout's marker-level +# setting takes precedence over the CLI flag for the tests that declare it. +BUILD_TEST_TIMEOUT_MARGIN_SECONDS = 60 + # ============================================================================= # Helper Functions @@ -95,7 +143,16 @@ def extract_profiles_from_compose(compose_path: Path) -> set[str]: class TestDockerBuild: """Tests for Docker image build process.""" + # OMN-15567: test_build_produces_reasonable_image_size (below) consumes + # built_test_image and has no function-level timeout marker of its own; + # this class-level marker is its fallback budget. See the module + # docstring "Cold-build timeout budget" section for why. + pytestmark = [ + pytest.mark.timeout(BUILD_TIMEOUT + BUILD_TEST_TIMEOUT_MARGIN_SECONDS) + ] + @pytest.mark.slow + @pytest.mark.timeout(BUILD_TIMEOUT + BUILD_TEST_TIMEOUT_MARGIN_SECONDS) def test_build_succeeds_with_public_deps( self, docker_available: bool, @@ -129,14 +186,10 @@ def test_build_succeeds_with_public_deps( env = os.environ.copy() env["DOCKER_BUILDKIT"] = "1" - result = subprocess.run( + result = _run_subprocess_with_group_kill( build_cmd, - capture_output=True, - text=True, timeout=BUILD_TIMEOUT, env=env, - check=False, - shell=False, ) assert result.returncode == 0, ( @@ -156,6 +209,7 @@ def test_build_succeeds_with_public_deps( ) @pytest.mark.slow + @pytest.mark.timeout(BUILD_TIMEOUT + BUILD_TEST_TIMEOUT_MARGIN_SECONDS) def test_build_uses_buildkit_cache_mounts( self, buildkit_available: bool, @@ -190,14 +244,10 @@ def test_build_uses_buildkit_cache_mounts( str(project_root), ] - first_result = subprocess.run( + first_result = _run_subprocess_with_group_kill( first_build_cmd, - capture_output=True, - text=True, timeout=BUILD_TIMEOUT, env=env, - check=False, - shell=False, ) assert first_result.returncode == 0, "First build failed" @@ -275,6 +325,15 @@ def test_build_produces_reasonable_image_size( class TestDockerSecurity: """Tests for Docker security properties.""" + # OMN-15567: one of this class's tests may be the first consumer of the + # module-scoped built_test_image fixture (order-dependent -- see the + # module docstring's "Cold-build timeout budget" section). This + # class-level marker guarantees the cold-build budget applies regardless + # of which one it is. + pytestmark = [ + pytest.mark.timeout(BUILD_TIMEOUT + BUILD_TEST_TIMEOUT_MARGIN_SECONDS) + ] + @pytest.mark.slow def test_container_runs_as_non_root_user( self, @@ -444,6 +503,12 @@ def test_sensitive_files_not_in_image( class TestDockerRuntime: """Tests for Docker container runtime behavior.""" + # OMN-15567: see TestDockerSecurity above -- same module-scoped + # built_test_image first-consumer hazard. + pytestmark = [ + pytest.mark.timeout(BUILD_TIMEOUT + BUILD_TEST_TIMEOUT_MARGIN_SECONDS) + ] + @pytest.mark.slow def test_container_starts_successfully( self, @@ -698,6 +763,12 @@ def test_graceful_shutdown_on_sigterm( class TestDockerHealthCheck: """Tests for Docker health check functionality.""" + # OMN-15567: see TestDockerSecurity above -- same module-scoped + # built_test_image first-consumer hazard. + pytestmark = [ + pytest.mark.timeout(BUILD_TIMEOUT + BUILD_TEST_TIMEOUT_MARGIN_SECONDS) + ] + @pytest.mark.slow def test_health_endpoint_accessible( self, @@ -1000,6 +1071,115 @@ def test_compose_defines_resource_reservations( # ============================================================================= +_PG_DSN = "postgresql://postgres:test@postgres:5432/omnibase_infra" +_INTEL_DSN = "postgresql://postgres:test@postgres:5432/omniintelligence" +_LOCAL_LAN_CIDR = ".".join(("192", "168", "86", "0")) + "/24" +_SECRET_RESOLVER_CONFIG_JSON = ( + '{"enable_convention_fallback":false,"mappings":[' + '{"logical_name":"llm.openrouter.api_key",' + '"source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},' + '{"logical_name":"llm.glm.api_key",' + '"source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},' + '{"logical_name":"llm.gemini.api_key",' + '"source":{"source_path":"GEMINI_API_KEY","source_type":"env"}}]}' +) +_SECRET_RESOLVER_CONFIG_PATH = "/app/data/delegation/secret_resolver.yaml" + +# OMN-15263: module-level so tests/ci/test_compose_required_env_coverage.py can +# extract this render fixture's env keys statically, the same way it extracts the +# other compose-render fixtures. Keep every `:?`-required var in +# docker/docker-compose.infra.yml represented here. +COMPOSE_CONFIG_RENDER_ENV: dict[str, str] = { + "POSTGRES_PASSWORD": "test", + "VALKEY_PASSWORD": "test", + "INFISICAL_ENCRYPTION_KEY": "0" * 64, + "INFISICAL_AUTH_SECRET": "test-auth-secret", + "OMNIBASE_INFRA_DB_URL": _PG_DSN, + "OMNIINTELLIGENCE_DB_URL": _INTEL_DSN, + "INFISICAL_DB_CONNECTION_URI": "postgresql://postgres:test@postgres:5432/infisical_db", + "INFISICAL_REDIS_URL": "redis://:test@valkey:6379", + "OMNIBASE_INFRA_AGENT_ACTIONS_POSTGRES_DSN": _PG_DSN, + "OMNIBASE_INFRA_SKILL_LIFECYCLE_POSTGRES_DSN": _PG_DSN, + # OMN-5240: context-audit-consumer requires its own DSN + "OMNIBASE_INFRA_CONTEXT_AUDIT_POSTGRES_DSN": _PG_DSN, + # OMN-3299: Redpanda removed from local compose; KAFKA_BOOTSTRAP_SERVERS + # now uses :? fail-fast — must be set explicitly for config validation. + "KAFKA_BOOTSTRAP_SERVERS": "localhost:19092", # kafka-fallback-ok — test fixture + # OMN-15173: dev-lane Redpanda advertise host now uses :? fail-fast + # (previously silently defaulted to localhost, breaking off-host clients). + "DEV_REDPANDA_ADVERTISE_HOST": "localhost", # kafka-fallback-ok — test fixture + "ARCH_GRAPH_BOLT_URI": "bolt://omnibase-infra-memgraph:7687", + # OMN-5439: Keycloak / ONEX service auth vars added with :? fail-fast + "ONEX_REGISTRATION_AUTO_ACK": "true", + "ONEX_SERVICE_CLIENT_SECRET": "test-service-secret", + "LINEAR_API_KEY": "test-linear-api-key", + "GITHUB_TOKEN": "test-github-token", + "DEPLOY_AGENT_HMAC_SECRET": "render-only-deploy-agent-hmac-secret", + # OMN-7979: LLM endpoint URLs added with :? fail-fast to + # activate PluginLlm in runtime containers. + "LLM_CODER_URL": "http://llm-coder.test:8000", + "LLM_CODER_FAST_URL": "http://llm-coder-fast.test:8001", + "LLM_EMBEDDING_URL": "http://llm-embed.test:8100", + "LLM_DEEPSEEK_R1_URL": "http://llm-r1.test:8101", + "BIFROST_LOCAL_CODER_ENDPOINT_URL": ( + "http://llm-coder.test:8000/v1/chat/completions" + ), + "BIFROST_LOCAL_REASONER_ENDPOINT_URL": ( + "http://llm-coder-fast.test:8001/v1/chat/completions" + ), + "BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL": ( + "http://llm-embed.test:8100/v1/chat/completions" + ), + "BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL": ( + "http://llm-r1.test:8101/v1/chat/completions" + ), + "LLM_GLM_URL": "http://llm-glm.test:8102", + "LLM_GLM_MODEL_NAME": "glm-4.5", + "LLM_GLM_API_KEY": "render-only-glm-api-key", + "GEMINI_API_KEY": "render-only-gemini-api-key", + "GOOGLE_API_KEY": "render-only-google-api-key", + "BIFROST_VERTEX_GEMINI_ENDPOINT_URL": ( + "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/" + "gen-lang-client-0084338881/locations/us-central1/endpoints/openapi/chat/completions" + ), + "GOOGLE_CLOUD_PROJECT": "gen-lang-client-0084338881", + "GOOGLE_CLOUD_LOCATION": "us-central1", + # OMN-10943: HTTP request signing and CIDR allowlist for the + # local LLM HTTP transport added with :? fail-fast. + "LOCAL_LLM_SHARED_SECRET": "render-only-local-llm-secret", + "LLM_ENDPOINT_CIDR_ALLOWLIST": _LOCAL_LAN_CIDR, + "LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST": "generativelanguage.googleapis.com,api.z.ai", + # OMN-11673: runtime policy contract vars are required by + # compose even when docker/runtime-policy.env is also loaded. + "AUXILIARY_SERVICES_OMNIMEMORY_ENABLED": "false", + "BIFROST_VERIFY_ENDPOINTS": "1", + "DEV_RUNTIME_EFFECTS_CAPABILITIES": "effects.consumer,market.skill-proof,runtime.effects", + "DEV_RUNTIME_EFFECTS_PORT": "8086", + "DEV_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_JSON": _SECRET_RESOLVER_CONFIG_JSON, + "DEV_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_PATH": _SECRET_RESOLVER_CONFIG_PATH, + "DEV_RUNTIME_MAIN_CAPABILITIES": "market.skill-proof,workflow.orchestration,runtime.main", + "DEV_RUNTIME_MAIN_PORT": "8085", + "DEV_RUNTIME_MAIN_PUBLISH_INTROSPECTION": "true", + "DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_JSON": _SECRET_RESOLVER_CONFIG_JSON, + "DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_PATH": _SECRET_RESOLVER_CONFIG_PATH, + "DEV_RUNTIME_WORKER_CAPABILITIES": "workflow.dispatch,contract.update,runtime.worker", + "DEV_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_JSON": _SECRET_RESOLVER_CONFIG_JSON, + "DEV_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_PATH": _SECRET_RESOLVER_CONFIG_PATH, + # OMN-14551: dev-lane flipped ON 2026-08-05; kept in sync with the + # rendered docker/runtime-policy.env value (this fixture only needs to + # satisfy the compose `:?` fail-fast, but tracking the live value avoids + # a misleading fixture). + "DEV_BOUNDARY_DLQ_ENABLED": "true", + # OMN-14968: runtime-worker's deploy.replicas is `:?`-required on the + # lane-prefixed policy value (it was a bare ${WORKER_REPLICAS:-0} that no + # surface exported, so the dev lane silently rendered zero replicas). + "DEV_WORKER_REPLICAS": "1", + "OMNIMEMORY_ENABLED": "false", + "OMNIMEMORY_MEMGRAPH_PORT": "7687", + "ONEX_ACTIVE_RUNTIME_PACKAGES": "omnibase_infra,omnimarket", +} + + @pytest.mark.integration class TestDockerComposeProfiles: """Tests for docker-compose profile configurations. @@ -1071,98 +1251,8 @@ def test_compose_config_valid( # All :? required vars must be set even for config validation; the PR # that removed nested expansion (OMN-3266) moved DSN/URL construction # out of compose into ~/.omnibase/.env, so these now use :? fail-fast. - _pg_dsn = "postgresql://postgres:test@postgres:5432/omnibase_infra" - _intel_dsn = "postgresql://postgres:test@postgres:5432/omniintelligence" - _local_lan_cidr = ".".join(("192", "168", "86", "0")) + "/24" - _secret_resolver_config_json = ( - '{"enable_convention_fallback":false,"mappings":[' - '{"logical_name":"llm.openrouter.api_key",' - '"source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},' - '{"logical_name":"llm.glm.api_key",' - '"source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},' - '{"logical_name":"llm.gemini.api_key",' - '"source":{"source_path":"GEMINI_API_KEY","source_type":"env"}}]}' - ) - _secret_resolver_config_path = "/app/data/delegation/secret_resolver.yaml" env = os.environ.copy() - env.update( - { - "POSTGRES_PASSWORD": "test", - "VALKEY_PASSWORD": "test", - "INFISICAL_ENCRYPTION_KEY": "0" * 64, - "INFISICAL_AUTH_SECRET": "test-auth-secret", - "OMNIBASE_INFRA_DB_URL": _pg_dsn, - "OMNIINTELLIGENCE_DB_URL": _intel_dsn, - "INFISICAL_DB_CONNECTION_URI": "postgresql://postgres:test@postgres:5432/infisical_db", - "INFISICAL_REDIS_URL": "redis://:test@valkey:6379", - "OMNIBASE_INFRA_AGENT_ACTIONS_POSTGRES_DSN": _pg_dsn, - "OMNIBASE_INFRA_SKILL_LIFECYCLE_POSTGRES_DSN": _pg_dsn, - # OMN-5240: context-audit-consumer requires its own DSN - "OMNIBASE_INFRA_CONTEXT_AUDIT_POSTGRES_DSN": _pg_dsn, - # OMN-3299: Redpanda removed from local compose; KAFKA_BOOTSTRAP_SERVERS - # now uses :? fail-fast — must be set explicitly for config validation. - "KAFKA_BOOTSTRAP_SERVERS": "localhost:19092", # kafka-fallback-ok — test fixture - "ARCH_GRAPH_BOLT_URI": "bolt://omnibase-infra-memgraph:7687", - # OMN-5439: Keycloak / ONEX service auth vars added with :? fail-fast - "ONEX_REGISTRATION_AUTO_ACK": "true", - "ONEX_SERVICE_CLIENT_SECRET": "test-service-secret", - "LINEAR_API_KEY": "test-linear-api-key", - "GITHUB_TOKEN": "test-github-token", - # OMN-7979: LLM endpoint URLs added with :? fail-fast to - # activate PluginLlm in runtime containers. - "LLM_CODER_URL": "http://llm-coder.test:8000", - "LLM_CODER_FAST_URL": "http://llm-coder-fast.test:8001", - "LLM_EMBEDDING_URL": "http://llm-embed.test:8100", - "LLM_DEEPSEEK_R1_URL": "http://llm-r1.test:8101", - "BIFROST_LOCAL_CODER_ENDPOINT_URL": ( - "http://llm-coder.test:8000/v1/chat/completions" - ), - "BIFROST_LOCAL_REASONER_ENDPOINT_URL": ( - "http://llm-coder-fast.test:8001/v1/chat/completions" - ), - "BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL": ( - "http://llm-embed.test:8100/v1/chat/completions" - ), - "BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL": ( - "http://llm-r1.test:8101/v1/chat/completions" - ), - "LLM_GLM_URL": "http://llm-glm.test:8102", - "LLM_GLM_MODEL_NAME": "glm-4.5", - "LLM_GLM_API_KEY": "render-only-glm-api-key", - "GEMINI_API_KEY": "render-only-gemini-api-key", - "GOOGLE_API_KEY": "render-only-google-api-key", - "BIFROST_VERTEX_GEMINI_ENDPOINT_URL": ( - "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/" - "gen-lang-client-0084338881/locations/us-central1/endpoints/openapi/chat/completions" - ), - "GOOGLE_CLOUD_PROJECT": "gen-lang-client-0084338881", - "GOOGLE_CLOUD_LOCATION": "us-central1", - # OMN-10943: HTTP request signing and CIDR allowlist for the - # local LLM HTTP transport added with :? fail-fast. - "LOCAL_LLM_SHARED_SECRET": "render-only-local-llm-secret", - "LLM_ENDPOINT_CIDR_ALLOWLIST": _local_lan_cidr, - "LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST": "generativelanguage.googleapis.com,api.z.ai", - # OMN-11673: runtime policy contract vars are required by - # compose even when docker/runtime-policy.env is also loaded. - "AUXILIARY_SERVICES_OMNIMEMORY_ENABLED": "false", - "BIFROST_VERIFY_ENDPOINTS": "1", - "DEV_RUNTIME_EFFECTS_CAPABILITIES": "effects.consumer,market.skill-proof,runtime.effects", - "DEV_RUNTIME_EFFECTS_PORT": "8086", - "DEV_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_JSON": _secret_resolver_config_json, - "DEV_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_PATH": _secret_resolver_config_path, - "DEV_RUNTIME_MAIN_CAPABILITIES": "market.skill-proof,workflow.orchestration,runtime.main", - "DEV_RUNTIME_MAIN_PORT": "8085", - "DEV_RUNTIME_MAIN_PUBLISH_INTROSPECTION": "true", - "DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_JSON": _secret_resolver_config_json, - "DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_PATH": _secret_resolver_config_path, - "DEV_RUNTIME_WORKER_CAPABILITIES": "workflow.dispatch,contract.update,runtime.worker", - "DEV_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_JSON": _secret_resolver_config_json, - "DEV_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_PATH": _secret_resolver_config_path, - "OMNIMEMORY_ENABLED": "false", - "OMNIMEMORY_MEMGRAPH_PORT": "7687", - "ONEX_ACTIVE_RUNTIME_PACKAGES": "omnibase_infra,omnimarket", - } - ) + env.update(COMPOSE_CONFIG_RENDER_ENV) result = subprocess.run( [ @@ -1195,6 +1285,12 @@ def test_compose_config_valid( class TestDockerImageLabels: """Tests for Docker image OCI labels.""" + # OMN-15567: see TestDockerSecurity above -- same module-scoped + # built_test_image first-consumer hazard. + pytestmark = [ + pytest.mark.timeout(BUILD_TIMEOUT + BUILD_TEST_TIMEOUT_MARGIN_SECONDS) + ] + @pytest.mark.slow def test_image_has_oci_labels( self, diff --git a/tests/integration/event_bus/test_kafka_concurrent_subscribe_startup.py b/tests/integration/event_bus/test_kafka_concurrent_subscribe_startup.py index fed1d71d2c..6c1ef1bee6 100644 --- a/tests/integration/event_bus/test_kafka_concurrent_subscribe_startup.py +++ b/tests/integration/event_bus/test_kafka_concurrent_subscribe_startup.py @@ -42,7 +42,9 @@ async def test_concurrent_distinct_subscriptions_start_consumers_in_parallel() - max_active_starts = 0 started: list[tuple[str, str]] = [] - async def fake_start(topic: str, group_id: str) -> None: + async def fake_start( + topic: str, group_id: str, *, auto_offset_reset_override: str | None = None + ) -> None: nonlocal active_starts, max_active_starts active_starts += 1 max_active_starts = max(max_active_starts, active_starts) @@ -80,7 +82,12 @@ async def test_concurrent_duplicate_subscription_starts_one_consumer() -> None: identity = make_test_node_identity("shared") started: list[tuple[str, str]] = [] - async def fake_start(start_topic: str, group_id: str) -> None: + async def fake_start( + start_topic: str, + group_id: str, + *, + auto_offset_reset_override: str | None = None, + ) -> None: await asyncio.sleep(0) bus._group_consumers[(start_topic, group_id)] = AsyncMock() bus._pending_consumer_keys.discard((start_topic, group_id)) diff --git a/tests/integration/gateway/test_bus_forwarder_contract_integration.py b/tests/integration/gateway/test_bus_forwarder_contract_integration.py index d2ab3d2d3a..86beb36103 100644 --- a/tests/integration/gateway/test_bus_forwarder_contract_integration.py +++ b/tests/integration/gateway/test_bus_forwarder_contract_integration.py @@ -3,16 +3,17 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable from dataclasses import dataclass -from typing import Any +from pathlib import Path from uuid import UUID, uuid4 import pytest +from omnibase_core.models.core.model_envelope_metadata import ModelEnvelopeMetadata +from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( + ModelGatewayCanaryConfig, ModelGatewayCloudBusConfig, - ModelGatewayEnvelope, ModelGatewayForwarderConfig, ModelGatewayMirrorTopics, ModelGatewayTenantIdentity, @@ -25,7 +26,7 @@ TENANT_ID = UUID("11111111-1111-1111-1111-111111111111") BROKER_PROVIDER_ID = UUID("22222222-2222-2222-2222-222222222222") -PRINCIPAL_ID = UUID("33333333-3333-3333-3333-333333333333") +PRINCIPAL_ID = "t-33333333333333333333333333333333" CORRELATION_ID = UUID("44444444-4444-4444-4444-444444444444") INBOUND_TOPIC = "onex.cmd.omnibase-infra.delegation-inference-request.v1" OUTBOUND_TOPIC = "onex.evt.omnibase-infra.inference-response.v1" @@ -43,30 +44,8 @@ class _Message: class _RecordingBus: def __init__(self) -> None: - self.subscriptions: dict[str, Callable[[Any], Awaitable[None]]] = {} - self.subscription_groups: dict[str, str] = {} self.published: list[_Message] = [] - async def subscribe( - self, - topic: str, - node_identity: object | None = None, - on_message: Callable[[Any], Awaitable[None]] | None = None, - *, - group_id: str | None = None, - **_kwargs: object, - ) -> Callable[[], Awaitable[None]]: - assert on_message is not None - assert group_id is not None - self.subscriptions[topic] = on_message - self.subscription_groups[topic] = group_id - - async def _unsubscribe() -> None: - self.subscriptions.pop(topic, None) - self.subscription_groups.pop(topic, None) - - return _unsubscribe - async def publish( self, topic: str, @@ -76,14 +55,16 @@ async def publish( ) -> None: self.published.append(_Message(topic, key, value, headers)) - async def emit(self, topic: str, envelope: ModelGatewayEnvelope) -> None: - await self.subscriptions[topic]( - _Message( - topic=topic, - key=b"tenant-key", - value=envelope.model_dump_json().encode("utf-8"), - headers={"traceparent": "00-test"}, - ) + def message( + self, + topic: str, + envelope: ModelEventEnvelope[dict[str, object]], + ) -> _Message: + return _Message( + topic=topic, + key=b"tenant-key", + value=envelope.model_dump_json().encode("utf-8"), + headers={"traceparent": "00-test"}, ) @@ -103,28 +84,35 @@ def _config() -> ModelGatewayForwarderConfig: client_secret_api_key_ref="infisical://gateway/redpanda-events", ), local_transport_flavor="containerized", + dedupe_store_path=Path.cwd() / "gateway-test.sqlite3", mirror_topics=ModelGatewayMirrorTopics( inbound=(INBOUND_TOPIC,), outbound=(OUTBOUND_TOPIC,), ), + canary=ModelGatewayCanaryConfig( + topic="onex.evt.omnibase-infra.gateway-canary.v1", + cadence_seconds=30, + produce_deadline_seconds=8, + readback_deadline_seconds=12, + ), ) -def _envelope(**overrides: object) -> ModelGatewayEnvelope: +def _envelope(**overrides: object) -> ModelEventEnvelope[dict[str, object]]: values = { - "tenant_id": TENANT_ID, - "tenant_slug": "acme", "envelope_id": uuid4(), "correlation_id": CORRELATION_ID, - "causation_id": None, "event_type": "LlmInferenceResponse", - "source_topic": OUTBOUND_TOPIC, - "wire_topic": "", - "canonical_topic": OUTBOUND_TOPIC, "payload": {"ok": True}, + "metadata": ModelEnvelopeMetadata( + tags={ + "source_tenant_id": str(TENANT_ID), + "source_tenant_principal_id": PRINCIPAL_ID, + } + ), } values.update(overrides) - return ModelGatewayEnvelope(**values) + return ModelEventEnvelope[dict[str, object]](**values) @pytest.mark.asyncio @@ -137,41 +125,36 @@ async def test_gateway_forwarder_preserves_envelope_across_both_bus_legs() -> No cloud_bus=cloud_bus, ) - await service.start() - await local_bus.emit(OUTBOUND_TOPIC, _envelope()) - await cloud_bus.emit( - WIRE_INBOUND_TOPIC, - _envelope( - event_type="DelegationInferenceRequest", - source_topic=WIRE_INBOUND_TOPIC, - wire_topic=WIRE_INBOUND_TOPIC, - canonical_topic=INBOUND_TOPIC, - ), - ) - - assert set(local_bus.subscriptions) == {OUTBOUND_TOPIC} - assert set(cloud_bus.subscriptions) == {WIRE_INBOUND_TOPIC} - assert local_bus.subscription_groups[OUTBOUND_TOPIC] == ( - "tenant-acme-gateway-forwarder-outbound" + await service.forward_outbound_message( + local_bus.message(OUTBOUND_TOPIC, _envelope()) ) - assert cloud_bus.subscription_groups[WIRE_INBOUND_TOPIC] == ( - "tenant-acme-gateway-forwarder-inbound" + await service.consume_inbound_message( + cloud_bus.message( + WIRE_INBOUND_TOPIC, + _envelope(event_type="DelegationInferenceRequest"), + ) ) outbound = cloud_bus.published[0] assert outbound.topic == WIRE_OUTBOUND_TOPIC assert outbound.key == b"tenant-key" assert outbound.headers == {"traceparent": "00-test"} - outbound_envelope = ModelGatewayEnvelope.model_validate_json(outbound.value) - assert outbound_envelope.wire_topic == WIRE_OUTBOUND_TOPIC - assert outbound_envelope.canonical_topic == OUTBOUND_TOPIC - assert outbound_envelope.tenant_id == TENANT_ID + outbound_envelope = ModelEventEnvelope[dict[str, object]].model_validate_json( + outbound.value + ) + assert outbound_envelope.metadata.tags["gateway_wire_topic"] == ( + WIRE_OUTBOUND_TOPIC + ) + assert outbound_envelope.metadata.tags["gateway_canonical_topic"] == OUTBOUND_TOPIC + assert outbound_envelope.metadata.tags["source_tenant_id"] == str(TENANT_ID) inbound = local_bus.published[0] assert inbound.topic == INBOUND_TOPIC assert inbound.key == b"tenant-key" assert inbound.headers == {"traceparent": "00-test"} - inbound_envelope = ModelGatewayEnvelope.model_validate_json(inbound.value) - assert inbound_envelope.source_topic == WIRE_INBOUND_TOPIC - assert inbound_envelope.canonical_topic == INBOUND_TOPIC - assert inbound_envelope.tenant_slug == "acme" + inbound_envelope = ModelEventEnvelope[dict[str, object]].model_validate_json( + inbound.value + ) + assert inbound_envelope.metadata.tags["gateway_wire_topic"] == WIRE_INBOUND_TOPIC + assert inbound_envelope.metadata.tags["gateway_canonical_topic"] == INBOUND_TOPIC + assert inbound_envelope.payload["tenant_id"] == "acme" diff --git a/tests/integration/gateway/test_gateway_offset_recovery.py b/tests/integration/gateway/test_gateway_offset_recovery.py new file mode 100644 index 0000000000..e5aa1b8f48 --- /dev/null +++ b/tests/integration/gateway/test_gateway_offset_recovery.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Real-broker proof of the OMN-15781 offset-policy fix. + +Drives the actual seam the gateway forwarder's legs run over +(``KafkaTransport`` against a live Kafka/Redpanda broker), not a mock: a +message is produced to a topic BEFORE any consumer with the gateway's +consumer-group name has ever joined that group, then a fresh +``KafkaTransport`` is started on that group with ``auto_offset_reset`` +matching the deployed gateway policy. With ``"earliest"`` (the OMN-15781 +fix, and the only value ``ModelGatewayForwarderRuntimeConfig`` now accepts) +the message MUST be delivered. With ``"latest"`` (the pre-fix +``beta-gateway-canary.yaml`` value) it is silently skipped -- the exact +mechanism that lost the 28 ``delegation-completed.v1`` events during the +2026-08-04 outage investigation (OMN-15742). + +Broker-gated: skipped when no broker is reachable (see +``tests.integration.transport._kafka_env``), matching the pattern used +there. Intended to run locally / on ``.201``/``.200`` against a live broker. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from uuid import uuid4 + +import pytest + +from omnibase_infra.event_bus.kafka_transport import KafkaTransport +from tests.integration.transport._kafka_env import kafka_available, transport_bootstrap + +pytestmark = [ + pytest.mark.integration, + pytest.mark.kafka, + pytest.mark.heavy, + pytest.mark.skipif( + not kafka_available(), + reason=( + "no Kafka broker reachable (set ONEX_TRANSPORT_KAFKA_BOOTSTRAP or " + "KAFKA_BOOTSTRAP_SERVERS to a live broker)" + ), + ), +] + + +@pytest.fixture +def bootstrap() -> str: + return transport_bootstrap() + + +@pytest.fixture +def topic() -> str: + # Uniquely named per test run (not delete/recreate) -- this suite runs + # against a shared LAN broker; auto-created via the producer's first + # ``send`` rather than deleting/recreating a fixed topic name. + return f"onex.transport.test.gateway-offset-recovery.{uuid4().hex}.v1" + + +async def _produce_one(bootstrap: str, *, topic: str, value: bytes) -> None: + producer = KafkaTransport.from_bootstrap(bootstrap) + await producer.start() + try: + await producer.send(topic, None, value, {}) + finally: + await producer.close() + + +@pytest.fixture +async def consumer_factory() -> AsyncIterator[list[KafkaTransport]]: + created: list[KafkaTransport] = [] + yield created + for transport in created: + await transport.close() + + +class TestGatewayOffsetRecoveryRealBroker: + """OMN-15781: the exact seam the gateway forwarder's legs run over.""" + + async def test_earliest_delivers_message_produced_before_group_joined( + self, + bootstrap: str, + topic: str, + consumer_factory: list[KafkaTransport], + ) -> None: + """The OMN-15781 fix: a fresh group on ``earliest`` sees the backlog. + + Mirrors a gateway leg restarting after an outage window (crash, + LeaveGroup, cold restart) during which the local publisher kept + producing: the consumer group is brand new (has never committed an + offset on this topic), and a message already sits at offset 0 + before the consumer ever calls ``start()``. + """ + group = f"gw-offset-recovery-earliest-{uuid4().hex}" + await _produce_one( + bootstrap, topic=topic, value=b"produced-before-group-joined" + ) + + consumer = KafkaTransport.from_bootstrap( + bootstrap, group=group, topics=[topic], auto_offset_reset="earliest" + ) + consumer_factory.append(consumer) + await consumer.start() + + messages = await consumer.poll(max_messages=10, timeout_ms=15000) + + assert len(messages) == 1 + assert messages[0].value == b"produced-before-group-joined" + + async def test_latest_silently_skips_message_produced_before_group_joined( + self, + bootstrap: str, + topic: str, + consumer_factory: list[KafkaTransport], + ) -> None: + """Documents the bug this fix removes from the deployed gateway path. + + Same scenario as above, but with the pre-fix + ``beta-gateway-canary.yaml`` value (``"latest"``): the message + produced before the group joined is never delivered. This is not a + delay -- ``poll()`` returns empty even after the full timeout, and + the message is gone from this group's perspective (its committed + offset lands after it). ``ModelGatewayForwarderRuntimeConfig`` now + refuses to resolve a runtime config with this value on either leg + (see ``tests/unit/runtime/test_gateway_forwarder_runtime.py``); this + test exercises ``KafkaTransport`` directly to prove the underlying + broker mechanics the validator exists to prevent. + """ + group = f"gw-offset-recovery-latest-{uuid4().hex}" + await _produce_one( + bootstrap, topic=topic, value=b"produced-before-group-joined" + ) + + consumer = KafkaTransport.from_bootstrap( + bootstrap, group=group, topics=[topic], auto_offset_reset="latest" + ) + consumer_factory.append(consumer) + await consumer.start() + + messages = await consumer.poll(max_messages=10, timeout_ms=5000) + + assert messages == [] diff --git a/tests/integration/infra/test_dev_runtime_compose_render.py b/tests/integration/infra/test_dev_runtime_compose_render.py new file mode 100644 index 0000000000..0a98de98bd --- /dev/null +++ b/tests/integration/infra/test_dev_runtime_compose_render.py @@ -0,0 +1,344 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Non-mutating compose render checks for the dev lane's silent-default holes. + +Two fixes of the same class are proven here — a base-compose var with a soft +`${VAR:-default}` that failed OPEN into a wrong-but-quiet render, replaced by +the lane-prefixed fail-closed `${DEV_...:?}` form: + +OMN-15173 (`DEV_REDPANDA_ADVERTISE_HOST`): the dev lane defaulted its Redpanda +advertise host to `localhost`, silently rendering an address unreachable by any +off-host client (CI runner, another machine). + +OMN-14968 (`DEV_WORKER_REPLICAS`): the `runtime-worker` deploy block resolved a +BARE `${WORKER_REPLICAS:-0}` that no surface exported, so the dev lane rendered +`replicas: 0`. `docker compose up -d --no-deps runtime-worker` then exited 0 +creating NOTHING, while `deploy-runtime.sh`'s `RUNTIME_SERVICES` / RT-6 deploy +readback requires a running container — so every dev-lane deploy aborted at the +readback and auto-restored. The lane-prefixed value is the ledgered policy +contract's (`DEV_WORKER_REPLICAS=1`, rendered from +`contracts/services/runtime_policy.contract.yaml`), matching what OMN-12988 / +OMN-12990 already did for the stability-test and prod overlays. + +This module only ever invokes `docker compose config` (a non-mutating render) +— it never brings up, restarts, or otherwise mutates any lane. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[3] +COMPOSE_FILE = REPO_ROOT / "docker" / "docker-compose.infra.yml" +_DEFAULT_POLICY_ENV_FILE = "docker/runtime-policy.env" +POLICY_ENV_PATH = REPO_ROOT / "docker" / "runtime-policy.env" + +# NOTE: docker-compose.infra.yml (bare, no overlay) is the dev lane's own +# compose file (scripts/deploy-runtime.sh: "Dev lane: infra.yml alone"). A +# `docker compose config` render interpolates every service's env block +# regardless of --profile, so every other :?-required var in the file must +# still be supplied here even though this suite only cares about +# DEV_REDPANDA_ADVERTISE_HOST. Kept in sync by the +# tests/ci/test_compose_required_env_coverage.py CI gate, which since OMN-15263 +# checks EVERY registered compose-render fixture (not just the one in +# tests/integration/docker/test_docker_integration.py). This module mirrors that +# fixture rather than importing it, matching the existing per-file convention +# used by test_prod_runtime_compose_render.py / +# test_stability_test_runtime_compose_render.py. +# +# DEV_REDPANDA_ADVERTISE_HOST is deliberately absent below and is registered in +# that gate as `intentionally_unset` for this module: supplying it here would +# make test_dev_redpanda_advertise_host_fails_fast_when_unset vacuous. +_PG_DSN = "postgresql://postgres:test@postgres:5432/omnibase_infra" +_INTEL_DSN = "postgresql://postgres:test@postgres:5432/omniintelligence" +_LOCAL_LAN_CIDR = ".".join(("192", "168", "86", "0")) + "/24" +_SECRET_RESOLVER_CONFIG_JSON = ( + '{"enable_convention_fallback":false,"mappings":[' + '{"logical_name":"llm.openrouter.api_key",' + '"source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}}]}' +) +_SECRET_RESOLVER_CONFIG_PATH = "/app/data/delegation/secret_resolver.yaml" + +# Every :?-required var in docker-compose.infra.yml EXCEPT +# DEV_REDPANDA_ADVERTISE_HOST, which each test sets (or omits) explicitly. +BASE_REQUIRED_ENV: dict[str, str] = { + "POSTGRES_PASSWORD": "test", + "VALKEY_PASSWORD": "test", + "INFISICAL_ENCRYPTION_KEY": "0" * 64, + "INFISICAL_AUTH_SECRET": "test-auth-secret", + "OMNIBASE_INFRA_DB_URL": _PG_DSN, + "OMNIINTELLIGENCE_DB_URL": _INTEL_DSN, + "INFISICAL_DB_CONNECTION_URI": "postgresql://postgres:test@postgres:5432/infisical_db", + "INFISICAL_REDIS_URL": "redis://:test@valkey:6379", + "OMNIBASE_INFRA_AGENT_ACTIONS_POSTGRES_DSN": _PG_DSN, + "OMNIBASE_INFRA_SKILL_LIFECYCLE_POSTGRES_DSN": _PG_DSN, + "OMNIBASE_INFRA_CONTEXT_AUDIT_POSTGRES_DSN": _PG_DSN, + "KAFKA_BOOTSTRAP_SERVERS": "localhost:19092", # kafka-fallback-ok — test fixture + "ARCH_GRAPH_BOLT_URI": "bolt://omnibase-infra-memgraph:7687", + "ONEX_REGISTRATION_AUTO_ACK": "true", + "ONEX_SERVICE_CLIENT_SECRET": "test-service-secret", + "LINEAR_API_KEY": "test-linear-api-key", + "GITHUB_TOKEN": "test-github-token", + "DEPLOY_AGENT_HMAC_SECRET": "render-only-deploy-agent-hmac-secret", + "LLM_CODER_URL": "http://llm-coder.test:8000", + "LLM_CODER_FAST_URL": "http://llm-coder-fast.test:8001", + "LLM_EMBEDDING_URL": "http://llm-embed.test:8100", + "LLM_DEEPSEEK_R1_URL": "http://llm-r1.test:8101", + "BIFROST_LOCAL_CODER_ENDPOINT_URL": "http://llm-coder.test:8000/v1/chat/completions", + "BIFROST_LOCAL_REASONER_ENDPOINT_URL": ( + "http://llm-coder-fast.test:8001/v1/chat/completions" + ), + "BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL": ( + "http://llm-embed.test:8100/v1/chat/completions" + ), + "BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL": "http://llm-r1.test:8101/v1/chat/completions", + "LLM_GLM_URL": "http://llm-glm.test:8102", + "LLM_GLM_MODEL_NAME": "glm-4.5", + "LLM_GLM_API_KEY": "render-only-glm-api-key", + "GEMINI_API_KEY": "render-only-gemini-api-key", + "GOOGLE_API_KEY": "render-only-google-api-key", + "BIFROST_VERTEX_GEMINI_ENDPOINT_URL": ( + "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/" + "gen-lang-client-0084338881/locations/us-central1/endpoints/openapi/chat/completions" + ), + "GOOGLE_CLOUD_PROJECT": "gen-lang-client-0084338881", + "GOOGLE_CLOUD_LOCATION": "us-central1", + "LOCAL_LLM_SHARED_SECRET": "render-only-local-llm-secret", + "LLM_ENDPOINT_CIDR_ALLOWLIST": _LOCAL_LAN_CIDR, + "LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST": "generativelanguage.googleapis.com,api.z.ai", + "AUXILIARY_SERVICES_OMNIMEMORY_ENABLED": "false", + "BIFROST_VERIFY_ENDPOINTS": "1", + "DEV_RUNTIME_EFFECTS_CAPABILITIES": "effects.consumer,market.skill-proof,runtime.effects", + "DEV_RUNTIME_EFFECTS_PORT": "8086", + "DEV_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_JSON": _SECRET_RESOLVER_CONFIG_JSON, + "DEV_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_PATH": _SECRET_RESOLVER_CONFIG_PATH, + "DEV_RUNTIME_MAIN_CAPABILITIES": "market.skill-proof,workflow.orchestration,runtime.main", + "DEV_RUNTIME_MAIN_PORT": "8085", + "DEV_RUNTIME_MAIN_PUBLISH_INTROSPECTION": "true", + "DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_JSON": _SECRET_RESOLVER_CONFIG_JSON, + "DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_PATH": _SECRET_RESOLVER_CONFIG_PATH, + "DEV_RUNTIME_WORKER_CAPABILITIES": "workflow.dispatch,contract.update,runtime.worker", + "DEV_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_JSON": _SECRET_RESOLVER_CONFIG_JSON, + "DEV_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_PATH": _SECRET_RESOLVER_CONFIG_PATH, + "OMNIMEMORY_ENABLED": "false", + "OMNIMEMORY_MEMGRAPH_PORT": "7687", + "ONEX_ACTIVE_RUNTIME_PACKAGES": "omnibase_infra,omnimarket", +} + +# RFC 5737 TEST-NET-2 documentation address — never a real host, avoids +# asserting against any live LAN/Tailscale identity. +_OFF_HOST_ADVERTISE_HOST = "198.51.100.50" + + +def _docker_compose_available() -> bool: + if shutil.which("docker") is None: + return False + result = subprocess.run( + ["docker", "compose", "version"], + check=False, + capture_output=True, + text=True, + ) + return result.returncode == 0 + + +def _render_env(**overrides: str) -> dict[str, str]: + env = { + "HOME": os.environ.get("HOME", ""), + "PATH": os.environ.get("PATH", ""), + "USER": os.environ.get("USER", ""), + **BASE_REQUIRED_ENV, + } + env.update(overrides) + return env + + +def _run_compose_config( + env: dict[str, str], + *, + policy_env_file: str = _DEFAULT_POLICY_ENV_FILE, + profile: str = "", +) -> subprocess.CompletedProcess[str]: + # NOTE: the default arm keeps the literal "--env-file", + # "docker/runtime-policy.env" pair on the command line, because + # tests/ci/test_compose_required_env_coverage.py discovers this fixture's + # env-file coverage by regex over that literal pair. Do not collapse the two + # arms into a single interpolated path. + command = ["docker", "compose"] + if policy_env_file == _DEFAULT_POLICY_ENV_FILE: + command += [ + "--env-file", + "docker/runtime-policy.env", + ] + else: + command += ["--env-file", policy_env_file] + command += [ + "-f", + str(COMPOSE_FILE), + ] + if profile: + command += ["--profile", profile] + command.append("config") + return subprocess.run( + command, + cwd=REPO_ROOT, + check=False, + capture_output=True, + env=env, + text=True, + timeout=60, + ) + + +pytestmark = pytest.mark.skipif( + not _docker_compose_available(), + reason="docker compose is required for non-mutating compose render validation", +) + + +@pytest.mark.integration +def test_dev_redpanda_advertise_host_fails_fast_when_unset() -> None: + """Unset DEV_REDPANDA_ADVERTISE_HOST must fail the compose render, never + silently render a localhost advertise address.""" + env = _render_env() + assert "DEV_REDPANDA_ADVERTISE_HOST" not in env + + result = _run_compose_config(env) + + assert result.returncode != 0, ( + "docker compose config unexpectedly succeeded with " + "DEV_REDPANDA_ADVERTISE_HOST unset:\n" + result.stdout + ) + assert "DEV_REDPANDA_ADVERTISE_HOST" in result.stderr + + +@pytest.mark.integration +def test_dev_redpanda_advertise_host_uses_explicit_value_when_set() -> None: + """An explicitly-set DEV_REDPANDA_ADVERTISE_HOST is honored verbatim — + never silently overridden with a localhost fallback.""" + env = _render_env(DEV_REDPANDA_ADVERTISE_HOST=_OFF_HOST_ADVERTISE_HOST) + + result = _run_compose_config(env) + + assert result.returncode == 0, f"docker compose config failed:\n{result.stderr}" + assert f"{_OFF_HOST_ADVERTISE_HOST}:19092" in result.stdout + assert f"{_OFF_HOST_ADVERTISE_HOST}:18082" in result.stdout + assert "localhost:19092" not in result.stdout + + +@pytest.mark.integration +def test_dev_lane_renders_one_runtime_worker_replica() -> None: + """OMN-14968: the dev lane must render `runtime-worker` with replicas == 1. + + The value is the ledgered policy contract's `DEV_WORKER_REPLICAS`, supplied + by `docker/runtime-policy.env`. A render of 0 reproduces the defect: compose + creates no container, `up` exits 0 with no output, and the RT-6 deploy + readback in `scripts/deploy-runtime.sh` then fails closed on an in-scope + service it can never resolve. + """ + env = _render_env(DEV_REDPANDA_ADVERTISE_HOST=_OFF_HOST_ADVERTISE_HOST) + + result = _run_compose_config(env, profile="runtime") + + assert result.returncode == 0, f"docker compose config failed:\n{result.stderr}" + rendered = yaml.safe_load(result.stdout) + worker = rendered["services"]["runtime-worker"] + assert worker["deploy"]["replicas"] == 1, ( + "dev-lane runtime-worker must render deploy.replicas == 1 (the ledgered " + f"DEV_WORKER_REPLICAS); got {worker['deploy']['replicas']!r}" + ) + + +@pytest.mark.integration +def test_dev_lane_delegation_routing_tiers_path_binding() -> None: + """OMN-15645: DELEGATION_ROUTING_TIERS_PATH must be bound on every runtime + service in the dev lane, to a fixed, non-version-embedded in-image path. + + omnimarket#2000 (OMN-15628) removed the packaged-default fallback for this + key in the delegation routing reducer's ``_get_config()`` singleton + (``resolve_required_path_config("DELEGATION_ROUTING_TIERS_PATH")`` — + omnimarket ``handler_delegation_routing.py:392-393``); an unbound key now + raises ``ProtocolConfigurationError`` at first config read instead of + silently defaulting. The bound value must never be a literal + ``python3.X`` site-packages path (a base-image Python version bump would + silently invalidate it) — ``docker/Dockerfile.runtime`` bakes the packaged + omnimarket ``routing_tiers.yaml`` into this exact fixed location at build + time via a glob-derived COPY, so the compose-declared value here is always + backed by a real file regardless of the interpreter minor version. + """ + env = _render_env(DEV_REDPANDA_ADVERTISE_HOST=_OFF_HOST_ADVERTISE_HOST) + + result = _run_compose_config(env, profile="runtime") + + assert result.returncode == 0, f"docker compose config failed:\n{result.stderr}" + rendered = yaml.safe_load(result.stdout) + services = rendered["services"] + + expected_path = "/app/config/delegation/routing_tiers.yaml" + for service_name in ("omninode-runtime", "runtime-effects", "runtime-worker"): + environment = services[service_name]["environment"] + assert environment.get("DELEGATION_ROUTING_TIERS_PATH") == expected_path, ( + f"Service '{service_name}' must bind DELEGATION_ROUTING_TIERS_PATH=" + f"{expected_path!r}; got " + f"{environment.get('DELEGATION_ROUTING_TIERS_PATH')!r}" + ) + assert "python3." not in environment.get("DELEGATION_ROUTING_TIERS_PATH", ""), ( + f"Service '{service_name}' binds a version-embedded python3.X literal " + "for DELEGATION_ROUTING_TIERS_PATH — the exact trap OMN-15628's " + "runtime self-heal exists to correct for a *stale* pin; the compose " + "default must be a stable, version-independent path instead." + ) + + # Services with no delegation-routing surface deliberately opt out (mirrors + # the BIFROST_CONTRACT_PATH opt-out pattern for the same two services). + for service_name in ("projection-api", "omninode-contract-resolver"): + environment = services[service_name]["environment"] + assert environment.get("DELEGATION_ROUTING_TIERS_PATH", "") == "", ( + f"Service '{service_name}' deliberately has no delegation-routing " + "surface and must not bind DELEGATION_ROUTING_TIERS_PATH; got " + f"{environment.get('DELEGATION_ROUTING_TIERS_PATH')!r}" + ) + + +@pytest.mark.integration +def test_dev_worker_replicas_fails_closed_when_policy_value_unset( + tmp_path: Path, +) -> None: + """OMN-14968 counter-test: an unset DEV_WORKER_REPLICAS must FAIL the render. + + This is the RED half of the fix. The old bare `${WORKER_REPLICAS:-0}` had no + exporter anywhere in the repo, so it always took the silent `0` branch and + the lane lost its worker with zero signal. The lane-prefixed `:?` form must + abort the render instead — never fall back to a replica count. + """ + policy_without_worker_replicas = tmp_path / "runtime-policy-no-worker.env" + policy_without_worker_replicas.write_text( + "\n".join( + line + for line in POLICY_ENV_PATH.read_text(encoding="utf-8").splitlines() + if not line.startswith("DEV_WORKER_REPLICAS=") + ) + + "\n", + encoding="utf-8", + ) + env = _render_env(DEV_REDPANDA_ADVERTISE_HOST=_OFF_HOST_ADVERTISE_HOST) + assert "DEV_WORKER_REPLICAS" not in env + + result = _run_compose_config( + env, + policy_env_file=str(policy_without_worker_replicas), + profile="runtime", + ) + + assert result.returncode != 0, ( + "docker compose config unexpectedly succeeded with DEV_WORKER_REPLICAS " + "unset — the silent-zero hole is back:\n" + result.stdout + ) + assert "DEV_WORKER_REPLICAS" in result.stderr + assert "replicas: 0" not in result.stdout diff --git a/tests/integration/infra/test_judge_compose_render.py b/tests/integration/infra/test_judge_compose_render.py index 2f3a7f278c..d27b6eef7c 100644 --- a/tests/integration/infra/test_judge_compose_render.py +++ b/tests/integration/infra/test_judge_compose_render.py @@ -23,7 +23,12 @@ # surfaces its `:?`-required vars even for services excluded from the judge # profile. These values are never used to run containers — `config` only. LAYERED_RENDER_DUMMY_ENV = { + # OMN-15263: `:?`-required in the base infra file since OMN-15173. The judge + # profile excludes the dev redpanda service, but compose still interpolates + # its `command:` block during the layered render. + "DEV_REDPANDA_ADVERTISE_HOST": "localhost", # kafka-fallback-ok — test fixture "GITHUB_TOKEN": "layered-render-dummy", + "DEPLOY_AGENT_HMAC_SECRET": "layered-render-dummy", "LINEAR_API_KEY": "layered-render-dummy", "ONEX_REGISTRATION_AUTO_ACK": "false", "ONEX_SERVICE_CLIENT_SECRET": "layered-render-dummy", @@ -344,6 +349,43 @@ def test_layered_render_isolates_every_judge_service_to_judge_network() -> None: assert published_ports.isdisjoint(DEV_OR_PROD_PUBLISHED_PORTS) +@pytest.mark.integration +def test_judge_lane_delegation_routing_tiers_path_binding() -> None: + """OMN-15645: DELEGATION_ROUTING_TIERS_PATH must reach the judge lane's + runtime services via the layered (deployed) shape, to a fixed, + non-version-embedded in-image path. + + The judge lane's own ``x-judge-runtime-env`` anchor does not declare this + key, so a *standalone* judge.yml render omits it — only the layered + base+overlay render (``deploy-runtime.sh``'s actual deployed shape) proves + the base file's ``x-runtime-env`` anchor binding survives into this lane. + See ``test_dev_lane_delegation_routing_tiers_path_binding`` in + ``test_dev_runtime_compose_render.py`` for the full seam citation. + """ + rendered_config = _layered_compose_config_json() + services = rendered_config["services"] + + expected_path = "/app/config/delegation/routing_tiers.yaml" + for service_name in ("omninode-runtime", "runtime-effects"): + environment = services[service_name]["environment"] + assert environment.get("DELEGATION_ROUTING_TIERS_PATH") == expected_path, ( + f"Service '{service_name}' must bind DELEGATION_ROUTING_TIERS_PATH=" + f"{expected_path!r}; got " + f"{environment.get('DELEGATION_ROUTING_TIERS_PATH')!r}" + ) + + # NOTE: omninode-contract-resolver is judge-profile-gated OUT_OF_SCOPE + # (never rendered under --profile judge), so only projection-api is + # checked here — see EXPECTED_RENDERED_SERVICES / OUT_OF_SCOPE_SERVICES + # above. + environment = services["projection-api"]["environment"] + assert environment.get("DELEGATION_ROUTING_TIERS_PATH", "") == "", ( + "Service 'projection-api' deliberately has no delegation-routing " + "surface and must not bind DELEGATION_ROUTING_TIERS_PATH; got " + f"{environment.get('DELEGATION_ROUTING_TIERS_PATH')!r}" + ) + + @pytest.mark.integration def test_judge_secret_refs_are_rendered_from_runtime_policy() -> None: rendered_config = _compose_config_json() diff --git a/tests/integration/infra/test_prod_runtime_compose_render.py b/tests/integration/infra/test_prod_runtime_compose_render.py index 4d0ffc455f..79bf9e1952 100644 --- a/tests/integration/infra/test_prod_runtime_compose_render.py +++ b/tests/integration/infra/test_prod_runtime_compose_render.py @@ -33,6 +33,7 @@ def _cidr(prefix: str, suffix: str) -> str: "INFISICAL_ENCRYPTION_KEY": "render-only-infisical-encryption-key-32", "INFISICAL_REDIS_URL": "redis://valkey:6379", "GITHUB_TOKEN": "render-only-github-token", + "DEPLOY_AGENT_HMAC_SECRET": "render-only-deploy-agent-hmac-secret", "LINEAR_API_KEY": "render-only-linear-api-key", "LLM_CODER_FAST_URL": _http_url("llm-coder-fast.invalid"), "LLM_CODER_URL": _http_url("llm-coder.invalid"), @@ -61,6 +62,12 @@ def _cidr(prefix: str, suffix: str) -> str: "ONEX_REGISTRATION_AUTO_ACK": "false", "ONEX_SERVICE_CLIENT_SECRET": "render-only-client-secret", "POSTGRES_PASSWORD": "postgres", + # OMN-15263: the base infra file's dev-lane advertise host is `:?`-required + # (OMN-15173) and compose interpolates it during this layered render even + # though the prod overlay replaces redpanda's `command:` wholesale. Render + # input only — `localhost` keeps the "did the overlay actually override?" + # leak detector below meaningful. + "DEV_REDPANDA_ADVERTISE_HOST": "localhost", # kafka-fallback-ok — test fixture "REDPANDA_ADVERTISE_HOST": "100.109.203.94", "PROD_REDPANDA_ADVERTISE_HOST": "192.168.86.201", "PROD_POSTGRES_EXTERNAL_PORT": "25436", @@ -136,3 +143,40 @@ def test_prod_redpanda_advertise_host_is_prod_specific() -> None: assert "192.168.86.201:49092" in redpanda_command assert "192.168.86.201:48082" in redpanda_command assert "100.109.203.94:49092" not in redpanda_command + + +@pytest.mark.integration +def test_prod_delegation_routing_tiers_path_binding() -> None: + """OMN-15645: DELEGATION_ROUTING_TIERS_PATH must be bound on every runtime + service in the prod lane, to a fixed, non-version-embedded in-image path. + + Config-level readback only, per the OMN-15645 prod boundary — this proves + the compose-declared binding without recreating any live prod container. + See ``test_dev_lane_delegation_routing_tiers_path_binding`` in + ``test_dev_runtime_compose_render.py`` for the full seam citation. + """ + rendered_config = _compose_config_json() + services = rendered_config["services"] + + expected_path = "/app/config/delegation/routing_tiers.yaml" + for service_name in ("omninode-runtime", "runtime-effects", "runtime-worker"): + environment = services[service_name]["environment"] + assert environment.get("DELEGATION_ROUTING_TIERS_PATH") == expected_path, ( + f"Service '{service_name}' must bind DELEGATION_ROUTING_TIERS_PATH=" + f"{expected_path!r}; got " + f"{environment.get('DELEGATION_ROUTING_TIERS_PATH')!r}" + ) + + # NOTE: omninode-contract-resolver is not rendered under --profile runtime + # for this lane (observed live via docker compose config, 2026-08-02); + # only projection-api is checked unconditionally. + for service_name in ("projection-api", "omninode-contract-resolver"): + service = services.get(service_name) + if service is None: + continue + environment = service["environment"] + assert environment.get("DELEGATION_ROUTING_TIERS_PATH", "") == "", ( + f"Service '{service_name}' deliberately has no delegation-routing " + "surface and must not bind DELEGATION_ROUTING_TIERS_PATH; got " + f"{environment.get('DELEGATION_ROUTING_TIERS_PATH')!r}" + ) diff --git a/tests/integration/infra/test_stability_test_runtime_compose_render.py b/tests/integration/infra/test_stability_test_runtime_compose_render.py index 2001dd0623..ce13fb8082 100644 --- a/tests/integration/infra/test_stability_test_runtime_compose_render.py +++ b/tests/integration/infra/test_stability_test_runtime_compose_render.py @@ -6,6 +6,7 @@ import json import os +import re import shutil import subprocess import sys @@ -19,6 +20,11 @@ "docker/docker-compose.infra.yml", "docker/docker-compose.stability-test.yml", ) +# OMN-14013: kept in sync with STABILITY_TEST_TOPIC_PARTITIONS_PER_SHARD in +# tests/unit/infra/test_stability_test_runtime_lane.py -- see that constant's +# docstring for why the durable committed value is raised above the base +# redpanda.yaml default (7000). +STABILITY_TEST_TOPIC_PARTITIONS_PER_SHARD = 15000 REQUIRED_RUNTIME_SERVICES = { "omninode-runtime", "runtime-effects", @@ -118,6 +124,7 @@ def _cidr(prefix: str, suffix: str) -> str: "INFISICAL_REDIS_URL": "redis://:render-only-valkey-password@valkey:6379", "CI_CALLBACK_TOKEN": "deploy-agent-compose-parse-only", "GITHUB_TOKEN": "render-only-github-token", + "DEPLOY_AGENT_HMAC_SECRET": "render-only-deploy-agent-hmac-secret", "KEYCLOAK_ADMIN_CLIENT_SECRET": "render-only-admin-client-secret", "LINEAR_API_KEY": "render-only-linear-api-key", "LINEAR_WEBHOOK_SECRET": "deploy-agent-compose-parse-only", @@ -150,6 +157,12 @@ def _cidr(prefix: str, suffix: str) -> str: "postgresql://postgres:postgres@postgres:5432/omnidash_analytics" ), "POSTGRES_PASSWORD": "postgres", + # OMN-15263: `:?`-required in the base infra file (OMN-15173) and + # interpolated by this layered render even though the stability overlay + # replaces redpanda's `command:`. `localhost` is deliberate: it keeps the + # `"localhost:19092" not in redpanda_command` overlay-leak assertion below + # able to catch a base command that stops being overridden. + "DEV_REDPANDA_ADVERTISE_HOST": "localhost", # kafka-fallback-ok — test fixture "REDPANDA_ADVERTISE_HOST": "192.168.86.201", "STABILITY_TEST_POSTGRES_EXTERNAL_PORT": "15436", "STABILITY_TEST_VALKEY_EXTERNAL_PORT": "26379", @@ -420,13 +433,53 @@ def test_stability_lane_render_contains_isolated_runtime_identity() -> None: ] == ("service_healthy") +@pytest.mark.integration +def test_stability_lane_delegation_routing_tiers_path_binding() -> None: + """OMN-15645: DELEGATION_ROUTING_TIERS_PATH must be bound on every runtime + service in the stability-test lane, to a fixed, non-version-embedded + in-image path. + + omnimarket#2000 (OMN-15628) removed the packaged-default fallback for this + key in the delegation routing reducer's ``_get_config()`` singleton; an + unbound key now raises ``ProtocolConfigurationError`` at first config read. + See ``test_dev_lane_delegation_routing_tiers_path_binding`` in + ``test_dev_runtime_compose_render.py`` for the full seam citation. + """ + rendered_config = _compose_config_json() + services = rendered_config["services"] + + expected_path = "/app/config/delegation/routing_tiers.yaml" + for service_name in REQUIRED_RUNTIME_SERVICES: + environment = services[service_name]["environment"] + assert environment.get("DELEGATION_ROUTING_TIERS_PATH") == expected_path, ( + f"Service '{service_name}' must bind DELEGATION_ROUTING_TIERS_PATH=" + f"{expected_path!r}; got " + f"{environment.get('DELEGATION_ROUTING_TIERS_PATH')!r}" + ) + + # NOTE: omninode-contract-resolver is not rendered under --profile runtime + # for this lane (observed live via docker compose config, 2026-08-02); + # only projection-api is checked unconditionally. + for service_name in ("projection-api", "omninode-contract-resolver"): + service = services.get(service_name) + if service is None: + continue + environment = service["environment"] + assert environment.get("DELEGATION_ROUTING_TIERS_PATH", "") == "", ( + f"Service '{service_name}' deliberately has no delegation-routing " + "surface and must not bind DELEGATION_ROUTING_TIERS_PATH; got " + f"{environment.get('DELEGATION_ROUTING_TIERS_PATH')!r}" + ) + + @pytest.mark.integration def test_stability_lane_render_pins_worker_replicas_to_one() -> None: """The stability worker must render with deploy.replicas == 1 (OMN-12988). - The base infra compose defaults runtime-worker to replicas 0 - (``${WORKER_REPLICAS:-0}``); without a hard pin in the stability override a - plain compose recreate silently drops the worker (4-container census). This + The base infra compose used to default runtime-worker to replicas 0 via a + bare ``${WORKER_REPLICAS:-0}`` (lane-prefixed and fail-closed since + OMN-14968); without a hard pin in the stability override a plain compose + recreate silently drops the worker (4-container census). This ratchet fails if the override regresses to 0 or to an env-interpolation default that resolves to anything other than 1. """ @@ -482,17 +535,51 @@ def test_stability_projection_api_has_separate_infra_and_analytics_dsns() -> Non @pytest.mark.integration -def test_stability_lane_render_inherits_failing_runtime_healthcheck() -> None: +def test_stability_lane_render_resolves_strict_semantic_healthcheck() -> None: + """The *rendered* lane must run the semantic probe with autoheal disarmed. + + OMN-15217. The unit test reads the overlay file; this reads what compose + actually resolves after merging base + overlay, which is the only surface + that can catch a merge-semantics mistake. Two merge behaviours make that + distinction load-bearing: + + * ``healthcheck`` is replaced wholesale, so a mis-authored override shows up + here as the inherited ``curl -sf`` probe rather than as a file diff. + * ``labels`` are *appended*, so the base service's ``autoheal=true`` survives + a plain ``labels:`` block. Only ``labels: !override`` disarms it, and the + overlay file alone cannot prove that — the parsed overlay looks identical + either way. + + Strict health plus autoheal is the harmful combination: semantic degradation + is typically restart-immune (contracts that fail to import will fail again), + so an armed autoheal would convert an honest unhealthy signal into a restart + loop and destroy the forensic state this lane exists to preserve. + """ rendered_config = _compose_config_json() services = rendered_config["services"] for service_name in REQUIRED_RUNTIME_SERVICES: - assert services[service_name]["healthcheck"]["test"] == [ + healthcheck = services[service_name]["healthcheck"] + + assert healthcheck["test"] == [ "CMD", - "curl", - "-sf", - "http://localhost:8085/health", - ] + "python", + "/usr/local/bin/onex-container-healthcheck", + "--degraded-policy", + "fail", + ], ( + f"{service_name}: rendered lane must run the strict semantic check; " + "the shallow curl probe passes a DEGRADED runtime (200 by design)" + ) + + assert _label_value(services[service_name], "autoheal") is None, ( + f"{service_name}: autoheal survived into the rendered lane — compose " + "appends label sequences, so `labels:` must be `labels: !override`. " + "Strict health + autoheal restart-loops a restart-immune defect." + ) + assert _label_value(services[service_name], "com.omninode.lane") == ( + "stability-test" + ) @pytest.mark.integration @@ -534,12 +621,35 @@ def test_stability_lane_render_does_not_expose_production_ports_or_services() -> assert "localhost:19092" not in redpanda_command assert "STABILITY_TEST_REDPANDA_ADVERTISE_HOST" not in redpanda_command assert "REDPANDA_ADVERTISE_HOST" not in redpanda_command + # OMN-14013: belt #2 (redpanda's own startup flag) must be present (this + # lane's `command: !override` previously dropped it entirely) and agree + # numerically with belt #3 below -- a substring check alone is not + # sufficient here since the rendered command string also carries this + # lane's own commented history of prior/stopgap cap values. + redpanda_set_flag_match = re.search( + r"topic_partitions_per_shard=(\d+)", redpanda_command + ) + assert redpanda_set_flag_match is not None, redpanda_command + assert int(redpanda_set_flag_match.group(1)) == ( + STABILITY_TEST_TOPIC_PARTITIONS_PER_SHARD + ) partition_cap_command = "\n".join(services["redpanda-partition-cap"]["command"]) assert "/usr/bin/rpk -X brokers=redpanda:9092" in partition_cap_command assert "admin.hosts=redpanda:9644" in partition_cap_command assert "topic_partitions_per_shard" in partition_cap_command - assert "7000" in partition_cap_command + # Extract the literal value passed to `rpk cluster config set`, not a bare + # substring match (this rendered command string also contains this lane's + # own comment mentioning superseded values -- see docker-compose.stability- + # test.yml's OMN-14013 comment block). + partition_cap_match = re.search( + r"cluster config set topic_partitions_per_shard\s+(\d+)", + partition_cap_command, + ) + assert partition_cap_match is not None, partition_cap_command + assert int(partition_cap_match.group(1)) == ( + STABILITY_TEST_TOPIC_PARTITIONS_PER_SHARD + ) assert "topic_memory_per_partition" in partition_cap_command assert "1048576" in partition_cap_command diff --git a/tests/integration/migrations/conftest.py b/tests/integration/migrations/conftest.py new file mode 100644 index 0000000000..7d603ee75d --- /dev/null +++ b/tests/integration/migrations/conftest.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Throwaway Postgres cluster shared by the migration live-apply proofs. + +Extracted from ``test_094_app_dashboard_role.py`` under OMN-15297, which needed +the same cluster to prove the app_dashboard *grant* chain rather than the role +migration alone. One copy, not two: a second hand-maintained copy of a fixture +whose whole job is to make security proofs honest is exactly how the two copies +drift apart and one of them quietly stops proving anything. + +Why an ephemeral cluster and not the shared local Postgres: these tests create, +reshape and drop cluster-wide ROLES and revoke database-level CONNECT. Doing +that against a shared lane would collide with anything else running there, and +doing it against a cloud/RDS instance is not something a test may do at all. +""" + +from __future__ import annotations + +import shutil +import socket +import subprocess +import tempfile +from collections.abc import Iterator +from pathlib import Path + +import psycopg2 +import pytest + +# initdb/pg_ctl/psql are the real production apply path's tools. When they are +# absent the live proofs SKIP rather than silently degrade to string matching — +# a skipped proof is visible, a downgraded one is not. +PG_TOOLS_MISSING = any( + shutil.which(tool) is None for tool in ("initdb", "pg_ctl", "psql") +) + + +def _free_port() -> int: + """Reserve a port number for the unix socket file name. + + ``listen_addresses=''`` below means the cluster never binds TCP, so this + number only ever names the socket file. It is still probed rather than + hardcoded so two modules (or two pytest-xdist workers) cannot collide on + the same socket path. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", 0)) + return int(probe.getsockname()[1]) + + +class EphemeralPostgres: + """A throwaway, superuser-owned Postgres cluster for one test.""" + + def __init__(self, socket_dir: str, port: int) -> None: + self.socket_dir = socket_dir + self.port = port + + def connect( + self, + *, + user: str = "postgres", + password: str | None = None, + dbname: str = "postgres", + ) -> psycopg2.extensions.connection: + return psycopg2.connect( + host=self.socket_dir, + port=self.port, + user=user, + password=password, + dbname=dbname, + ) + + def psql( + self, + *args: str, + user: str = "postgres", + dbname: str = "postgres", + ) -> subprocess.CompletedProcess[str]: + """Apply SQL the same way the real migration runner does. + + ``run-forward-migrations.sh`` invokes each file as + ``psql -v ON_ERROR_STOP=1 -f `` — matching that invocation + (rather than executing the SQL text through a driver call) is what + makes this an honest reproduction of the production apply path, and is + what makes ``\\connect`` directives inside a migration behave the way + they do in production. + """ + return subprocess.run( + [ + "psql", + "-h", + self.socket_dir, + "-p", + str(self.port), + "-U", + user, + "-d", + dbname, + *args, + ], + capture_output=True, + text=True, + check=False, + ) + + +@pytest.fixture +def ephemeral_postgres() -> Iterator[EphemeralPostgres]: + if PG_TOOLS_MISSING: + pytest.skip( + "initdb/pg_ctl/psql not on PATH — cannot spin up an ephemeral " + "Postgres cluster for the live-apply proof" + ) + + scratch = tempfile.mkdtemp(prefix="onexpg_") + data_dir = Path(scratch) / "data" + log_file = Path(scratch) / "server.log" + port = _free_port() + + init = subprocess.run( + ["initdb", "-D", str(data_dir), "-U", "postgres", "--auth=trust", "--no-sync"], + capture_output=True, + text=True, + check=False, + ) + if init.returncode != 0: + shutil.rmtree(scratch, ignore_errors=True) + pytest.fail(f"initdb failed for the ephemeral test cluster: {init.stderr}") + + start = subprocess.run( + [ + "pg_ctl", + "-D", + str(data_dir), + "-o", + f"-k {scratch} -p {port} -c listen_addresses=", + "-l", + str(log_file), + "-w", + "-t", + "30", + "start", + ], + capture_output=True, + text=True, + check=False, + ) + if start.returncode != 0: + log_text = log_file.read_text() if log_file.exists() else "" + shutil.rmtree(scratch, ignore_errors=True) + pytest.fail( + f"pg_ctl start failed for the ephemeral test cluster: " + f"{start.stderr}\n{log_text}" + ) + + try: + yield EphemeralPostgres(socket_dir=scratch, port=port) + finally: + subprocess.run( + ["pg_ctl", "-D", str(data_dir), "-m", "fast", "stop"], + capture_output=True, + text=True, + check=False, + ) + shutil.rmtree(scratch, ignore_errors=True) diff --git a/tests/integration/migrations/cutover/__init__.py b/tests/integration/migrations/cutover/__init__.py new file mode 100644 index 0000000000..ed35317bbe --- /dev/null +++ b/tests/integration/migrations/cutover/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""PostgreSQL integration proof for OMN-15420.""" diff --git a/tests/integration/migrations/cutover/test_cutover_receipts_postgres16.py b/tests/integration/migrations/cutover/test_cutover_receipts_postgres16.py new file mode 100644 index 0000000000..90d1a010d6 --- /dev/null +++ b/tests/integration/migrations/cutover/test_cutover_receipts_postgres16.py @@ -0,0 +1,817 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Real PostgreSQL 16 receipt, journal, and rollback proof for OMN-15420.""" + +from __future__ import annotations + +import hashlib +import shutil +import socket +import subprocess +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta +from itertools import pairwise +from pathlib import Path +from uuid import NAMESPACE_URL, UUID, uuid4, uuid5 + +import asyncpg +import pytest + +from omnibase_infra.migration.cutover import ( + CutoverCoordinator, + ModelControlPlaneDeltaEvidence, + ModelCutoverContinuityEvidence, + ModelCutoverFamilyContract, + ModelCutoverJournalRequest, + ModelPostgresEvidenceQuerySet, + ModelProjectionReplayEvidence, + ModelReverseDeltaEntry, + ModelReverseDeltaProof, + PostgresTransformationEvidenceCollector, + RepositoryPostgresCutoverJournal, +) +from omnibase_infra.migration.cutover.enums import ( + EnumCutoverEventKind, + EnumCutoverFamilyKind, + EnumCutoverFamilyStatus, + EnumPostCheckpointMode, + EnumReceiptStatus, + EnumReverseDeltaOperation, +) + +pytestmark = [ + pytest.mark.integration, + pytest.mark.postgres, + pytest.mark.serial, + pytest.mark.asyncio(loop_scope="module"), +] + + +def _postgres_bin_dir() -> Path | None: + initdb = shutil.which("initdb") + candidates = [Path(initdb).parent] if initdb else [] + candidates.extend( + sorted(Path("/opt/homebrew/opt").glob("postgresql@*/bin"), reverse=True) + ) + candidates.extend(sorted(Path("/usr/lib/postgresql").glob("*/bin"), reverse=True)) + for candidate in candidates: + if not all((candidate / name).is_file() for name in ("initdb", "pg_ctl")): + continue + version = subprocess.run( + [str(candidate / "postgres"), "--version"], + capture_output=True, + text=True, + check=False, + ).stdout + if " 16." in version: + return candidate + return None + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +@pytest.fixture(scope="module") +def postgres_dsn(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: + bin_dir = _postgres_bin_dir() + if bin_dir is None: + pytest.skip("PostgreSQL 16 initdb/pg_ctl are unavailable") + root = tmp_path_factory.mktemp("omn15420-pg16") + data = root / "data" + port = _free_port() + init = subprocess.run( + [ + str(bin_dir / "initdb"), + "-D", + str(data), + "-U", + "postgres", + "--auth=trust", + "--no-sync", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if init.returncode != 0: + pytest.fail(f"PostgreSQL 16 initdb failed: {init.stderr}") + start = subprocess.run( + [ + str(bin_dir / "pg_ctl"), + "-D", + str(data), + "-o", + f"-F -h 127.0.0.1 -p {port}", + "-l", + str(root / "postgres.log"), + "-w", + "start", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if start.returncode != 0: + postgres_log = root / "postgres.log" + log_text = ( + postgres_log.read_text(encoding="utf-8", errors="replace") + if postgres_log.is_file() + else "" + ) + pytest.skip( + "PostgreSQL 16 binaries are present but an ephemeral cluster could " + f"not start; pg_ctl stderr={start.stderr!r}; postgres.log={log_text!r}" + ) + try: + yield f"postgresql://postgres@127.0.0.1:{port}/postgres" + finally: + subprocess.run( + [ + str(bin_dir / "pg_ctl"), + "-D", + str(data), + "-m", + "immediate", + "-w", + "stop", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + +def _hash(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest() + + +def _family_id(family_key: str) -> UUID: + return uuid5(NAMESPACE_URL, f"omninode-cutover:{family_key}") + + +def _contract( + family_key: str, + kind: EnumCutoverFamilyKind, + mode: EnumPostCheckpointMode, + source_evidence_contract_hash: str, + target_evidence_contract_hash: str, +) -> ModelCutoverFamilyContract: + return ModelCutoverFamilyContract( + family_id=_family_id(family_key), + family_key=family_key, + family_kind=kind, + source_binding_ref="application.legacy", + target_binding_ref="application.target", + source_evidence_contract_hash=source_evidence_contract_hash, + target_evidence_contract_hash=target_evidence_contract_hash, + post_checkpoint_mode=mode, + reverse_delta_contract_ref=( + "contracts/reverse-delta/usage.yaml" + if mode is EnumPostCheckpointMode.REVERSE_DELTA + else "" + ), + forward_fix_runbook_ref=( + "runbooks/control-plane-forward-fix.md" + if mode is EnumPostCheckpointMode.FORWARD_FIX_ONLY + else "" + ), + dual_write_max_seconds=30, + observation_window_seconds=2, + ) + + +def _source_queries() -> ModelPostgresEvidenceQuerySet: + return ModelPostgresEvidenceQuerySet( + label="legacy-transformed", + keys_sql=""" +SELECT mapping.tenant_uuid::text || ':' || usage.id::text +FROM legacy_fixture.usage +JOIN legacy_fixture.tenant_mapping mapping + ON mapping.legacy_slug = usage.tenant_slug +""", + rows_sql=""" +SELECT jsonb_build_array(usage.id, mapping.tenant_uuid, usage.amount)::text +FROM legacy_fixture.usage +JOIN legacy_fixture.tenant_mapping mapping + ON mapping.legacy_slug = usage.tenant_slug +""", + foreign_keys_sql=""" +SELECT 'tenant_id->tenants.id:' || constraint_row.confdeltype::text +FROM pg_constraint constraint_row +WHERE constraint_row.conrelid = 'legacy_fixture.usage'::regclass + AND constraint_row.contype = 'f' +""", + sequences_sql="SELECT 'usage_id:' || last_value::text FROM legacy_fixture.usage_id_seq", + owners_sql=""" +SELECT 'usage:' || role_row.rolname +FROM pg_class class_row +JOIN pg_roles role_row ON role_row.oid = class_row.relowner +WHERE class_row.oid = 'legacy_fixture.usage'::regclass +""", + grants_sql=""" +SELECT lower(grantee) || ':' || lower(privilege_type) +FROM information_schema.role_table_grants +WHERE table_schema = 'legacy_fixture' + AND table_name = 'usage' + AND grantee = 'tenant_writer' +""", + policies_sql=""" +SELECT policy_row.polname || ':' || policy_row.polpermissive::text || ':' || + policy_row.polcmd::text || ':' || + pg_get_expr(policy_row.polqual, policy_row.polrelid) +FROM pg_policy policy_row +WHERE policy_row.polrelid = 'legacy_fixture.usage'::regclass +""", + views_functions_sql=""" +SELECT 'usage_view:' || + (coalesce(class_row.reloptions, ARRAY[]::text[]) @> + ARRAY['security_invoker=true'])::text +FROM pg_class class_row +WHERE class_row.oid = 'legacy_fixture.usage_view'::regclass +UNION ALL +SELECT 'usage_count:' || procedure_row.prosecdef::text +FROM pg_proc procedure_row +WHERE procedure_row.oid = 'legacy_fixture.usage_count()'::regprocedure +""", + dependencies_sql=""" +SELECT 'usage_count' +WHERE to_regprocedure('legacy_fixture.usage_count()') IS NOT NULL +UNION ALL +SELECT 'usage_view' +WHERE to_regclass('legacy_fixture.usage_view') IS NOT NULL +""", + collisions_sql=""" +WITH transformed AS ( + SELECT mapping.tenant_uuid::text || ':' || usage.id::text AS canonical_key + FROM legacy_fixture.usage + JOIN legacy_fixture.tenant_mapping mapping + ON mapping.legacy_slug = usage.tenant_slug +) +SELECT canonical_key +FROM transformed +GROUP BY canonical_key +HAVING count(*) > 1 +""", + ) + + +def _target_queries() -> ModelPostgresEvidenceQuerySet: + return ModelPostgresEvidenceQuerySet( + label="target", + keys_sql="SELECT tenant_id::text || ':' || id::text FROM tenant.usage", + rows_sql="SELECT jsonb_build_array(id, tenant_id, amount)::text FROM tenant.usage", + foreign_keys_sql=""" +SELECT 'tenant_id->tenants.id:' || constraint_row.confdeltype::text +FROM pg_constraint constraint_row +WHERE constraint_row.conrelid = 'tenant.usage'::regclass + AND constraint_row.contype = 'f' +""", + sequences_sql="SELECT 'usage_id:' || last_value::text FROM tenant.usage_id_seq", + owners_sql=""" +SELECT 'usage:' || role_row.rolname +FROM pg_class class_row +JOIN pg_roles role_row ON role_row.oid = class_row.relowner +WHERE class_row.oid = 'tenant.usage'::regclass +""", + grants_sql=""" +SELECT lower(grantee) || ':' || lower(privilege_type) +FROM information_schema.role_table_grants +WHERE table_schema = 'tenant' + AND table_name = 'usage' + AND grantee = 'tenant_writer' +""", + policies_sql=""" +SELECT policy_row.polname || ':' || policy_row.polpermissive::text || ':' || + policy_row.polcmd::text || ':' || + pg_get_expr(policy_row.polqual, policy_row.polrelid) +FROM pg_policy policy_row +WHERE policy_row.polrelid = 'tenant.usage'::regclass +""", + views_functions_sql=""" +SELECT 'usage_view:' || + (coalesce(class_row.reloptions, ARRAY[]::text[]) @> + ARRAY['security_invoker=true'])::text +FROM pg_class class_row +WHERE class_row.oid = 'tenant.usage_view'::regclass +UNION ALL +SELECT 'usage_count:' || procedure_row.prosecdef::text +FROM pg_proc procedure_row +WHERE procedure_row.oid = 'tenant.usage_count()'::regprocedure +""", + dependencies_sql=""" +SELECT 'usage_count' +WHERE to_regprocedure('tenant.usage_count()') IS NOT NULL +UNION ALL +SELECT 'usage_view' +WHERE to_regclass('tenant.usage_view') IS NOT NULL +""", + collisions_sql=""" +SELECT tenant_id::text || ':' || id::text AS canonical_key +FROM tenant.usage +GROUP BY tenant_id, id +HAVING count(*) > 1 +""", + ) + + +async def _seed_transformed_family(connection: asyncpg.Connection) -> None: + await connection.execute( + """ +CREATE ROLE tenant_writer NOLOGIN; +CREATE SCHEMA legacy_fixture; +CREATE SCHEMA tenant; +CREATE TABLE legacy_fixture.tenants (slug TEXT PRIMARY KEY); +CREATE TABLE legacy_fixture.tenant_mapping ( + legacy_slug TEXT PRIMARY KEY REFERENCES legacy_fixture.tenants(slug), + tenant_uuid UUID NOT NULL UNIQUE +); +CREATE TABLE legacy_fixture.usage ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + tenant_slug TEXT NOT NULL REFERENCES legacy_fixture.tenants(slug), + amount INTEGER NOT NULL +); +CREATE TABLE tenant.tenants (id UUID PRIMARY KEY); +CREATE TABLE tenant.usage ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + tenant_id UUID NOT NULL REFERENCES tenant.tenants(id), + amount INTEGER NOT NULL +); +INSERT INTO legacy_fixture.tenants VALUES ('alpha'), ('beta'); +INSERT INTO legacy_fixture.tenant_mapping VALUES + ('alpha', '00000000-0000-0000-0000-000000000001'), + ('beta', '00000000-0000-0000-0000-000000000002'); +INSERT INTO legacy_fixture.usage (id, tenant_slug, amount) VALUES + (1, 'alpha', 10), (2, 'beta', 20); +INSERT INTO tenant.tenants VALUES + ('00000000-0000-0000-0000-000000000001'), + ('00000000-0000-0000-0000-000000000002'); +INSERT INTO tenant.usage (id, tenant_id, amount) VALUES + (1, '00000000-0000-0000-0000-000000000001', 10), + (2, '00000000-0000-0000-0000-000000000002', 20); +SELECT setval('legacy_fixture.usage_id_seq', 2, true); +SELECT setval('tenant.usage_id_seq', 2, true); +GRANT SELECT ON legacy_fixture.usage TO tenant_writer; +GRANT SELECT ON tenant.usage TO tenant_writer; +ALTER TABLE legacy_fixture.usage ENABLE ROW LEVEL SECURITY; +ALTER TABLE legacy_fixture.usage FORCE ROW LEVEL SECURITY; +ALTER TABLE tenant.usage ENABLE ROW LEVEL SECURITY; +ALTER TABLE tenant.usage FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON legacy_fixture.usage USING (true); +CREATE POLICY tenant_isolation ON tenant.usage USING (true); +CREATE VIEW legacy_fixture.usage_view WITH (security_invoker=true) + AS SELECT * FROM legacy_fixture.usage; +CREATE VIEW tenant.usage_view WITH (security_invoker=true) + AS SELECT * FROM tenant.usage; +CREATE FUNCTION legacy_fixture.usage_count() RETURNS BIGINT + LANGUAGE SQL STABLE AS 'SELECT count(*) FROM legacy_fixture.usage'; +CREATE FUNCTION tenant.usage_count() RETURNS BIGINT + LANGUAGE SQL STABLE AS 'SELECT count(*) FROM tenant.usage'; +REVOKE ALL ON FUNCTION legacy_fixture.usage_count() FROM PUBLIC; +REVOKE ALL ON FUNCTION tenant.usage_count() FROM PUBLIC; +""" + ) + + +async def test_postgres16_receipts_journal_and_rollback_boundaries( + postgres_dsn: str, +) -> None: + connection = await asyncpg.connect(postgres_dsn) + try: + await _seed_transformed_family(connection) + repository = RepositoryPostgresCutoverJournal(connection) + await repository.initialize() + await repository.initialize() + coordinator = CutoverCoordinator(repository) + collector = PostgresTransformationEvidenceCollector(connection) + source, target = await collector.collect_pair( + _source_queries(), + _target_queries(), + ) + + projection = _contract( + "tenant.usage", + EnumCutoverFamilyKind.PROJECTION, + EnumPostCheckpointMode.REVERSE_DELTA, + source.evidence_contract_hash, + target.evidence_contract_hash, + ) + control = _contract( + "tenant.control-plane", + EnumCutoverFamilyKind.CONTROL_PLANE, + EnumPostCheckpointMode.FORWARD_FIX_ONLY, + source.evidence_contract_hash, + target.evidence_contract_hash, + ) + await coordinator.register_family(projection) + await coordinator.register_family(control) + now = datetime.now(UTC) - timedelta(minutes=1) + + projection_continuity = ModelCutoverContinuityEvidence( + projection_replays=( + ModelProjectionReplayEvidence( + projection_id=uuid5( + NAMESPACE_URL, + "projection:usage_projection", + ), + projection_label="usage_projection", + projection_version="v7", + topic="onex.evt.usage-recorded.v1", + partition=0, + source_offset=42, + target_offset=42, + ), + ) + ) + projection_receipt = await coordinator.reconcile( + projection, + source, + target, + projection_continuity, + ) + assert projection_receipt.status is EnumReceiptStatus.PASS + + digest = _hash("control-plane") + control_continuity = ModelCutoverContinuityEvidence( + control_plane_delta=ModelControlPlaneDeltaEvidence( + snapshot_id=uuid4(), + source_snapshot_hash=digest, + target_snapshot_hash=digest, + final_delta_id=uuid4(), + source_final_delta_hash=digest, + target_final_delta_hash=digest, + source_watermark="42", + target_watermark="42", + ) + ) + old_control_receipt = await coordinator.reconcile( + control, + source, + target, + control_continuity, + ) + with pytest.raises(asyncpg.ForeignKeyViolationError): + await connection.execute( + """ +INSERT INTO omninode_internal.cutover_journal + (event_id, family_id, sequence, event_kind, request_json, receipt_id, + previous_event_hash, event_hash, occurred_at) +VALUES ($1, $2, 99, 'backfill_completed', '{}'::jsonb, $3, + repeat('0', 64), repeat('f', 64), clock_timestamp()) +""", + uuid4(), + projection.family_id, + old_control_receipt.receipt_id, + ) + failed_target = target.model_copy(update={"owners": ("usage:wrong_owner",)}) + failed_control_receipt = await coordinator.reconcile( + control, + source, + failed_target, + control_continuity, + ) + assert failed_control_receipt.status is EnumReceiptStatus.FAIL + assert (await repository.get_state(control.family_id)).status is ( + EnumCutoverFamilyStatus.BLOCKED + ) + assert (await repository.get_state(projection.family_id)).status is ( + EnumCutoverFamilyStatus.READY + ) + with pytest.raises(ValueError, match="silent fallback"): + await coordinator.append( + control.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.BACKFILL_STARTED, + occurred_at=now, + evidence_ref="proof/control/backfill", + ), + ) + with pytest.raises(ValueError, match="postdate the failure"): + await coordinator.append( + control.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.MISMATCH_RESOLVED, + occurred_at=now, + evidence_ref="proof/control/stale-pass-replay", + receipt_id=old_control_receipt.receipt_id, + ), + ) + + control_receipt = await coordinator.reconcile( + control, + source, + target, + control_continuity, + ) + await coordinator.append( + control.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.MISMATCH_RESOLVED, + occurred_at=now, + evidence_ref="proof/control/mismatch-resolved", + receipt_id=control_receipt.receipt_id, + ), + ) + + assert ( + await coordinator.evaluate_direct_rollback(projection.family_id) + ).allowed + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.BACKFILL_STARTED, + occurred_at=now, + evidence_ref="proof/projection/backfill-start", + ), + ) + with pytest.raises(ValueError, match="precedes the durable prior event"): + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.BACKFILL_COMPLETED, + occurred_at=now - timedelta(seconds=1), + evidence_ref="proof/projection/time-travel", + receipt_id=projection_receipt.receipt_id, + ), + ) + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.BACKFILL_COMPLETED, + occurred_at=now + timedelta(seconds=1), + evidence_ref="proof/projection/backfill-complete", + receipt_id=projection_receipt.receipt_id, + ), + ) + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.DUAL_WRITE_STARTED, + occurred_at=now + timedelta(seconds=2), + evidence_ref="proof/projection/dual-write-telemetry", + dual_write_expires_at=now + timedelta(seconds=12), + ), + ) + dual_write_rollback = await coordinator.evaluate_direct_rollback( + projection.family_id + ) + assert not dual_write_rollback.allowed + assert "dual-write" in dual_write_rollback.reason + with pytest.raises(ValueError, match="must end"): + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.FINAL_DELTA_APPLIED, + occurred_at=now + timedelta(seconds=3), + evidence_ref="proof/projection/final-delta-early", + receipt_id=projection_receipt.receipt_id, + ), + ) + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.DUAL_WRITE_ENDED, + occurred_at=now + timedelta(seconds=4), + evidence_ref="proof/projection/dual-write-ended", + ), + ) + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.FINAL_DELTA_APPLIED, + occurred_at=now + timedelta(seconds=5), + evidence_ref="proof/projection/final-delta", + receipt_id=projection_receipt.receipt_id, + ), + ) + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.WRITER_CHECKPOINT, + occurred_at=now + timedelta(seconds=6), + evidence_ref="proof/projection/writer-checkpoint", + receipt_id=projection_receipt.receipt_id, + source_binding_ref=projection.source_binding_ref, + target_binding_ref=projection.target_binding_ref, + ), + ) + before_write = await coordinator.evaluate_direct_rollback(projection.family_id) + assert before_write.allowed and before_write.direct_dsn_rollback + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.APPLICATION_PATH_WRITE_PROVEN, + occurred_at=now + timedelta(seconds=7), + evidence_ref="proof/projection/real-application-path-write", + database_ref="application", + principal="tenant_projection_writer", + schema_ref="tenant", + target_sequence=7, + ), + ) + after_write = await coordinator.evaluate_direct_rollback(projection.family_id) + assert not after_write.allowed + assert "complete reverse delta" in after_write.reason + with pytest.raises(ValueError, match="after target write refused"): + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.PRE_CHECKPOINT_ROLLBACK, + occurred_at=now + timedelta(seconds=8), + evidence_ref="proof/projection/unsafe-direct-rollback", + ), + ) + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.READER_CUTOVER, + occurred_at=now + timedelta(seconds=8), + evidence_ref="proof/projection/reader-cutover", + ), + ) + observation_ends_at = now + timedelta(seconds=11) + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.OBSERVATION_WINDOW_STARTED, + occurred_at=now + timedelta(seconds=9), + evidence_ref="proof/projection/observation-start", + observation_ends_at=observation_ends_at, + ), + ) + with pytest.raises(ValueError, match="has not reached its deadline"): + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.OBSERVATION_WINDOW_COMPLETED, + occurred_at=now + timedelta(seconds=10), + evidence_ref="proof/projection/observation-too-early", + ), + ) + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.OBSERVATION_WINDOW_COMPLETED, + occurred_at=observation_ends_at, + evidence_ref="proof/projection/observation-complete", + ), + ) + quiescence = await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.WRITER_QUIESCED, + occurred_at=now + timedelta(seconds=12), + evidence_ref="proof/projection/writer-quiesced", + target_sequence=8, + ), + ) + reverse_reconciliation_receipt = await coordinator.reconcile( + projection, + source, + target, + projection_continuity, + ) + entries = tuple( + ModelReverseDeltaEntry( + entry_id=uuid4(), + family_id=projection.family_id, + target_sequence=sequence, + relation="tenant.usage", + operation=EnumReverseDeltaOperation.INSERT, + primary_key_hash=_hash(f"pk-{sequence}"), + before_image_hash=_hash(f"before-{sequence}"), + after_image_hash=_hash(f"after-{sequence}"), + inverse_artifact_ref=f"proof/reverse-delta/{sequence}", + ) + for sequence in (7, 8) + ) + reverse_proof = ModelReverseDeltaProof( + proof_id=uuid4(), + family_id=projection.family_id, + start_sequence=7, + end_sequence=8, + entries=entries, + quiescence_event_id=quiescence.event_id, + reconciliation_receipt_id=reverse_reconciliation_receipt.receipt_id, + behavioral_readback_ref="proof/reverse-delta/behavioral-readback", + proven_at=datetime.now(UTC), + ) + await coordinator.record_reverse_delta(reverse_proof) + assert not ( + await coordinator.evaluate_direct_rollback(projection.family_id) + ).allowed + await coordinator.append( + projection.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.REVERSE_DELTA_PROVEN, + occurred_at=datetime.now(UTC), + evidence_ref="proof/reverse-delta/complete", + receipt_id=reverse_reconciliation_receipt.receipt_id, + reverse_delta_proof_id=reverse_proof.proof_id, + ), + ) + reverse_allowed = await coordinator.evaluate_direct_rollback( + projection.family_id + ) + assert reverse_allowed.allowed and reverse_allowed.direct_dsn_rollback + assert reverse_allowed.reverse_delta_proof_id == reverse_proof.proof_id + + await coordinator.append( + control.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.BACKFILL_STARTED, + occurred_at=now, + evidence_ref="proof/control/backfill-start", + ), + ) + await coordinator.append( + control.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.BACKFILL_COMPLETED, + occurred_at=now + timedelta(seconds=1), + evidence_ref="proof/control/backfill-complete", + receipt_id=control_receipt.receipt_id, + ), + ) + await coordinator.append( + control.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.FINAL_DELTA_APPLIED, + occurred_at=now + timedelta(seconds=2), + evidence_ref="proof/control/final-delta", + receipt_id=control_receipt.receipt_id, + ), + ) + await coordinator.append( + control.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.WRITER_CHECKPOINT, + occurred_at=now + timedelta(seconds=3), + evidence_ref="proof/control/checkpoint", + receipt_id=control_receipt.receipt_id, + source_binding_ref=control.source_binding_ref, + target_binding_ref=control.target_binding_ref, + ), + ) + await coordinator.append( + control.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.APPLICATION_PATH_WRITE_PROVEN, + occurred_at=now + timedelta(seconds=4), + evidence_ref="proof/control/real-application-path-write", + database_ref="application", + principal="onex_api", + schema_ref="tenant", + target_sequence=1, + ), + ) + forward_fix_receipt = await coordinator.reconcile( + control, + source, + target, + control_continuity, + ) + await coordinator.append( + control.family_id, + ModelCutoverJournalRequest( + kind=EnumCutoverEventKind.FORWARD_FIX_RECORDED, + occurred_at=now + timedelta(seconds=5), + evidence_ref="proof/control/forward-fix", + receipt_id=forward_fix_receipt.receipt_id, + ), + ) + forward_only = await coordinator.evaluate_direct_rollback(control.family_id) + assert not forward_only.allowed + assert "forward-fix-only" in forward_only.reason + + journal_rows = await connection.fetch( + """ +SELECT sequence, previous_event_hash, event_hash +FROM omninode_internal.cutover_journal +WHERE family_id = $1 +ORDER BY sequence +""", + projection.family_id, + ) + assert journal_rows[0]["previous_event_hash"] == "0" * 64 + for previous, current in pairwise(journal_rows): + assert current["previous_event_hash"] == previous["event_hash"] + finally: + await connection.close() + + # Reconnect to prove the journal projection is durable, not process memory. + reopened = await asyncpg.connect(postgres_dsn) + try: + state = await RepositoryPostgresCutoverJournal(reopened).get_state( + _family_id("tenant.usage") + ) + assert state.verified_reverse_delta_proof_id is not None + assert state.observation_ends_at == observation_ends_at + assert state.last_sequence > 0 + finally: + await reopened.close() diff --git a/tests/integration/migrations/test_094_app_dashboard_role.py b/tests/integration/migrations/test_094_app_dashboard_role.py index 49da431d22..ec6581b8e5 100644 --- a/tests/integration/migrations/test_094_app_dashboard_role.py +++ b/tests/integration/migrations/test_094_app_dashboard_role.py @@ -11,17 +11,17 @@ from __future__ import annotations -import shutil +import re import subprocess -import tempfile import uuid -from collections.abc import Iterator from pathlib import Path import psycopg2 import psycopg2.errors import pytest +from tests.integration.migrations.conftest import EphemeralPostgres + REPO_ROOT = Path(__file__).parent.parent.parent.parent MIGRATION_FILE = ( REPO_ROOT @@ -56,12 +56,56 @@ def test_094_enforces_flags_on_preexisting_role() -> None: """ALTER ROLE must re-assert the flags — presence is not the property.""" sql = MIGRATION_FILE.read_text() - assert "ALTER ROLE app_dashboard" in sql - # The ALTER block (after the guarded CREATE) must carry both - # security-critical negations. - alter_block = sql.split("ALTER ROLE app_dashboard", 1)[1] - assert "NOSUPERUSER" in alter_block - assert "NOBYPASSRLS" in alter_block + assert "ALTER ROLE app_dashboard NOSUPERUSER NOBYPASSRLS NOREPLICATION" in sql, ( + "the security-critical negations must still be issued when pg_roles " + "shows an actual escalation" + ) + + +@pytest.mark.integration +def test_094_never_revokes_the_deployment_owned_login_attach() -> None: + """OMN-15343: LOGIN is create-time only, never re-asserted. + + The LOGIN + password attach is a deployment-owned, operator-gated step + (AWS Secrets Manager, OMN-14899). On the cloud instance app_dashboard + already carries LOGIN (live readback 2026-07-29: rolcanlogin = t), so a + blanket ``ALTER ROLE app_dashboard NOLOGIN`` would break the dashboard's + runtime connection as a side effect of recording a migration. + """ + sql = MIGRATION_FILE.read_text() + executable = "\n".join( + line for line in sql.splitlines() if not line.lstrip().startswith("--") + ) + for statement in re.findall(r"ALTER ROLE app_dashboard[^';]*", executable): + assert "NOLOGIN" not in statement, ( + "NOLOGIN belongs to CREATE ROLE only; re-asserting it on a " + f"pre-existing role revokes a deployment-owned attach: {statement!r}" + ) + + +@pytest.mark.integration +def test_094_gates_every_privileged_statement_on_an_observed_divergence() -> None: + """No unconditional ALTER ROLE may survive. + + An unconditional ALTER is a privilege demand made on every apply, which is + why 094 could not run under the RDS-shaped master (OMN-14899 follow-up) and + then could not run at all under the ordinary service role the k8s Job uses + on the managed instance (OMN-15343). Every ALTER must live inside a DO + block that first read pg_roles. + """ + sql = MIGRATION_FILE.read_text() + executable = [ + line for line in sql.splitlines() if not line.lstrip().startswith("--") + ] + for line in executable: + stripped = line.strip() + if stripped.startswith("ALTER ROLE"): + msg = ( + "top-level (ungated) ALTER ROLE found; it must be inside a DO " + f"block gated on a pg_roles read: {stripped!r}" + ) + raise AssertionError(msg) + assert executable # the file is not empty @pytest.mark.integration @@ -113,133 +157,13 @@ def test_094_rollback_drops_role_and_grants() -> None: # superuser and not a string match. # ============================================================================= -_PG_TOOLS_MISSING = any( - shutil.which(tool) is None for tool in ("initdb", "pg_ctl", "psql") -) - - -class _EphemeralPostgres: - """A throwaway, superuser-owned Postgres 16 cluster for one test. - - Spun up via initdb/pg_ctl into a scratch directory — never the shared - local docker Postgres and never any cloud/RDS/staging database — so a - test can freely create, reshape, and drop roles without risking a - shared-state collision or requiring cloud credentials. - """ - - def __init__(self, socket_dir: str, port: int) -> None: - self.socket_dir = socket_dir - self.port = port - - def connect( - self, - *, - user: str = "postgres", - password: str | None = None, - dbname: str = "postgres", - ) -> psycopg2.extensions.connection: - return psycopg2.connect( - host=self.socket_dir, - port=self.port, - user=user, - password=password, - dbname=dbname, - ) - - def psql( - self, *args: str, user: str = "postgres" - ) -> subprocess.CompletedProcess[str]: - """Apply SQL the same way the real migration runner does. - - run-forward-migrations.sh invokes each file as - ``psql -v ON_ERROR_STOP=1 -f `` — matching that invocation - (rather than executing the SQL text through a driver call) is what - makes this an honest reproduction of the production apply path. - """ - return subprocess.run( - [ - "psql", - "-h", - self.socket_dir, - "-p", - str(self.port), - "-U", - user, - "-d", - "postgres", - *args, - ], - capture_output=True, - text=True, - check=False, - ) - - -@pytest.fixture -def ephemeral_postgres() -> Iterator[_EphemeralPostgres]: - if _PG_TOOLS_MISSING: - pytest.skip( - "initdb/pg_ctl/psql not on PATH — cannot spin up an ephemeral " - "Postgres cluster for the live-connection proof" - ) - - scratch = tempfile.mkdtemp(prefix="pg094_") - data_dir = Path(scratch) / "data" - log_file = Path(scratch) / "server.log" - # Arbitrary; listen_addresses='' below means no TCP bind, so this port - # is only ever used to name the unix socket file and never contended. - port = 55491 - - init = subprocess.run( - ["initdb", "-D", str(data_dir), "-U", "postgres", "--auth=trust", "--no-sync"], - capture_output=True, - text=True, - check=False, - ) - if init.returncode != 0: - shutil.rmtree(scratch, ignore_errors=True) - pytest.fail(f"initdb failed for the ephemeral test cluster: {init.stderr}") - - start = subprocess.run( - [ - "pg_ctl", - "-D", - str(data_dir), - "-o", - f"-k {scratch} -p {port} -c listen_addresses=", - "-l", - str(log_file), - "-w", - "-t", - "30", - "start", - ], - capture_output=True, - text=True, - check=False, - ) - if start.returncode != 0: - log_text = log_file.read_text() if log_file.exists() else "" - shutil.rmtree(scratch, ignore_errors=True) - pytest.fail( - f"pg_ctl start failed for the ephemeral test cluster: " - f"{start.stderr}\n{log_text}" - ) - - try: - yield _EphemeralPostgres(socket_dir=scratch, port=port) - finally: - subprocess.run( - ["pg_ctl", "-D", str(data_dir), "-m", "fast", "stop"], - capture_output=True, - text=True, - check=False, - ) - shutil.rmtree(scratch, ignore_errors=True) +# The throwaway-cluster harness moved to tests/integration/migrations/conftest.py +# under OMN-15297, which needed the same cluster to prove the app_dashboard +# GRANT chain rather than the role migration alone. One copy, not two. def _apply_094_as_rds_shaped_role( - pg: _EphemeralPostgres, + pg: EphemeralPostgres, ) -> subprocess.CompletedProcess[str]: """Create an RDS-master-shaped role and apply 094 through it. @@ -267,7 +191,7 @@ def _apply_094_as_rds_shaped_role( @pytest.mark.integration @pytest.mark.postgres def test_094_alter_role_succeeds_under_rds_shaped_master_role( - ephemeral_postgres: _EphemeralPostgres, + ephemeral_postgres: EphemeralPostgres, ) -> None: """094 must apply cleanly through an RDS-shaped (non-superuser) role. @@ -307,10 +231,177 @@ def test_094_alter_role_succeeds_under_rds_shaped_master_role( assert rolcanlogin is False +# ----------------------------------------------------------------------------- +# OMN-15343 — the ordinary-service-role apply path (the live cloud case) +# +# The k8s migration Job (omninode_infra k8s/migrations/omnibase-infra-migrate +# .yaml) escalated role DDL to `-U postgres`. The managed RDS instance has no +# such role, so deploy run 30406741279 died at connect time on THIS file, at the +# last flat migration, before the node loop ran. The runner now resolves the +# execution identity per POSTGRES_TARGET and on RDS uses the ordinary +# per-database role — which holds no CREATEROLE. This file must therefore be a +# true no-op under that role when app_dashboard is already in the required +# state, or it can never be recorded and every migration behind it stays +# blocked. +# +# `app_dashboard` on the live instance carries LOGIN (readback 2026-07-29: +# rolcanlogin = t) with every other flag already correct, so that is the exact +# state reproduced below. +# ----------------------------------------------------------------------------- + +ORDINARY_ROLE = "omn15343_ordinary" + + +def _seed_live_cloud_role_state(pg: EphemeralPostgres) -> None: + """app_dashboard as it exists on the managed instance, plus a service role + shaped like the one the Job connects as (LOGIN, no CREATEROLE).""" + bootstrap = pg.connect() + bootstrap.autocommit = True + with bootstrap.cursor() as cur: + cur.execute( + "CREATE ROLE app_dashboard WITH LOGIN NOSUPERUSER NOBYPASSRLS " + "NOCREATEDB NOCREATEROLE NOREPLICATION" + ) + cur.execute( + f"CREATE ROLE {ORDINARY_ROLE} WITH LOGIN NOSUPERUSER NOBYPASSRLS " + "NOCREATEDB NOCREATEROLE NOREPLICATION" + ) + bootstrap.close() + + +def _login_flag(pg: EphemeralPostgres) -> bool: + conn = pg.connect() + conn.autocommit = True + with conn.cursor() as cur: + cur.execute("SELECT rolcanlogin FROM pg_roles WHERE rolname = 'app_dashboard'") + row = cur.fetchone() + conn.close() + assert row is not None + return bool(row[0]) + + +@pytest.mark.integration +@pytest.mark.postgres +def test_094_applies_under_an_ordinary_role_when_the_role_already_exists( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """GREEN: the live cloud state, applied by the identity the Job actually + connects as. Must exit 0 so the runner records it legitimately.""" + _seed_live_cloud_role_state(ephemeral_postgres) + + result = ephemeral_postgres.psql( + "-v", "ON_ERROR_STOP=1", "-f", str(MIGRATION_FILE), user=ORDINARY_ROLE + ) + + assert result.returncode == 0, ( + "094 must be a true no-op under a non-CREATEROLE role when " + "app_dashboard is already in the required state, or the k8s Job can " + f"never record it on RDS (OMN-15343).\npsql stderr:\n{result.stderr}" + ) + assert _login_flag(ephemeral_postgres) is True, ( + "the deployment-owned LOGIN attach must survive the apply" + ) + + +@pytest.mark.integration +@pytest.mark.postgres +def test_094_pre_fix_shape_fails_under_an_ordinary_role( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """RED baseline: the pre-OMN-15343 shape of this file — an unconditional + CREATE plus an unconditional ALTER — is refused by the same role in the same + state, so the GREEN above is not vacuous. + + Derived as the minimal pre-fix statements rather than a copy of the old + file: what is under test is that UNGATED role DDL cannot run here at all. + """ + _seed_live_cloud_role_state(ephemeral_postgres) + + for statement in ( + "CREATE ROLE app_dashboard WITH NOLOGIN NOSUPERUSER NOBYPASSRLS", + "ALTER ROLE app_dashboard NOLOGIN NOCREATEDB NOCREATEROLE", + ): + refused = ephemeral_postgres.psql( + "-v", "ON_ERROR_STOP=1", "-c", statement, user=ORDINARY_ROLE + ) + assert refused.returncode != 0, ( + f"expected {statement!r} to be refused for a non-CREATEROLE role" + ) + assert "permission denied" in refused.stderr.lower(), refused.stderr + + assert _login_flag(ephemeral_postgres) is True + + +@pytest.mark.integration +@pytest.mark.postgres +def test_094_still_refuses_when_the_role_is_absent_and_cannot_be_created( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """Fail-closed: "no-op when already correct" must not become "succeed when + the role is missing". A migration that did not achieve its effect must not + exit 0, because the runner records anything that exits 0. + """ + bootstrap = ephemeral_postgres.connect() + bootstrap.autocommit = True + with bootstrap.cursor() as cur: + cur.execute( + f"CREATE ROLE {ORDINARY_ROLE} WITH LOGIN NOSUPERUSER NOBYPASSRLS " + "NOCREATEDB NOCREATEROLE NOREPLICATION" + ) + bootstrap.close() + + result = ephemeral_postgres.psql( + "-v", "ON_ERROR_STOP=1", "-f", str(MIGRATION_FILE), user=ORDINARY_ROLE + ) + + assert result.returncode != 0, ( + "with app_dashboard absent and no CREATEROLE, 094 must fail loudly:\n" + + result.stdout + ) + assert "permission denied to create role" in result.stderr, result.stderr + + verify = ephemeral_postgres.connect() + verify.autocommit = True + with verify.cursor() as cur: + cur.execute("SELECT count(*) FROM pg_roles WHERE rolname = 'app_dashboard'") + (count,) = cur.fetchone() + verify.close() + assert count == 0 + + +@pytest.mark.integration +@pytest.mark.postgres +def test_094_still_corrects_an_escalated_preexisting_role( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """Gating on divergence must not weaken the security invariant: a role that + actually carries BYPASSRLS is still corrected when the executing identity + can do it.""" + bootstrap = ephemeral_postgres.connect() + bootstrap.autocommit = True + with bootstrap.cursor() as cur: + cur.execute("CREATE ROLE app_dashboard WITH LOGIN BYPASSRLS") + bootstrap.close() + + result = ephemeral_postgres.psql("-v", "ON_ERROR_STOP=1", "-f", str(MIGRATION_FILE)) + assert result.returncode == 0, result.stderr + + verify = ephemeral_postgres.connect() + verify.autocommit = True + with verify.cursor() as cur: + cur.execute( + "SELECT rolbypassrls, rolsuper, rolreplication FROM pg_roles " + "WHERE rolname = 'app_dashboard'" + ) + row = cur.fetchone() + verify.close() + assert row == (False, False, False), row + + @pytest.mark.integration @pytest.mark.postgres def test_094_denies_read_without_tenant_context_through_real_connection( - ephemeral_postgres: _EphemeralPostgres, + ephemeral_postgres: EphemeralPostgres, ) -> None: """The connecting role, not the RLS policy, must be the enforced boundary. diff --git a/tests/integration/migrations/test_097_app_dashboard_connect_omn15297.py b/tests/integration/migrations/test_097_app_dashboard_connect_omn15297.py new file mode 100644 index 0000000000..3d3bcaff76 --- /dev/null +++ b/tests/integration/migrations/test_097_app_dashboard_connect_omn15297.py @@ -0,0 +1,449 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""OMN-15297 — the app_dashboard GRANT CHAIN must yield a usable connection. + +THE DEFECT (live readback, 2026-07-28, `.201` dev lane, db `omnidash_analytics`) + + FATAL: permission denied for database "omnidash_analytics" + DETAIL: User does not have CONNECT privilege. + +094 creates the role. 0023 grants ``USAGE ON SCHEMA public`` and ``SELECT`` on +the two RLS-covered delegation tables. Nothing in the chain ever grants +``CONNECT ON DATABASE``. On a stock Postgres ``CONNECT`` is held by ``PUBLIC``, +so the gap is LATENT — every existing test passes and the role appears to work. +On a database where ``PUBLIC``'s CONNECT has been revoked (the dev lane today, +and the explicit target state of OMN-15355) the role cannot open a session at +all, and every grant behind it is unreachable. + +WHY THIS FILE IS SHAPED THE WAY IT IS +------------------------------------- +The chain is DISCOVERED, not enumerated (``_app_dashboard_grant_chain``). That +is deliberate and it is what makes the RED honest: with the fix absent this +module still collects, still applies a complete and self-consistent migration +chain, and fails on the OBSERVED PRIVILEGE — not on a missing file. A test that +red-fails with ``FileNotFoundError: 097_...sql`` proves only that a file is +absent; it cannot distinguish "not written yet" from "written and wrong", which +is the distinction the RED is supposed to establish. + +Every assertion here is driven through a real client connection authenticated +AS ``app_dashboard`` against a real cluster, applied through the same +``psql -v ON_ERROR_STOP=1 -f`` invocation ``scripts/run-forward-migrations.sh`` +uses. String-matching the SQL text would restate the migration rather than test +it. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import psycopg2 +import pytest + +from tests.integration.migrations.conftest import EphemeralPostgres + +REPO_ROOT = Path(__file__).resolve().parents[3] +FORWARD_DIR = REPO_ROOT / "docker" / "migrations" / "forward" +NODE_DELEGATION_DIR = FORWARD_DIR / "nodes" / "node_projection_delegation" +RLS_MIGRATION = NODE_DELEGATION_DIR / "0023_delegation_rls_tenant_isolation.sql" + +# The database the dashboard read path connects to. Named here rather than +# derived so the test states the seam value it is asserting against, matching +# `APPLICATION_DATABASE_PHYSICAL_NAME` in omnibase_infra.topology. +ANALYTICS_DB = "omnidash_analytics" +READ_ROLE = "app_dashboard" + + +def _executable_sql(path: Path) -> str: + """The migration text with ``--`` comment lines stripped. + + Comments mention roles constantly (096 names app_dashboard only to point at + this very ticket). Discovery has to read what the file DOES. + """ + return "\n".join( + line + for line in path.read_text().splitlines() + if not line.lstrip().startswith("--") + ) + + +def _app_dashboard_grant_chain() -> list[Path]: + """Every top-level forward migration that acts on ``app_dashboard``. + + Sorted by filename, which is the order the runner applies them in. + """ + return sorted( + path for path in FORWARD_DIR.glob("*.sql") if READ_ROLE in _executable_sql(path) + ) + + +def _apply( + pg: EphemeralPostgres, path: Path, *, dbname: str +) -> subprocess.CompletedProcess[str]: + result = pg.psql("-v", "ON_ERROR_STOP=1", "-f", str(path), dbname=dbname) + assert result.returncode == 0, ( + f"{path.name} failed to apply against {dbname}:\n{result.stderr}" + ) + return result + + +def _seed_analytics_database(pg: EphemeralPostgres) -> None: + """Reproduce the lane condition the defect needs, and only that. + + Two facts matter and both are reproduced from live state rather than + invented: + + * ``omnidash_analytics`` exists and ``PUBLIC`` has NO ``CONNECT`` on it. + This is the dev-lane state recorded on OMN-15297 and the declared target + state of OMN-15355 (revoke PUBLIC CONNECT). It is what turns the missing + grant from latent into fatal. + * The two tables 0023 covers exist, owned by the migration role (never by + ``app_dashboard`` — an owner is exempt from RLS and the proof would be + vacuous). + """ + bootstrap = pg.connect() + bootstrap.autocommit = True + with bootstrap.cursor() as cur: + cur.execute(f'CREATE DATABASE "{ANALYTICS_DB}"') + cur.execute(f'REVOKE CONNECT ON DATABASE "{ANALYTICS_DB}" FROM PUBLIC') + bootstrap.close() + + analytics = pg.connect(dbname=ANALYTICS_DB) + analytics.autocommit = True + with analytics.cursor() as cur: + cur.execute( + "CREATE TABLE delegation_events (" + " event_id TEXT PRIMARY KEY," + " tenant_id TEXT NOT NULL" + ")" + ) + cur.execute( + "CREATE TABLE delegation_budget_state (" + " budget_key TEXT PRIMARY KEY," + " tenant_id TEXT NOT NULL" + ")" + ) + # A tenant-stamped table with NO tenant_isolation policy. Nothing in the + # chain may grant SELECT on it: readable-before-policy is the exact + # ordering hazard migration 094's header calls out. + cur.execute( + "CREATE TABLE uncovered_projection (" + " row_id TEXT PRIMARY KEY," + " tenant_id TEXT NOT NULL" + ")" + ) + analytics.close() + + +def _attach_deployment_owned_login(pg: EphemeralPostgres) -> None: + """The LOGIN + password attach the migrations deliberately do not carry. + + 094 states this explicitly: credential material never lives in a migration, + the attach is deployment-owned (AWS Secrets Manager on the cloud path). The + test performs it here because a role that cannot log in cannot demonstrate + whether CONNECT is the thing that is missing. + """ + bootstrap = pg.connect() + bootstrap.autocommit = True + with bootstrap.cursor() as cur: + cur.execute(f"ALTER ROLE {READ_ROLE} WITH LOGIN PASSWORD 'omn15297-ephemeral'") + bootstrap.close() + + +def _apply_full_chain(pg: EphemeralPostgres) -> None: + """Apply the chain exactly as the runner does. + + Top-level forward migrations run against ``POSTGRES_DB``; node migrations + run against ``NODE_POSTGRES_DB`` (compose sets ``omnidash_analytics``). Any + file needing the analytics context switches with its own ``\\connect`` + directive, which is why these go through psql rather than a driver. + """ + for migration in _app_dashboard_grant_chain(): + _apply(pg, migration, dbname="postgres") + _apply(pg, RLS_MIGRATION, dbname=ANALYTICS_DB) + + +def _database_privilege(pg: EphemeralPostgres, privilege: str) -> bool: + conn = pg.connect() + conn.autocommit = True + with conn.cursor() as cur: + cur.execute( + "SELECT has_database_privilege(%s, %s, %s)", + (READ_ROLE, ANALYTICS_DB, privilege), + ) + row = cur.fetchone() + conn.close() + assert row is not None + return bool(row[0]) + + +# ============================================================================= +# The seam: the chain must leave app_dashboard able to open a session. +# ============================================================================= + + +@pytest.mark.integration +@pytest.mark.postgres +def test_grant_chain_leaves_app_dashboard_with_connect_on_the_analytics_database( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """``has_database_privilege(app_dashboard, omnidash_analytics, CONNECT)``. + + This is OMN-15297's first acceptance test, stated against the catalog + rather than against the migration text. + """ + pg = ephemeral_postgres + _seed_analytics_database(pg) + _apply_full_chain(pg) + + assert _database_privilege(pg, "CONNECT"), ( + "the app_dashboard grant chain applied cleanly and still leaves the " + f"role without CONNECT on {ANALYTICS_DB}. PUBLIC's CONNECT is revoked " + "on this database (the .201 dev-lane state, and the declared target " + "state of OMN-15355), so the role cannot open a session at all and " + "every USAGE/SELECT grant behind it is unreachable. Chain applied: " + f"{[p.name for p in _app_dashboard_grant_chain()]} + {RLS_MIGRATION.name}" + ) + + +@pytest.mark.integration +@pytest.mark.postgres +def test_app_dashboard_can_actually_open_a_session_and_read_its_own_tenant( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """The catalog check above, driven through a real client connection. + + ``has_database_privilege`` is the catalog's opinion; this is the observed + behaviour of the connection path the dashboard actually uses. Both are kept + because they fail differently: a catalog-only assertion cannot see a + ``pg_hba``/session-level refusal, and a connect-only assertion cannot say + which privilege was missing. + """ + pg = ephemeral_postgres + _seed_analytics_database(pg) + _apply_full_chain(pg) + _attach_deployment_owned_login(pg) + + seed = pg.connect(dbname=ANALYTICS_DB) + seed.autocommit = True + with seed.cursor() as cur: + cur.execute( + "INSERT INTO delegation_events (event_id, tenant_id) VALUES (%s, %s)", + ("omn15297-a", "tenant-a"), + ) + seed.close() + + try: + reader = pg.connect( + user=READ_ROLE, password="omn15297-ephemeral", dbname=ANALYTICS_DB + ) + except psycopg2.OperationalError as exc: # pragma: no cover - the RED path + pytest.fail( + f"{READ_ROLE} could not open a session on {ANALYTICS_DB} after the " + f"full grant chain applied cleanly: {exc}" + ) + + reader.autocommit = True + with reader.cursor() as cur: + cur.execute("SELECT current_user, session_user") + identity = cur.fetchone() + cur.execute("SET app.tenant_id = 'tenant-a'") + cur.execute("SELECT count(*) FROM delegation_events") + visible = cur.fetchone() + reader.close() + + assert identity == (READ_ROLE, READ_ROLE) + assert visible is not None and visible[0] == 1, ( + "app_dashboard connected but could not read its own tenant's row — " + "CONNECT is granted and the SELECT/USAGE half of the chain is not" + ) + + +@pytest.mark.integration +@pytest.mark.postgres +def test_grant_chain_is_idempotent_on_reapply( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """OMN-15297 acceptance 2. Re-running the runner must not error. + + ``_apply`` asserts returncode 0 on every file, so a second full pass is the + assertion. Applied twice rather than once because the migration ledger does + not protect a file that has been edited and re-run by hand on a lane. + """ + pg = ephemeral_postgres + _seed_analytics_database(pg) + _apply_full_chain(pg) + _apply_full_chain(pg) + + assert _database_privilege(pg, "CONNECT") + + +@pytest.mark.integration +@pytest.mark.postgres +def test_chain_grants_exactly_connect_on_the_database_and_nothing_more( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """The database-level ACL entry for app_dashboard must be exactly CONNECT. + + ``CREATE ON DATABASE`` lets a role make schemas, and a role that creates + objects OWNS them — and an owner is exempt from row-level security, FORCE + included. Widening this grant would reopen the bypass the whole epic exists + to close, so the ceiling is pinned rather than assumed. + + Read from ``pg_database.datacl`` filtered to this grantee, NOT from + ``has_database_privilege``: the effective-privilege function also reports + privileges inherited from ``PUBLIC`` (``TEMPORARY`` is a stock Postgres + PUBLIC default), so it cannot distinguish what this chain granted from what + the cluster already gave everyone. Revoking PUBLIC's remaining defaults is + OMN-15355's blast radius, deliberately not this ticket's. + """ + pg = ephemeral_postgres + _seed_analytics_database(pg) + _apply_full_chain(pg) + + conn = pg.connect() + conn.autocommit = True + with conn.cursor() as cur: + cur.execute( + "SELECT coalesce(array_agg(acl.privilege_type ORDER BY acl.privilege_type), " + "ARRAY[]::text[]) " + "FROM pg_database d, aclexplode(d.datacl) acl " + "WHERE d.datname = %s AND acl.grantee = %s::regrole::oid", + (ANALYTICS_DB, READ_ROLE), + ) + row = cur.fetchone() + conn.close() + + assert row is not None + assert list(row[0]) == ["CONNECT"], ( + f"database-level grants to {READ_ROLE} on {ANALYTICS_DB} are {row[0]}; " + "the read path needs CONNECT and nothing else — CREATE would let the " + "role own objects, and an owner is exempt from row-level security" + ) + + +@pytest.mark.integration +@pytest.mark.postgres +def test_chain_never_grants_select_on_a_table_without_a_tenant_policy( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """Readable-before-policy is the ordering hazard, and it is fail-closed. + + ``uncovered_projection`` carries ``tenant_id`` but has no RLS and no + ``tenant_isolation`` policy. If any migration in the chain reaches for a + blanket ``GRANT SELECT ON ALL TABLES``, this table becomes readable with no + tenant predicate at all — a cross-tenant read that every RLS test in the + repo would still report green, because none of them look at this table. + """ + pg = ephemeral_postgres + _seed_analytics_database(pg) + _apply_full_chain(pg) + + conn = pg.connect(dbname=ANALYTICS_DB) + conn.autocommit = True + with conn.cursor() as cur: + cur.execute( + "SELECT has_table_privilege(%s, 'public.uncovered_projection', 'SELECT')", + (READ_ROLE,), + ) + uncovered = cur.fetchone() + cur.execute( + "SELECT has_table_privilege(%s, 'public.delegation_events', 'SELECT')", + (READ_ROLE,), + ) + covered = cur.fetchone() + conn.close() + + assert uncovered is not None and uncovered[0] is False, ( + "app_dashboard can SELECT a table that has no tenant_isolation policy " + "— the grant chain must never make a table readable before its policy " + "exists" + ) + assert covered is not None and covered[0] is True + + +@pytest.mark.integration +@pytest.mark.postgres +def test_chain_never_grants_write_on_the_rls_covered_read_tables( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """app_dashboard is the READ role. INSERT/UPDATE/DELETE are not its shape. + + 096 had to re-narrow role_omnidash after a blanket grant silently re-added + DELETE to three FORCE-RLS tables. Same class, pinned here before it can + happen to this role. + """ + pg = ephemeral_postgres + _seed_analytics_database(pg) + _apply_full_chain(pg) + + conn = pg.connect(dbname=ANALYTICS_DB) + conn.autocommit = True + granted: dict[tuple[str, str], bool] = {} + with conn.cursor() as cur: + for table in ("delegation_events", "delegation_budget_state"): + for privilege in ("INSERT", "UPDATE", "DELETE", "TRUNCATE"): + cur.execute( + "SELECT has_table_privilege(%s, %s, %s)", + (READ_ROLE, f"public.{table}", privilege), + ) + row = cur.fetchone() + assert row is not None + granted[(table, privilege)] = bool(row[0]) + conn.close() + + offenders = [key for key, held in granted.items() if held] + assert not offenders, ( + f"app_dashboard holds write privileges it must not: {offenders}" + ) + + +@pytest.mark.integration +@pytest.mark.postgres +def test_chain_leaves_the_read_role_non_owner_and_non_bypassing( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """The role flags and ownership, re-read after the grant chain. + + 094 asserts these at role-creation time. They are re-asserted AFTER the + grants because the grant chain is the thing this ticket edits, and a grant + file that reached for ownership or an ALTER ROLE would make every policy in + OMN-14894 inert without failing any existing test. + """ + pg = ephemeral_postgres + _seed_analytics_database(pg) + _apply_full_chain(pg) + + cluster = pg.connect() + cluster.autocommit = True + with cluster.cursor() as cur: + cur.execute( + "SELECT rolsuper, rolbypassrls, rolcreatedb, rolcreaterole, rolreplication " + "FROM pg_roles WHERE rolname = %s", + (READ_ROLE,), + ) + flags = cur.fetchone() + cluster.close() + + assert flags is not None, f"{READ_ROLE} was not created by the chain" + assert flags == (False, False, False, False, False), ( + f"{READ_ROLE} carries an escalated flag after the grant chain: {flags}" + ) + + analytics = pg.connect(dbname=ANALYTICS_DB) + analytics.autocommit = True + with analytics.cursor() as cur: + cur.execute( + "SELECT tablename FROM pg_tables " + "WHERE schemaname = 'public' AND tableowner = %s", + (READ_ROLE,), + ) + owned = [row[0] for row in cur.fetchall()] + analytics.close() + + assert not owned, ( + f"{READ_ROLE} owns table(s) {owned} — an owner is exempt from row-level " + "security (FORCE included), so every tenant_isolation policy on them is " + "inert and any 'clean under RLS' reading is a false clean" + ) diff --git a/tests/integration/migrations/test_098_omninode_internal_schema_omn15359.py b/tests/integration/migrations/test_098_omninode_internal_schema_omn15359.py new file mode 100644 index 0000000000..b1aa453572 --- /dev/null +++ b/tests/integration/migrations/test_098_omninode_internal_schema_omn15359.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""OMN-15359 — the `omninode_internal` schema must exist, additively, in +`omnidash_analytics`. + +THE GAP (live evidence, OMN-15426 readback, rolling ledger 2026-08-03T19:2xZ) + + `handler_wiring.py` auto-wiring issues schema-qualified SQL against + contract-declared `db_io.schema=omninode_internal` for the 41-table + `omninode_runtime` domain. The schema itself has never been created + anywhere in the migration corpus -- every table lands, unqualified, in + `public`. A grant was withheld because the target relation would not + resolve: "relation does not exist", not a permission error. + +This proof applies `098_create_omninode_internal_schema.sql` through the same +`psql -v ON_ERROR_STOP=1 -f` invocation `scripts/run-forward-migrations.sh` +uses (matching the pattern of `test_097_app_dashboard_connect_omn15297.py`), +against a real ephemeral cluster, and asserts real catalog state -- not string +matching the SQL text. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.integration.migrations.conftest import EphemeralPostgres + +pytestmark = pytest.mark.integration + +REPO_ROOT = Path(__file__).resolve().parents[3] +FORWARD_DIR = REPO_ROOT / "docker" / "migrations" / "forward" +ROLLBACK_DIR = REPO_ROOT / "docker" / "migrations" / "rollback" +MIGRATION = FORWARD_DIR / "098_create_omninode_internal_schema.sql" +ROLLBACK = ROLLBACK_DIR / "rollback_098_create_omninode_internal_schema.sql" +ANALYTICS_DB = "omnidash_analytics" +TARGET_SCHEMA = "omninode_internal" + + +def _apply( + pg: EphemeralPostgres, path: Path, *, dbname: str = "postgres" +) -> subprocess.CompletedProcess[str]: + result = pg.psql("-v", "ON_ERROR_STOP=1", "-f", str(path), dbname=dbname) + assert result.returncode == 0, ( + f"{path.name} failed to apply against {dbname}:\n{result.stderr}" + ) + return result + + +def _seed_analytics_database(pg: EphemeralPostgres) -> None: + bootstrap = pg.connect() + bootstrap.autocommit = True + with bootstrap.cursor() as cur: + cur.execute(f'CREATE DATABASE "{ANALYTICS_DB}"') + bootstrap.close() + + +def _schema_exists(pg: EphemeralPostgres) -> bool: + conn = pg.connect(dbname=ANALYTICS_DB) + conn.autocommit = True + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = %s", + (TARGET_SCHEMA,), + ) + exists = cur.fetchone() is not None + conn.close() + return exists + + +def _table_count_in_schema(pg: EphemeralPostgres) -> int: + conn = pg.connect(dbname=ANALYTICS_DB) + conn.autocommit = True + with conn.cursor() as cur: + cur.execute( + "SELECT count(*) FROM information_schema.tables WHERE table_schema = %s", + (TARGET_SCHEMA,), + ) + (count,) = cur.fetchone() + conn.close() + return int(count) + + +def test_098_creates_the_schema_in_omnidash_analytics( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """The migration's only mutation: `omninode_internal` starts absent, ends present.""" + _seed_analytics_database(ephemeral_postgres) + assert not _schema_exists(ephemeral_postgres), ( + "test setup invariant: the schema must not pre-exist" + ) + + _apply(ephemeral_postgres, MIGRATION) + + assert _schema_exists(ephemeral_postgres) + + +def test_098_is_additive_only_zero_tables_created( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """No table lands in the new schema -- this migration only builds the target.""" + _seed_analytics_database(ephemeral_postgres) + _apply(ephemeral_postgres, MIGRATION) + + assert _table_count_in_schema(ephemeral_postgres) == 0 + + +def test_098_is_idempotent_on_reapply(ephemeral_postgres: EphemeralPostgres) -> None: + """Safe to re-run: `CREATE SCHEMA IF NOT EXISTS` on an already-applied lane.""" + _seed_analytics_database(ephemeral_postgres) + _apply(ephemeral_postgres, MIGRATION) + _apply(ephemeral_postgres, MIGRATION) + + assert _schema_exists(ephemeral_postgres) + + +def test_098_rollback_drops_the_empty_schema( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """Rollback undoes exactly what 098 did, on a lane where it is still empty.""" + _seed_analytics_database(ephemeral_postgres) + _apply(ephemeral_postgres, MIGRATION) + assert _schema_exists(ephemeral_postgres) + + _apply(ephemeral_postgres, ROLLBACK) + + assert not _schema_exists(ephemeral_postgres) + + +def test_098_rollback_refuses_once_a_table_has_landed( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """RESTRICT, not CASCADE: rollback must fail closed once data has moved in. + + Proves the rollback file's own safety claim rather than trusting its + comment: this is the exact scenario -- a later migration has copied a + table into the schema -- the RESTRICT clause exists to protect against. + """ + _seed_analytics_database(ephemeral_postgres) + _apply(ephemeral_postgres, MIGRATION) + + conn = ephemeral_postgres.connect(dbname=ANALYTICS_DB) + conn.autocommit = True + with conn.cursor() as cur: + cur.execute(f"CREATE TABLE {TARGET_SCHEMA}.copied_family (id TEXT PRIMARY KEY)") + conn.close() + + result = ephemeral_postgres.psql( + "-v", "ON_ERROR_STOP=1", "-f", str(ROLLBACK), dbname="postgres" + ) + assert result.returncode != 0 + assert "depends on" in result.stderr or "cannot drop" in result.stderr + assert _schema_exists(ephemeral_postgres), ( + "RESTRICT must have refused the drop; the schema must still be present" + ) diff --git a/tests/integration/migrations/test_099_omninode_internal_live_events_omn15359.py b/tests/integration/migrations/test_099_omninode_internal_live_events_omn15359.py new file mode 100644 index 0000000000..f202d78fa8 --- /dev/null +++ b/tests/integration/migrations/test_099_omninode_internal_live_events_omn15359.py @@ -0,0 +1,298 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""OMN-15359 -- physically create omninode_internal.live_events. + +THE GAP (live evidence, mergesweep-0809-projplane ground phase + adversarial +verify, rolling ledger 2026-08-09) + + `098_create_omninode_internal_schema.sql` created the empty + `omninode_internal` schema. `node_projection_live_events/contract.yaml` + declares `db_io.db_tables[0].schema: omninode_internal`, and + `handler_wiring._resolve_projection_database_target` uses that + contract-declared schema literally as the SQL write target -- so the + runtime has been issuing `INSERT INTO omninode_internal.live_events` + since before 098 merged. No migration had ever physically created that + relation, so every insert failed `UndefinedTable` + (`relation "omninode_internal.live_events" does not exist`), live- + reconfirmed on onex-dev at ~10s cadence. + +This proof applies `099_create_omninode_internal_live_events.sql` through the +same `psql -v ON_ERROR_STOP=1 -f` invocation `scripts/run-forward-migrations.sh` +uses (matching `test_098_omninode_internal_schema_omn15359.py`), against a +real ephemeral cluster seeded with representative pre-existing +`public.live_events` rows, and asserts real catalog + row-level state -- not +string matching the SQL text. + +OMN-15838 AMENDMENT: 099 originally also transform-copied every row from +`public.live_events` into `omninode_internal.live_events` in the same file, +then RAISE EXCEPTIONed unless the two tables' row counts matched exactly. +That reconciliation was a non-atomic race on any lane with a concurrent +writer to `public.live_events` (the stability-test lane runs at ~24 +writes/min): the INSERT...SELECT snapshots the source, then two separate +follow-up `SELECT count(*)` statements re-read both tables, so any row +committed to the source in between makes the counts diverge and the DO block +RAISE deterministically -- not intermittently. Because +`scripts/run-forward-migrations.sh` runs with `ON_ERROR_STOP=1`, that RAISE +aborted the whole migration run before it was ever recorded applied, so every +subsequent refresh retried and failed identically (963965 vs 963977 rows, +OMN-15838). Data delivery for this table is independently owned by the +node-owned replacement migration +(`docker/migrations/forward/nodes/node_projection_live_events/ +0002_create_omninode_internal_live_events.sql`, OMN-15819), so 099's own copy +of that logic was also redundant on top of being racy. 099 now only creates +schema shape (table, grants, indexes) and asserts it -- it never reads +`public.live_events` at all. The row-copy/idempotent-copy/reconciliation +tests that covered the removed logic are replaced below by tests proving the +removed behavior stays removed: 099 no longer copies rows, and a diverging +public/internal row count no longer fails the migration. +""" + +from __future__ import annotations + +import subprocess +import uuid +from pathlib import Path + +import pytest + +from tests.integration.migrations.conftest import EphemeralPostgres + +pytestmark = pytest.mark.integration + +REPO_ROOT = Path(__file__).resolve().parents[3] +FORWARD_DIR = REPO_ROOT / "docker" / "migrations" / "forward" +ROLLBACK_DIR = REPO_ROOT / "docker" / "migrations" / "rollback" +SCHEMA_MIGRATION = FORWARD_DIR / "098_create_omninode_internal_schema.sql" +MIGRATION = FORWARD_DIR / "099_create_omninode_internal_live_events.sql" +ROLLBACK = ROLLBACK_DIR / "rollback_099_create_omninode_internal_live_events.sql" +ANALYTICS_DB = "omnidash_analytics" + + +def _apply( + pg: EphemeralPostgres, path: Path, *, dbname: str = "postgres" +) -> subprocess.CompletedProcess[str]: + result = pg.psql("-v", "ON_ERROR_STOP=1", "-f", str(path), dbname=dbname) + return result + + +def _apply_ok( + pg: EphemeralPostgres, path: Path, *, dbname: str = "postgres" +) -> subprocess.CompletedProcess[str]: + result = _apply(pg, path, dbname=dbname) + assert result.returncode == 0, ( + f"{path.name} failed to apply against {dbname}:\n{result.stderr}" + ) + return result + + +def _seed_analytics_database_with_source_table(pg: EphemeralPostgres) -> None: + """Build omnidash_analytics with a public.live_events shaped and seeded + exactly like the live table (0000_create_live_events.sql shape).""" + bootstrap = pg.connect() + bootstrap.autocommit = True + with bootstrap.cursor() as cur: + cur.execute(f'CREATE DATABASE "{ANALYTICS_DB}"') + bootstrap.close() + + conn = pg.connect(dbname=ANALYTICS_DB) + conn.autocommit = True + with conn.cursor() as cur: + cur.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto") + cur.execute( + """ + CREATE TABLE public.live_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + event_id TEXT UNIQUE NOT NULL, + type TEXT NOT NULL DEFAULT 'ACTION', + timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), + source TEXT NOT NULL DEFAULT 'platform', + topic TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + payload TEXT NOT NULL DEFAULT '{}', + correlation_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + for i in range(25): + cur.execute( + """ + INSERT INTO public.live_events + (event_id, type, source, topic, summary, payload, correlation_id) + VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + ( + f"evt-{i:04d}-{uuid.uuid4()}", + "ACTION" if i % 2 == 0 else "ERROR", + "platform", + f"onex.evt.omnimarket.projection-live-events-applied.v1#{i}", + f"summary {i}", + f'{{"i": {i}}}', + str(uuid.uuid4()) if i % 3 == 0 else None, + ), + ) + conn.close() + + +def _table_exists(pg: EphemeralPostgres, schema: str, table: str) -> bool: + conn = pg.connect(dbname=ANALYTICS_DB) + conn.autocommit = True + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM information_schema.tables " + "WHERE table_schema = %s AND table_name = %s", + (schema, table), + ) + exists = cur.fetchone() is not None + conn.close() + return exists + + +def _row_count(pg: EphemeralPostgres, schema: str, table: str) -> int: + conn = pg.connect(dbname=ANALYTICS_DB) + conn.autocommit = True + with conn.cursor() as cur: + cur.execute(f'SELECT count(*) FROM "{schema}"."{table}"') # noqa: S608 + (count,) = cur.fetchone() + conn.close() + return int(count) + + +def _insert_out_of_band_row(pg: EphemeralPostgres, event_id: str) -> None: + """Insert directly into omninode_internal.live_events with an event_id + that does not exist in public.live_events -- simulating the dual-write + bleed OMN-15838 observed (a row landing in one table but not the other).""" + conn = pg.connect(dbname=ANALYTICS_DB) + conn.autocommit = True + with conn.cursor() as cur: + cur.execute( + "INSERT INTO omninode_internal.live_events (event_id, topic) " + "VALUES (%s, %s)", + (event_id, "onex.evt.platform.node-heartbeat.v1"), + ) + conn.close() + + +@pytest.fixture +def seeded_pg( + ephemeral_postgres: EphemeralPostgres, +) -> EphemeralPostgres: + _seed_analytics_database_with_source_table(ephemeral_postgres) + _apply_ok(ephemeral_postgres, SCHEMA_MIGRATION) + return ephemeral_postgres + + +def test_099_creates_omninode_internal_live_events( + seeded_pg: EphemeralPostgres, +) -> None: + """RED-before: absent before 099; created (additively) by 099.""" + assert not _table_exists(seeded_pg, "omninode_internal", "live_events"), ( + "test setup invariant: the physical table must not pre-exist" + ) + + _apply_ok(seeded_pg, MIGRATION) + + assert _table_exists(seeded_pg, "omninode_internal", "live_events") + + +def test_099_preserves_public_live_events(seeded_pg: EphemeralPostgres) -> None: + """OMN-15359 AC: source relation must survive until parity is reproven.""" + before_count = _row_count(seeded_pg, "public", "live_events") + + _apply_ok(seeded_pg, MIGRATION) + + assert _table_exists(seeded_pg, "public", "live_events") + assert _row_count(seeded_pg, "public", "live_events") == before_count + + +def test_099_does_not_copy_any_rows_even_when_source_has_data( + seeded_pg: EphemeralPostgres, +) -> None: + """OMN-15838: 099 is schema-shape + grants only. It must never read + public.live_events, regardless of how many rows the source carries.""" + src_count = _row_count(seeded_pg, "public", "live_events") + assert src_count == 25, "test setup invariant" + + _apply_ok(seeded_pg, MIGRATION) + + assert _row_count(seeded_pg, "omninode_internal", "live_events") == 0 + + +def test_099_succeeds_when_public_and_internal_row_counts_diverge( + seeded_pg: EphemeralPostgres, +) -> None: + """OMN-15838 regression: the removed transform-copy/reconciliation block + used to RAISE EXCEPTION whenever public.live_events and + omninode_internal.live_events had different row counts -- exactly the + condition the stability-test lane hit under concurrent writes (963965 vs + 963977). Manufacture that divergence directly (an out-of-band row in the + destination with no counterpart in the source, mirroring the dual-write + bleed) and prove a re-apply of 099 still succeeds and leaves the + divergence untouched -- it no longer compares the two tables at all. + """ + _apply_ok(seeded_pg, MIGRATION) + _insert_out_of_band_row(seeded_pg, f"evt-out-of-band-{uuid.uuid4()}") + _insert_out_of_band_row(seeded_pg, f"evt-out-of-band-{uuid.uuid4()}") + + src_count = _row_count(seeded_pg, "public", "live_events") + dst_count_before = _row_count(seeded_pg, "omninode_internal", "live_events") + assert dst_count_before != src_count, "test setup invariant: counts diverge" + + result = _apply(seeded_pg, MIGRATION) + + assert result.returncode == 0, ( + f"099 must not fail on a diverging row count:\n{result.stderr}" + ) + assert _row_count(seeded_pg, "omninode_internal", "live_events") == ( + dst_count_before + ), "099 must not mutate existing rows in either table on re-apply" + + +def test_099_schema_shape_is_idempotent_on_reapply( + seeded_pg: EphemeralPostgres, +) -> None: + _apply_ok(seeded_pg, MIGRATION) + _apply_ok(seeded_pg, MIGRATION) + + assert _table_exists(seeded_pg, "omninode_internal", "live_events") + + +def test_099_rollback_drops_the_table_and_preserves_the_source( + seeded_pg: EphemeralPostgres, +) -> None: + _apply_ok(seeded_pg, MIGRATION) + assert _table_exists(seeded_pg, "omninode_internal", "live_events") + + _apply_ok(seeded_pg, ROLLBACK, dbname="postgres") + + assert not _table_exists(seeded_pg, "omninode_internal", "live_events") + assert _table_exists(seeded_pg, "public", "live_events") + assert _row_count(seeded_pg, "public", "live_events") == 25 + + +def test_099_succeeds_when_public_live_events_does_not_exist( + ephemeral_postgres: EphemeralPostgres, +) -> None: + """Matches omnibase_infra's standalone "Migration Integration Test" CI gate, + which applies only the numbered top-level docker/migrations/forward/*.sql + files (not the nodes/ subtree that vendors public.live_events's own + CREATE TABLE) against a fresh database. public.live_events genuinely does + not exist in that scope -- 099 must still succeed, creating + omninode_internal.live_events empty (as it always does now, OMN-15838) + rather than failing UndefinedTable. + """ + bootstrap = ephemeral_postgres.connect() + bootstrap.autocommit = True + with bootstrap.cursor() as cur: + cur.execute(f'CREATE DATABASE "{ANALYTICS_DB}"') + bootstrap.close() + + _apply_ok(ephemeral_postgres, SCHEMA_MIGRATION) + + assert not _table_exists(ephemeral_postgres, "public", "live_events") + + _apply_ok(ephemeral_postgres, MIGRATION) + + assert _table_exists(ephemeral_postgres, "omninode_internal", "live_events") + assert _row_count(ephemeral_postgres, "omninode_internal", "live_events") == 0 diff --git a/tests/integration/migrations/test_application_migration_ledger_omn15413.py b/tests/integration/migrations/test_application_migration_ledger_omn15413.py new file mode 100644 index 0000000000..542159ec4b --- /dev/null +++ b/tests/integration/migrations/test_application_migration_ledger_omn15413.py @@ -0,0 +1,1276 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""PostgreSQL 16 proof for OMN-15413 ledger selection and import semantics. + +OMN-15695 extends this module with the adopt/convert proof for the +pre-OMN-15413 ``public.schema_migrations(migration_id, applied_at, checksum, +source_set)`` node ledger written by the historical runner into the application +database. Operator ruling 2026-08-04: adopt that exact shape non-destructively; +every other unrecognized shape stays fail-closed. +""" + +# ruff: noqa: S608 -- all interpolated SQL values are checked-in manifest data. + +from __future__ import annotations + +import hashlib +import os +import shutil +import socket +import subprocess +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.integration, pytest.mark.postgres, pytest.mark.serial] + +REPO_ROOT = Path(__file__).resolve().parents[3] +LEDGER_DIR = REPO_ROOT / "docker" / "migrations" / "forward" / "_ledger" +MANIFEST = LEDGER_DIR / "application-migrations.tsv" +LEGACY_NODE_DECLARATIONS = LEDGER_DIR / "legacy-node-migrations.tsv" +BOOTSTRAP = LEDGER_DIR / "bootstrap.sql" +RUNNER = REPO_ROOT / "scripts" / "run-forward-migrations.sh" + + +def _postgres_bin_dir() -> Path | None: + initdb = shutil.which("initdb") + candidates = [Path(initdb).parent] if initdb else [] + candidates.extend( + sorted(Path("/opt/homebrew/opt").glob("postgresql@*/bin"), reverse=True) + ) + candidates.extend(sorted(Path("/usr/lib/postgresql").glob("*/bin"), reverse=True)) + for candidate in candidates: + required = ("initdb", "pg_ctl", "psql") + if not all((candidate / binary).is_file() for binary in required): + continue + version = subprocess.run( + [str(candidate / "psql"), "--version"], + capture_output=True, + text=True, + check=False, + ).stdout + if " 16." in version: + return candidate + return None + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +@dataclass(frozen=True) +class Pg16Cluster: + bin_dir: Path + port: int + + @property + def psql(self) -> str: + return str(self.bin_dir / "psql") + + def command( + self, + database: str, + *arguments: str, + input_text: str | None = None, + check: bool = True, + ) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + self.psql, + "-X", + "-q", + "-h", + "127.0.0.1", + "-p", + str(self.port), + "-U", + "postgres", + "-d", + database, + "-v", + "ON_ERROR_STOP=1", + *arguments, + ], + input=input_text, + capture_output=True, + text=True, + timeout=30, + check=check, + env={**os.environ, "PGPASSWORD": ""}, + ) + + def sql(self, database: str, statement: str) -> str: + return self.command(database, "-At", "-c", statement).stdout.strip() + + def create_database(self, database: str) -> None: + self.command("postgres", "-c", f"CREATE DATABASE {database}") + + +@pytest.fixture(scope="module") +def pg16(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Pg16Cluster]: + bin_dir = _postgres_bin_dir() + if bin_dir is None: + pytest.skip("PostgreSQL 16 initdb/pg_ctl/psql are unavailable") + + cluster_dir = tmp_path_factory.mktemp("omn15413-pg16") + data_dir = cluster_dir / "data" + port = _free_port() + init = subprocess.run( + [ + str(bin_dir / "initdb"), + "-D", + str(data_dir), + "-U", + "postgres", + "--auth=trust", + "--no-sync", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if init.returncode != 0: + pytest.fail(f"PostgreSQL 16 initdb failed: {init.stderr}") + + start = subprocess.run( + [ + str(bin_dir / "pg_ctl"), + "-D", + str(data_dir), + "-o", + f"-F -h 127.0.0.1 -p {port}", + "-l", + str(cluster_dir / "postgres.log"), + "-w", + "start", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if start.returncode != 0: + postgres_log = cluster_dir / "postgres.log" + log_text = ( + postgres_log.read_text(encoding="utf-8", errors="replace") + if postgres_log.is_file() + else "" + ) + pytest.skip( + "PostgreSQL 16 binaries are present but an ephemeral cluster could " + f"not start; pg_ctl stderr={start.stderr!r}; postgres.log={log_text!r}" + ) + + try: + yield Pg16Cluster(bin_dir=bin_dir, port=port) + finally: + subprocess.run( + [ + str(bin_dir / "pg_ctl"), + "-D", + str(data_dir), + "-m", + "immediate", + "-w", + "stop", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + +def _declarations() -> list[list[str]]: + return [line.split("\t") for line in MANIFEST.read_text().splitlines()] + + +def _run_bootstrap( + pg16: Pg16Cluster, database: str +) -> subprocess.CompletedProcess[str]: + create_manifest = """ +CREATE TEMP TABLE onex_application_migration_manifest ( + artifact_path TEXT NOT NULL, + migration_stream TEXT NOT NULL, + owner TEXT NOT NULL, + domain TEXT NOT NULL, + version TEXT NOT NULL, + checksum TEXT NOT NULL, + PRIMARY KEY (artifact_path), + UNIQUE (migration_stream, domain, version) +) +""" + return pg16.command( + database, + "-c", + create_manifest, + "-c", + """ +CREATE TEMP TABLE onex_legacy_node_migration_declarations ( + migration_stream TEXT NOT NULL, + owner TEXT NOT NULL, + domain TEXT NOT NULL, + version TEXT NOT NULL PRIMARY KEY, + source_checksum TEXT NOT NULL, + ticket TEXT NOT NULL +) +""", + "-c", + ( + "\\copy onex_application_migration_manifest " + f"FROM '{MANIFEST}' WITH (FORMAT text, DELIMITER E'\\t')" + ), + "-c", + ( + "\\copy onex_legacy_node_migration_declarations " + f"FROM '{LEGACY_NODE_DECLARATIONS}' WITH (FORMAT text, DELIMITER E'\\t')" + ), + "-f", + str(BOOTSTRAP), + check=False, + ) + + +def _seed_sources( + pg16: Pg16Cluster, + database: str, + *, + node_version: str, + node_checksum: str, + omnimarket: list[str] | None = None, +) -> None: + omnimarket_sql = "" + if omnimarket is not None: + artifact, _, _, _, version, checksum = omnimarket + node_name = artifact.split("/")[1] + filename = artifact.split("/")[2] + assert version == f"node:{node_name}:{filename}" + omnimarket_sql = f""" +CREATE TABLE public.omnimarket_schema_migrations ( + id SERIAL PRIMARY KEY, + node_name TEXT NOT NULL, + version TEXT NOT NULL, + filename TEXT NOT NULL, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (node_name, version) +); +INSERT INTO public.omnimarket_schema_migrations + (node_name, version, filename, checksum, applied_at) +VALUES + ('{node_name}', '{filename}', '{filename}', '{checksum}', + TIMESTAMPTZ '2026-01-03 00:00:00+00'); +""" + pg16.command( + database, + "-f", + "-", + input_text=f""" +CREATE TABLE public.schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL +); +INSERT INTO public.schema_migrations VALUES + ('0001_filename_only.sql', TIMESTAMPTZ '2026-01-01 00:00:00+00'); +CREATE TABLE public.node_schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now(), + checksum TEXT NOT NULL +); +INSERT INTO public.node_schema_migrations VALUES + ('{node_version}', TIMESTAMPTZ '2026-01-02 00:00:00+00', '{node_checksum}'); +{omnimarket_sql} +""", + ) + + +def test_selects_one_checksum_ledger_and_imports_sources_twice( + pg16: Pg16Cluster, +) -> None: + database = "omn15413_green" + pg16.create_database(database) + first, second = _declarations()[:2] + _seed_sources( + pg16, + database, + node_version=first[4], + node_checksum=first[5], + omnimarket=second, + ) + node_oid = pg16.sql( + database, "SELECT 'public.node_schema_migrations'::regclass::oid" + ) + filename_oid = pg16.sql( + database, "SELECT 'public.schema_migrations'::regclass::oid" + ) + omnimarket_oid = pg16.sql( + database, "SELECT 'public.omnimarket_schema_migrations'::regclass::oid" + ) + + signatures: list[str] = [] + for _ in range(2): + run = _run_bootstrap(pg16, database) + assert run.returncode == 0, run.stderr + signatures.append( + pg16.sql( + database, + """ +SELECT string_agg( + migration_stream || '|' || owner || '|' || domain || '|' || version || + '|' || checksum || '|' || checksum_kind || '|' || applied_at::text || + '|' || provenance, + E'\n' ORDER BY migration_stream, domain, version +) +FROM platform_catalog.schema_migrations +""", + ) + ) + + assert signatures[0] == signatures[1] + assert len(signatures[0].splitlines()) == 3 + assert ( + "legacy:filename-only|legacy:filename-only|legacy_unclassified" + in (signatures[0]) + ) + assert signatures[0].count("|content_sha256|") == 2 + assert signatures[0].count("|legacy_attestation|") == 1 + assert ( + pg16.sql(database, "SELECT 'platform_catalog.schema_migrations'::regclass::oid") + == node_oid + ) + assert ( + pg16.sql(database, "SELECT 'public.schema_migrations'::regclass::oid") + == filename_oid + ) + assert ( + pg16.sql( + database, "SELECT 'public.omnimarket_schema_migrations'::regclass::oid" + ) + == omnimarket_oid + ) + assert ( + pg16.sql( + database, "SELECT to_regclass('public.node_schema_migrations') IS NULL" + ) + == "t" + ) + assert ( + pg16.sql( + database, + "SELECT count(*) FROM public.schema_migrations " + "WHERE filename = '0001_filename_only.sql'", + ) + == "1" + ) + + +@pytest.mark.parametrize( + ("case_name", "version_factory", "checksum_factory", "signature"), + [ + ( + "checksum_conflict", + lambda row: row[4], + lambda row: "0" * 64, + "conflicting migration checksum", + ), + ( + "unknown_stream", + lambda row: "node:unknown:0001.sql", + lambda row: row[5], + "unknown migration stream/domain", + ), + ], +) +def test_checksum_conflict_and_unknown_stream_are_atomic_reds( + pg16: Pg16Cluster, + case_name: str, + version_factory: Callable[[list[str]], str], + checksum_factory: Callable[[list[str]], str], + signature: str, +) -> None: + database = f"omn15413_{case_name}" + pg16.create_database(database) + first = _declarations()[0] + version = version_factory(first) + checksum = checksum_factory(first) + _seed_sources(pg16, database, node_version=version, node_checksum=checksum) + + run = _run_bootstrap(pg16, database) + + assert run.returncode != 0 + assert signature in run.stderr + assert ( + pg16.sql( + database, "SELECT to_regclass('public.node_schema_migrations') IS NOT NULL" + ) + == "t" + ) + assert ( + pg16.sql( + database, + "SELECT to_regclass('platform_catalog.schema_migrations') IS NULL", + ) + == "t" + ) + + +def test_node_and_omnimarket_double_declaration_is_atomic_red( + pg16: Pg16Cluster, +) -> None: + database = "omn15413_double_declaration" + pg16.create_database(database) + first = _declarations()[0] + _seed_sources( + pg16, + database, + node_version=first[4], + node_checksum=first[5], + omnimarket=first, + ) + + run = _run_bootstrap(pg16, database) + + assert run.returncode != 0 + assert "double migration declaration" in run.stderr + assert ( + pg16.sql( + database, "SELECT to_regclass('public.node_schema_migrations') IS NOT NULL" + ) + == "t" + ) + assert ( + pg16.sql( + database, + "SELECT to_regclass('platform_catalog.schema_migrations') IS NULL", + ) + == "t" + ) + + +def _synthetic_migration_tree(tmp_path: Path) -> tuple[Path, str, str]: + migrations_dir = tmp_path / "forward" + ledger_dir = migrations_dir / "_ledger" + node_dir = migrations_dir / "nodes" / "node_example" + ledger_dir.mkdir(parents=True) + node_dir.mkdir(parents=True) + shutil.copy2(BOOTSTRAP, ledger_dir / "bootstrap.sql") + + (migrations_dir / "000_db_metadata.sql").write_text( + """ +CREATE TABLE IF NOT EXISTS public.db_metadata ( + id BOOLEAN PRIMARY KEY, + migrations_complete BOOLEAN NOT NULL DEFAULT FALSE, + runner_completed_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +INSERT INTO public.db_metadata (id) VALUES (TRUE) ON CONFLICT (id) DO NOTHING; +""", + encoding="utf-8", + ) + node_file = node_dir / "0001_create_example.sql" + node_file.write_text( + "CREATE TABLE IF NOT EXISTS public.omn15413_example (id INTEGER PRIMARY KEY);\n", + encoding="utf-8", + ) + checksum = hashlib.sha256(node_file.read_bytes()).hexdigest() + version = "node:node_example:0001_create_example.sql" + (ledger_dir / "application-migrations.tsv").write_text( + "\t".join( + ( + "nodes/node_example/0001_create_example.sql", + "node:node_example", + "node:node_example", + "omninode_internal", + version, + checksum, + ) + ) + + "\n", + encoding="utf-8", + ) + (ledger_dir / "application-migration-blocks.tsv").write_text("", encoding="utf-8") + (ledger_dir / "legacy-node-migrations.tsv").write_text("", encoding="utf-8") + (ledger_dir / "cloud-migration-aliases.tsv").write_text( + "20260101_cloud\t20260101_cloud.sql\n", encoding="utf-8" + ) + # OMN-15349: the runner unconditionally requires the single-sourced + # operator fence manifest under MIGRATIONS_DIR; none of this synthetic + # tree's ids need to be fenced, so an empty-baseline manifest suffices. + (migrations_dir / "fenced-node-migrations.yaml").write_text( + "fenced_node_migrations: []\n", encoding="utf-8" + ) + return migrations_dir, version, checksum + + +def _run_forward_runner( + pg16: Pg16Cluster, + migrations_dir: Path, + *, + service_database: str, + application_database: str, + cloud_database: str, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["sh", str(RUNNER)], + capture_output=True, + text=True, + timeout=60, + check=False, + env={ + **os.environ, + "PATH": ( + f"{pg16.bin_dir}:/opt/homebrew/bin:/usr/local/bin:" + "/usr/bin:/bin:/usr/sbin:/sbin" + ), + "POSTGRES_HOST": "127.0.0.1", + "POSTGRES_PORT": str(pg16.port), + "POSTGRES_USER": "postgres", + "POSTGRES_PASSWORD": "", + "POSTGRES_DB": service_database, + "NODE_POSTGRES_DB": application_database, + "OMNINODE_CLOUD_HISTORY_DB": cloud_database, + "MIGRATIONS_DIR": str(migrations_dir), + "PG_WAIT_RETRIES": "5", + "MIGRATION_LOCK_WAIT_SECONDS": "10", + }, + ) + + +def test_real_runner_fresh_install_twice_on_postgresql_16( + pg16: Pg16Cluster, tmp_path: Path +) -> None: + migrations_dir, version, checksum = _synthetic_migration_tree(tmp_path) + service_database = "omn15413_runner_fresh_service" + application_database = "omn15413_runner_fresh_app" + cloud_database = "omn15413_runner_fresh_cloud" + for database in (service_database, application_database, cloud_database): + pg16.create_database(database) + + runs = [ + _run_forward_runner( + pg16, + migrations_dir, + service_database=service_database, + application_database=application_database, + cloud_database=cloud_database, + ) + for _ in range(2) + ] + + for run in runs: + assert run.returncode == 0, f"{run.stdout}\n{run.stderr}" + assert "Sentinel set. Migration gate will report HEALTHY." in run.stdout + assert ( + "Complete: 0 infra applied, 1 infra skipped; 0 node applied, 1 node skipped" + in (runs[1].stdout) + ) + assert ( + pg16.sql( + application_database, + "SELECT checksum FROM platform_catalog.schema_migrations " + f"WHERE version = '{version}'", + ) + == checksum + ) + assert ( + pg16.sql( + application_database, + "SELECT count(*) FROM platform_catalog.schema_migrations", + ) + == "1" + ) + + +def test_real_runner_imports_sanitized_legacy_history_twice_on_postgresql_16( + pg16: Pg16Cluster, tmp_path: Path +) -> None: + migrations_dir, version, checksum = _synthetic_migration_tree(tmp_path) + service_database = "omn15413_runner_legacy_service" + application_database = "omn15413_runner_legacy_app" + cloud_database = "omn15413_runner_legacy_cloud" + for database in (service_database, application_database, cloud_database): + pg16.create_database(database) + + pg16.command( + application_database, + "-f", + "-", + input_text=f""" +CREATE TABLE public.schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL +); +INSERT INTO public.schema_migrations VALUES + ('0001_legacy_dashboard.sql', TIMESTAMPTZ '2026-01-01 00:00:00+00'); +CREATE TABLE public.node_schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL, + checksum TEXT NOT NULL +); +INSERT INTO public.node_schema_migrations VALUES + ('{version}', TIMESTAMPTZ '2026-01-02 00:00:00+00', '{checksum}'); +""", + ) + pg16.command( + cloud_database, + "-f", + "-", + input_text=""" +CREATE TABLE public.schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL, + checksum TEXT +); +INSERT INTO public.schema_migrations VALUES + ('20260101_cloud.sql', TIMESTAMPTZ '2026-01-03 00:00:00+00', NULL); +CREATE TABLE public.migrations_log ( + id SERIAL PRIMARY KEY, + migration_name TEXT NOT NULL, + direction TEXT NOT NULL, + executed_at TIMESTAMPTZ NOT NULL, + notes TEXT, + UNIQUE (migration_name, direction) +); +INSERT INTO public.migrations_log + (migration_name, direction, executed_at, notes) +VALUES + ('20260101_cloud', 'forward', TIMESTAMPTZ '2026-01-03 00:00:00+00', + 'synthetic fixture'); +""", + ) + source_signature = pg16.sql( + cloud_database, + "SELECT version || '|' || coalesce(checksum, '') || '|' || " + "applied_at::text FROM public.schema_migrations", + ) + + signatures: list[str] = [] + for _ in range(2): + run = _run_forward_runner( + pg16, + migrations_dir, + service_database=service_database, + application_database=application_database, + cloud_database=cloud_database, + ) + assert run.returncode == 0, f"{run.stdout}\n{run.stderr}" + signatures.append( + pg16.sql( + application_database, + "SELECT string_agg(migration_stream || '|' || owner || '|' || " + "domain || '|' || version || '|' || checksum || '|' || " + "checksum_kind || '|' || applied_at::text || '|' || provenance, " + "E'\\n' ORDER BY migration_stream, domain, version) " + "FROM platform_catalog.schema_migrations", + ) + ) + + assert signatures[0] == signatures[1] + assert len(signatures[0].splitlines()) == 3 + assert "omninode-cloud|service:onex_api|legacy_unclassified" in signatures[0] + assert ";migrations_log:20260101_cloud" in signatures[0] + assert ( + pg16.sql( + cloud_database, + "SELECT version || '|' || coalesce(checksum, '') || '|' || " + "applied_at::text FROM public.schema_migrations", + ) + == source_signature + ) + + +def test_cloud_log_only_alias_is_a_real_runner_red( + pg16: Pg16Cluster, tmp_path: Path +) -> None: + migrations_dir, _, _ = _synthetic_migration_tree(tmp_path) + service_database = "omn15413_log_only_service" + application_database = "omn15413_log_only_app" + cloud_database = "omn15413_log_only_cloud" + for database in (service_database, application_database, cloud_database): + pg16.create_database(database) + pg16.command( + cloud_database, + "-f", + "-", + input_text=""" +CREATE TABLE public.schema_migrations ( + version TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL, + checksum TEXT +); +CREATE TABLE public.migrations_log ( + migration_name TEXT NOT NULL, + direction TEXT NOT NULL, + executed_at TIMESTAMPTZ NOT NULL +); +INSERT INTO public.migrations_log VALUES + ('20260101_cloud', 'forward', TIMESTAMPTZ '2026-01-03 00:00:00+00'); +""", + ) + + run = _run_forward_runner( + pg16, + migrations_dir, + service_database=service_database, + application_database=application_database, + cloud_database=cloud_database, + ) + + assert run.returncode != 0 + assert "log-only alias cannot be imported as applied" in run.stderr + assert pg16.sql(cloud_database, "SELECT count(*) FROM public.migrations_log") == "1" + + +def test_duplicate_cloud_versions_are_a_real_runner_red( + pg16: Pg16Cluster, tmp_path: Path +) -> None: + migrations_dir, _, _ = _synthetic_migration_tree(tmp_path) + service_database = "omn15413_duplicate_service" + application_database = "omn15413_duplicate_app" + cloud_database = "omn15413_duplicate_cloud" + for database in (service_database, application_database, cloud_database): + pg16.create_database(database) + pg16.command( + cloud_database, + "-f", + "-", + input_text=""" +CREATE TABLE public.schema_migrations ( + version TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL, + checksum TEXT +); +INSERT INTO public.schema_migrations VALUES + ('duplicate.sql', TIMESTAMPTZ '2026-01-03 00:00:00+00', NULL), + ('duplicate.sql', TIMESTAMPTZ '2026-01-04 00:00:00+00', NULL); +""", + ) + + run = _run_forward_runner( + pg16, + migrations_dir, + service_database=service_database, + application_database=application_database, + cloud_database=cloud_database, + ) + + assert run.returncode != 0 + assert "duplicate migration version in import" in run.stderr + assert ( + pg16.sql(cloud_database, "SELECT count(*) FROM public.schema_migrations") == "2" + ) + + +# --------------------------------------------------------------------------- +# OMN-15695: adopt/convert the pre-OMN-15413 migration_id node ledger. +# +# The live dev-lane application database (omnidash_analytics) carries 80 rows +# written by the historical runner as +# public.schema_migrations(migration_id, applied_at, checksum, source_set) +# with checksum='applied-by-runner' and source_set='node'. That relation is +# the predecessor NODE ledger of the application database, not the service +# ledger, so the fail-closed migration_id arm was a false negative for it. +# --------------------------------------------------------------------------- + +LIVE_APPLIED_AT = "2026-07-31 06:44:27.696803+00" + +LEDGER_SIGNATURE_SQL = """ +SELECT string_agg( + migration_stream || '|' || owner || '|' || domain || '|' || version || + '|' || checksum || '|' || checksum_kind || '|' || applied_at::text || + '|' || provenance, + E'\n' ORDER BY migration_stream, domain, version +) +FROM platform_catalog.schema_migrations +""" + + +def _seed_migration_id_ledger( + pg16: Pg16Cluster, + database: str, + rows: list[tuple[str, str, str]], + *, + applied_at: str = LIVE_APPLIED_AT, +) -> None: + """Create the historical runner ledger shape verbatim and seed ``rows``. + + ``rows`` are ``(migration_id, checksum, source_set)`` triples. + """ + values = ",\n ".join( + f"('{migration_id}', TIMESTAMPTZ '{applied_at}', '{checksum}', '{source_set}')" + for migration_id, checksum, source_set in rows + ) + pg16.command( + database, + "-f", + "-", + input_text=f""" +CREATE TABLE public.schema_migrations ( + migration_id TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + checksum TEXT NOT NULL, + source_set TEXT NOT NULL +); +INSERT INTO public.schema_migrations + (migration_id, applied_at, checksum, source_set) +VALUES + {values}; +""", + ) + + +def _live_shaped_rows(count: int = 80) -> list[tuple[str, str, str]]: + """Reproduce the live dev-lane ledger: N declared node ids, no byte evidence.""" + return [(row[4], "applied-by-runner", "node") for row in _declarations()[:count]] + + +def _canonical_rows(pg16: Pg16Cluster, database: str) -> list[list[str]]: + dumped = pg16.sql( + database, + "SELECT version || E'\\t' || migration_stream || E'\\t' || owner || " + "E'\\t' || domain || E'\\t' || checksum || E'\\t' || checksum_kind || " + "E'\\t' || provenance FROM platform_catalog.schema_migrations " + "ORDER BY version", + ) + return [line.split("\t") for line in dumped.splitlines()] + + +def test_migration_id_node_ledger_is_adopted_twice(pg16: Pg16Cluster) -> None: + database = "omn15695_adopt" + pg16.create_database(database) + declarations = _declarations()[:80] + assert len(declarations) == 80 + _seed_migration_id_ledger(pg16, database, _live_shaped_rows()) + source_oid = pg16.sql(database, "SELECT 'public.schema_migrations'::regclass::oid") + + signatures: list[str] = [] + for _ in range(2): + run = _run_bootstrap(pg16, database) + assert run.returncode == 0, f"{run.stdout}\n{run.stderr}" + signatures.append(pg16.sql(database, LEDGER_SIGNATURE_SQL)) + + assert signatures[0] == signatures[1] + assert len(signatures[0].splitlines()) == 80 + assert ( + pg16.sql(database, "SELECT count(*) FROM platform_catalog.schema_migrations") + == "80" + ) + assert ( + pg16.sql( + database, + "SELECT count(*) FROM platform_catalog.schema_migrations " + "WHERE checksum_kind = 'content_sha256'", + ) + == "80" + ) + # applied_at is preserved verbatim: this is the ruling's history clause. + assert ( + pg16.sql( + database, + "SELECT count(*) FROM platform_catalog.schema_migrations " + f"WHERE applied_at = TIMESTAMPTZ '{LIVE_APPLIED_AT}'", + ) + == "80" + ) + + expected = {row[4]: (row[1], row[2], row[3], row[5]) for row in declarations} + observed = _canonical_rows(pg16, database) + assert len(observed) == 80 + for version, stream, owner, domain, checksum, kind, provenance in observed: + assert expected[version] == (stream, owner, domain, checksum) + assert kind == "content_sha256" + assert provenance == ( + f"adopted:{database}:public.schema_migrations:migration_id:" + f"{version}:raw-checksum=applied-by-runner" + ) + + # The source relation is never renamed, updated, or deleted. + assert ( + pg16.sql(database, "SELECT 'public.schema_migrations'::regclass::oid") + == source_oid + ) + assert pg16.sql(database, "SELECT count(*) FROM public.schema_migrations") == "80" + + +def test_pr_review_bot_bypass_log_adopts_cleanly_omn15717(pg16: Pg16Cluster) -> None: + """Reproduces the exact OMN-15717 live failure and proves the fix. + + Before the OMN-15717 declaration was added, a database carrying the + legacy runner's row for + ``node:node_pr_review_bot:001_create_review_bot_bypass_log.sql`` failed + bootstrap.sql with "unknown migration stream/domain: adopted node + version ... has no checked-in declaration" (the live + refresh_stability_lane.sh forensic log, bootstrap.sql:673). This test + seeds that exact legacy row and asserts bootstrap.sql now adopts it + without error, is idempotent, and classifies it omninode_internal (R-q: + bookkeeping/audit-log state, not tenant workload data). + """ + database = "omn15717_pr_review_bot_adopt" + pg16.create_database(database) + version = "node:node_pr_review_bot:001_create_review_bot_bypass_log.sql" + _seed_migration_id_ledger(pg16, database, [(version, "applied-by-runner", "node")]) + + for _ in range(2): + run = _run_bootstrap(pg16, database) + assert run.returncode == 0, f"{run.stdout}\n{run.stderr}" + + row = pg16.sql( + database, + "SELECT migration_stream || E'\\t' || owner || E'\\t' || domain || E'\\t' || " + "checksum_kind || E'\\t' || provenance FROM platform_catalog.schema_migrations " + f"WHERE version = '{version}'", + ) + stream, owner, domain, checksum_kind, provenance = row.split("\t") + assert stream == "node:node_pr_review_bot" + assert owner == "node:node_pr_review_bot" + assert domain == "omninode_internal" + assert checksum_kind == "content_sha256" + assert provenance == ( + f"adopted:{database}:public.schema_migrations:migration_id:" + f"{version}:raw-checksum=applied-by-runner" + ) + # Legacy source row is preserved verbatim, never rewritten or deleted. + assert pg16.sql(database, "SELECT count(*) FROM public.schema_migrations") == "1" + + +def test_historical_projection_delegation_rows_adopt_cleanly_omn15717( + pg16: Pg16Cluster, +) -> None: + """The complete stability predecessor-ledger corpus is idempotently adopted. + + These historical identities have no current vendored SQL artifact, so they + can only enter the canonical ledger through the checked-in historical-node + declaration table. Their ``hotfix-applied-by-codex`` values are source + records, not file hashes; bootstrap records a deterministic legacy + attestation and therefore cannot let either row satisfy an active-file + migration probe. + """ + database = "omn15717_legacy_projection_delegation" + pg16.create_database(database) + versions = ( + "node:node_projection_delegation:0014_create_live_event_projection_view.sql", + "node:node_projection_delegation:0015_create_generation_dashboard_views.sql", + ) + _seed_migration_id_ledger( + pg16, + database, + [(version, "hotfix-applied-by-codex", "node") for version in versions], + ) + + signatures: list[str] = [] + for _ in range(2): + run = _run_bootstrap(pg16, database) + assert run.returncode == 0, f"{run.stdout}\n{run.stderr}" + signatures.append(pg16.sql(database, LEDGER_SIGNATURE_SQL)) + + assert signatures[0] == signatures[1] + rows = _canonical_rows(pg16, database) + assert len(rows) == 2 + for version, stream, owner, domain, checksum, kind, provenance in rows: + assert version in versions + assert stream == "node:node_projection_delegation" + assert owner == stream + assert domain == "omninode_internal" + assert len(checksum) == 64 + assert kind == "legacy_attestation" + assert provenance == ( + f"legacy-adopted:{database}:public.schema_migrations:migration_id:" + f"{version}:raw-checksum=hotfix-applied-by-codex:ticket=OMN-15717" + ) + assert pg16.sql(database, "SELECT count(*) FROM public.schema_migrations") == "2" + + +def test_historical_projection_delegation_source_checksum_mismatch_is_atomic_red( + pg16: Pg16Cluster, +) -> None: + database = "omn15717_legacy_projection_mismatch" + pg16.create_database(database) + _seed_migration_id_ledger( + pg16, + database, + [ + ( + "node:node_projection_delegation:0014_create_live_event_projection_view.sql", + "applied-by-runner", + "node", + ) + ], + ) + + run = _run_bootstrap(pg16, database) + + assert run.returncode != 0 + assert "conflicting migration checksum" in run.stderr + assert ( + pg16.sql( + database, "SELECT to_regclass('platform_catalog.schema_migrations') IS NULL" + ) + == "t" + ) + + +def test_adopted_ledger_makes_the_real_runner_skip( + pg16: Pg16Cluster, tmp_path: Path +) -> None: + migrations_dir, version, checksum = _synthetic_migration_tree(tmp_path) + service_database = "omn15695_runner_adopt_service" + application_database = "omn15695_runner_adopt_app" + cloud_database = "omn15695_runner_adopt_cloud" + for database in (service_database, application_database, cloud_database): + pg16.create_database(database) + _seed_migration_id_ledger( + pg16, application_database, [(version, "applied-by-runner", "node")] + ) + + runs = [ + _run_forward_runner( + pg16, + migrations_dir, + service_database=service_database, + application_database=application_database, + cloud_database=cloud_database, + ) + for _ in range(2) + ] + + for run in runs: + assert run.returncode == 0, f"{run.stdout}\n{run.stderr}" + # The FIRST run is the load-bearing no-re-application proof. + assert "0 node applied, 1 node skipped" in run.stdout + assert f"skip {version} (already applied)" in run.stdout + assert ( + pg16.sql( + application_database, + "SELECT checksum FROM platform_catalog.schema_migrations " + f"WHERE version = '{version}'", + ) + == checksum + ) + + +def test_service_owned_migration_id_ledger_still_fails_closed( + pg16: Pg16Cluster, +) -> None: + database = "omn15695_service_only" + pg16.create_database(database) + _seed_migration_id_ledger( + pg16, + database, + [ + ("docker/000_db_metadata.sql", "applied-by-runner", "docker"), + ("docker/001_initial.sql", "skip-manifest", "docker"), + ], + ) + + run = _run_bootstrap(pg16, database) + + assert run.returncode != 0 + assert ( + "unknown migration stream: service-owned migration_id ledger cannot be " + "selected for the application database" in run.stderr + ) + assert ( + pg16.sql( + database, "SELECT to_regclass('platform_catalog.schema_migrations') IS NULL" + ) + == "t" + ) + assert pg16.sql(database, "SELECT count(*) FROM public.schema_migrations") == "2" + + +def test_mixed_service_and_node_rows_partition(pg16: Pg16Cluster) -> None: + database = "omn15695_mixed" + pg16.create_database(database) + first = _declarations()[0] + _seed_migration_id_ledger( + pg16, + database, + [ + ("docker/000_db_metadata.sql", "applied-by-runner", "docker"), + (first[4], "applied-by-runner", "node"), + ], + ) + + run = _run_bootstrap(pg16, database) + + assert run.returncode == 0, f"{run.stdout}\n{run.stderr}" + assert ( + pg16.sql(database, "SELECT version FROM platform_catalog.schema_migrations") + == first[4] + ) + assert pg16.sql(database, "SELECT count(*) FROM public.schema_migrations") == "2" + assert ( + pg16.sql( + database, + "SELECT count(*) FROM platform_catalog.schema_migrations " + "WHERE version LIKE 'docker/%'", + ) + == "0" + ) + + +@pytest.mark.parametrize( + ("case_name", "migration_id_factory", "source_set"), + [ + ("unknown_source_set", lambda row: row[4], "cloud"), + ("bare_filename_identity", lambda row: "0001_bare.sql", "node"), + ], +) +def test_unrecognized_migration_id_rows_are_atomic_red( + pg16: Pg16Cluster, + case_name: str, + migration_id_factory: Callable[[list[str]], str], + source_set: str, +) -> None: + database = f"omn15695_{case_name}" + pg16.create_database(database) + first = _declarations()[0] + _seed_migration_id_ledger( + pg16, + database, + [(migration_id_factory(first), "applied-by-runner", source_set)], + ) + + run = _run_bootstrap(pg16, database) + + assert run.returncode != 0 + assert "unrecognized migration_id rows" in run.stderr + assert ( + pg16.sql( + database, "SELECT to_regclass('platform_catalog.schema_migrations') IS NULL" + ) + == "t" + ) + assert pg16.sql(database, "SELECT count(*) FROM public.schema_migrations") == "1" + + +def test_undeclared_node_version_is_atomic_red(pg16: Pg16Cluster) -> None: + database = "omn15695_undeclared" + pg16.create_database(database) + _seed_migration_id_ledger( + pg16, database, [("node:unknown:0001.sql", "applied-by-runner", "node")] + ) + + run = _run_bootstrap(pg16, database) + + assert run.returncode != 0 + assert "unknown migration stream/domain" in run.stderr + assert ( + pg16.sql( + database, "SELECT to_regclass('platform_catalog.schema_migrations') IS NULL" + ) + == "t" + ) + assert pg16.sql(database, "SELECT count(*) FROM public.schema_migrations") == "1" + + +@pytest.mark.parametrize( + ("case_name", "checksum"), + [("hex_conflict", "0" * 64), ("non_sentinel", "migrated-by-hand")], +) +def test_conflicting_adoption_checksums_are_atomic_reds( + pg16: Pg16Cluster, case_name: str, checksum: str +) -> None: + database = f"omn15695_{case_name}" + pg16.create_database(database) + first = _declarations()[0] + _seed_migration_id_ledger(pg16, database, [(first[4], checksum, "node")]) + + run = _run_bootstrap(pg16, database) + + assert run.returncode != 0 + assert "conflicting migration checksum" in run.stderr + assert ( + pg16.sql( + database, "SELECT to_regclass('platform_catalog.schema_migrations') IS NULL" + ) + == "t" + ) + assert pg16.sql(database, "SELECT count(*) FROM public.schema_migrations") == "1" + + +def test_adoption_is_idempotent_beside_a_populated_canonical_ledger( + pg16: Pg16Cluster, +) -> None: + """Regression for the double-declaration guard and the filename-import guard. + + The source relation is deliberately preserved, so a second bootstrap sees a + checksum-capable ``public.schema_migrations`` beside a populated canonical + ledger. Neither the ledger-selection guard nor the filename import may + treat that as a double declaration. + """ + database = "omn15695_idempotent" + pg16.create_database(database) + _seed_migration_id_ledger(pg16, database, _live_shaped_rows(5)) + + first_run = _run_bootstrap(pg16, database) + assert first_run.returncode == 0, f"{first_run.stdout}\n{first_run.stderr}" + + second_run = _run_bootstrap(pg16, database) + + assert second_run.returncode == 0, f"{second_run.stdout}\n{second_run.stderr}" + assert "double migration declaration" not in second_run.stderr + assert "remains beside the canonical ledger" not in second_run.stderr + assert ( + pg16.sql(database, "SELECT count(*) FROM platform_catalog.schema_migrations") + == "5" + ) + + +def test_tampered_adopted_row_is_a_double_declaration_red( + pg16: Pg16Cluster, +) -> None: + database = "omn15695_tampered" + pg16.create_database(database) + first = _declarations()[0] + _seed_migration_id_ledger(pg16, database, [(first[4], "applied-by-runner", "node")]) + first_run = _run_bootstrap(pg16, database) + assert first_run.returncode == 0, f"{first_run.stdout}\n{first_run.stderr}" + pg16.command( + database, + "-c", + "UPDATE platform_catalog.schema_migrations " + "SET provenance = provenance || ':tampered'", + ) + + run = _run_bootstrap(pg16, database) + + assert run.returncode != 0 + assert "double migration declaration for version" in run.stderr + + +def test_unknown_public_ledger_shape_still_fails_closed(pg16: Pg16Cluster) -> None: + """A shape that is neither filename, version, migration_id, nor node stays RED.""" + database = "omn15695_unknown_shape" + pg16.create_database(database) + pg16.command( + database, + "-f", + "-", + input_text=""" +CREATE TABLE public.schema_migrations ( + migration_id TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL, + checksum TEXT NOT NULL, + source_set TEXT NOT NULL, + extra_column TEXT NOT NULL +); +""", + ) + + run = _run_bootstrap(pg16, database) + + assert run.returncode != 0 + assert "unknown migration ledger shape" in run.stderr + assert ( + pg16.sql( + database, "SELECT to_regclass('platform_catalog.schema_migrations') IS NULL" + ) + == "t" + ) diff --git a/tests/integration/migrations/test_node_migration_discovery_applies.py b/tests/integration/migrations/test_node_migration_discovery_applies.py index 8594c0a478..5bd896e546 100644 --- a/tests/integration/migrations/test_node_migration_discovery_applies.py +++ b/tests/integration/migrations/test_node_migration_discovery_applies.py @@ -49,6 +49,7 @@ RUNNER = REPO_ROOT / "scripts" / "run-forward-migrations.sh" SAVINGS_BASE = FORWARD_DIR / "074_create_savings_estimates.sql" INFRA_FLAT_076 = FORWARD_DIR / "076_add_savings_estimate_provenance.sql" +FENCE_MANIFEST = FORWARD_DIR / "fenced-node-migrations.yaml" OMNIBASE_ENV = Path.home() / ".omnibase" / ".env" @@ -195,6 +196,9 @@ def test_node_migration_discovery_applies_views(tmp_path: Path) -> None: mig_dir.mkdir() shutil.copy(SAVINGS_BASE, mig_dir / SAVINGS_BASE.name) shutil.copytree(NODES_DIR, mig_dir / "nodes") + # OMN-15349: the runner now unconditionally requires the single-sourced + # operator fence manifest to exist under MIGRATIONS_DIR. + shutil.copy(FENCE_MANIFEST, mig_dir / FENCE_MANIFEST.name) env = _psql_env(config) env["MIGRATIONS_DIR"] = str(mig_dir) diff --git a/tests/integration/migrations/test_node_migration_shape_drift_omn15376.py b/tests/integration/migrations/test_node_migration_shape_drift_omn15376.py new file mode 100644 index 0000000000..1724dcd786 --- /dev/null +++ b/tests/integration/migrations/test_node_migration_shape_drift_omn15376.py @@ -0,0 +1,671 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Execution proof for the node-migration shape-drift class (OMN-15376). + +Runs the REAL vendored SQL through the SAME psql invocation the deploy-time +runners use (``psql -v ON_ERROR_STOP=1 -f ``) against a REAL Postgres, on +two paths: + +* **fresh** — empty database, the tables do not exist. +* **drifted** — every table the corpus creates already exists carrying ONLY its + first declared column, which is the shape class the live failure evidences: + ``relation "llm_cost_aggregates" already exists, skipping`` followed by + ``ERROR: column "aggregation_key" does not exist`` (deploy-onex-dev run + 30418878385, ``0001_create_llm_cost_aggregates.sql:64``). + +Three claims are proven by execution, not by inspection: + +1. **RED** — the migration WITHOUT its reconciliation block fails on the drifted + shape with the exact live error. The unfixed variant is derived by deleting + the ``BEGIN/END OMN-15376 shape reconciliation`` region from the real file, + so the RED is against "exists but wrong", never a hand-written surrogate. +2. **GREEN** — the real file succeeds on the same drifted shape. +3. **CONVERGENCE** — after the whole corpus applies on both paths, the two + schemas are byte-identical (columns + types + nullability + defaults + + constraints + indexes + views + triggers + RLS flags). A fix that merely + stops erroring, while leaving the drifted table a different shape from a + fresh one, would pass 1 and 2 and fail this. + +Fenced ids (OMN-14974 / OMN-15313 / OMN-15335) are excluded exactly as both +runners exclude them; the list is read from ``scripts/run-forward-migrations.sh`` +rather than restated. + +Postgres source, in order: an already-running server named by +``OMNIBASE_INFRA_DB_URL`` / ``POSTGRES_HOST`` (this is how the CI +``migration-integration`` job supplies one), else a hermetic ephemeral cluster +from local ``initdb``. Skips only when neither exists. + +Run: uv run pytest tests/integration/migrations/test_node_migration_shape_drift_omn15376.py -v + +Ticket: OMN-15376 (class), OMN-15302 (second live instance) +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import tempfile +import uuid +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import unquote, urlparse + +import pytest + +from tests.helpers.util_migration_shape import ( + fenced_migration_ids, + guarded_create_tables, + node_migration_files, +) + +pytestmark = [pytest.mark.integration, pytest.mark.postgres, pytest.mark.serial] + +RECONCILIATION_BEGIN = "-- ---- BEGIN OMN-15376 shape reconciliation:" +RECONCILIATION_END = "-- ---- END OMN-15376 shape reconciliation:" + +# Roles the corpus GRANTs to. They are environment-provisioned in every real +# lane (forward migration 094 / the RDS bootstrap), so the fixture provisions +# them too -- otherwise this suite would prove role tolerance, not shape drift. +SEED_ROLES = ("app_dashboard", "role_omnidash", "omninodeadmin") + +# The two live instances, pinned to the error text and file line the deploy +# printed. A change that moves either line without updating this is a signal. +LIVE_SIGNATURES = ( + pytest.param( + "node_projection_cost_summary", + "0001_create_llm_cost_aggregates.sql", + "llm_cost_aggregates", + "aggregation_key", + id="OMN-15376-llm_cost_aggregates.aggregation_key", + ), + pytest.param( + "node_projection_baselines", + "0001_create_baselines_tables.sql", + "baselines_comparisons", + "snapshot_id", + id="OMN-15302-baselines_comparisons.snapshot_id", + ), +) + + +def _pg_bin(name: str) -> str | None: + found = shutil.which(name) + if found: + return found + for prefix in sorted(Path("/opt/homebrew/opt").glob("postgresql@*"), reverse=True): + candidate = prefix / "bin" / name + if candidate.exists(): + return str(candidate) + return None + + +_PSQL = _pg_bin("psql") +_INITDB = _pg_bin("initdb") +_PG_CTL = _pg_bin("pg_ctl") + +# When a Postgres has been handed to this suite deliberately (the CI +# migration-integration job sets this), a skip is a vacuous green, so every +# skip path below becomes a hard failure instead. +_REQUIRE_PG = os.environ.get("OMN15376_REQUIRE_PG") == "1" + + +def _unavailable(reason: str) -> None: + if _REQUIRE_PG: + raise AssertionError(f"OMN15376_REQUIRE_PG=1 but {reason}") + pytest.skip(reason) + + +if _PSQL is None: # pragma: no cover - environment dependent + if os.environ.get("OMN15376_REQUIRE_PG") == "1": + raise AssertionError("OMN15376_REQUIRE_PG=1 but psql is not available") + pytest.skip("psql not available", allow_module_level=True) + + +@dataclass(frozen=True) +class Server: + """Connection coordinates for whichever Postgres this run got.""" + + host: str + port: str + user: str + password: str + + def env(self) -> dict[str, str]: + merged = dict(os.environ) + merged["PGPASSWORD"] = self.password + return merged + + +def _server_from_env() -> Server | None: + dsn = os.environ.get("OMNIBASE_INFRA_DB_URL", "") + if dsn: + parsed = urlparse(dsn) + if parsed.hostname: + return Server( + host=parsed.hostname, + port=str(parsed.port or 5432), + user=unquote(parsed.username or "postgres"), + password=unquote(parsed.password or ""), + ) + host = os.environ.get("POSTGRES_HOST") + if host: + return Server( + host=host, + port=os.environ.get("POSTGRES_PORT", "5432"), + user=os.environ.get("POSTGRES_USER", "postgres"), + password=os.environ.get("POSTGRES_PASSWORD", ""), + ) + return None + + +@pytest.fixture(scope="module") +def server() -> Iterator[Server]: + """A Postgres to talk to: the CI service if present, else a temp cluster.""" + external = _server_from_env() + if external is not None: + probe = subprocess.run( + [ + _PSQL, + "-X", + "-h", + external.host, + "-p", + external.port, + "-U", + external.user, + "-d", + "postgres", + "-tAc", + "SELECT 1", + ], + env=external.env(), + capture_output=True, + text=True, + check=False, + ) + if probe.returncode == 0: + yield external + return + + if _INITDB is None or _PG_CTL is None: # pragma: no cover + _unavailable( + "no reachable Postgres (OMNIBASE_INFRA_DB_URL/POSTGRES_HOST) and no " + "local initdb to build an ephemeral cluster" + ) + + root = Path(tempfile.mkdtemp(prefix="omn15376-pg-")) + sock = root / "sock" + sock.mkdir() + data = root / "data" + subprocess.run( + [_INITDB, "-D", str(data), "-U", "postgres", "-A", "trust"], + check=True, + capture_output=True, + ) + subprocess.run( + [ + _PG_CTL, + "-D", + str(data), + "-l", + str(root / "postgres.log"), + "-o", + f"-k {sock} -h '' -c listen_addresses=''", + "-w", + "start", + ], + check=True, + capture_output=True, + ) + try: + yield Server(host=str(sock), port="5432", user="postgres", password="") + finally: + subprocess.run( + [_PG_CTL, "-D", str(data), "-m", "immediate", "stop"], + check=False, + capture_output=True, + ) + shutil.rmtree(root, ignore_errors=True) + + +def _psql(srv: Server, database: str, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + _PSQL, + "-X", + "-q", + "-h", + srv.host, + "-p", + srv.port, + "-U", + srv.user, + "-d", + database, + *args, + ], + env=srv.env(), + capture_output=True, + text=True, + check=False, + ) + + +def _psql_script( + srv: Server, database: str, sql: str, **variables: str +) -> subprocess.CompletedProcess[str]: + """Run SQL from stdin so psql :'var' interpolation applies. + + ``psql -c`` sends its argument to the server verbatim and does NOT expand + ``:'var'`` (verified: ``syntax error at or near ":"``), so anything that + needs a value substituted has to arrive on stdin. + """ + args = [ + _PSQL, + "-X", + "-q", + "-t", + "-A", + "-h", + srv.host, + "-p", + srv.port, + "-U", + srv.user, + "-d", + database, + ] + for name, value in variables.items(): + args += ["-v", f"{name}={value}"] + args += ["-f", "-"] + return subprocess.run( + args, input=sql, env=srv.env(), capture_output=True, text=True, check=False + ) + + +def _new_database(srv: Server) -> str: + name = f"omn15376_{uuid.uuid4().hex[:12]}" + created = _psql(srv, "postgres", "-c", f"CREATE DATABASE {name}") + assert created.returncode == 0, created.stderr + for role in SEED_ROLES: + # Roles are CLUSTER-wide; a shared CI server may already have them. + # Passed as a psql variable rather than interpolated into the SQL text. + exists = _psql_script( + srv, + "postgres", + "SELECT 1 FROM pg_roles WHERE rolname = :'role'", + role=role, + ) + if "1" not in exists.stdout: + _psql_script( + srv, + "postgres", + 'CREATE ROLE :"role" NOSUPERUSER NOBYPASSRLS;', + role=role, + ) + return name + + +def _drop_database(srv: Server, name: str) -> None: + _psql(srv, "postgres", "-c", f"DROP DATABASE IF EXISTS {name} WITH (FORCE)") + + +# FENCE-PARITY GAP, found by running this suite (OMN-15379): +# omninode_infra's k8s Job runner fences SEVEN node ids; this repo's +# scripts/run-forward-migrations.sh fences SIX -- it is missing +# node:node_projection_registration:0002_node_service_registry_tenant_rls.sql. +# 0002 ALTERs node_service_registry, which only the FENCED 0000 creates, so on +# any lane where that table does not already exist the compose runner skips 0000 +# and then dies on 0002 with `relation "node_service_registry" does not exist`. +# That is a real wall and a separate ticket; it is NOT the shape-drift class and +# must not be what makes this suite red. Excluded here with its ticket, not +# silently: the fence itself is operator-gated and is not edited from this lane. +_K8S_ONLY_FENCED = frozenset( + {"node:node_projection_registration:0002_node_service_registry_tenant_rls.sql"} +) + + +def _corpus() -> list[tuple[str, Path]]: + fenced = fenced_migration_ids() | _K8S_ONLY_FENCED + return [ + (migration_id, path) + for migration_id, path in node_migration_files() + if migration_id not in fenced + ] + + +def _strip_reconciliation(sql: str) -> str: + """Delete every BEGIN/END reconciliation region -- the pre-fix variant.""" + pattern = re.compile( + re.escape(RECONCILIATION_BEGIN) + + r".*?" + + re.escape(RECONCILIATION_END) + + r"[^\n]*\n", + re.S, + ) + stripped, count = pattern.subn("", sql) + assert count > 0, "no reconciliation region found — RED would be vacuous" + return stripped + + +def _drift_seed_statements() -> list[str]: + """``CREATE TABLE`` for every corpus table, carrying only its first column.""" + statements: list[str] = [] + for _migration_id, path in _corpus(): + for table in guarded_create_tables(path.read_text(encoding="utf-8")): + if not table.columns or table.columns[0].generated: + continue + first = table.columns[0] + statements.append( + f"CREATE TABLE IF NOT EXISTS {table.qualified_name} " + f"({first.seed_ddl_fragment()});" + ) + return statements + + +def _apply_corpus(srv: Server, database: str) -> list[tuple[str, str]]: + return _apply_corpus_files(srv, database, _corpus()) + + +def _apply_corpus_files( + srv: Server, database: str, files: list[tuple[str, Path]] +) -> list[tuple[str, str]]: + failures: list[tuple[str, str]] = [] + for migration_id, path in files: + result = _psql(srv, database, "-v", "ON_ERROR_STOP=1", "-f", str(path)) + if result.returncode != 0: + output = result.stdout + result.stderr + line = next( + (ln for ln in output.splitlines() if "ERROR:" in ln), output[-400:] + ) + failures.append((migration_id, line)) + return failures + + +_SCHEMA_SNAPSHOT_SQL = """ +SELECT 'COL|'||c.relname||'|'||a.attname||'|' + ||format_type(a.atttypid, a.atttypmod)||'|'||a.attnotnull::text||'|' + ||coalesce(pg_get_expr(d.adbin, d.adrelid), '')||'|'||a.attgenerated::text +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = 'public' +JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped +LEFT JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum +WHERE c.relkind IN ('r', 'p', 'v', 'm') +UNION ALL +SELECT 'CON|'||c.relname||'|'||con.contype::text||'|' + ||pg_get_constraintdef(con.oid) +FROM pg_constraint con +JOIN pg_class c ON c.oid = con.conrelid +JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = 'public' +UNION ALL +SELECT 'IDX|'||tablename||'|'||indexdef FROM pg_indexes WHERE schemaname = 'public' +UNION ALL +SELECT 'VIEW|'||viewname||'|'||md5(definition) FROM pg_views WHERE schemaname = 'public' +UNION ALL +SELECT 'TRG|'||c.relname||'|'||t.tgname +FROM pg_trigger t +JOIN pg_class c ON c.oid = t.tgrelid +JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = 'public' +WHERE NOT t.tgisinternal +UNION ALL +SELECT 'RLS|'||c.relname||'|'||c.relrowsecurity::text||'|' + ||c.relforcerowsecurity::text +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = 'public' +WHERE c.relkind = 'r' +UNION ALL +SELECT 'POL|'||tablename||'|'||policyname||'|'||coalesce(qual, '') +FROM pg_policies WHERE schemaname = 'public' +ORDER BY 1 +""" + + +def _schema_snapshot(srv: Server, database: str) -> list[str]: + result = _psql( + srv, database, "-t", "-A", "-v", "ON_ERROR_STOP=1", "-c", _SCHEMA_SNAPSHOT_SQL + ) + assert result.returncode == 0, result.stderr + return sorted(line for line in result.stdout.splitlines() if line.strip()) + + +_CREATE_TYPE_RE = re.compile(r"CREATE\s+TYPE\s+([A-Za-z0-9_]+)\s+AS\s+", re.I) + + +def _migration_path(node: str, filename: str) -> Path: + return ( + Path(__file__).resolve().parents[3] + / "docker" + / "migrations" + / "forward" + / "nodes" + / node + / filename + ) + + +def _drifted_table_ddl(path: Path, table: str, omit_column: str) -> str: + """DDL for a pre-existing table that predates ``path``. + + Carries every declared column except the witness, and except any column + whose type this very migration defines -- a table that predates the + migration cannot be using an enum the migration itself creates. + """ + sql = path.read_text(encoding="utf-8") + own_types = {name.lower() for name in _CREATE_TYPE_RE.findall(sql)} + target = {t.bare_name: t for t in guarded_create_tables(sql)}[table] + kept = [ + column + for column in target.columns + if column.name.strip('"') != omit_column + and column.type_text.split("(")[0].strip().lower() not in own_types + and not column.generated + ] + assert kept, f"{table}: drift seed would be empty" + fragments = ", ".join(column.seed_ddl_fragment() for column in kept) + return f"CREATE TABLE {target.qualified_name} ({fragments});" + + +@pytest.mark.parametrize(("node", "filename", "table", "column"), LIVE_SIGNATURES) +def test_unfixed_migration_is_red_on_the_drifted_shape( + server: Server, node: str, filename: str, table: str, column: str +) -> None: + """RED: strip the reconciliation, seed the drift, get the live error back.""" + path = _migration_path(node, filename) + declared = { + c.name.strip('"') + for t in guarded_create_tables(path.read_text("utf-8")) + if t.bare_name == table + for c in t.columns + } + assert column in declared, f"{column} is not declared by {table} - premise moved" + + database = _new_database(server) + try: + seeded = _psql( + server, + database, + "-v", + "ON_ERROR_STOP=1", + "-c", + _drifted_table_ddl(path, table, column), + ) + assert seeded.returncode == 0, seeded.stderr + + unfixed = Path(tempfile.mkdtemp(prefix="omn15376-red-")) / filename + unfixed.write_text( + _strip_reconciliation(path.read_text("utf-8")), encoding="utf-8" + ) + result = _psql(server, database, "-v", "ON_ERROR_STOP=1", "-f", str(unfixed)) + output = result.stdout + result.stderr + + assert result.returncode != 0, ( + f"the UNFIXED {node}/{filename} succeeded against a table missing " + f"{column} — the RED premise is vacuous" + ) + assert f'column "{column}" does not exist' in output, output + assert f'relation "{table}" already exists, skipping' in output, output + finally: + _drop_database(server, database) + + +@pytest.mark.parametrize(("node", "filename", "table", "column"), LIVE_SIGNATURES) +def test_fixed_migration_is_green_on_the_same_drifted_shape( + server: Server, node: str, filename: str, table: str, column: str +) -> None: + """GREEN: the real file converges the same drifted table and exits 0.""" + path = _migration_path(node, filename) + database = _new_database(server) + try: + seeded = _psql( + server, + database, + "-v", + "ON_ERROR_STOP=1", + "-c", + _drifted_table_ddl(path, table, column), + ) + assert seeded.returncode == 0, seeded.stderr + result = _psql(server, database, "-v", "ON_ERROR_STOP=1", "-f", str(path)) + assert result.returncode == 0, result.stdout + result.stderr + + # Passed as psql variables, never interpolated into the SQL text. + present = _psql_script( + server, + database, + "SELECT count(*) FROM information_schema.columns " + "WHERE table_name = :'tbl' AND column_name = :'col';", + tbl=table, + col=column, + ) + assert present.stdout.strip() == "1", present.stdout + finally: + _drop_database(server, database) + + +def test_reconciliation_preserves_pre_existing_rows(server: Server) -> None: + """A drifted table with data keeps every row; nothing is dropped.""" + path = _migration_path( + "node_projection_cost_summary", "0001_create_llm_cost_aggregates.sql" + ) + database = _new_database(server) + try: + created = _psql( + server, + database, + "-v", + "ON_ERROR_STOP=1", + "-c", + _drifted_table_ddl(path, "llm_cost_aggregates", "aggregation_key"), + ) + assert created.returncode == 0, created.stderr + seeded = _psql( + server, + database, + "-v", + "ON_ERROR_STOP=1", + "-c", + "INSERT INTO llm_cost_aggregates (total_cost_usd, total_tokens, " + "call_count) VALUES (1.5, 10, 2), (2.5, 20, 4);", + ) + assert seeded.returncode == 0, seeded.stderr + + # aggregation_key is NOT NULL with no DEFAULT: on a POPULATED drifted + # table it cannot be converged without inventing data, so the migration + # must refuse LOUDLY and name the conflict rather than guess. + result = _psql(server, database, "-v", "ON_ERROR_STOP=1", "-f", str(path)) + output = result.stdout + result.stderr + assert result.returncode != 0, output + assert "OMN-15376" in output, output + assert "aggregation_key" in output, output + assert "data ruling" in output, output + + rows = _psql( + server, + database, + "-t", + "-A", + "-v", + "ON_ERROR_STOP=1", + "-c", + "SELECT count(*) FROM llm_cost_aggregates", + ) + assert rows.stdout.strip() == "2", rows.stdout + finally: + _drop_database(server, database) + + +def test_whole_corpus_converges_from_drifted_shapes(server: Server) -> None: + """The load-bearing claim: fresh and drifted end at the SAME schema.""" + corpus = _corpus() + seeds = _drift_seed_statements() + assert len(corpus) >= 60, len(corpus) + assert len(seeds) >= 40, len(seeds) + + fresh_db = _new_database(server) + drift_db = _new_database(server) + try: + fresh_failures = _apply_corpus(server, fresh_db) + assert not fresh_failures, fresh_failures + + for statement in seeds: + seeded = _psql(server, drift_db, "-v", "ON_ERROR_STOP=1", "-c", statement) + assert seeded.returncode == 0, f"{statement}\n{seeded.stderr}" + drift_failures = _apply_corpus(server, drift_db) + assert not drift_failures, drift_failures + + fresh_schema = _schema_snapshot(server, fresh_db) + drift_schema = _schema_snapshot(server, drift_db) + only_fresh = [row for row in fresh_schema if row not in set(drift_schema)] + only_drift = [row for row in drift_schema if row not in set(fresh_schema)] + assert not only_fresh and not only_drift, ( + f"fresh-only={only_fresh[:20]}\ndrift-only={only_drift[:20]}" + ) + finally: + _drop_database(server, fresh_db) + _drop_database(server, drift_db) + + +def test_fresh_path_schema_is_unchanged_by_the_reconciliation(server: Server) -> None: + """The reconciliation is a strict no-op on an empty database. + + This is the claim that keeps the blast radius honest: 46 vendored migrations + gained ~5k lines of guarded DDL, and every one of those lines must be + inert when the table is being created fresh. Proven by applying the corpus + twice on two empty databases -- once with the BEGIN/END reconciliation + regions stripped out, once as committed -- and asserting the resulting + schemas are identical. Nothing downstream of a projection table (golden + chains, handler column expectations, dashboard reads) can have moved if + this holds. + """ + corpus = _corpus() + stripped_dir = Path(tempfile.mkdtemp(prefix="omn15376-nofix-")) + stripped: list[tuple[str, Path]] = [] + stripped_count = 0 + for migration_id, path in corpus: + original = path.read_text(encoding="utf-8") + target = stripped_dir / path.parent.name / path.name + target.parent.mkdir(parents=True, exist_ok=True) + if RECONCILIATION_BEGIN in original: + target.write_text(_strip_reconciliation(original), encoding="utf-8") + stripped_count += 1 + else: + target.write_text(original, encoding="utf-8") + stripped.append((migration_id, target)) + assert stripped_count >= 40, stripped_count + + with_fix_db = _new_database(server) + without_fix_db = _new_database(server) + try: + assert not _apply_corpus(server, with_fix_db) + assert not _apply_corpus_files(server, without_fix_db, stripped) + assert _schema_snapshot(server, with_fix_db) == _schema_snapshot( + server, without_fix_db + ) + finally: + _drop_database(server, with_fix_db) + _drop_database(server, without_fix_db) + shutil.rmtree(stripped_dir, ignore_errors=True) diff --git a/tests/integration/nodes/test_runtime_manifest_reducer_integration.py b/tests/integration/nodes/test_runtime_manifest_reducer_integration.py index 7274b0531c..afff330408 100644 --- a/tests/integration/nodes/test_runtime_manifest_reducer_integration.py +++ b/tests/integration/nodes/test_runtime_manifest_reducer_integration.py @@ -45,6 +45,13 @@ / "forward" / "079_create_runtime_manifests.sql" ) +_ATTACH_MIGRATION_PATH = ( + _REPO_ROOT + / "docker" + / "migrations" + / "forward" + / "095_add_attach_readiness_to_runtime_manifests.sql" +) def _valid_payload( @@ -98,6 +105,10 @@ def test_sql_template_names_every_payload_column() -> None: Drift here would silently drop fields on insert or break with a Postgres column-mismatch error after deploy. + + ``attach_readiness`` (OMN-15512) is the one payload field that is NOT a + column: the handler flattens the aggregate across four columns, so it is + asserted through ``_ATTACH_READINESS_COLUMNS`` below rather than by name. """ expected_columns = { "runtime_profile", @@ -118,6 +129,46 @@ def test_sql_template_names_every_payload_column() -> None: for col in expected_columns: assert col.lower() in sql, f"SQL template missing column {col!r}" + flattened = {"attach_readiness"} + modelled = set(ModelPayloadInsertRuntimeManifest.model_fields) - {"intent_type"} + assert modelled == expected_columns | flattened, ( + "payload model fields drifted from the SQL column set — add the column " + "(or extend the flattened set with a stated reason)" + ) + + +# OMN-15512: the four columns the attach-readiness aggregate flattens into. +_ATTACH_READINESS_COLUMNS = ( + "attach_state", + "attach_required_contracts", + "attach_attached_contracts", + "attach_not_ready_contracts", +) + + +def test_sql_template_writes_the_attach_readiness_columns() -> None: + """The blocker set must actually be written, not merely modelled (OMN-15512).""" + sql = SQL_INSERT_RUNTIME_MANIFEST.lower() + for col in _ATTACH_READINESS_COLUMNS: + assert col in sql, f"SQL template missing column {col!r}" + + +def test_attach_readiness_columns_exist_in_a_migration() -> None: + """Every column the INSERT names must be created by a forward migration. + + Migration 079 creates the table; 095 adds the attach-readiness columns. + A column present in the INSERT but absent from DDL fails only at runtime, + after deploy. + """ + if not _MIGRATION_PATH.is_file() or not _ATTACH_MIGRATION_PATH.is_file(): + pytest.skip("migration files not present") + ddl = ( + _MIGRATION_PATH.read_text(encoding="utf-8") + + _ATTACH_MIGRATION_PATH.read_text(encoding="utf-8") + ).lower() + for col in _ATTACH_READINESS_COLUMNS: + assert col in ddl, f"no migration creates column {col!r}" + def test_migration_creates_table_and_unique_index() -> None: """The handler uses ON CONFLICT DO NOTHING on (runtime_profile, topology_hash, started_at). diff --git a/tests/integration/projectors/test_handler_wiring_projection_dispatch.py b/tests/integration/projectors/test_handler_wiring_projection_dispatch.py index c33126176f..5df68743f4 100644 --- a/tests/integration/projectors/test_handler_wiring_projection_dispatch.py +++ b/tests/integration/projectors/test_handler_wiring_projection_dispatch.py @@ -17,9 +17,17 @@ from omnibase_infra.runtime.auto_wiring.handler_wiring import ( _make_projection_dispatch_callback, ) +from tests.helpers.application_db_topology import projection_database_target + + +@pytest.fixture(autouse=True) +def _configured_projection_dsn(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OMNIDASH_ANALYTICS_DB_URL", "postgresql://fixture") + monkeypatch.setenv("OMNINODE_INTERNAL_DB_URL", "postgresql://fixture") + _PATCH_BUILD_ADAPTER = ( - "omnibase_infra.runtime.auto_wiring.handler_wiring._build_sync_db_adapter" + "omnibase_infra.runtime.auto_wiring.handler_wiring._build_projection_db_adapter" ) _PATCH_ENVIRON_GET = "omnibase_infra.runtime.auto_wiring.handler_wiring.os.environ.get" @@ -39,7 +47,9 @@ def handle(self, input_data: dict) -> dict: received.append(dict(input_data)) return {"rows_upserted": 1} - db_tables = [{"name": "node_service_registry", "database": "omnidash_analytics"}] + db_tables = projection_database_target( + "node_service_registry", schema="omninode_internal" + ) handler = FakeProjectionHandler() callback = _make_projection_dispatch_callback( handler, db_tables, ("onex.evt.platform.node-heartbeat.v1",) @@ -86,7 +96,7 @@ def handle(self, input_data: dict) -> dict: received.append(dict(input_data)) return {"rows_upserted": 1} - db_tables = [{"name": "delegation_events", "database": "omnidash_analytics"}] + db_tables = projection_database_target("delegation_events") handler = FakeDelegationHandler() callback = _make_projection_dispatch_callback( handler, db_tables, ("onex.evt.omniclaude.task-delegated.v1",) @@ -119,11 +129,10 @@ def handle(self, input_data: dict) -> dict: @pytest.mark.integration -def test_projection_dispatch_bridge_no_call_when_db_url_missing() -> None: - """Projection handler is NOT called when OMNIDASH_ANALYTICS_DB_URL is unset. - - Verifies no silent error occurs — optional projection handler is skipped. - """ +def test_projection_dispatch_bridge_rejects_missing_db_url_at_wiring( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A missing required DSN fails before the projection can dispatch.""" call_count = [0] class FakeProjectionHandler: @@ -131,16 +140,11 @@ def handle(self, input_data: dict) -> dict: call_count[0] += 1 return {} - db_tables = [{"name": "node_service_registry", "database": "omnidash_analytics"}] + db_tables = projection_database_target("delegation_events", schema="tenant") handler = FakeProjectionHandler() - callback = _make_projection_dispatch_callback(handler, db_tables, ()) - - envelope = MagicMock() - envelope.topic = "onex.evt.platform.node-heartbeat.v1" - envelope.payload = {} + monkeypatch.delenv("OMNIDASH_ANALYTICS_DB_URL") - with patch(_PATCH_ENVIRON_GET, return_value=""): - result = asyncio.run(callback(envelope)) + with pytest.raises(ValueError, match="tenant_projection"): + _make_projection_dispatch_callback(handler, db_tables, ()) - assert result is None assert call_count[0] == 0, "Handler must not be called when DB URL is absent" diff --git a/tests/integration/projectors/test_projection_terminal_event_emission.py b/tests/integration/projectors/test_projection_terminal_event_emission.py index 84fda81167..177b2a67db 100644 --- a/tests/integration/projectors/test_projection_terminal_event_emission.py +++ b/tests/integration/projectors/test_projection_terminal_event_emission.py @@ -21,14 +21,21 @@ ProjectionDispatchSinks, _make_projection_dispatch_callback, ) +from tests.helpers.application_db_topology import projection_database_target + + +@pytest.fixture(autouse=True) +def _configured_projection_dsn(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OMNIDASH_ANALYTICS_DB_URL", "postgresql://fixture") + _PATCH_BUILD_ADAPTER = ( - "omnibase_infra.runtime.auto_wiring.handler_wiring._build_sync_db_adapter" + "omnibase_infra.runtime.auto_wiring.handler_wiring._build_projection_db_adapter" ) _PATCH_ENVIRON_GET = "omnibase_infra.runtime.auto_wiring.handler_wiring.os.environ.get" TERMINAL_TOPIC = "onex.evt.omnimarket.projection-delegation-applied.v1" -DB_TABLES = [{"name": "delegation_events", "database": "omnidash_analytics"}] +DB_TARGET = projection_database_target("delegation_events") SUBSCRIBE_TOPICS = ("onex.evt.omniclaude.task-delegated.v1",) @@ -53,7 +60,7 @@ async def publish(self, topic: str, key: object, value: bytes) -> None: callback = _make_projection_dispatch_callback( FakeDelegationHandler(), - DB_TABLES, + DB_TARGET, SUBSCRIBE_TOPICS, sinks=ProjectionDispatchSinks( event_bus=FakeEventBus(), @@ -102,7 +109,7 @@ async def publish(self, topic: str, key: object, value: bytes) -> None: callback = _make_projection_dispatch_callback( FailingHandler(), - DB_TABLES, + DB_TARGET, SUBSCRIBE_TOPICS, sinks=ProjectionDispatchSinks( event_bus=FakeEventBus(), @@ -151,7 +158,7 @@ def handle(self, input_data: dict) -> dict: callback = _make_projection_dispatch_callback( FakeDelegationHandler(), - DB_TABLES, + DB_TARGET, SUBSCRIBE_TOPICS, sinks=ProjectionDispatchSinks( event_bus=None, diff --git a/tests/integration/registration/test_registration_orchestrator_all_topics_subscribed.py b/tests/integration/registration/test_registration_orchestrator_all_topics_subscribed.py index 0053494ff5..1c80e75f46 100644 --- a/tests/integration/registration/test_registration_orchestrator_all_topics_subscribed.py +++ b/tests/integration/registration/test_registration_orchestrator_all_topics_subscribed.py @@ -5,17 +5,15 @@ Regression test for OMN-9413: contract.yaml declares 7 subscribe_topics but only 3 consumer groups were wired at runtime. -Root cause: the _prepare_handler_wiring special-case for node_registration_orchestrator -skips all handler dispatchers (RESOLVED_VIA_LOCAL_OWNERSHIP_SKIP) so the generic -contract auto-wiring path owns topic subscription. This test verifies that all 7 -declared subscribe_topics receive Kafka consumer subscriptions via -subscribe_wired_contract_topics. +The generic contract auto-wiring path owns these topic subscriptions. This test +verifies that all 7 declared subscribe_topics receive Kafka consumer +subscriptions via subscribe_wired_contract_topics. """ from __future__ import annotations from pathlib import Path -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest @@ -69,10 +67,8 @@ async def test_registration_orchestrator_all_declared_topics_get_subscribed( ) -> None: """All 7 contract subscribe_topics must receive a Kafka subscription. - This is the regression test for OMN-9413. wire_from_manifest skips - handler dispatchers for the orchestrator (all handlers are - RESOLVED_VIA_LOCAL_OWNERSHIP_SKIP) but subscribe_wired_contract_topics - must still subscribe every declared topic. + This is the regression test for OMN-9413. The generic auto-wiring path must + subscribe every declared topic after registering the contract's dispatchers. """ assert _CONTRACT_PATH.exists(), f"Contract not found: {_CONTRACT_PATH}" manifest = discover_contracts_from_paths([_CONTRACT_PATH]) @@ -87,6 +83,10 @@ async def test_registration_orchestrator_all_declared_topics_get_subscribed( dispatch_engine = MagicMock() dispatch_engine.is_frozen = True dispatch_engine._routes = {} + dispatch_engine.dispatch_scoped = AsyncMock(return_value=None) + dispatch_engine.validate_contract_dispatcher_scope = MagicMock( + side_effect=lambda _contract_name, dispatcher_ids: frozenset(dispatcher_ids) + ) monkeypatch.setenv("RUNTIME_PROFILE", "main") diff --git a/tests/integration/runtime/conftest.py b/tests/integration/runtime/conftest.py new file mode 100644 index 0000000000..f0d8c4aba1 --- /dev/null +++ b/tests/integration/runtime/conftest.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Re-export the ephemeral Postgres fixture for tests under this directory. + +``tests/integration/migrations/conftest.py`` owns the real fixture +(``EphemeralPostgres`` / ``ephemeral_postgres``); pytest only auto-discovers +fixtures from a file's own directory and its ancestors, not sibling +directories, so a live-database test under ``tests/integration/runtime/`` +needs this thin re-export to see it. +""" + +from __future__ import annotations + +from tests.integration.migrations.conftest import ( + EphemeralPostgres, + ephemeral_postgres, +) diff --git a/tests/integration/runtime/test_async_container_pre_resolution_integration.py b/tests/integration/runtime/test_async_container_pre_resolution_integration.py index 4586edbe7e..da2845ff0e 100644 --- a/tests/integration/runtime/test_async_container_pre_resolution_integration.py +++ b/tests/integration/runtime/test_async_container_pre_resolution_integration.py @@ -101,6 +101,14 @@ async def test_async_pre_resolution_succeeds_when_sync_raises_runtime_error() -> dispatch_engine._container = None dispatch_engine.register_dispatcher = MagicMock() dispatch_engine.register_route = MagicMock() + # OMN-15474: contract-scoped subscription proves its dispatcher scope + # against the live engine. A bare MagicMock iterates EMPTY, which reads as + # a zero-owner scope and refuses the attach; mirror the real engine by + # echoing the scope back. + dispatch_engine.dispatch_scoped = AsyncMock() + dispatch_engine.validate_contract_dispatcher_scope = MagicMock( + side_effect=lambda _contract_name, dispatcher_ids: frozenset(dispatcher_ids) + ) dispatch_engine.freeze = MagicMock() with patch( @@ -170,6 +178,12 @@ async def handle(self, envelope: object) -> None: dispatch_engine = MagicMock() dispatch_engine._routes = {} dispatch_engine._container = None + # OMN-15474: see the note above — echo the scope back so the scoped-attach + # check sees a real owner set instead of MagicMock's empty iteration. + dispatch_engine.dispatch_scoped = AsyncMock() + dispatch_engine.validate_contract_dispatcher_scope = MagicMock( + side_effect=lambda _contract_name, dispatcher_ids: frozenset(dispatcher_ids) + ) with patch( "omnibase_infra.runtime.auto_wiring.handler_wiring._import_handler_class", diff --git a/tests/integration/runtime/test_attach_readiness_publish_seam.py b/tests/integration/runtime/test_attach_readiness_publish_seam.py new file mode 100644 index 0000000000..7efdaffef0 --- /dev/null +++ b/tests/integration/runtime/test_attach_readiness_publish_seam.py @@ -0,0 +1,589 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Cross-boundary regression: the NOT-READY blocker set must LEAVE the runtime. + +OMN-15512. The defect this guards is NOT "``ModelRuntimeAttachReadiness`` computes +the wrong thing" — it computes the right thing and always did (OMN-13237). The +defect is that the aggregate never left ``service_kernel``: it was a local +variable consumed by exactly one ``logger.info``, so the only way to read the +blocker set was ``ssh`` + ``docker logs omninode-runtime | grep NOT-READY``, +which is literally how the parent OMN-15508 had to be diagnosed. + +So a unit test on ``ModelRuntimeAttachReadiness.from_results`` would be vacuous +here — it would pass both before and after the fix. These tests drive the whole +chain end to end with a contract whose required topic is ABSENT: + + real wire_from_manifest + -> real subscribe_wired_contract_topics (provision -> confirm-ready -> attach) + -> real publish_runtime_manifest (the function the kernel calls) + -> the envelope captured off a recording bus + -> real ModelPayloadInsertRuntimeManifest coercion of that wire payload + -> real HandlerPostgresRuntimeManifestInsert SQL arguments + +and assert the absent topic is NAMED at the far end. Everything in that chain is +the artifact that runs; the only doubles are the event bus, the topic +provisioner, and the asyncpg pool — the three genuine external boundaries. + +RED-before proof lives in ``TestGuardActuallyGuards``: the same chain run +WITHOUT threading the aggregate reproduces the pre-fix shape (payload +``attach_readiness is None``, SQL writes ``'unknown'`` / ``[]``) and fails every +assertion the fixed path makes. That discriminates against exists-but-wrong +rather than against a surrogate. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import TYPE_CHECKING +from unittest.mock import AsyncMock, MagicMock, patch +from uuid import UUID, uuid4 + +import pytest + +from omnibase_infra.event_bus.enum_contract_attach_status import ( + EnumContractAttachStatus, +) +from omnibase_infra.event_bus.enum_runtime_readiness_state import ( + EnumRuntimeReadinessState, +) +from omnibase_infra.event_bus.enum_topic_readiness_failure_reason import ( + EnumTopicReadinessFailureReason, +) +from omnibase_infra.event_bus.enum_topic_readiness_status import ( + EnumTopicReadinessStatus, +) +from omnibase_infra.event_bus.model_contract_attach_result import ( + ModelContractAttachResult, +) +from omnibase_infra.event_bus.model_runtime_attach_readiness import ( + ModelRuntimeAttachReadiness, +) +from omnibase_infra.event_bus.model_topic_readiness_config import ( + ModelTopicReadinessConfig, +) +from omnibase_infra.event_bus.model_topic_readiness_failure import ( + ModelTopicReadinessFailure, +) +from omnibase_infra.event_bus.model_topic_set_readiness import ModelTopicSetReadiness +from omnibase_infra.nodes.node_runtime_manifest_reducer.handlers.handler_postgres_runtime_manifest_insert import ( + SQL_INSERT_RUNTIME_MANIFEST, + HandlerPostgresRuntimeManifestInsert, +) +from omnibase_infra.nodes.node_runtime_manifest_reducer.models.model_payload_insert_runtime_manifest import ( + ModelPayloadInsertRuntimeManifest, +) +from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + subscribe_wired_contract_topics, + wire_from_manifest, +) +from omnibase_infra.runtime.auto_wiring.models import ( + ModelAutoWiringManifest, + ModelContractVersion, + ModelDiscoveredContract, + ModelEventBusWiring, + ModelHandlerRef, + ModelHandlerRouting, + ModelHandlerRoutingEntry, +) +from omnibase_infra.runtime.manifest_builder import publish_runtime_manifest +from omnibase_infra.runtime.message_dispatch_engine import MessageDispatchEngine +from omnibase_infra.runtime.models.model_runtime_manifest_published import ( + ModelRuntimeManifestPublished, +) + +if TYPE_CHECKING: + from omnibase_infra.runtime.auto_wiring.report import ModelAutoWiringReport + +pytestmark = pytest.mark.integration + +# The contract that CAN attach, and the one whose required topic is absent. +ATTACHING_CONTRACT = "node_alpha" +ATTACHING_TOPIC = "onex.evt.omnibase-infra.seam-alpha.v1" +BLOCKED_CONTRACT = "node_beta" +BLOCKED_TOPIC = "onex.evt.omnibase-infra.seam-beta-absent.v1" + +MANIFEST_TOPIC = "onex.evt.omnibase-infra.runtime-manifest-published.v1" + + +# --------------------------------------------------------------------------- +# Boundary doubles: bus, provisioner, asyncpg pool. Nothing else is faked. +# --------------------------------------------------------------------------- + + +class _AbsentTopicProvisioner: + """Topic provisioner where ``absent_topics`` never converge. + + Mirrors the live failure: the runtime creates the topic, then the metadata + readiness confirm does not converge for it, so the consumer attach is + skipped and the contract is recorded NOT_READY (OMN-13237). + """ + + def __init__(self, *, absent_topics: frozenset[str]) -> None: + self._absent = absent_topics + + async def ensure_topic_exists( + self, + topic_name: str, + spec: object | None = None, + correlation_id: UUID | None = None, + ) -> bool: + return True + + async def confirm_topics_ready( + self, + topics: Sequence[str], + *, + expected_specs: Mapping[str, object] | None = None, + config: ModelTopicReadinessConfig | None = None, + correlation_id: UUID | None = None, + ) -> ModelTopicSetReadiness: + unready = [t for t in topics if t in self._absent] + if not unready: + return ModelTopicSetReadiness( + topics=tuple(topics), + status=EnumTopicReadinessStatus.READY, + ready_topics=tuple(topics), + attempts=1, + ) + return ModelTopicSetReadiness( + topics=tuple(topics), + status=EnumTopicReadinessStatus.NOT_READY, + ready_topics=tuple(t for t in topics if t not in unready), + failures=tuple( + ModelTopicReadinessFailure( + topic=t, + reason=EnumTopicReadinessFailureReason.TOPIC_ABSENT, + ) + for t in unready + ), + attempts=1, + ) + + +class _RecordingBus: + """Event bus double that captures every published envelope.""" + + def __init__(self) -> None: + self.published: list[tuple[str, object]] = [] + + async def subscribe( + self, + *, + topic: str, + node_identity: object, + on_message: object, + ) -> object: + async def _unsub() -> None: + return None + + return _unsub + + async def publish_envelope( + self, + envelope: object, + topic: str, + *, + key: bytes | None = None, + ) -> None: + self.published.append((topic, envelope)) + + +def _make_pool() -> MagicMock: + pool = MagicMock() + conn = AsyncMock() + record = MagicMock() + record.__getitem__ = MagicMock(side_effect=lambda k: 1 if k == "id" else None) + conn.fetchrow = AsyncMock(return_value=record) + ctx = AsyncMock() + ctx.__aenter__ = AsyncMock(return_value=conn) + ctx.__aexit__ = AsyncMock(return_value=False) + pool.acquire = MagicMock(return_value=ctx) + pool._test_conn = conn + return pool + + +# --------------------------------------------------------------------------- +# Chain drivers +# --------------------------------------------------------------------------- + + +def _contract(name: str, topic: str) -> ModelDiscoveredContract: + return ModelDiscoveredContract( + name=name, + node_type="ORCHESTRATOR_GENERIC", + contract_version=ModelContractVersion(major=1, minor=0, patch=0), + contract_path=Path("/fake/contract.yaml"), + entry_point_name=name, + package_name="test-package", + event_bus=ModelEventBusWiring(subscribe_topics=(topic,), publish_topics=()), + handler_routing=ModelHandlerRouting( + routing_strategy="payload_type_match", + handlers=( + ModelHandlerRoutingEntry( + handler=ModelHandlerRef(name="FakeHandler", module="fake.module"), + event_model=None, + operation=None, + ), + ), + ), + ) + + +def _fake_handler_cls() -> type: + class FakeHandler: + async def handle(self, envelope: object) -> None: + return None + + return FakeHandler + + +async def _run_boot_interleave() -> tuple[ + ModelAutoWiringManifest, + ModelAutoWiringReport, + _RecordingBus, + ModelRuntimeAttachReadiness, +]: + """Run the REAL provision -> confirm-ready -> attach interleave. + + ``BLOCKED_TOPIC`` never converges, so ``BLOCKED_CONTRACT`` is recorded + NOT_READY exactly as it is on a cold lane. + """ + manifest = ModelAutoWiringManifest( + contracts=( + _contract(ATTACHING_CONTRACT, ATTACHING_TOPIC), + _contract(BLOCKED_CONTRACT, BLOCKED_TOPIC), + ) + ) + engine = MessageDispatchEngine() + bus = _RecordingBus() + attach_results: list[ModelContractAttachResult] = [] + + with patch( + "omnibase_infra.runtime.auto_wiring.handler_wiring._import_handler_class", + return_value=_fake_handler_cls(), + ): + report = await wire_from_manifest( + manifest, + engine, + event_bus=bus, + environment="local", + subscribe_immediately=False, + ) + await subscribe_wired_contract_topics( + manifest=manifest, + report=report, + dispatch_engine=engine, + event_bus=bus, + environment="local", + provisioner=_AbsentTopicProvisioner( + absent_topics=frozenset({BLOCKED_TOPIC}) + ), + readiness_config=ModelTopicReadinessConfig( + max_concurrent_contract_attach=1 + ), + attach_results_out=attach_results, + ) + + readiness = ModelRuntimeAttachReadiness.from_results(tuple(attach_results)) + return manifest, report, bus, readiness + + +async def _publish( + *, + thread_readiness: bool, +) -> tuple[_RecordingBus, ModelRuntimeManifestPublished]: + """Drive the boot interleave then the REAL kernel publish function. + + ``thread_readiness=False`` reproduces the pre-OMN-15512 shape, where the + aggregate was computed and then dropped. + """ + manifest, report, bus, readiness = await _run_boot_interleave() + published = await publish_runtime_manifest( + event_bus=bus, + report=report, + manifest=manifest, + runtime_profile="seam-test", + topic=MANIFEST_TOPIC, + correlation_id=uuid4(), + image_digest=None, + attach_readiness=readiness if thread_readiness else None, + ) + return bus, published + + +def _captured_manifest_payload(bus: _RecordingBus) -> dict[str, object]: + """The wire-shape payload as a consumer off the topic would see it.""" + manifest_events = [e for t, e in bus.published if t == MANIFEST_TOPIC] + assert len(manifest_events) == 1, ( + f"expected exactly one runtime-manifest publish, got {len(manifest_events)}" + ) + envelope = manifest_events[0] + dumped = envelope.model_dump(mode="json") # type: ignore[attr-defined] + payload = dumped["payload"] + assert isinstance(payload, dict) + return payload + + +async def _sql_args_for(payload: dict[str, object]) -> tuple[object, ...]: + """Coerce the wire payload into the intent model and run the real handler. + + ``ModelPayloadInsertRuntimeManifest`` is ``extra="forbid"``: if the producer + key name and the consumer field name ever drift apart, this coercion raises + instead of silently dropping the blocker set. + """ + intent_payload = ModelPayloadInsertRuntimeManifest( + runtime_profile=str(payload["runtime_profile"]), + contract_hash=str(payload["contract_hash"]), + topology_hash=str(payload["topology_hash"]), + manifest_hash=str(payload["topology_hash"]), + contracts=payload["contracts"], # type: ignore[arg-type] + owned_command_topics=payload["owned_command_topics"], # type: ignore[arg-type] + subscribed_event_topics=payload["subscribed_event_topics"], # type: ignore[arg-type] + handlers=payload["handlers"], # type: ignore[arg-type] + skipped_contracts=payload["skipped_contracts"], # type: ignore[arg-type] + failed_contracts=payload["failed_contracts"], # type: ignore[arg-type] + ownership_violations=payload["ownership_violations"], # type: ignore[arg-type] + image_digest=None, + started_at=payload["started_at"], # type: ignore[arg-type] + attach_readiness=payload["attach_readiness"], # type: ignore[arg-type] + ) + pool = _make_pool() + result = await HandlerPostgresRuntimeManifestInsert(pool).handle( + intent_payload, uuid4() + ) + assert result.success is True, result.error + args: tuple[object, ...] = pool._test_conn.fetchrow.call_args[0] + assert args[0] == SQL_INSERT_RUNTIME_MANIFEST + return args + + +# --------------------------------------------------------------------------- +# The published envelope carries the blocker set (OMN-15512 AC1 / AC4) +# --------------------------------------------------------------------------- + + +class TestPublishedEnvelopeNamesTheAbsentTopic: + @pytest.mark.asyncio + async def test_blocked_contract_is_not_ready_in_published_payload(self) -> None: + """AC4: the contract whose required topic is absent appears, by name.""" + bus, _published = await _publish(thread_readiness=True) + payload = _captured_manifest_payload(bus) + + readiness = payload["attach_readiness"] + assert readiness is not None, ( + "attach_readiness missing from the published payload — the aggregate " + "did not leave service_kernel, which IS the OMN-15512 defect" + ) + assert isinstance(readiness, dict) + + blocked = [ + r for r in readiness["results"] if r["contract_name"] == BLOCKED_CONTRACT + ] + assert len(blocked) == 1, ( + f"{BLOCKED_CONTRACT} absent from the published NOT-READY set: " + f"{readiness['results']}" + ) + assert blocked[0]["status"] == EnumContractAttachStatus.NOT_READY.value + + @pytest.mark.asyncio + async def test_published_blocker_names_the_failing_topic(self) -> None: + """AC1: the failing TOPIC survives, not just the contract name. + + A blocker set that says "node_beta did not attach" without naming the + topic is not a replacement for the log grep — the operator still has to + go read logs to find out which topic. The topic string is the payload. + """ + bus, _published = await _publish(thread_readiness=True) + payload = _captured_manifest_payload(bus) + readiness = payload["attach_readiness"] + assert isinstance(readiness, dict) + + blocked = next( + r for r in readiness["results"] if r["contract_name"] == BLOCKED_CONTRACT + ) + assert blocked["readiness"] is not None, ( + "readiness detail was dropped in serialization — the topic-level " + "failure is exactly what makes this queryable" + ) + failing_topics = {f["topic"] for f in blocked["readiness"]["failures"]} + assert BLOCKED_TOPIC in failing_topics, ( + f"absent topic {BLOCKED_TOPIC!r} not named in the published blocker; " + f"got {failing_topics}" + ) + assert ( + blocked["readiness"]["status"] == EnumTopicReadinessStatus.NOT_READY.value + ) + + @pytest.mark.asyncio + async def test_attaching_contract_is_not_in_the_blocker_set(self) -> None: + """The blocker set is the NON-attached subset, not every contract.""" + bus, _published = await _publish(thread_readiness=True) + payload = _captured_manifest_payload(bus) + readiness = payload["attach_readiness"] + assert isinstance(readiness, dict) + + names = {r["contract_name"] for r in readiness["results"]} + assert BLOCKED_CONTRACT in names + assert ATTACHING_CONTRACT not in names + + @pytest.mark.asyncio + async def test_counts_are_whole_walk_not_blocker_length(self) -> None: + """Narrowing ``results`` must not corrupt the counts. + + ``required - attached == len(results)`` is the invariant that lets a + reader reconstruct the full picture from the narrowed copy. + """ + bus, published = await _publish(thread_readiness=True) + payload = _captured_manifest_payload(bus) + readiness = payload["attach_readiness"] + assert isinstance(readiness, dict) + + assert readiness["required_contracts"] == 2 + assert readiness["attached_contracts"] == 1 + assert readiness["state"] == EnumRuntimeReadinessState.DEGRADED.value + assert ( + readiness["required_contracts"] - readiness["attached_contracts"] + == len(readiness["results"]) + == 1 + ) + assert published.attach_readiness is not None + assert published.attach_readiness.attached_contracts == 1 + + @pytest.mark.asyncio + async def test_payload_stays_wire_compatible_with_the_base_manifest(self) -> None: + """Additive only: every pre-existing manifest key is still present. + + ``node_redeploy_orchestrator`` (omnimarket) also subscribes to this + topic. Adding a key must not remove or rename one. + """ + bus, _published = await _publish(thread_readiness=True) + payload = _captured_manifest_payload(bus) + + for key in ( + "runtime_profile", + "contracts", + "owned_command_topics", + "subscribed_event_topics", + "handlers", + "skipped_contracts", + "failed_contracts", + "ownership_violations", + "image_digest", + "started_at", + "contract_hash", + "topology_hash", + ): + assert key in payload, f"published payload lost base manifest key {key!r}" + + @pytest.mark.asyncio + async def test_attach_readiness_does_not_move_the_topology_hash(self) -> None: + """The dedup/drift hashes must not shift because of this field. + + ``runtime_manifests`` dedups on ``topology_hash``; if the new field fed + the hash, every historical comparison would break. + """ + _bus_with, published_with = await _publish(thread_readiness=True) + _bus_without, published_without = await _publish(thread_readiness=False) + assert published_with.topology_hash == published_without.topology_hash + assert published_with.contract_hash == published_without.contract_hash + + +# --------------------------------------------------------------------------- +# The blocker set survives into the projection write (OMN-15512 AC2) +# --------------------------------------------------------------------------- + + +class TestProjectionWriteCarriesTheBlockerSet: + @pytest.mark.asyncio + async def test_sql_args_carry_state_and_counts(self) -> None: + bus, _published = await _publish(thread_readiness=True) + args = await _sql_args_for(_captured_manifest_payload(bus)) + + assert args[14] == EnumRuntimeReadinessState.DEGRADED.value # attach_state + assert args[15] == 2 # attach_required_contracts + assert args[16] == 1 # attach_attached_contracts + + @pytest.mark.asyncio + async def test_sql_args_carry_the_named_topic(self) -> None: + """The end of the chain: the absent topic reaches the DB write.""" + bus, _published = await _publish(thread_readiness=True) + args = await _sql_args_for(_captured_manifest_payload(bus)) + + blockers = json.loads(str(args[17])) # attach_not_ready_contracts + assert len(blockers) == 1 + assert blockers[0]["contract_name"] == BLOCKED_CONTRACT + failing_topics = {f["topic"] for f in blockers[0]["readiness"]["failures"]} + assert BLOCKED_TOPIC in failing_topics + + @pytest.mark.asyncio + async def test_migration_declares_every_column_the_sql_writes(self) -> None: + """Guard the SQL <-> DDL seam; a missing column fails only in prod.""" + repo_root = Path(__file__).resolve().parents[3] + migration = ( + repo_root + / "docker" + / "migrations" + / "forward" + / "095_add_attach_readiness_to_runtime_manifests.sql" + ) + assert migration.is_file(), f"migration not found: {migration}" + ddl = migration.read_text(encoding="utf-8").lower() + sql = SQL_INSERT_RUNTIME_MANIFEST.lower() + for column in ( + "attach_state", + "attach_required_contracts", + "attach_attached_contracts", + "attach_not_ready_contracts", + ): + assert column in sql, f"INSERT does not write {column!r}" + assert f"add column if not exists {column}" in ddl, ( + f"migration 095 does not add {column!r}" + ) + + +# --------------------------------------------------------------------------- +# RED-before: the same chain without the fix fails every assertion above +# --------------------------------------------------------------------------- + + +class TestGuardActuallyGuards: + """Prove the tests discriminate rather than passing on anything. + + ``thread_readiness=False`` is the pre-OMN-15512 code path: the aggregate is + computed by the interleave and then not handed to the publisher. + """ + + @pytest.mark.asyncio + async def test_unthreaded_publish_has_no_attach_readiness(self) -> None: + bus, published = await _publish(thread_readiness=False) + payload = _captured_manifest_payload(bus) + assert payload["attach_readiness"] is None + assert published.attach_readiness is None + + @pytest.mark.asyncio + async def test_unthreaded_projection_write_names_nothing(self) -> None: + bus, _published = await _publish(thread_readiness=False) + args = await _sql_args_for(_captured_manifest_payload(bus)) + + # 'unknown', NOT 'ready': a boot that reported nothing must never read + # as a boot where everything attached. + assert args[14] == "unknown" + assert args[15] == 0 + assert args[16] == 0 + assert json.loads(str(args[17])) == [] + assert BLOCKED_TOPIC not in str(args[17]) + + @pytest.mark.asyncio + async def test_the_interleave_itself_was_never_the_defect(self) -> None: + """In-memory computation was already correct — that is the point. + + This is the assertion a naive unit test would have made, and it passes + both before and after the fix. It is kept only to document why it is + NOT sufficient evidence for this ticket. + """ + _manifest, _report, _bus, readiness = await _run_boot_interleave() + by_name = {r.contract_name: r for r in readiness.results} + assert by_name[BLOCKED_CONTRACT].status is EnumContractAttachStatus.NOT_READY + assert by_name[ATTACHING_CONTRACT].status is EnumContractAttachStatus.ATTACHED diff --git a/tests/integration/runtime/test_contract_scoped_dispatch.py b/tests/integration/runtime/test_contract_scoped_dispatch.py new file mode 100644 index 0000000000..10cbf706a3 --- /dev/null +++ b/tests/integration/runtime/test_contract_scoped_dispatch.py @@ -0,0 +1,1500 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Cross-boundary regression for contract-scoped Kafka dispatch (OMN-15474). + +Two contracts may intentionally consume the same topic under distinct consumer +groups (for example, an orchestrator and a projection). Each Kafka callback +must dispatch only to the handlers registered by the contract that owns that +callback. Calling the process-global dispatch fan-out from both callbacks +executes every matching handler twice. +""" + +from __future__ import annotations + +import json +from collections.abc import Awaitable, Callable, Mapping, Sequence +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from omnibase_infra.event_bus.enum_contract_attach_status import ( + EnumContractAttachStatus, +) +from omnibase_infra.event_bus.enum_topic_readiness_status import ( + EnumTopicReadinessStatus, +) +from omnibase_infra.event_bus.model_contract_attach_result import ( + ModelContractAttachResult, +) +from omnibase_infra.event_bus.model_topic_set_readiness import ( + ModelTopicSetReadiness, +) +from omnibase_infra.models import ModelNodeIdentity +from omnibase_infra.runtime.auto_wiring import ( + ModelAutoWiringManifest, + ModelAutoWiringReport, + ModelContractVersion, + ModelDiscoveredContract, + ModelEventBusWiring, + ModelHandlerRef, + ModelHandlerRouting, + ModelHandlerRoutingEntry, + subscribe_wired_contract_topics, + wire_from_manifest, +) +from omnibase_infra.runtime.message_dispatch_engine import MessageDispatchEngine + +if TYPE_CHECKING: + from uuid import UUID + + from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope + from omnibase_infra.models.dispatch.model_dispatch_result import ( + ModelDispatchResult, + ) + from omnibase_infra.topics.model_topic_spec import ModelTopicSpec + +pytestmark = pytest.mark.integration + +_SHARED_TOPIC = "onex.cmd.omn15474.shared-command.v1" + + +class HandlerContractA: + """First owning handler on the shared topic.""" + + calls = 0 + + async def handle(self, envelope: object) -> None: + del envelope + type(self).calls += 1 + + +class HandlerContractB: + """Second owning handler on the shared topic.""" + + calls = 0 + + async def handle(self, envelope: object) -> None: + del envelope + type(self).calls += 1 + + +class _RecordingApplier: + def __init__(self) -> None: + self.results: list[ModelDispatchResult] = [] + + async def apply( + self, + result: ModelDispatchResult | None, + correlation_id: UUID | None = None, + ) -> None: + del correlation_id + if result is not None: + self.results.append(result) + + +class _RecordingBus: + """Kafka-boundary double preserving every distinct consumer group.""" + + def __init__(self) -> None: + self.subscriptions: list[ + tuple[str, ModelNodeIdentity, Callable[..., Awaitable[None]]] + ] = [] + + async def subscribe( + self, + *, + topic: str, + node_identity: ModelNodeIdentity, + on_message: Callable[..., Awaitable[None]], + ) -> object: + self.subscriptions.append((topic, node_identity, on_message)) + + async def _unsubscribe() -> None: + return None + + return _unsubscribe + + async def deliver_to_every_group( + self, envelope: ModelEventEnvelope[object] + ) -> None: + message = type( + "KafkaMessage", (), {"value": envelope.model_dump_json().encode()} + )() + for _topic, _identity, callback in self.subscriptions: + await callback(message) + + +class _RecordingReadyProvisioner: + """Provisioner double that exposes any pre-validation side effect.""" + + def __init__(self) -> None: + self.ensure_calls: list[str] = [] + self.confirm_calls: list[tuple[str, ...]] = [] + + async def ensure_topic_exists( + self, + topic_name: str, + spec: ModelTopicSpec | None = None, + correlation_id: UUID | None = None, + ) -> bool: + del spec, correlation_id + self.ensure_calls.append(topic_name) + return True + + async def confirm_topics_ready( + self, + topics: Sequence[str], + *, + expected_specs: Mapping[str, ModelTopicSpec] | None = None, + config: object | None = None, + correlation_id: UUID | None = None, + ) -> ModelTopicSetReadiness: + del expected_specs, config, correlation_id + normalized_topics = tuple(topics) + self.confirm_calls.append(normalized_topics) + return ModelTopicSetReadiness( + topics=normalized_topics, + status=EnumTopicReadinessStatus.READY, + ready_topics=normalized_topics, + attempts=1, + ) + + +class _ProtocolOnlyDispatchEngine: + """Published SPI shape without the opt-in scoped-dispatch capability.""" + + is_frozen = True + + async def dispatch( + self, + topic: str, + envelope: ModelEventEnvelope[object], + ) -> ModelDispatchResult | None: + del topic, envelope + return None + + async def dispatch_with_transaction( + self, + *, + topic: str, + envelope: ModelEventEnvelope[object], + tx: object, + ) -> ModelDispatchResult | None: + del topic, envelope, tx + return None + + +def _contract(name: str, handler_name: str) -> ModelDiscoveredContract: + return _contract_with_handlers(name, (handler_name,)) + + +def _contract_with_handlers( + name: str, + handler_names: tuple[str, ...], + *, + topics: tuple[str, ...] = (_SHARED_TOPIC,), +) -> ModelDiscoveredContract: + return ModelDiscoveredContract( + name=name, + node_type="ORCHESTRATOR_GENERIC", + contract_version=ModelContractVersion(major=1, minor=0, patch=0), + contract_path=Path(f"/tmp/{name}/contract.yaml"), # noqa: S108 + entry_point_name=name, + package_name="omn15474-fixture", + event_bus=ModelEventBusWiring(subscribe_topics=topics), + handler_routing=ModelHandlerRouting( + routing_strategy="operation_match", + handlers=tuple( + ModelHandlerRoutingEntry( + handler=ModelHandlerRef(name=handler_name, module=__name__), + event_model=None, + ) + for handler_name in handler_names + ), + ), + ) + + +def _revalidate_report_with_scopes( + report: ModelAutoWiringReport, + scopes_by_contract: Mapping[str, tuple[str, ...]], +) -> ModelAutoWiringReport: + """Forge only through the public serialization/validation boundary.""" + payload = json.loads(report.model_dump_json()) + for result in payload["results"]: + contract_name = result["contract_name"] + if contract_name in scopes_by_contract: + result["dispatchers_registered"] = list(scopes_by_contract[contract_name]) + return ModelAutoWiringReport.model_validate_json(json.dumps(payload)) + + +@pytest.mark.asyncio +async def test_shared_topic_dispatch_is_scoped_to_each_contract_owner() -> None: + """One broker record executes each owning handler/applier exactly once.""" + from datetime import UTC, datetime + from uuid import uuid4 + + from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope + + HandlerContractA.calls = 0 + HandlerContractB.calls = 0 + contract_a = _contract("node_contract_a", "HandlerContractA") + contract_b = _contract("node_contract_b", "HandlerContractB") + manifest = ModelAutoWiringManifest(contracts=(contract_a, contract_b), errors=()) + engine = MessageDispatchEngine() + bus = _RecordingBus() + applier_a = _RecordingApplier() + applier_b = _RecordingApplier() + appliers = { + contract_a.name: applier_a, + contract_b.name: applier_b, + } + + report = await wire_from_manifest( + manifest, + engine, + event_bus=bus, + environment="test", + subscribe_immediately=False, + result_appliers_by_contract=appliers, + ) + engine.freeze() + subscriptions = await subscribe_wired_contract_topics( + manifest, + report, + engine, + bus, + "test", + appliers, + ) + + assert subscriptions == { + contract_a.name: (_SHARED_TOPIC,), + contract_b.name: (_SHARED_TOPIC,), + } + assert len(bus.subscriptions) == 2 + assert { + identity.node_name for _topic, identity, _callback in bus.subscriptions + } == {contract_a.name, contract_b.name} + + envelope = ModelEventEnvelope[object]( + payload={"prompt": "one broker record"}, + correlation_id=uuid4(), + envelope_timestamp=datetime.now(UTC), + event_type="omn15474.shared-command", + source_tool="contract-scoped-dispatch-test", + ) + await bus.deliver_to_every_group(envelope) + + assert HandlerContractA.calls == 1 + assert HandlerContractB.calls == 1 + assert len(applier_a.results) == 1 + assert len(applier_b.results) == 1 + + result_by_contract = {result.contract_name: result for result in report.results} + dispatcher_a = result_by_contract[contract_a.name].dispatchers_registered[0] + dispatcher_b = result_by_contract[contract_b.name].dispatchers_registered[0] + assert applier_a.results[0].dispatcher_id == dispatcher_a + assert applier_b.results[0].dispatcher_id == dispatcher_b + + +@pytest.mark.asyncio +async def test_immediate_subscriptions_preserve_contract_scope() -> None: + """The pre-freeze immediate attach path captures the same exact scope.""" + from datetime import UTC, datetime + from uuid import uuid4 + + from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope + + HandlerContractA.calls = 0 + HandlerContractB.calls = 0 + contract_a = _contract("node_immediate_a", "HandlerContractA") + contract_b = _contract("node_immediate_b", "HandlerContractB") + manifest = ModelAutoWiringManifest(contracts=(contract_a, contract_b), errors=()) + engine = MessageDispatchEngine() + bus = _RecordingBus() + applier_a = _RecordingApplier() + applier_b = _RecordingApplier() + + report = await wire_from_manifest( + manifest, + engine, + event_bus=bus, + environment="test", + result_appliers_by_contract={ + contract_a.name: applier_a, + contract_b.name: applier_b, + }, + ) + engine.freeze() + + assert len(bus.subscriptions) == 2 + assert all( + result.topics_subscribed == (_SHARED_TOPIC,) for result in report.results + ) + + envelope = ModelEventEnvelope[object]( + payload={"prompt": "immediate path"}, + correlation_id=uuid4(), + envelope_timestamp=datetime.now(UTC), + event_type="omn15474.shared-command", + source_tool="contract-scoped-dispatch-test", + ) + await bus.deliver_to_every_group(envelope) + + assert HandlerContractA.calls == 1 + assert HandlerContractB.calls == 1 + assert len(applier_a.results) == 1 + assert len(applier_b.results) == 1 + + +@pytest.mark.parametrize("subscribe_immediately", [True, False]) +@pytest.mark.asyncio +async def test_cross_contract_derived_id_collision_fails_before_manifest_commit( + monkeypatch: pytest.MonkeyPatch, + *, + subscribe_immediately: bool, +) -> None: + """A batch-wide derived-ID collision cannot expose partial live state.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring import handler_wiring + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _derive_dispatcher_id, + _derive_handler_entry_key, + _derive_route_id, + ) + + contract_a = _contract("node.alpha", "HandlerContractA") + contract_b = _contract("node", "alpha.HandlerContractA") + manifest = ModelAutoWiringManifest(contracts=(contract_a, contract_b), errors=()) + handler_key_a = _derive_handler_entry_key( + contract_a.handler_routing.handlers[0] # type: ignore[union-attr] + ) + handler_key_b = _derive_handler_entry_key( + contract_b.handler_routing.handlers[0] # type: ignore[union-attr] + ) + assert _derive_dispatcher_id(contract_a.name, handler_key_a) == ( + _derive_dispatcher_id(contract_b.name, handler_key_b) + ) + assert _derive_route_id(contract_a.name, handler_key_a, _SHARED_TOPIC) == ( + _derive_route_id(contract_b.name, handler_key_b, _SHARED_TOPIC) + ) + + monkeypatch.setattr( + handler_wiring, + "_import_handler_class", + lambda _module, _name: HandlerContractA, + ) + engine = MessageDispatchEngine() + bus = _RecordingBus() + + with pytest.raises(ModelOnexError, match="duplicate prepared dispatcher IDs"): + await wire_from_manifest( + manifest, + engine, + event_bus=bus, + environment="test", + subscribe_immediately=subscribe_immediately, + ) + + assert engine.dispatcher_count == 0 + assert engine.route_count == 0 + assert bus.subscriptions == [] + + +@pytest.mark.parametrize("subscribe_immediately", [True, False]) +@pytest.mark.asyncio +async def test_normalized_route_collision_fails_before_manifest_commit( + *, + subscribe_immediately: bool, +) -> None: + """Topic normalization collisions are rejected before engine mutation.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _derive_handler_entry_key, + _derive_route_id, + ) + + topics = ( + "onex.cmd.foo-bar.name.v1", + "onex.cmd.foo.bar-name.v1", + ) + contract = _contract_with_handlers( + "node_normalized_route_collision", + ("HandlerContractA",), + topics=topics, + ) + handler_key = _derive_handler_entry_key( + contract.handler_routing.handlers[0] # type: ignore[union-attr] + ) + assert _derive_route_id(contract.name, handler_key, topics[0]) == _derive_route_id( + contract.name, + handler_key, + topics[1], + ) + engine = MessageDispatchEngine() + bus = _RecordingBus() + + with pytest.raises(ModelOnexError, match="duplicate prepared route IDs"): + await wire_from_manifest( + ModelAutoWiringManifest(contracts=(contract,), errors=()), + engine, + event_bus=bus, + environment="test", + subscribe_immediately=subscribe_immediately, + ) + + assert engine.dispatcher_count == 0 + assert engine.route_count == 0 + assert bus.subscriptions == [] + + +@pytest.mark.asyncio +async def test_duplicate_manifest_names_fail_at_wire_entry() -> None: + """Immediate wiring rejects typed name collisions before preparation.""" + from omnibase_core.models.errors import ModelOnexError + + HandlerContractA.calls = 0 + HandlerContractB.calls = 0 + contract_a = _contract("node_wire_duplicate_a", "HandlerContractA") + contract_b = _contract("node_wire_duplicate_b", "HandlerContractB") + original_manifest = ModelAutoWiringManifest( + contracts=(contract_a, contract_b), + errors=(), + ) + payload = json.loads(original_manifest.model_dump_json()) + payload["contracts"][1]["name"] = contract_a.name + duplicate_manifest = ModelAutoWiringManifest.model_validate_json( + json.dumps(payload) + ) + engine = MessageDispatchEngine() + bus = _RecordingBus() + applier = _RecordingApplier() + caught: ModelOnexError | None = None + + try: + await wire_from_manifest( + duplicate_manifest, + engine, + event_bus=bus, + environment="test", + result_appliers_by_contract={contract_a.name: applier}, + ) + except ModelOnexError as exc: + caught = exc + + assert caught is not None, ( + "wire_from_manifest accepted a serialized/Pydantic HandlerA/HandlerB " + "manifest name collision " + f"(dispatchers={engine.dispatcher_count}, subscriptions={len(bus.subscriptions)})" + ) + assert "duplicate manifest contract names" in str(caught) + assert engine.dispatcher_count == 0 + assert bus.subscriptions == [] + assert HandlerContractA.calls == 0 + assert HandlerContractB.calls == 0 + assert applier.results == [] + + +@pytest.mark.asyncio +async def test_wired_subscription_without_dispatcher_scope_fails_closed() -> None: + """A wired callback can never silently regain process-global fan-out.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _subscribe_contract_topics, + ) + + with pytest.raises(ModelOnexError, match="missing its dispatcher scope"): + await _subscribe_contract_topics( + contract=_contract("node_missing_scope", "HandlerContractA"), + dispatch_engine=MessageDispatchEngine(), + event_bus=_RecordingBus(), + environment="test", + ) + + +@pytest.mark.asyncio +async def test_engine_without_scoped_dispatch_fails_before_subscribe() -> None: + """Protocol-only engines cannot discover incompatibility after consuming.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _subscribe_contract_topics, + ) + + bus = _RecordingBus() + + with pytest.raises(ModelOnexError, match="scoped dispatch capability"): + await _subscribe_contract_topics( + contract=_contract("node_protocol_only", "HandlerContractA"), + dispatch_engine=_ProtocolOnlyDispatchEngine(), + event_bus=bus, + environment="test", + allowed_dispatcher_ids={"dispatcher.protocol-only"}, + ) + + assert bus.subscriptions == [] + + +@pytest.mark.asyncio +async def test_revalidated_cross_contract_scope_fails_before_provisioning() -> None: + """A typed report cannot assign one dispatcher to two contract owners.""" + from omnibase_core.models.errors import ModelOnexError + + contract_a = _contract("node_forged_owner_a", "HandlerContractA") + contract_b = _contract("node_forged_owner_b", "HandlerContractB") + manifest = ModelAutoWiringManifest(contracts=(contract_a, contract_b), errors=()) + engine = MessageDispatchEngine() + bus = _RecordingBus() + report = await wire_from_manifest( + manifest, + engine, + event_bus=bus, + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + + by_name = {result.contract_name: result for result in report.results} + dispatcher_a = by_name[contract_a.name].dispatchers_registered[0] + forged_report = _revalidate_report_with_scopes( + report, + { + contract_a.name: (dispatcher_a,), + contract_b.name: (dispatcher_a,), + }, + ) + provisioner = _RecordingReadyProvisioner() + + with pytest.raises(ModelOnexError, match="multiple contracts"): + await subscribe_wired_contract_topics( + manifest, + forged_report, + engine, + bus, + "test", + provisioner=provisioner, + ) + + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + + +@pytest.mark.parametrize("scope_variant", ["proper_subset", "empty"]) +@pytest.mark.asyncio +async def test_revalidated_initial_scope_must_equal_live_owner_set( + scope_variant: str, +) -> None: + """A typed initial report cannot suppress an owner's registered handlers.""" + from omnibase_core.models.errors import ModelOnexError + + contract = _contract_with_handlers( + "node_initial_complete_scope", + ("HandlerContractA", "HandlerContractB"), + ) + manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) + engine = MessageDispatchEngine() + report = await wire_from_manifest( + manifest, + engine, + event_bus=_RecordingBus(), + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + owned_scope = report.results[0].dispatchers_registered + assert len(owned_scope) == 2 + forged_scope = owned_scope[:1] if scope_variant == "proper_subset" else () + forged_report = _revalidate_report_with_scopes( + report, + {contract.name: forged_scope}, + ) + bus = _RecordingBus() + provisioner = _RecordingReadyProvisioner() + attach_results: list[ModelContractAttachResult] = [] + + with pytest.raises(ModelOnexError, match="complete live owner set"): + await subscribe_wired_contract_topics( + manifest, + forged_report, + engine, + bus, + "test", + provisioner=provisioner, + attach_results_out=attach_results, + ) + + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + assert attach_results == [] + + +@pytest.mark.asyncio +async def test_revalidated_duplicate_report_names_fail_before_side_effects() -> None: + """Typed duplicate result identities cannot schedule a contract twice.""" + from omnibase_core.models.errors import ModelOnexError + + contract = _contract("node_duplicate_report", "HandlerContractA") + manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) + engine = MessageDispatchEngine() + report = await wire_from_manifest( + manifest, + engine, + event_bus=_RecordingBus(), + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + payload = json.loads(report.model_dump_json()) + payload["results"].append(dict(payload["results"][0])) + duplicate_report = ModelAutoWiringReport.model_validate_json(json.dumps(payload)) + bus = _RecordingBus() + provisioner = _RecordingReadyProvisioner() + attach_results: list[ModelContractAttachResult] = [] + + with pytest.raises(ModelOnexError, match="duplicate report contract names"): + await subscribe_wired_contract_topics( + manifest, + duplicate_report, + engine, + bus, + "test", + provisioner=provisioner, + attach_results_out=attach_results, + ) + + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + assert attach_results == [] + + +@pytest.mark.asyncio +async def test_revalidated_duplicate_manifest_names_fail_before_side_effects() -> None: + """Two different handlers cannot collapse under one manifest identity.""" + from omnibase_core.models.errors import ModelOnexError + + contract_a = _contract("node_manifest_a", "HandlerContractA") + contract_b = _contract("node_manifest_b", "HandlerContractB") + original_manifest = ModelAutoWiringManifest( + contracts=(contract_a, contract_b), + errors=(), + ) + engine = MessageDispatchEngine() + report = await wire_from_manifest( + original_manifest, + engine, + event_bus=_RecordingBus(), + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + payload = json.loads(original_manifest.model_dump_json()) + payload["contracts"][1]["name"] = contract_a.name + duplicate_manifest = ModelAutoWiringManifest.model_validate_json( + json.dumps(payload) + ) + bus = _RecordingBus() + provisioner = _RecordingReadyProvisioner() + attach_results: list[ModelContractAttachResult] = [] + + with pytest.raises(ModelOnexError, match="duplicate manifest contract names"): + await subscribe_wired_contract_topics( + duplicate_manifest, + report, + engine, + bus, + "test", + provisioner=provisioner, + attach_results_out=attach_results, + ) + + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + assert attach_results == [] + + +@pytest.mark.asyncio +async def test_initial_contract_names_must_be_canonical_and_bijective() -> None: + """Whitespace aliases, unexpected names, and missing names all fail synchronously. + + OMN-15474 ruling 4 (restored by OMN-15621 after PR #2609 narrowed this to a + report-subset-of-manifest check): the report must be an EXACT bijection of + the manifest, both directions. A report naming a contract the manifest + never declared is an identity error (``unexpected_in_report``), and a + report that OMITS a manifest contract is equally an identity error + (``missing_from_report``) — "this contract has no verdict" is exactly the + state that let a contract's events reach a process-global dispatch. + """ + from omnibase_core.models.errors import ModelOnexError + + contract_a = _contract("node_identity_a", "HandlerContractA") + contract_b = _contract("node_identity_b", "HandlerContractB") + manifest = ModelAutoWiringManifest(contracts=(contract_a, contract_b), errors=()) + engine = MessageDispatchEngine() + report = await wire_from_manifest( + manifest, + engine, + event_bus=_RecordingBus(), + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + + noncanonical_payload = json.loads(report.model_dump_json()) + noncanonical_payload["results"][0]["contract_name"] = f" {contract_a.name} " + noncanonical_report = ModelAutoWiringReport.model_validate_json( + json.dumps(noncanonical_payload) + ) + unexpected_payload = json.loads(report.model_dump_json()) + unexpected_payload["results"][0]["contract_name"] = "node_identity_absent" + unexpected_report = ModelAutoWiringReport.model_validate_json( + json.dumps(unexpected_payload) + ) + partial_payload = json.loads(report.model_dump_json()) + partial_payload["results"] = partial_payload["results"][:1] + partial_report = ModelAutoWiringReport.model_validate_json( + json.dumps(partial_payload) + ) + noncanonical_manifest_payload = json.loads(manifest.model_dump_json()) + noncanonical_manifest_payload["contracts"][0]["name"] = f" {contract_a.name} " + noncanonical_manifest = ModelAutoWiringManifest.model_validate_json( + json.dumps(noncanonical_manifest_payload) + ) + + for forged_manifest, forged_report, error_match in ( + ( + noncanonical_manifest, + report, + "noncanonical manifest contract names", + ), + (manifest, noncanonical_report, "noncanonical report contract names"), + (manifest, unexpected_report, "unexpected_in_report"), + (manifest, partial_report, "missing_from_report"), + ): + bus = _RecordingBus() + provisioner = _RecordingReadyProvisioner() + attach_results: list[ModelContractAttachResult] = [] + with pytest.raises(ModelOnexError, match=error_match): + await subscribe_wired_contract_topics( + forged_manifest, + forged_report, + engine, + bus, + "test", + provisioner=provisioner, + attach_results_out=attach_results, + ) + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + assert attach_results == [] + + +@pytest.mark.asyncio +async def test_revalidated_wrong_registered_owner_fails_before_provisioning() -> None: + """A unique registered ID still cannot be reassigned to another contract.""" + from omnibase_core.models.errors import ModelOnexError + + contract_a = _contract("node_wrong_owner_a", "HandlerContractA") + contract_b = _contract("node_wrong_owner_b", "HandlerContractB") + manifest = ModelAutoWiringManifest(contracts=(contract_a, contract_b), errors=()) + engine = MessageDispatchEngine() + bus = _RecordingBus() + report = await wire_from_manifest( + manifest, + engine, + event_bus=bus, + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + by_name = {result.contract_name: result for result in report.results} + dispatcher_b = by_name[contract_b.name].dispatchers_registered[0] + forged_report = _revalidate_report_with_scopes( + report, + { + contract_a.name: (dispatcher_b,), + contract_b.name: (), + }, + ) + provisioner = _RecordingReadyProvisioner() + + with pytest.raises(ModelOnexError, match="not owned by contract"): + await subscribe_wired_contract_topics( + manifest, + forged_report, + engine, + bus, + "test", + provisioner=provisioner, + ) + + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + + +@pytest.mark.asyncio +async def test_revalidated_unknown_report_scope_fails_before_provisioning() -> None: + """Every dispatcher cited by a typed wiring report must exist now.""" + from omnibase_core.models.errors import ModelOnexError + + contract = _contract("node_forged_unknown", "HandlerContractA") + manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) + engine = MessageDispatchEngine() + bus = _RecordingBus() + report = await wire_from_manifest( + manifest, + engine, + event_bus=bus, + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + forged_report = _revalidate_report_with_scopes( + report, + {contract.name: ("dispatcher.forged-unknown",)}, + ) + provisioner = _RecordingReadyProvisioner() + + with pytest.raises(ModelOnexError, match="not registered on this engine"): + await subscribe_wired_contract_topics( + manifest, + forged_report, + engine, + bus, + "test", + provisioner=provisioner, + ) + + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + + +@pytest.mark.asyncio +async def test_direct_unknown_scope_fails_before_subscribe() -> None: + """The lowest direct subscription boundary also checks membership.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _subscribe_contract_topics, + ) + + engine = MessageDispatchEngine() + engine.freeze() + bus = _RecordingBus() + + with pytest.raises(ModelOnexError, match="not registered on this engine"): + await _subscribe_contract_topics( + contract=_contract("node_direct_unknown", "HandlerContractA"), + dispatch_engine=engine, + event_bus=bus, + environment="test", + allowed_dispatcher_ids={"dispatcher.direct-unknown"}, + ) + + assert bus.subscriptions == [] + + +@pytest.mark.parametrize("scope_variant", ["proper_subset", "empty"]) +@pytest.mark.asyncio +async def test_direct_scope_must_equal_live_owner_set(scope_variant: str) -> None: + """The direct subscription seam requires the complete immutable owner set.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _subscribe_contract_topics, + ) + + contract = _contract_with_handlers( + "node_direct_complete_scope", + ("HandlerContractA", "HandlerContractB"), + ) + manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) + engine = MessageDispatchEngine() + report = await wire_from_manifest( + manifest, + engine, + event_bus=_RecordingBus(), + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + owned_scope = report.results[0].dispatchers_registered + forged_scope = owned_scope[:1] if scope_variant == "proper_subset" else () + bus = _RecordingBus() + + with pytest.raises( + ModelOnexError, + match=r"complete live owner set|empty or invalid dispatcher scope", + ): + await _subscribe_contract_topics( + contract=contract, + dispatch_engine=engine, + event_bus=bus, + environment="test", + allowed_dispatcher_ids=forged_scope, + ) + + assert bus.subscriptions == [] + + +@pytest.mark.asyncio +async def test_revalidated_not_ready_unknown_scope_fails_before_provisioning() -> None: + """A persisted NOT_READY scope is revalidated against the live engine.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + reattach_not_ready_contracts, + ) + + contract = _contract("node_not_ready_unknown", "HandlerContractA") + manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) + engine = MessageDispatchEngine() + bus = _RecordingBus() + await wire_from_manifest( + manifest, + engine, + event_bus=bus, + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + forged_result = ModelContractAttachResult.model_validate_json( + json.dumps( + { + "contract_name": contract.name, + "status": EnumContractAttachStatus.NOT_READY.value, + "dispatcher_ids": ["dispatcher.not-ready-unknown"], + } + ) + ) + provisioner = _RecordingReadyProvisioner() + + with pytest.raises(ModelOnexError, match="not registered on this engine"): + await reattach_not_ready_contracts( + manifest, + (forged_result,), + engine, + bus, + "test", + provisioner=provisioner, + ) + + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + + +@pytest.mark.parametrize("scope_variant", ["proper_subset", "empty"]) +@pytest.mark.asyncio +async def test_revalidated_not_ready_scope_must_equal_live_owner_set( + scope_variant: str, +) -> None: + """A persisted retry scope cannot omit any dispatcher owned by its contract.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + reattach_not_ready_contracts, + ) + + contract = _contract_with_handlers( + "node_not_ready_complete_scope", + ("HandlerContractA", "HandlerContractB"), + ) + manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) + engine = MessageDispatchEngine() + report = await wire_from_manifest( + manifest, + engine, + event_bus=_RecordingBus(), + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + owned_scope = report.results[0].dispatchers_registered + forged_scope = owned_scope[:1] if scope_variant == "proper_subset" else () + forged_result = ModelContractAttachResult.model_validate_json( + json.dumps( + { + "contract_name": contract.name, + "status": EnumContractAttachStatus.NOT_READY.value, + "dispatcher_ids": list(forged_scope), + } + ) + ) + bus = _RecordingBus() + provisioner = _RecordingReadyProvisioner() + + with pytest.raises( + ModelOnexError, + match=r"complete live owner set|empty or invalid dispatcher scope", + ): + await reattach_not_ready_contracts( + manifest, + (forged_result,), + engine, + bus, + "test", + provisioner=provisioner, + ) + + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + + +@pytest.mark.asyncio +async def test_not_ready_cross_contract_scope_fails_before_provisioning() -> None: + """Reattach rejects one dispatcher persisted under two contracts.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + reattach_not_ready_contracts, + ) + + contract_a = _contract("node_not_ready_owner_a", "HandlerContractA") + contract_b = _contract("node_not_ready_owner_b", "HandlerContractB") + manifest = ModelAutoWiringManifest(contracts=(contract_a, contract_b), errors=()) + engine = MessageDispatchEngine() + bus = _RecordingBus() + report = await wire_from_manifest( + manifest, + engine, + event_bus=bus, + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + by_name = {result.contract_name: result for result in report.results} + shared_dispatcher = by_name[contract_a.name].dispatchers_registered[0] + results = tuple( + ModelContractAttachResult.model_validate_json( + json.dumps( + { + "contract_name": contract_name, + "status": EnumContractAttachStatus.NOT_READY.value, + "dispatcher_ids": [shared_dispatcher], + } + ) + ) + for contract_name in (contract_a.name, contract_b.name) + ) + provisioner = _RecordingReadyProvisioner() + + with pytest.raises(ModelOnexError, match="multiple contracts"): + await reattach_not_ready_contracts( + manifest, + results, + engine, + bus, + "test", + provisioner=provisioner, + ) + + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + + +@pytest.mark.asyncio +async def test_duplicate_typed_not_ready_names_fail_before_side_effects() -> None: + """Persisted duplicate identities cannot schedule two reattach attempts.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + reattach_not_ready_contracts, + ) + + contract = _contract("node_duplicate_not_ready", "HandlerContractA") + manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) + engine = MessageDispatchEngine() + report = await wire_from_manifest( + manifest, + engine, + event_bus=_RecordingBus(), + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + dispatcher_id = report.results[0].dispatchers_registered[0] + serialized_result = json.dumps( + { + "contract_name": contract.name, + "status": EnumContractAttachStatus.NOT_READY.value, + "dispatcher_ids": [dispatcher_id], + } + ) + duplicate_results = tuple( + ModelContractAttachResult.model_validate_json(serialized_result) + for _ in range(2) + ) + bus = _RecordingBus() + provisioner = _RecordingReadyProvisioner() + + with pytest.raises(ModelOnexError, match="duplicate NOT_READY contract names"): + await reattach_not_ready_contracts( + manifest, + duplicate_results, + engine, + bus, + "test", + provisioner=provisioner, + ) + + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + + +@pytest.mark.asyncio +async def test_duplicate_initial_not_ready_fails_before_loop_side_effects() -> None: + """The public reconciliation loop validates before dict collapse or sleep.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + run_not_ready_reconciliation_loop, + ) + + contract = _contract("node_duplicate_initial_not_ready", "HandlerContractA") + manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) + engine = MessageDispatchEngine() + report = await wire_from_manifest( + manifest, + engine, + event_bus=_RecordingBus(), + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + baseline_dispatcher_count = engine.dispatcher_count + dispatcher_id = report.results[0].dispatchers_registered[0] + serialized_result = json.dumps( + { + "contract_name": contract.name, + "status": EnumContractAttachStatus.NOT_READY.value, + "dispatcher_ids": [dispatcher_id], + } + ) + duplicate_initial_not_ready = tuple( + ModelContractAttachResult.model_validate_json(serialized_result) + for _ in range(2) + ) + bus = _RecordingBus() + provisioner = _RecordingReadyProvisioner() + applier = _RecordingApplier() + sleep_calls: list[float] = [] + attempt_results: list[tuple[ModelContractAttachResult, ...]] = [] + returned_results: tuple[ModelContractAttachResult, ...] = () + caught: ModelOnexError | None = None + + async def _record_sleep(seconds: float) -> None: + sleep_calls.append(seconds) + + def _record_attempt( + _subscribed: dict[str, tuple[str, ...]], + results: tuple[ModelContractAttachResult, ...], + ) -> None: + attempt_results.append(results) + + try: + returned_results = await run_not_ready_reconciliation_loop( + manifest, + duplicate_initial_not_ready, + engine, + bus, + "test", + {contract.name: applier}, + provisioner=provisioner, + max_attempts=1, + on_attempt=_record_attempt, + sleep=_record_sleep, + ) + except ModelOnexError as exc: + caught = exc + + assert caught is not None, ( + "reconciliation accepted duplicate typed initial NOT_READY identities " + f"(sleep={sleep_calls}, ensure={provisioner.ensure_calls}, " + f"confirm={provisioner.confirm_calls}, subscriptions={len(bus.subscriptions)}, " + f"results={returned_results})" + ) + assert "duplicate NOT_READY contract names" in str(caught) + assert engine.dispatcher_count == baseline_dispatcher_count + assert sleep_calls == [] + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + assert returned_results == () + assert attempt_results == [] + assert applier.results == [] + + +@pytest.mark.asyncio +async def test_forged_initial_not_ready_scope_fails_before_loop_sleep() -> None: + """The public retry wrapper validates live scope before its first sleep.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + run_not_ready_reconciliation_loop, + ) + + contract = _contract("node_forged_initial_scope", "HandlerContractA") + manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) + engine = MessageDispatchEngine() + await wire_from_manifest( + manifest, + engine, + event_bus=_RecordingBus(), + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + baseline_dispatcher_count = engine.dispatcher_count + forged_result = ModelContractAttachResult.model_validate_json( + json.dumps( + { + "contract_name": contract.name, + "status": EnumContractAttachStatus.NOT_READY.value, + "dispatcher_ids": ["dispatcher.forged-before-sleep"], + } + ) + ) + bus = _RecordingBus() + provisioner = _RecordingReadyProvisioner() + applier = _RecordingApplier() + sleep_calls: list[float] = [] + attempt_results: list[tuple[ModelContractAttachResult, ...]] = [] + + async def _record_sleep(seconds: float) -> None: + sleep_calls.append(seconds) + + def _record_attempt( + _subscribed: dict[str, tuple[str, ...]], + results: tuple[ModelContractAttachResult, ...], + ) -> None: + attempt_results.append(results) + + with pytest.raises(ModelOnexError, match="not registered on this engine"): + await run_not_ready_reconciliation_loop( + manifest, + (forged_result,), + engine, + bus, + "test", + {contract.name: applier}, + provisioner=provisioner, + max_attempts=1, + on_attempt=_record_attempt, + sleep=_record_sleep, + ) + + assert engine.dispatcher_count == baseline_dispatcher_count + assert sleep_calls == [] + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + assert attempt_results == [] + assert applier.results == [] + + +@pytest.mark.asyncio +async def test_not_ready_names_must_be_canonical_manifest_subset() -> None: + """Reattach rejects whitespace aliases and names absent from the manifest.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + reattach_not_ready_contracts, + ) + + contract = _contract("node_not_ready_identity", "HandlerContractA") + manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) + engine = MessageDispatchEngine() + report = await wire_from_manifest( + manifest, + engine, + event_bus=_RecordingBus(), + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + dispatcher_id = report.results[0].dispatchers_registered[0] + + for contract_name, error_match in ( + (f" {contract.name} ", "noncanonical NOT_READY contract names"), + ("node_not_in_manifest", "NOT_READY.*manifest contract-name mismatch"), + ): + forged_result = ModelContractAttachResult.model_validate_json( + json.dumps( + { + "contract_name": contract_name, + "status": EnumContractAttachStatus.NOT_READY.value, + "dispatcher_ids": [dispatcher_id], + } + ) + ) + bus = _RecordingBus() + provisioner = _RecordingReadyProvisioner() + with pytest.raises(ModelOnexError, match=error_match): + await reattach_not_ready_contracts( + manifest, + (forged_result,), + engine, + bus, + "test", + provisioner=provisioner, + ) + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + + +@pytest.mark.asyncio +async def test_not_ready_wrong_registered_owner_fails_before_provisioning() -> None: + """A reattach scope must retain its original live contract provenance.""" + from omnibase_core.models.errors import ModelOnexError + from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + reattach_not_ready_contracts, + ) + + contract_a = _contract("node_not_ready_wrong_a", "HandlerContractA") + contract_b = _contract("node_not_ready_wrong_b", "HandlerContractB") + manifest = ModelAutoWiringManifest(contracts=(contract_a, contract_b), errors=()) + engine = MessageDispatchEngine() + bus = _RecordingBus() + report = await wire_from_manifest( + manifest, + engine, + event_bus=bus, + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + by_name = {result.contract_name: result for result in report.results} + dispatcher_b = by_name[contract_b.name].dispatchers_registered[0] + forged_result = ModelContractAttachResult.model_validate_json( + json.dumps( + { + "contract_name": contract_a.name, + "status": EnumContractAttachStatus.NOT_READY.value, + "dispatcher_ids": [dispatcher_b], + } + ) + ) + provisioner = _RecordingReadyProvisioner() + + with pytest.raises(ModelOnexError, match="not owned by contract"): + await reattach_not_ready_contracts( + manifest, + (forged_result,), + engine, + bus, + "test", + provisioner=provisioner, + ) + + assert provisioner.ensure_calls == [] + assert provisioner.confirm_calls == [] + assert bus.subscriptions == [] + + +@pytest.mark.asyncio +async def test_one_contract_may_own_two_unique_dispatchers() -> None: + """Ownership uniqueness is per dispatcher, not one dispatcher per contract.""" + from datetime import UTC, datetime + from uuid import uuid4 + + from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope + + HandlerContractA.calls = 0 + HandlerContractB.calls = 0 + contract = _contract_with_handlers( + "node_two_dispatchers", + ("HandlerContractA", "HandlerContractB"), + ) + manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) + engine = MessageDispatchEngine() + bus = _RecordingBus() + report = await wire_from_manifest( + manifest, + engine, + event_bus=bus, + environment="test", + subscribe_immediately=False, + ) + engine.freeze() + + owned_dispatchers = report.results[0].dispatchers_registered + assert len(owned_dispatchers) == 2 + assert len(set(owned_dispatchers)) == 2 + subscriptions = await subscribe_wired_contract_topics( + manifest, + report, + engine, + bus, + "test", + ) + assert subscriptions == {contract.name: (_SHARED_TOPIC,)} + assert len(bus.subscriptions) == 1 + + envelope = ModelEventEnvelope[object]( + payload={"prompt": "one contract, two dispatchers"}, + correlation_id=uuid4(), + envelope_timestamp=datetime.now(UTC), + event_type="omn15474.shared-command", + source_tool="contract-scoped-dispatch-test", + ) + await bus.deliver_to_every_group(envelope) + + assert HandlerContractA.calls == 1 + assert HandlerContractB.calls == 1 + + +@pytest.mark.asyncio +async def test_dispatch_engine_rejects_empty_contract_scope() -> None: + """An explicitly scoped dispatch cannot use an empty allowlist.""" + from datetime import UTC, datetime + from uuid import uuid4 + + from omnibase_core.models.errors import ModelOnexError + from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope + + engine = MessageDispatchEngine() + engine.freeze() + envelope = ModelEventEnvelope[object]( + payload={}, + correlation_id=uuid4(), + envelope_timestamp=datetime.now(UTC), + event_type="omn15474.shared-command", + source_tool="contract-scoped-dispatch-test", + ) + + with pytest.raises(ModelOnexError, match="at least one allowed dispatcher"): + await engine.dispatch( + _SHARED_TOPIC, + envelope, + allowed_dispatcher_ids=frozenset(), + ) + + +@pytest.mark.asyncio +async def test_dispatch_engine_rejects_unknown_contract_scope() -> None: + """A stale or forged dispatcher ID cannot silently route process-wide.""" + from datetime import UTC, datetime + from uuid import uuid4 + + from omnibase_core.models.errors import ModelOnexError + from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope + + engine = MessageDispatchEngine() + engine.freeze() + envelope = ModelEventEnvelope[object]( + payload={}, + correlation_id=uuid4(), + envelope_timestamp=datetime.now(UTC), + event_type="omn15474.shared-command", + source_tool="contract-scoped-dispatch-test", + ) + + with pytest.raises(ModelOnexError, match="not registered on this engine"): + await engine.dispatch_scoped( + _SHARED_TOPIC, + envelope, + allowed_dispatcher_ids={"dispatcher.unknown"}, + ) diff --git a/tests/integration/runtime/test_delegation_dispatch_port_handler_compat.py b/tests/integration/runtime/test_delegation_dispatch_port_handler_compat.py new file mode 100644 index 0000000000..9c748ed133 --- /dev/null +++ b/tests/integration/runtime/test_delegation_dispatch_port_handler_compat.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Consumer-handler compatibility for the runtime-owned delegation port.""" + +from __future__ import annotations + +import inspect +from datetime import UTC, datetime +from uuid import uuid4 + +import pytest + +from omnibase_core.models.dispatch.model_dispatch_bus_command import ( + ModelDispatchBusCommand, +) +from omnibase_core.models.dispatch.model_dispatch_bus_terminal_result import ( + ModelDispatchBusTerminalResult, +) +from omnibase_infra.runtime.protocols.protocol_delegation_dispatch_port import ( + ProtocolDelegationDispatchPort, +) +from omnibase_infra.runtime.runtime_local_ingress import ModelRuntimeLocalIngressRoute +from omnibase_infra.runtime.service_delegation_dispatch_port import ( + RuntimeDelegationDispatchPort, +) + +pytestmark = pytest.mark.integration + + +def _delegation_route() -> ModelRuntimeLocalIngressRoute: + return ModelRuntimeLocalIngressRoute( + node_name="node_delegation_orchestrator", + contract_name="node_delegation_orchestrator", + command_topic="onex.cmd.omnibase-infra.delegation-request.v1", + event_type="omnibase-infra.delegation-request", + terminal_event="onex.evt.omnibase-infra.delegation-completed.v1", + terminal_events=( + "onex.evt.omnibase-infra.delegation-completed.v1", + "onex.evt.omnibase-infra.delegation-failed.v1", + ), + contract_path="/contracts/omnimarket/node_delegation_orchestrator/contract.yaml", + package_name="omnimarket", + ) + + +def test_runtime_port_exposes_consumer_handler_optional_parameters() -> None: + """The injected implementation and its protocol evolve as one boundary.""" + for dispatch_method in ( + ProtocolDelegationDispatchPort.dispatch, + RuntimeDelegationDispatchPort.dispatch, + ): + parameters = inspect.signature(dispatch_method).parameters + assert parameters["max_tokens"].annotation in {"int | None", int | None} + assert parameters["backend_id"].default is None + assert parameters["response_contract"].default is None + assert parameters["system_prompt"].default is None + assert parameters["temperature"].default is None + assert parameters["response_format"].default is None + + +@pytest.mark.asyncio +async def test_absent_consumer_features_dispatch_through_runtime_bus( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Always-supplied None kwargs must reach the existing Pattern-B route.""" + route = _delegation_route() + captured_commands: list[ModelDispatchBusCommand] = [] + + class FakePatternBBroker: + def __init__(self, *_args: object, **_kwargs: object) -> None: + pass + + async def dispatch_request( + self, command: ModelDispatchBusCommand + ) -> tuple[ModelRuntimeLocalIngressRoute, ModelDispatchBusTerminalResult]: + captured_commands.append(command) + return route, ModelDispatchBusTerminalResult( + correlation_id=command.correlation_id, + status="completed", + payload={"content": "workflow-ok"}, + completed_at=datetime.now(UTC), + ) + + monkeypatch.setattr( + "omnibase_infra.runtime.service_delegation_dispatch_port.RuntimePatternBBroker", + FakePatternBBroker, + ) + port = RuntimeDelegationDispatchPort( + event_bus=object(), # type: ignore[arg-type] + routes={"delegation.orchestrate": route}, + ) + + result = await port.dispatch( + prompt="workflow probe", + task_type="reasoning", + correlation_id=uuid4(), + max_tokens=None, + source_file_path=None, + source_session_id=None, + wait=True, + quality_contract_mode="extend_task_class", + acceptance_criteria=(), + tenant_id=None, + backend_id=None, + response_contract=None, + system_prompt=None, + temperature=None, + response_format=None, + ) + + assert result["status"] == "completed" + assert captured_commands[0].payload["prompt"] == "workflow probe" + assert "max_tokens" not in captured_commands[0].payload + assert "backend_id" not in captured_commands[0].payload + assert "response_contract" not in captured_commands[0].payload + assert "system_prompt" not in captured_commands[0].payload + assert "temperature" not in captured_commands[0].payload + assert "response_format" not in captured_commands[0].payload + + +@pytest.mark.asyncio +async def test_metered_terminal_cost_crosses_the_runtime_consumer_boundary( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A metered workflow terminal reaches the consumer as measured actual cost.""" + route = _delegation_route() + + class FakePatternBBroker: + def __init__(self, *_args: object, **_kwargs: object) -> None: + pass + + async def dispatch_request( + self, command: ModelDispatchBusCommand + ) -> tuple[ModelRuntimeLocalIngressRoute, ModelDispatchBusTerminalResult]: + return route, ModelDispatchBusTerminalResult( + correlation_id=command.correlation_id, + status="completed", + payload={ + "model_used": "gemini-2.5-flash", + "prompt_tokens": 115, + "completion_tokens": 130, + "final_attempt_cost": 0.00137, + "cumulative_attempt_cost": 0.00182, + }, + completed_at=datetime.now(UTC), + ) + + monkeypatch.setattr( + "omnibase_infra.runtime.service_delegation_dispatch_port.RuntimePatternBBroker", + FakePatternBBroker, + ) + port = RuntimeDelegationDispatchPort( + event_bus=object(), # type: ignore[arg-type] + routes={"delegation.orchestrate": route}, + ) + + result = await port.dispatch( + prompt="metered workflow probe", + task_type="reasoning", + correlation_id=uuid4(), + max_tokens=None, + source_file_path=None, + source_session_id=None, + wait=True, + quality_contract_mode="extend_task_class", + acceptance_criteria=(), + tenant_id=None, + backend_id=None, + response_contract=None, + ) + + assert result["cost_usd"] == pytest.approx(0.00182) diff --git a/tests/integration/runtime/test_dynamic_contract_registration_e2e.py b/tests/integration/runtime/test_dynamic_contract_registration_e2e.py index 282aa8eede..987d0422f3 100644 --- a/tests/integration/runtime/test_dynamic_contract_registration_e2e.py +++ b/tests/integration/runtime/test_dynamic_contract_registration_e2e.py @@ -20,6 +20,7 @@ import pytest +from omnibase_infra.enums.enum_message_category import EnumMessageCategory from omnibase_infra.event_bus.event_bus_inmemory import EventBusInmemory from omnibase_infra.runtime.enums.enum_materialization_rejection import ( EnumMaterializationRejection, @@ -36,7 +37,7 @@ # --------------------------------------------------------------------------- # Minimal contract YAML for a Noop in-process handler. -# handler_class in metadata so ContractYamlParser can extract it. +# handler_class is top-level because ModelHandlerContract now owns it directly. # NOTE: topic strings here are contract-declared per ONEX convention. # --------------------------------------------------------------------------- @@ -52,12 +53,12 @@ input_model: omnibase_infra.models.types.JsonDict output_model: omnibase_core.models.dispatch.model_handler_output.ModelHandlerOutput description: E2E test for dynamic contract registration -metadata: - handler_class: tests.fixtures.handler_noop.HandlerNoop -event_bus: - subscribe_topics: - - onex.evt.test.e2e-dynamic.v1 - publish_topics: [] +handler_class: tests.fixtures.handler_noop.HandlerNoop +metadata: {} +yaml_consumed_events: + - event_type: onex.evt.test.e2e-dynamic.v1 + handler_function: handle +yaml_published_events: [] handler_routing: version: major: 1 @@ -68,6 +69,7 @@ - handler: name: HandlerNoop module: tests.fixtures.handler_noop + topic: onex.evt.test.e2e-dynamic.v1 """ _EVIL_MODULE_CONTRACT_YAML = ( @@ -163,6 +165,46 @@ async def test_register_then_materialize_wires_dispatcher_and_topic() -> None: assert "onex.evt.test.e2e-dynamic.v1" in result.subscribed_topics +@pytest.mark.asyncio +async def test_existing_contract_owner_rejects_before_dynamic_engine_mutation() -> None: + """A stale same-owner dispatcher must fail before dynamic commit side effects.""" + source = _make_source() + engine = _make_engine() + bus = _make_bus() + + async def stale_handler(envelope: object) -> None: + del envelope + + engine.register_dispatcher( + dispatcher_id="stale-node-e2e-dynamic-dispatcher", + dispatcher=stale_handler, + category=EnumMessageCategory.EVENT, + owner_contract_name="node_e2e_dynamic", + ) + engine.freeze() + source.on_contract_registered( + node_name="node_e2e_dynamic", + contract_yaml=_NOOP_CONTRACT_YAML, + correlation_id=uuid4(), + ) + + dispatchers_before = frozenset(engine._dispatchers) + routes_before = frozenset(engine._routes) + topics_before = await bus.get_topics() + + result = await source.materialize_cached_contract( + node_name="node_e2e_dynamic", + dispatch_engine=engine, + event_bus=bus, + environment="test", + ) + + assert result.status == EnumMaterializationStatus.REJECTED + assert frozenset(engine._dispatchers) == dispatchers_before + assert frozenset(engine._routes) == routes_before + assert await bus.get_topics() == topics_before + + # --------------------------------------------------------------------------- # Test: idempotency # --------------------------------------------------------------------------- diff --git a/tests/integration/runtime/test_golden_chain_live_runtime.py b/tests/integration/runtime/test_golden_chain_live_runtime.py index e7b05f522c..4120e1c17e 100644 --- a/tests/integration/runtime/test_golden_chain_live_runtime.py +++ b/tests/integration/runtime/test_golden_chain_live_runtime.py @@ -89,6 +89,7 @@ def _kafka_available() -> bool: pytestmark = [ pytest.mark.integration, + pytest.mark.kafka, pytest.mark.external, pytest.mark.skipif( not _kafka_available(), diff --git a/tests/integration/runtime/test_handler_wiring_async_incompat_quarantine_integration.py b/tests/integration/runtime/test_handler_wiring_async_incompat_quarantine_integration.py index fe2a160a01..523e9e491e 100644 --- a/tests/integration/runtime/test_handler_wiring_async_incompat_quarantine_integration.py +++ b/tests/integration/runtime/test_handler_wiring_async_incompat_quarantine_integration.py @@ -22,7 +22,6 @@ import pytest -from omnibase_core.errors.error_service_resolution import ServiceResolutionError from omnibase_infra.protocols import ProtocolEventBusLike from omnibase_infra.runtime.auto_wiring.enum_quarantine_reason import ( EnumQuarantineReason, @@ -92,6 +91,10 @@ def _make_dispatch_engine() -> MagicMock: engine._container = None engine.register_dispatcher = MagicMock() engine.register_route = MagicMock() + engine.dispatch_scoped = AsyncMock() + engine.validate_contract_dispatcher_scope = MagicMock( + side_effect=lambda _contract_name, dispatcher_ids: frozenset(dispatcher_ids) + ) engine.freeze = MagicMock() return engine @@ -205,9 +208,8 @@ async def test_mixed_contract_wires_good_quarantines_bad() -> None: The contract outcome is WIRED (at least one live handler); quarantined_handlers has exactly one entry for _BadHandler; total_failed == 0. - _GoodHandler is zero-arg: the container raises ServiceResolutionError (not an - asyncio error) so wiring falls through to the zero-arg construction path, which - succeeds cleanly. + The container returns a concrete _GoodHandler while the bad handler raises the + exact async-incompatibility signal, so the WIRED outcome cannot be vacuous. """ class _BadHandler: @@ -226,18 +228,15 @@ async def handle(self, envelope: object) -> None: ) manifest = ModelAutoWiringManifest(contracts=(contract,), errors=()) - # Container raises ServiceResolutionError for good handler (triggers zero-arg - # fallback) and asyncio RuntimeError for bad handler (triggers quarantine). + # Resolve the good handler through the container so this test owns a real + # dispatcher; the bad handler still triggers quarantine. async def _get_service_async(cls: type) -> object: if cls is _GoodHandler: - raise ServiceResolutionError("no registration for _GoodHandler") + return _GoodHandler() raise RuntimeError(_ASYNCIO_RUN_MSG) - def _get_service(cls: type) -> object: - raise ServiceResolutionError("sync path not used") - container = MagicMock() - container.get_service = MagicMock(side_effect=_get_service) + container.get_service = MagicMock(side_effect=AssertionError("sync path not used")) container.get_service_async = AsyncMock(side_effect=_get_service_async) event_bus = MagicMock(spec=ProtocolEventBusLike) @@ -268,9 +267,10 @@ def _import_side_effect(module: str, name: str) -> type: report.quarantined_handlers[0].reason == EnumQuarantineReason.ASYNC_INCOMPATIBLE ) assert "_BadHandler" in report.quarantined_handlers[0].handler_name - # Contract must be WIRED (good handler resolved via zero-arg path) + # Contract must be WIRED (good handler resolved through the container). wired_results = [r for r in report.results if r.outcome == EnumWiringOutcome.WIRED] assert len(wired_results) == 1, f"Expected 1 WIRED result, got {report.results}" + assert len(wired_results[0].dispatchers_registered) == 1 @pytest.mark.integration diff --git a/tests/integration/runtime/test_live_events_projection_write_path_omn15359.py b/tests/integration/runtime/test_live_events_projection_write_path_omn15359.py new file mode 100644 index 0000000000..82c6821352 --- /dev/null +++ b/tests/integration/runtime/test_live_events_projection_write_path_omn15359.py @@ -0,0 +1,395 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""OMN-15359 cross-boundary regression: the live_events projection write path +against a REAL physical omninode_internal.live_events table. + +THE GAP THIS CLOSES (flagged by the mergesweep-0809-projplane-verify +adversarial verify, rolling ledger 2026-08-09T18:45Z) + + Every prior OMN-15359 proof for this table was one of: + (a) a unit test resolving ``_resolve_projection_database_target`` + against a topology fixture with NO live database at all, or + (b) an integration test that PATCHES ``_build_projection_db_adapter`` + with a ``FakeDb``/``MagicMock`` (e.g. + ``tests/integration/runtime/test_projection_handler_db_injection_integration.py``), + which proves the dispatch *bridge* shape but never touches real SQL. + + Neither proves the thing that was actually broken: that + ``handler_wiring._resolve_projection_database_target`` resolves the + contract-declared ``schema: omninode_internal`` write target, that the + REAL ``ProjectionDatabaseOperations`` adapter (unpatched, real psycopg2, + real connection-identity attestation) issues + ``INSERT INTO "omninode_internal"."live_events"`` against a database where + that relation actually exists, and that the row is durably readable back + from that exact physical location afterward. + +THE SECOND GAP THIS CLOSES (projplane-slice-verify grant-gap, same ledger +entry, live-verified on a real RDS snapshot 2026-08-10) + + This file's own fixture used to hand-issue the write-path role's grants + (``CREATE ROLE ... LOGIN`` + ``GRANT ... ON omninode_internal.live_events``) + directly in Python, independent of migration 099. That MASKED a real + defect: 099 created the physical table but never granted anything on it, + so on live RDS ``omninode_runtime`` carried zero privileges on + ``omninode_internal.live_events`` and no CREATE on the schema -- + deploying 099 as shipped would have converted the original UndefinedTable + failure into InsufficientPrivilege on the very first write. The fixture's + hand-issued GRANT proved the dispatch/adapter code path while silently + supplying the one thing production would not have had. + + 099 now issues the grant itself (guarded role creation + GRANT USAGE ON + SCHEMA + GRANT SELECT/INSERT/UPDATE ON the table, derived from + ``src/omnibase_infra/topology/instances/*.yaml``). The fixture below no + longer grants anything -- it applies 098 + 099 exactly as production does + and only attaches LOGIN + a password afterward, mirroring the same + deployment-owned credential-attach step 094/096 already document for + app_dashboard/role_omnidash (a migration never carries LOGIN + password; + that stays an operator-gated step, never re-asserted by a migration re-run). + If 099 stopped granting privileges, this fixture would surface it as a + real ``psycopg2.errors.InsufficientPrivilege`` on first write, not a + silently-passing test. + +THIS TEST + + Drives a real event through the REAL infra dispatch bridge + (``_make_projection_dispatch_callback`` -> real, unpatched + ``_build_projection_db_adapter`` -> real ``ProjectionDatabaseOperations``) + against an ephemeral, real PostgreSQL 16 cluster that has had + 098 + 099 applied via the same ``psql -f`` path production uses, connecting + as the REAL ``omninode_runtime`` role 099 itself creates and grants -- + LOGIN is attached afterward as the one deployment-owned step a migration + never performs (mirroring ``docker/domain-adapter-proof/prove.py``'s + real-role-identity pattern). Nothing about the DB adapter, the role, or + its grants is mocked or hand-issued outside the migration. The handler + under test is a minimal stand-in (the real ``HandlerProjectionLiveEvents`` + lives in the omnimarket repo and is not importable here) that performs the + exact ``_db.upsert("live_events", "event_id", row)`` call the golden path + in ``node_projection_live_events/contract.yaml`` documents -- the infra + dispatch/adapter code under test is 100% real either way. + + Assertions: + 1. The row lands in AND reads back from ``omninode_internal.live_events`` + -- verified through an INDEPENDENT, freshly-opened connection (not + reusing the writer's connection), proving the write is durably + committed to the real physical location, not merely visible in-session. + 2. ``physical_grant_schema_for_table('omninode_internal', 'live_events')`` + agrees with the resolved write-target schema post-migration (both + 'omninode_internal', with live_events removed from the physical + bridge in this same PR). + 3. The EFFECTIVE grant set 099 produced on a fresh cluster (queried from + ``information_schema.role_table_grants``, not asserted from memory) + equals the set DERIVED from the shipped topology declaration + (``application_topology()``'s ``principals.omninode_runtime.grants`` + entry for this table) -- exactly, neither a subset nor a superset. +""" + +from __future__ import annotations + +import asyncio +import subprocess +import uuid +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import psycopg2 +import pytest + +from omnibase_core.enums.enum_database_grant_object_type import ( + EnumDatabaseGrantObjectType, +) +from omnibase_core.models.contracts.subcontracts.model_db_table_declaration import ( + ModelDbTableDeclaration, +) +from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _make_projection_dispatch_callback, + _resolve_projection_database_target, +) +from omnibase_infra.topology.physical_schema_mapping import ( + physical_grant_schema_for_table, +) +from tests.helpers.application_db_topology import application_topology +from tests.integration.migrations.conftest import EphemeralPostgres + +pytestmark = pytest.mark.integration + +REPO_ROOT = Path(__file__).resolve().parents[3] +FORWARD_DIR = REPO_ROOT / "docker" / "migrations" / "forward" +SCHEMA_MIGRATION = FORWARD_DIR / "098_create_omninode_internal_schema.sql" +TABLE_MIGRATION = FORWARD_DIR / "099_create_omninode_internal_live_events.sql" +ANALYTICS_DB = "omnidash_analytics" +INTERNAL_ROLE = "omninode_runtime" +ROLE_PASSWORD = "live-events-write-path-proof-only" # pragma: allowlist secret + + +def _apply(pg: EphemeralPostgres, path: Path) -> None: + result = pg.psql("-v", "ON_ERROR_STOP=1", "-f", str(path), dbname="postgres") + assert result.returncode == 0, f"{path.name} failed to apply:\n{result.stderr}" + + +def _admin_dsn(pg: EphemeralPostgres) -> str: + return f"host={pg.socket_dir} port={pg.port} user=postgres dbname={ANALYTICS_DB}" + + +def _internal_role_dsn(pg: EphemeralPostgres) -> str: + return ( + f"host={pg.socket_dir} port={pg.port} user={INTERNAL_ROLE} " + f"password={ROLE_PASSWORD} dbname={ANALYTICS_DB}" + ) + + +@pytest.fixture +def live_events_pg(ephemeral_postgres: EphemeralPostgres) -> EphemeralPostgres: + """Ephemeral cluster with 098 + 099 applied EXACTLY as production applies + them -- no hand-issued role or grant. 099 itself now guard-creates + ``omninode_runtime`` (NOLOGIN) and issues every grant this fixture used to + supply by hand; the only thing added here is the LOGIN + password attach, + which mirrors the same deployment-owned credential step 094/096 already + document (a migration never carries LOGIN + password -- that stays an + operator-gated attach, on RDS via Secrets Manager, here via a direct + ALTER ROLE). If 099 regressed to granting nothing again, the write-path + test below would fail with a real InsufficientPrivilege, not silently + pass.""" + bootstrap = ephemeral_postgres.connect() + bootstrap.autocommit = True + with bootstrap.cursor() as cur: + cur.execute(f'CREATE DATABASE "{ANALYTICS_DB}"') + bootstrap.close() + + # 099 transform-copies FROM public.live_events -- seed the (empty) source + # relation with the same shape production carries, matching + # docker/migrations/forward/nodes/node_projection_live_events/0000_create_live_events.sql. + # This fixture intentionally starts empty: the copy-migration's own + # correctness (count/key/hash reconciliation over real rows) is proven in + # tests/integration/migrations/test_099_omninode_internal_live_events_omn15359.py; + # this file's job is the write-PATH seam, not the historical-row copy. + seed = psycopg2.connect(_admin_dsn(ephemeral_postgres)) + seed.autocommit = True + with seed.cursor() as cur: + cur.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto") + cur.execute( + """ + CREATE TABLE public.live_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + event_id TEXT UNIQUE NOT NULL, + type TEXT NOT NULL DEFAULT 'ACTION', + timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), + source TEXT NOT NULL DEFAULT 'platform', + topic TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + payload TEXT NOT NULL DEFAULT '{}', + correlation_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + seed.close() + + _apply(ephemeral_postgres, SCHEMA_MIGRATION) + _apply(ephemeral_postgres, TABLE_MIGRATION) + + # The ONE deployment-owned step a migration never performs: attaching + # LOGIN + a real password to the role 099 already created NOLOGIN. No + # privilege statement runs here -- everything omninode_runtime can do + # against omninode_internal.live_events was granted by 099 itself. + admin = psycopg2.connect(_admin_dsn(ephemeral_postgres)) + admin.autocommit = True + with admin.cursor() as cur: + cur.execute(f"ALTER ROLE {INTERNAL_ROLE} LOGIN PASSWORD %s", (ROLE_PASSWORD,)) + admin.close() + return ephemeral_postgres + + +def _read_back_independently( + pg: EphemeralPostgres, event_id: str +) -> tuple[str, str, str] | None: + """Open a brand-new connection (never the writer's) and read the row back + directly from the physical location -- proves durable commit, not merely + in-session visibility.""" + conn = psycopg2.connect(_admin_dsn(pg)) + conn.autocommit = True + try: + with conn.cursor() as cur: + cur.execute( + "SELECT event_id, topic, summary " + "FROM omninode_internal.live_events WHERE event_id = %s", + (event_id,), + ) + row = cur.fetchone() + return tuple(row) if row is not None else None + finally: + conn.close() + + +class _StandInProjectionHandler: + """Minimal stand-in for HandlerProjectionLiveEvents (omnimarket repo, not + importable from omnibase_infra). Exercises the same + ``_db.upsert("live_events", "event_id", row)`` call the node's golden + path documents -- everything downstream of this call (the adapter, the + connection, the SQL, the real Postgres cluster) is 100% real.""" + + def handle(self, input_data: dict[str, Any]) -> dict[str, Any]: + db = input_data.pop("_db") + input_data.pop("_event_type", None) + row = { + "event_id": input_data["event_id"], + "topic": input_data.get("topic", ""), + "summary": input_data.get("summary", ""), + "source": input_data.get("source", "platform"), + "type": input_data.get("type", "ACTION"), + "payload": input_data.get("payload", "{}"), + } + assert db.upsert("live_events", "event_id", row) is True + return {"rows_upserted": 1} + + +def test_live_events_write_lands_in_and_reads_back_from_omninode_internal( + live_events_pg: EphemeralPostgres, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OMNINODE_INTERNAL_DB_URL", _internal_role_dsn(live_events_pg)) + + declaration = ModelDbTableDeclaration( + name="live_events", + database_ref="application", + schema="omninode_internal", + migration="docker/migrations/forward/099_create_omninode_internal_live_events.sql", + access="write", + role="live_events", + ) + target = _resolve_projection_database_target((declaration,), application_topology()) + + assert target.table_targets[0].schema == "omninode_internal" + assert target.table_targets[0].write_binding is not None + assert ( + target.table_targets[0].write_binding.binding_ref == "omninode_runtime_service" + ) + + handler = _StandInProjectionHandler() + callback = _make_projection_dispatch_callback( + handler, target, ("onex.evt.platform.node-heartbeat.v1",) + ) + + event_id = f"evt-write-path-{uuid.uuid4()}" + envelope = MagicMock() + envelope.topic = "onex.evt.platform.node-heartbeat.v1" + envelope.payload = { + "event_id": event_id, + "topic": "onex.evt.platform.node-heartbeat.v1", + "summary": "cross-boundary write-path proof", + "source": "platform", + "type": "ACTION", + "payload": "{}", + } + + asyncio.run(callback(envelope)) + + row = _read_back_independently(live_events_pg, event_id) + assert row is not None, ( + "row did not land in omninode_internal.live_events -- the real " + "dispatch/adapter write path failed silently" + ) + assert row[0] == event_id + assert row[1] == "onex.evt.platform.node-heartbeat.v1" + assert row[2] == "cross-boundary write-path proof" + + +def test_grant_derivation_schema_agrees_with_the_insert_target_schema() -> None: + """Post-099 + bridge removal: the grant-privilege check and the real SQL + INSERT target must resolve to the identical physical schema -- the exact + seam a bridge-set omission would silently reintroduce. Pure topology + resolution, no live database required; paired here with the write-path + test above so both halves of the seam are proven in one file.""" + declaration = ModelDbTableDeclaration( + name="live_events", + database_ref="application", + schema="omninode_internal", + migration="docker/migrations/forward/099_create_omninode_internal_live_events.sql", + access="write", + role="live_events", + ) + target = _resolve_projection_database_target((declaration,), application_topology()) + insert_target_schema = target.table_targets[0].schema + + grant_check_schema = physical_grant_schema_for_table( + "omninode_internal", "live_events" + ) + + assert grant_check_schema == "omninode_internal" + assert grant_check_schema == insert_target_schema + + +def _declared_live_events_privileges() -> set[str]: + """Derive the expected omninode_runtime privilege set on + omninode_internal.live_events from the shipped topology declaration -- + never hand-typed. Fails loudly (KeyError/StopIteration) if the shipped + instance ever drops the grant this migration is supposed to carry, + instead of silently asserting an empty/stale set.""" + database = application_topology().databases["application"] + grant = next( + g + for g in database.principals["omninode_runtime"].grants + if g.object_type is EnumDatabaseGrantObjectType.TABLE + and g.schema == "omninode_internal" + and "live_events" in g.objects + ) + return {privilege.value.upper() for privilege in grant.privileges} + + +def test_099_effective_grants_match_the_shipped_topology_declaration_exactly( + live_events_pg: EphemeralPostgres, +) -> None: + """The grant-gap repair itself, proven end to end: what 099 ACTUALLY + grants on a fresh cluster (queried from live catalog state, not asserted + from memory) must equal -- not merely include -- what the shipped + topology declares for omninode_runtime on this table. A superset would + hide an over-grant (e.g. a stray DELETE); a subset would reproduce the + exact InsufficientPrivilege defect this migration exists to close. + """ + expected = _declared_live_events_privileges() + assert expected == {"SELECT", "INSERT", "UPDATE"}, ( + "topology declaration drifted from what this test assumed; re-derive " + "before trusting the comparison below" + ) + + conn = psycopg2.connect(_admin_dsn(live_events_pg)) + conn.autocommit = True + try: + with conn.cursor() as cur: + cur.execute( + "SELECT privilege_type FROM information_schema.role_table_grants " + "WHERE table_schema = %s AND table_name = %s AND grantee = %s", + ("omninode_internal", "live_events", INTERNAL_ROLE), + ) + effective = {row[0] for row in cur.fetchall()} + finally: + conn.close() + + assert effective == expected, ( + f"099's effective grant on omninode_internal.live_events for " + f"{INTERNAL_ROLE} is {effective!r}, the shipped topology declares " + f"{expected!r} -- these must match exactly" + ) + + +def test_099_grants_schema_usage_on_omninode_internal( + live_events_pg: EphemeralPostgres, +) -> None: + """USAGE ON SCHEMA is a separate ACL from the TABLE grant above and is not + visible in role_table_grants -- without it every table grant is inert + (Postgres refuses to even resolve the table name for a role with no + schema USAGE). Queried from has_schema_privilege, live catalog state.""" + conn = psycopg2.connect(_admin_dsn(live_events_pg)) + conn.autocommit = True + try: + with conn.cursor() as cur: + cur.execute( + "SELECT has_schema_privilege(%s, 'omninode_internal', 'USAGE')", + (INTERNAL_ROLE,), + ) + (has_usage,) = cur.fetchone() + finally: + conn.close() + + assert has_usage is True diff --git a/tests/integration/runtime/test_manifest_pool_injection_integration.py b/tests/integration/runtime/test_manifest_pool_injection_integration.py index 5600c64d28..35e4ea3695 100644 --- a/tests/integration/runtime/test_manifest_pool_injection_integration.py +++ b/tests/integration/runtime/test_manifest_pool_injection_integration.py @@ -14,6 +14,7 @@ import pytest +from omnibase_infra.runtime.auto_wiring.discovery import discover_contracts_from_paths from omnibase_infra.runtime.auto_wiring.handler_wiring import wire_from_manifest from omnibase_infra.runtime.auto_wiring.models import ( ModelAutoWiringManifest, @@ -26,6 +27,27 @@ ) from omnibase_infra.runtime.service_kernel import _build_runtime_handler_dependencies +_GATEWAY_RESOLVER_CONFIG = """\ +enable_convention_fallback: false +mappings: + - logical_name: gateway.attach.keycloak.issuer + source: {source_type: env, source_path: TEST_ISSUER} + - logical_name: gateway.attach.keycloak.introspection + source: {source_type: env, source_path: TEST_INTROSPECTION} + - logical_name: gateway.attach.keycloak.admin_client_credentials.client_id + source: {source_type: env, source_path: TEST_CLIENT_ID} + - logical_name: gateway.attach.keycloak.admin_client_credentials.client_secret + source: {source_type: env, source_path: TEST_CLIENT_SECRET} + - logical_name: gateway.attach.keycloak.jwks + source: {source_type: env, source_path: TEST_JWKS} +""" + + +def _write_gateway_resolver_config(tmp_path: Path) -> Path: + config_path = tmp_path / "secret_resolver.yaml" + config_path.write_text(_GATEWAY_RESOLVER_CONFIG, encoding="utf-8") + return config_path + def _make_pool_backed_contract( *, @@ -223,6 +245,93 @@ def test_runtime_handler_dependencies_include_dlq_replay_when_kafka_configured() assert consumer.config.dlq_topic == "onex.dlq.omnibase-infra.events.v1" +@pytest.mark.integration +def test_runtime_handler_dependencies_share_gateway_state_and_resolver( + tmp_path: Path, +) -> None: + """All gateway lifecycle operations use one session authority and resolver.""" + config_path = _write_gateway_resolver_config(tmp_path) + + result = _build_runtime_handler_dependencies( + None, + gateway_secret_resolver_config_path=config_path, + ) + + assert result is not None + attach = result["HandlerGatewayAttach"] + heartbeat = result["HandlerGatewayHeartbeat"] + detach = result["HandlerGatewayDetach"] + assert attach["config"] is heartbeat["config"] is detach["config"] + assert attach["session_store"] is heartbeat["session_store"] + assert attach["session_store"] is detach["session_store"] + assert attach["secret_resolver"] is heartbeat["secret_resolver"] + assert attach["secret_resolver"] is detach["secret_resolver"] + + +@pytest.mark.integration +def test_gateway_runtime_dependencies_fail_closed_on_invalid_config( + tmp_path: Path, +) -> None: + """A missing deploy-rendered resolver artifact cannot silently disable auth.""" + from omnibase_infra.errors import ProtocolConfigurationError + + with pytest.raises(ProtocolConfigurationError, match="valid rendered"): + _build_runtime_handler_dependencies( + None, + gateway_secret_resolver_config_path=tmp_path / "missing.yaml", + ) + + +@pytest.mark.integration +def test_gateway_runtime_dependencies_fail_closed_on_missing_mapping( + tmp_path: Path, +) -> None: + """A partial resolver artifact cannot defer an auth failure to first traffic.""" + from omnibase_infra.errors import ProtocolConfigurationError + + config_path = tmp_path / "secret_resolver.yaml" + config_path.write_text( + "enable_convention_fallback: false\nmappings: []\n", + encoding="utf-8", + ) + + with pytest.raises(ProtocolConfigurationError, match="missing explicit"): + _build_runtime_handler_dependencies( + None, + gateway_secret_resolver_config_path=config_path, + ) + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_real_gateway_contract_wires_in_strict_mode( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The shipped contract and all real handlers satisfy strict boot wiring.""" + from omnibase_infra.runtime.message_dispatch_engine import MessageDispatchEngine + + contract_path = ( + Path(__file__).parents[3] + / "src/omnibase_infra/nodes/node_gateway_attach_effect/contract.yaml" + ) + config_path = _write_gateway_resolver_config(tmp_path) + monkeypatch.setenv("ONEX_WIRING_STRICT_MODE", "1") + + report = await wire_from_manifest( + discover_contracts_from_paths([contract_path]), + MessageDispatchEngine(), + materialized_explicit_dependencies=_build_runtime_handler_dependencies( + None, + gateway_secret_resolver_config_path=config_path, + ), + ) + + assert report.total_failed == 0 + assert report.total_wired == 1 + assert len(report.results[0].wirings) == 3 + + @pytest.mark.integration @pytest.mark.asyncio async def test_kernel_runtime_dependencies_cover_dlq_replay_handler() -> None: diff --git a/tests/integration/runtime/test_projection_dispatch_tenant_authority_context.py b/tests/integration/runtime/test_projection_dispatch_tenant_authority_context.py new file mode 100644 index 0000000000..d33c439458 --- /dev/null +++ b/tests/integration/runtime/test_projection_dispatch_tenant_authority_context.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Live-dispatch regression for projection tenant authority (OMN-15421).""" + +from __future__ import annotations + +from unittest.mock import patch +from uuid import uuid4 + +import pytest + +from omnibase_core.models.contracts.subcontracts.model_db_table_declaration import ( + ModelDbTableDeclaration, +) +from omnibase_core.models.dispatch.model_dispatch_route import ModelDispatchRoute +from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope +from omnibase_infra.enums import EnumDispatchStatus, EnumMessageCategory +from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _make_projection_dispatch_callback, + _resolve_projection_database_target, +) +from omnibase_infra.runtime.dispatch_envelope_context import ( + bind_projection_tenant_authority, +) +from omnibase_infra.runtime.message_dispatch_engine import MessageDispatchEngine +from tests.helpers.application_db_topology import ( + application_topology, + projection_database_target, +) +from tests.helpers.projection_tenant_authority import signed_tenant_authority_fixture + +pytestmark = pytest.mark.integration + +TOPIC = "onex.evt.platform.tenant-proof.v1" + + +class _Cursor: + def __init__( + self, + calls: list[tuple[str, object]], + principal: str = "tenant_projection_writer", + ) -> None: + self._calls = calls + self._principal = principal + + def __enter__(self) -> _Cursor: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, params: object = None) -> None: + self._calls.append((sql, params)) + + def fetchone(self) -> tuple[str, str]: + return (self._principal, "omnidash_analytics") + + +class _Connection: + closed = False + + def __init__( + self, + calls: list[tuple[str, object]], + principal: str = "tenant_projection_writer", + ) -> None: + self.autocommit = True + self._calls = calls + self._principal = principal + self.close_calls = 0 + + def cursor(self, *_args: object, **_kwargs: object) -> _Cursor: + return _Cursor(self._calls, self._principal) + + def commit(self) -> None: + return None + + def rollback(self) -> None: + return None + + def close(self) -> None: + self.close_calls += 1 + self.closed = True + + +class _TenantProjectionHandler: + def handle(self, input_data: dict[str, object]) -> dict[str, int]: + database = input_data["_db"] + database.upsert( # type: ignore[union-attr] + "delegation_events", + "correlation_id", + {"correlation_id": input_data["_envelope_id"], "value": "dispatched"}, + ) + return {"rows_upserted": 1} + + +class _InternalProjectionHandler: + def handle(self, input_data: dict[str, object]) -> dict[str, int]: + database = input_data["_db"] + database.upsert( # type: ignore[union-attr] + "generation_events", + "correlation_id", + {"correlation_id": input_data["_envelope_id"], "status": "complete"}, + ) + return {"rows_upserted": 1} + + +async def test_dispatch_engine_keeps_verified_authority_out_of_band( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """JSON materialization preserves capability and typed transport identity.""" + tenant_id = uuid4() + correlation_id = uuid4() + calls: list[tuple[str, object]] = [] + connection = _Connection(calls) + monkeypatch.setenv("OMNIDASH_ANALYTICS_DB_URL", "postgresql://fixture") + callback = _make_projection_dispatch_callback( + _TenantProjectionHandler(), + projection_database_target("delegation_events", schema="tenant"), + (TOPIC,), + ) + engine = MessageDispatchEngine() + engine.register_dispatcher( + dispatcher_id="tenant-projection-proof", + dispatcher=callback, + category=EnumMessageCategory.EVENT, + ) + engine.register_route( + ModelDispatchRoute( + route_id="tenant-projection-proof", + topic_pattern=TOPIC, + message_category=EnumMessageCategory.EVENT, + dispatcher_id="tenant-projection-proof", + ) + ) + engine.freeze() + envelope = ModelEventEnvelope[dict[str, object]]( + payload={"value": "dispatched", "tenant_id": str(uuid4())}, + correlation_id=correlation_id, + event_type=TOPIC, + ) + authority = signed_tenant_authority_fixture( + tenant_id, + event_envelope=envelope, + ).verify() + + with ( + bind_projection_tenant_authority(authority), + patch("psycopg2.connect", return_value=connection), + ): + result = await engine.dispatch(topic=TOPIC, envelope=envelope) + + assert result.status == EnumDispatchStatus.SUCCESS + assert calls[0] == ("SELECT current_user, current_database()", None) + assert calls[1] == ( + "SELECT set_config(%s, %s, true)", + ("app.tenant_id", str(tenant_id)), + ) + assert 'INSERT INTO "tenant"."delegation_events"' in calls[2][0] + assert calls[2][1]["correlation_id"] == envelope.envelope_id # type: ignore[index] + assert calls[2][1]["tenant_id"] == tenant_id # type: ignore[index] + assert connection.close_calls == 1 + + +async def test_dispatch_without_verified_capability_fails_before_connect( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OMNIDASH_ANALYTICS_DB_URL", "postgresql://fixture") + callback = _make_projection_dispatch_callback( + _TenantProjectionHandler(), + projection_database_target("delegation_events", schema="tenant"), + (TOPIC,), + ) + envelope = ModelEventEnvelope[dict[str, object]]( + payload={"tenant_id": str(uuid4())}, + event_type=TOPIC, + ) + + with patch("psycopg2.connect") as connect: + await callback(envelope) + + connect.assert_not_called() + + +async def test_mixed_target_internal_operation_does_not_resolve_tenant_authority( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tables = ( + ModelDbTableDeclaration( + name="delegation_events", + database_ref="application", + schema="tenant", + migration="proof/tenant.sql", + access="read_write", + role="tenant", + ), + ModelDbTableDeclaration( + name="generation_events", + database_ref="application", + schema="omninode_internal", + migration="proof/internal.sql", + access="read_write", + role="internal", + ), + ) + target = _resolve_projection_database_target(tables, application_topology()) + monkeypatch.setenv("OMNIDASH_ANALYTICS_DB_URL", "postgresql://tenant") + monkeypatch.setenv("OMNINODE_INTERNAL_DB_URL", "postgresql://internal") + callback = _make_projection_dispatch_callback( + _InternalProjectionHandler(), target, (TOPIC,) + ) + calls: list[tuple[str, object]] = [] + connection = _Connection(calls, principal="omninode_runtime") + envelope = ModelEventEnvelope[dict[str, object]]( + payload={"status": "complete"}, + correlation_id=uuid4(), + event_type=TOPIC, + ) + + with patch("psycopg2.connect", return_value=connection) as connect: + await callback(envelope) + + connect.assert_called_once_with("postgresql://internal") + assert all("set_config" not in sql for sql, _params in calls) + assert any('"omninode_internal"."generation_events"' in sql for sql, _ in calls) diff --git a/tests/integration/runtime/test_projection_handler_db_injection_integration.py b/tests/integration/runtime/test_projection_handler_db_injection_integration.py index 2aad7d8132..63e1d3f92d 100644 --- a/tests/integration/runtime/test_projection_handler_db_injection_integration.py +++ b/tests/integration/runtime/test_projection_handler_db_injection_integration.py @@ -19,10 +19,15 @@ import pytest +from omnibase_core.models.contracts.subcontracts.model_db_ownership_subcontract import ( + ModelDbOwnershipSubcontract, +) +from omnibase_core.models.contracts.subcontracts.model_db_table_declaration import ( + ModelDbTableDeclaration, +) from omnibase_infra.runtime.auto_wiring.handler_wiring import ( - _build_sync_db_adapter, + _build_projection_db_adapter, _make_projection_dispatch_callback, - _read_db_io_tables, ) from omnibase_infra.runtime.auto_wiring.models import ( ModelContractVersion, @@ -32,24 +37,32 @@ ModelHandlerRouting, ModelHandlerRoutingEntry, ) +from tests.helpers.application_db_topology import projection_database_target + + +@pytest.fixture(autouse=True) +def _configured_projection_dsn(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OMNIDASH_ANALYTICS_DB_URL", "postgresql://fixture") + monkeypatch.setenv("OMNINODE_INTERNAL_DB_URL", "postgresql://fixture") + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -def _make_contract(tmp_path: Path, db_tables_yaml: str = "") -> ModelDiscoveredContract: +def _make_contract( + tmp_path: Path, *, declare_db_io: bool = False +) -> ModelDiscoveredContract: """Write a minimal contract.yaml and return a ModelDiscoveredContract.""" - db_io_block = f"db_io:\n db_tables:\n{db_tables_yaml}" if db_tables_yaml else "" contract_path = tmp_path / "contract.yaml" contract_path.write_text( - f"name: projection_registration\n" - f"node_type: reducer\n" - f"contract_version: {{major: 1, minor: 0, patch: 0}}\n" - f"{db_io_block}\n" - f"event_bus:\n" - f" subscribe_topics:\n" - f" - onex.evt.platform.node-heartbeat.v1\n" + "name: projection_registration\n" + "node_type: reducer\n" + "contract_version: {major: 1, minor: 0, patch: 0}\n" + "event_bus:\n" + " subscribe_topics:\n" + " - onex.evt.platform.node-heartbeat.v1\n" ) return ModelDiscoveredContract( name="projection_registration", @@ -73,6 +86,22 @@ def _make_contract(tmp_path: Path, db_tables_yaml: str = "") -> ModelDiscoveredC ), ), ), + db_io=( + ModelDbOwnershipSubcontract( + db_tables=[ + ModelDbTableDeclaration( + name="node_service_registry", + database_ref="application", + schema="tenant", + migration="tests/node_service_registry.sql", + access="read_write", + role="service_registry", + ) + ] + ) + if declare_db_io + else None + ), ) @@ -105,7 +134,9 @@ def upsert(self, table: str, key: str, row: dict) -> bool: def query(self, table: str, filters: dict | None = None) -> list: return [] - db_tables = [{"name": "node_service_registry", "database": "omnidash_analytics"}] + db_tables = projection_database_target( + "node_service_registry", schema="omninode_internal" + ) handler = FakeProjectionHandler() callback = _make_projection_dispatch_callback( handler, db_tables, ("onex.evt.platform.node-heartbeat.v1",) @@ -118,7 +149,7 @@ def query(self, table: str, filters: dict | None = None) -> list: fake_db = FakeDb() with patch( - "omnibase_infra.runtime.auto_wiring.handler_wiring._build_sync_db_adapter", + "omnibase_infra.runtime.auto_wiring.handler_wiring._build_projection_db_adapter", return_value=fake_db, ): with patch( @@ -161,7 +192,7 @@ def query(self, table: str, filters: dict | None = None) -> list: callback = _make_projection_dispatch_callback( FakeProjectionHandler(), - [{"name": "node_service_registry", "database": "omnidash_analytics"}], + projection_database_target("node_service_registry", schema="omninode_internal"), ("onex.evt.platform.node-heartbeat.v1",), ) @@ -170,7 +201,7 @@ def query(self, table: str, filters: dict | None = None) -> list: envelope.payload = {"service_name": "runtime-host", "health_status": "healthy"} with patch( - "omnibase_infra.runtime.auto_wiring.handler_wiring._build_sync_db_adapter", + "omnibase_infra.runtime.auto_wiring.handler_wiring._build_projection_db_adapter", return_value=FakeDb(), ): with patch( @@ -212,7 +243,7 @@ def query(self, table: str, filters: dict | None = None) -> list: callback = _make_projection_dispatch_callback( FakeProjectionHandler(), - [{"name": "node_service_registry", "database": "omnidash_analytics"}], + projection_database_target("node_service_registry", schema="omninode_internal"), ( "onex.evt.platform.node-introspection.v1", "onex.evt.platform.node-heartbeat.v1", @@ -225,7 +256,7 @@ def query(self, table: str, filters: dict | None = None) -> list: envelope.payload = {"service_name": "runtime-host", "health_status": "healthy"} with patch( - "omnibase_infra.runtime.auto_wiring.handler_wiring._build_sync_db_adapter", + "omnibase_infra.runtime.auto_wiring.handler_wiring._build_projection_db_adapter", return_value=FakeDb(), ): with patch( @@ -267,7 +298,7 @@ def query(self, table: str, filters: dict | None = None) -> list: callback = _make_projection_dispatch_callback( FakeProjectionHandler(), - [{"name": "node_service_registry", "database": "omnidash_analytics"}], + projection_database_target("node_service_registry", schema="omninode_internal"), ( "onex.evt.platform.node-introspection.v1", "onex.evt.platform.node-heartbeat.v1", @@ -284,7 +315,7 @@ def query(self, table: str, filters: dict | None = None) -> list: } with patch( - "omnibase_infra.runtime.auto_wiring.handler_wiring._build_sync_db_adapter", + "omnibase_infra.runtime.auto_wiring.handler_wiring._build_projection_db_adapter", return_value=FakeDb(), ): with patch( @@ -326,7 +357,7 @@ def query(self, table: str, filters: dict | None = None) -> list: callback = _make_projection_dispatch_callback( FakeProjectionHandler(), - [{"name": "node_service_registry", "database": "omnidash_analytics"}], + projection_database_target("node_service_registry", schema="omninode_internal"), ( "onex.evt.platform.node-introspection.v1", "onex.evt.platform.node-heartbeat.v1", @@ -344,7 +375,7 @@ def query(self, table: str, filters: dict | None = None) -> list: } with patch( - "omnibase_infra.runtime.auto_wiring.handler_wiring._build_sync_db_adapter", + "omnibase_infra.runtime.auto_wiring.handler_wiring._build_projection_db_adapter", return_value=FakeDb(), ): with patch( @@ -363,28 +394,23 @@ def test_wire_handler_entry_uses_projection_path_when_db_io_declared( tmp_path: Path, ) -> None: """_wire_handler_entry selects projection callback (not standard) when contract has db_io.""" - contract = _make_contract( - tmp_path, - db_tables_yaml=" - name: node_service_registry\n database: omnidash_analytics\n", - ) + contract = _make_contract(tmp_path, declare_db_io=True) - # _read_db_io_tables should find the declared table - tables = _read_db_io_tables(contract.contract_path) - assert len(tables) == 1 - assert tables[0]["database"] == "omnidash_analytics" + assert contract.db_io is not None + assert contract.db_io.db_tables[0].database_ref == "application" + assert contract.db_io.db_tables[0].schema == "tenant" @pytest.mark.integration def test_wire_handler_entry_uses_standard_path_when_no_db_io(tmp_path: Path) -> None: """_wire_handler_entry uses standard envelope path when contract has no db_io.""" - contract = _make_contract(tmp_path, db_tables_yaml="") - tables = _read_db_io_tables(contract.contract_path) - assert tables == [] + contract = _make_contract(tmp_path) + assert contract.db_io is None @pytest.mark.integration -def test_projection_callback_no_op_when_db_url_missing(tmp_path: Path) -> None: - """Projection callback returns None without calling handler when DB URL unset.""" +def test_projection_callback_rejects_missing_db_url_at_wiring(tmp_path: Path) -> None: + """Projection wiring fails before dispatch when a required DSN is unset.""" call_count = [0] class CountingHandler: @@ -392,20 +418,17 @@ def handle(self, input_data: dict) -> dict: call_count[0] += 1 return {} - db_tables = [{"name": "node_service_registry", "database": "omnidash_analytics"}] - callback = _make_projection_dispatch_callback(CountingHandler(), db_tables, ()) - - envelope = MagicMock() - envelope.topic = "onex.evt.platform.node-heartbeat.v1" - envelope.payload = {} - with patch( "omnibase_infra.runtime.auto_wiring.handler_wiring.os.environ.get", return_value="", ): - result = asyncio.run(callback(envelope)) + with pytest.raises(ValueError, match="tenant_projection"): + _make_projection_dispatch_callback( + CountingHandler(), + projection_database_target("delegation_events", schema="tenant"), + (), + ) - assert result is None assert call_count[0] == 0 @@ -428,10 +451,13 @@ def __enter__(self) -> FakeCursor: def __exit__(self, *args: object) -> None: return None - def execute(self, sql: str, params: object) -> None: + def execute(self, sql: str, params: object | None = None) -> None: captured_execute["sql"] = sql captured_execute["params"] = params + def fetchone(self) -> tuple[str, str]: + return ("omninode_runtime", "omnidash_analytics") + class FakeConnection: closed = False autocommit = False @@ -439,7 +465,23 @@ class FakeConnection: def cursor(self, *args: object, **kwargs: object) -> FakeCursor: return FakeCursor() - fake_extras = types.SimpleNamespace(Json=FakeJson, RealDictCursor=object) + # OMN-15301: the adapter now runs each statement inside an explicit + # tenant-scoped transaction, so the double must model the transaction + # control every real psycopg2 connection has. + def commit(self) -> None: + captured_execute["committed"] = True + + def rollback(self) -> None: + captured_execute["rolled_back"] = True + + def close(self) -> None: + self.closed = True + + fake_extras = types.SimpleNamespace( + Json=FakeJson, + RealDictCursor=object, + register_uuid=lambda: None, + ) fake_psycopg2 = types.SimpleNamespace( connect=lambda dsn: FakeConnection(), extras=fake_extras, @@ -448,7 +490,13 @@ def cursor(self, *args: object, **kwargs: object) -> FakeCursor: monkeypatch.setitem(sys.modules, "psycopg2", fake_psycopg2) monkeypatch.setitem(sys.modules, "psycopg2.extras", fake_extras) - adapter = _build_sync_db_adapter("postgresql://example") + target = projection_database_target("swarm_runs", schema="omninode_internal") + adapter = _build_projection_db_adapter( + {"omninode_runtime_service": "postgresql://example"}, + target, + None, + None, + ) result = adapter.upsert( "swarm_runs", "run_id", @@ -467,3 +515,8 @@ def cursor(self, *args: object, **kwargs: object) -> FakeCursor: assert params["machines_used"] == ["worker-a", "worker-b"] assert isinstance(params["metadata"], FakeJson) assert params["metadata"].value == {"source": "integration-test"} + # OMN-15421: this is an internal-domain operation, so it uses ordinary + # autocommit and never enters the tenant transaction helper. + assert "set_config" not in str(captured_execute["sql"]) + assert "committed" not in captured_execute + assert "rolled_back" not in captured_execute diff --git a/tests/integration/runtime/test_savings_estimator_subscription_group.py b/tests/integration/runtime/test_savings_estimator_subscription_group.py index 2b35adc5b4..b1fa624959 100644 --- a/tests/integration/runtime/test_savings_estimator_subscription_group.py +++ b/tests/integration/runtime/test_savings_estimator_subscription_group.py @@ -6,10 +6,20 @@ import ast from pathlib import Path +from uuid import uuid4 import pytest +from omnibase_infra.event_bus.event_bus_kafka import EventBusKafka +from omnibase_infra.event_bus.models.config import ModelKafkaEventBusConfig +from omnibase_infra.models import ModelNodeIdentity +from omnibase_infra.services.observability.savings_estimation.config import ( + ConfigSavingsEstimation, +) +from omnibase_infra.utils import compute_consumer_group_id + SERVICE_KERNEL_PATH = Path("src/omnibase_infra/runtime/service_kernel.py") +TEST_BOOTSTRAP_SERVERS = "localhost:9092" def _load_service_kernel_ast() -> ast.Module: @@ -25,30 +35,128 @@ def _call_name(node: ast.AST) -> str: @pytest.mark.integration -def test_savings_estimator_subscribe_calls_include_group_id() -> None: - """Savings estimator subscriptions must be consumer-group owned.""" +def test_savings_estimator_subscribe_calls_use_canonical_identity() -> None: + """Savings subscriptions derive their groups from a typed node identity.""" tree = _load_service_kernel_ast() - subscribe_calls = [ + savings_identity_assignments = [ node for node in ast.walk(tree) - if isinstance(node, ast.Call) and _call_name(node.func) == "subscribe" + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "_savings_node_identity" + for target in node.targets + ) + and isinstance(node.value, ast.Call) + and _call_name(node.value.func) == "ModelNodeIdentity" + ] + assert len(savings_identity_assignments) == 1 + identity_call = savings_identity_assignments[0].value + assert isinstance(identity_call, ast.Call) + identity_keywords = { + keyword.arg: keyword.value for keyword in identity_call.keywords + } + assert isinstance(identity_keywords["env"], ast.Name) + assert identity_keywords["env"].id == "environment" + assert isinstance(identity_keywords["node_name"], ast.Constant) + assert identity_keywords["node_name"].value == "savings-estimator" + assert isinstance(identity_keywords["service"], ast.BoolOp) + assert isinstance(identity_keywords["service"].op, ast.Or) + assert isinstance(identity_keywords["service"].values[0], ast.Attribute) + assert identity_keywords["service"].values[0].attr == "name" + assert isinstance(identity_keywords["service"].values[1], ast.Constant) + assert identity_keywords["service"].values[1].value == "onex-kernel" + assert isinstance(identity_keywords["version"], ast.Constant) + assert identity_keywords["version"].value == "v1" + + savings_input_loops = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.For) + and isinstance(node.target, ast.Name) + and node.target.id == "_input_topic" ] + assert len(savings_input_loops) == 1 savings_subscribe_calls = [ node - for node in subscribe_calls + for node in ast.walk(savings_input_loops[0]) + if isinstance(node, ast.Call) and _call_name(node.func) == "subscribe" if any( - keyword.arg == "group_id" - and isinstance(keyword.value, ast.JoinedStr) - and any( - isinstance(part, ast.Constant) and part.value == "savings-estimator." - for part in keyword.value.values - ) + keyword.arg == "node_identity" + and isinstance(keyword.value, ast.Name) + and keyword.value.id == "_savings_node_identity" for keyword in node.keywords ) ] assert len(savings_subscribe_calls) == 1 + assert all( + keyword.arg != "group_id" for keyword in savings_subscribe_calls[0].keywords + ) + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_savings_estimator_topics_keep_distinct_effective_consumer_groups() -> ( + None +): + """Canonical base identity remains isolated per topic at the Kafka boundary.""" + savings_config = ConfigSavingsEstimation( + kafka_bootstrap_servers=TEST_BOOTSTRAP_SERVERS, + ) + identity = ModelNodeIdentity( + env="dev", + service="onex-kernel", + node_name="savings-estimator", + version="v1", + ) + base_group_id = compute_consumer_group_id(identity) + event_bus = EventBusKafka( + config=ModelKafkaEventBusConfig( + bootstrap_servers=TEST_BOOTSTRAP_SERVERS, + environment="dev", + ) + ) + event_bus._started = True + effective_group_ids: dict[str, str] = {} + captured_auto_offset_reset_overrides: dict[str, str | None] = {} + + async def capture_consumer_start( + topic: str, + group_id: str, + *, + auto_offset_reset_override: str | None = None, + ) -> None: + effective_group_ids[topic] = event_bus._resolve_effective_group_id( + group_id, + topic, + uuid4(), + (topic, group_id), + ) + captured_auto_offset_reset_overrides[topic] = auto_offset_reset_override + event_bus._pending_consumer_keys.discard((topic, group_id)) + + async def on_message(_message: object) -> None: + return None + + event_bus._start_consumer_for_topic_unlocked = capture_consumer_start # type: ignore[method-assign] + + for topic in savings_config.consumed_topics: + await event_bus.subscribe( + topic, + node_identity=identity, + on_message=on_message, + ) + + assert len(effective_group_ids) >= 3 + assert len(set(effective_group_ids.values())) == len(effective_group_ids) + for topic, effective_group_id in effective_group_ids.items(): + assert effective_group_id == f"{base_group_id}.__t.{topic}" + registered_group_ids = { + group_id + for group_id, _subscription_id, _callback in event_bus._subscribers[topic] + } + assert registered_group_ids == {base_group_id} @pytest.mark.integration diff --git a/tests/integration/runtime/test_seam_correlation_authority_omn15546.py b/tests/integration/runtime/test_seam_correlation_authority_omn15546.py new file mode 100644 index 0000000000..9b60f692a0 --- /dev/null +++ b/tests/integration/runtime/test_seam_correlation_authority_omn15546.py @@ -0,0 +1,307 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Cross-boundary seam test for OMN-15546 — ingress owns correlation identity. + +The local ingress assigns the correlation id when it ACCEPTS a `/skill` request. +Every command it dispatches, and every field inside that command's payload, must +carry that id BYTE-IDENTICAL. Nothing downstream may mint one. + +The measured defect (onex-dev, 2026-07-30): one request entered with outer +correlation ``a4740001-…`` and the returned typed payload carried +``f34eea98-…``. ``RuntimeHostProcess._dispatch_local_ingress_request`` normalized +the outer UUID and stamped it on ``ModelDispatchBusCommand.correlation_id``, but +validated the route payload WITHOUT passing that authority in — so a typed input +model declaring ``correlation_id: UUID = Field(default_factory=uuid4)`` minted a +SECOND id for the absent key. Envelope identity and domain identity then diverge +for the rest of the request: the handler, the response payload and every +projection row use the minted id, while the caller awaits a terminal on the +outer one. + +This drives the REAL host seam — ``_dispatch_local_ingress_request`` through to +the ``ModelDispatchBusCommand`` the Pattern B broker actually receives — and +asserts, per the ticket's acceptance criteria: + + command.correlation_id == UUID(command.payload["correlation_id"]) == outer + +A unit test on ``validate_runtime_local_ingress_payload`` does NOT satisfy this: +the defect is that the host never handed the validator its authority, so a test +that calls the validator with the correlation already supplied cannot observe +the bug at all. + +Shares ``tests/fixtures/seams/dispatch_correlation/accepted_command.json`` with +OMN-15474 (byte-identical file) — one accepted command, one correlation +authority, asserted at two different seams. + +Merge-train order: OMN-15474 lands BEFORE this. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock +from uuid import UUID, uuid4 + +import pytest +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_core.models.dispatch.model_dispatch_bus_command import ( + ModelDispatchBusCommand, +) +from omnibase_core.models.dispatch.model_dispatch_bus_terminal_result import ( + ModelDispatchBusTerminalResult, +) +from omnibase_infra.runtime.runtime_host_process import RuntimeHostProcess +from omnibase_infra.runtime.runtime_local_ingress import ModelRuntimeLocalIngressRoute +from tests.helpers.runtime_helpers import make_runtime_config + +pytestmark = pytest.mark.integration + +SEAM_FIXTURE = ( + Path(__file__).resolve().parents[2] + / "fixtures" + / "seams" + / "dispatch_correlation" + / "accepted_command.json" +) + +_COMMAND_NAME = "delegate_skill" +_MODULE = __name__ + + +class ModelSeamDelegateRequest(BaseModel): + """Stands in for market's ``ModelDelegateSkillRequest``. + + The one property that matters is reproduced exactly: ``correlation_id`` has + a ``default_factory=uuid4``, so an ABSENT key silently mints a new identity + rather than failing. That is the whole defect surface. + """ + + model_config = ConfigDict(extra="forbid") + + correlation_id: UUID = Field(default_factory=uuid4) + task_type: str = "test" + prompt: str = "" + + +class ModelSeamNoCorrelationRequest(BaseModel): + """Control: a route model with NO correlation field must be left alone.""" + + model_config = ConfigDict(extra="forbid") + + task_type: str = "test" + prompt: str = "" + + +def _route(input_model_name: str | None) -> ModelRuntimeLocalIngressRoute: + return ModelRuntimeLocalIngressRoute( + node_name="node_delegate_skill_orchestrator", + contract_name=_COMMAND_NAME, + command_topic="onex.cmd.omnibase-infra.delegation-request.v1", + event_type="omnibase-infra.delegation-request", + terminal_event="onex.evt.omnibase-infra.delegation-completed.v1", + contract_path="/tmp/node_delegate_skill_orchestrator/contract.yaml", # noqa: S108 + package_name="omnibase-infra", + input_model_module=_MODULE if input_model_name else None, + input_model_name=input_model_name, + ) + + +class _CapturingBroker: + """Records the exact ModelDispatchBusCommand the host dispatches.""" + + def __init__(self, route: ModelRuntimeLocalIngressRoute) -> None: + self._route = route + self.commands: list[ModelDispatchBusCommand] = [] + + async def dispatch_request( + self, command: ModelDispatchBusCommand + ) -> tuple[ModelRuntimeLocalIngressRoute, ModelDispatchBusTerminalResult]: + self.commands.append(command) + return self._route, ModelDispatchBusTerminalResult( + status="completed", + payload={"status": "complete"}, + correlation_id=command.correlation_id, + ) + + +def _host(route: ModelRuntimeLocalIngressRoute) -> tuple[RuntimeHostProcess, Any]: + process = RuntimeHostProcess( + config=make_runtime_config(local_ingress={"enabled": True}), + dispatch_engine=AsyncMock(), + ) + process._is_running = True + process._local_ingress_routes = {_COMMAND_NAME: route} + broker = _CapturingBroker(route) + process._pattern_b_broker = broker + return process, broker + + +async def _dispatch( + process: RuntimeHostProcess, payload: dict[str, object], correlation_id: UUID +) -> Any: + from omnibase_infra.runtime.runtime_local_ingress import ( + ModelLocalRuntimeIngressRequest, + ) + + return await process._dispatch_local_ingress_request( + ModelLocalRuntimeIngressRequest( + command_name=_COMMAND_NAME, + payload=payload, + correlation_id=correlation_id, + ) + ) + + +@pytest.mark.asyncio +async def test_seam_ingress_correlation_is_authoritative_in_dispatched_command() -> ( + None +): + """Omitted payload correlation_id must be the INGRESS id, never a minted one. + + This is the RED case: with a ``default_factory=uuid4`` input model and no + ``correlation_id`` in the payload, the pre-fix host validated the payload + without its authority and the model minted a second id. Envelope identity + (``command.correlation_id``) and domain identity + (``command.payload["correlation_id"]``) then disagree. + """ + seam = json.loads(SEAM_FIXTURE.read_text(encoding="utf-8")) + outer = UUID(seam["ingress_correlation_id"]) + payload = { + k: v + for k, v in seam["envelope"]["payload"].items() + if k in {"task_type", "prompt"} + } + assert "correlation_id" not in payload, ( + "seam precondition: the payload must OMIT correlation_id — that is the " + "case where default_factory mints a second identity" + ) + + process, broker = _host(_route("ModelSeamDelegateRequest")) + response = await _dispatch(process, payload, outer) + + assert response.ok is True, f"dispatch failed: {response.error}" + assert len(broker.commands) == 1 + command = broker.commands[0] + + # Envelope identity is the ingress id. + assert command.correlation_id == outer, ( + f"command envelope correlation {command.correlation_id} != ingress {outer}" + ) + + # Domain identity is the SAME id, byte-identical — not a minted sibling. + assert isinstance(command.payload, dict) + payload_correlation = command.payload.get("correlation_id") + assert payload_correlation is not None, ( + "the typed payload carries no correlation_id at all — ingress authority " + "was not injected" + ) + assert UUID(str(payload_correlation)) == outer, ( + "OMN-15546 split identity: the dispatched command's envelope carries " + f"{command.correlation_id}, but its typed payload carries " + f"{payload_correlation}. The input model's default_factory=uuid4 minted " + "a SECOND correlation id because the host validated the payload without " + "passing the authoritative ingress correlation in." + ) + assert str(payload_correlation) == str(outer), ( + "byte-identical means byte-identical: the payload representation " + f"{payload_correlation!s} differs textually from the ingress {outer!s}" + ) + + +@pytest.mark.asyncio +async def test_seam_conflicting_payload_correlation_is_rejected_before_dispatch() -> ( + None +): + """A payload correlation that CONFLICTS with the ingress must not dispatch. + + Silently overwriting the caller's value would be as wrong as minting one: + the caller believes it supplied an identity that the runtime discarded. The + ticket's fix contract requires a typed ``validation_error`` and the broker + NOT called. + """ + seam = json.loads(SEAM_FIXTURE.read_text(encoding="utf-8")) + outer = UUID(seam["ingress_correlation_id"]) + conflicting = uuid4() + assert conflicting != outer + + process, broker = _host(_route("ModelSeamDelegateRequest")) + response = await _dispatch( + process, + {"task_type": "test", "prompt": "x", "correlation_id": str(conflicting)}, + outer, + ) + + assert response.ok is False, ( + "a payload correlation_id conflicting with the ingress authority was " + "ACCEPTED; one request now has two claimed identities" + ) + assert response.error is not None + assert response.error.code == "validation_error", ( + f"expected typed validation_error, got {response.error.code}" + ) + assert broker.commands == [], ( + "fail-closed means the broker is never reached: a conflicting-identity " + "command was dispatched anyway" + ) + + +@pytest.mark.asyncio +async def test_seam_matching_payload_correlation_is_accepted() -> None: + """Supplying the SAME id is legal and must normalize, not conflict.""" + seam = json.loads(SEAM_FIXTURE.read_text(encoding="utf-8")) + outer = UUID(seam["ingress_correlation_id"]) + + process, broker = _host(_route("ModelSeamDelegateRequest")) + response = await _dispatch( + process, + {"task_type": "test", "prompt": "x", "correlation_id": str(outer)}, + outer, + ) + + assert response.ok is True, f"matching correlation rejected: {response.error}" + command = broker.commands[0] + assert command.correlation_id == outer + assert isinstance(command.payload, dict) + assert UUID(str(command.payload["correlation_id"])) == outer + + +@pytest.mark.asyncio +async def test_seam_model_without_correlation_field_is_untouched() -> None: + """Control: never inject an unknown key into an ``extra=forbid`` model. + + An over-broad fix that stamped ``correlation_id`` onto every route model + would hard-fail every ``extra=forbid`` input model that does not declare it. + """ + seam = json.loads(SEAM_FIXTURE.read_text(encoding="utf-8")) + outer = UUID(seam["ingress_correlation_id"]) + + process, broker = _host(_route("ModelSeamNoCorrelationRequest")) + response = await _dispatch(process, {"task_type": "test", "prompt": "x"}, outer) + + assert response.ok is True, ( + "a route model without a correlation_id field was broken by the " + f"authority injection: {response.error}" + ) + command = broker.commands[0] + assert command.correlation_id == outer + assert isinstance(command.payload, dict) + assert "correlation_id" not in command.payload, ( + "correlation_id was injected into a model that does not declare it" + ) + + +@pytest.mark.asyncio +async def test_seam_route_without_input_model_preserves_raw_payload() -> None: + """Control: an untyped route must keep today's pass-through behavior.""" + seam = json.loads(SEAM_FIXTURE.read_text(encoding="utf-8")) + outer = UUID(seam["ingress_correlation_id"]) + + process, broker = _host(_route(None)) + response = await _dispatch(process, {"dry_run": True}, outer) + + assert response.ok is True, f"untyped route broke: {response.error}" + command = broker.commands[0] + assert command.correlation_id == outer + assert command.payload == {"dry_run": True} diff --git a/tests/integration/runtime/test_seam_dispatch_correlation_omn15474.py b/tests/integration/runtime/test_seam_dispatch_correlation_omn15474.py new file mode 100644 index 0000000000..2a7f5e8090 --- /dev/null +++ b/tests/integration/runtime/test_seam_dispatch_correlation_omn15474.py @@ -0,0 +1,385 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Cross-boundary seam test for OMN-15474 — one accepted command, one dispatch. + +This is the regression test OMN-15474 acceptance criterion 5 requires: it drives +a real ingress-ACCEPTED command through the ACTUAL auto-wiring subscription +boundary (``wire_from_manifest`` -> ``event_bus.subscribe`` -> the wiring +callback -> ``MessageDispatchEngine.dispatch_scoped`` -> the result applier), +not a unit test on one handler. The live defect lives at the wiring/subscription +boundary, so a handler-level test cannot see it. + +Observed live shape (OMN-15474, pod ``omninode-runtime-7765f8977f-ggjv7``, +correlation ``a4000001-0000-4000-8000-000000000001``):: + + [WIRING-CALLBACK] Deserialized envelope … topic=…delegation-request.v1 + [WIRING-CALLBACK] Dispatching to engine … topic=…delegation-request.v1 + [WIRING-CALLBACK] Deserialized envelope … topic=…delegation-request.v1 <-- SECOND + [WIRING-CALLBACK] Dispatching to engine … topic=…delegation-request.v1 <-- SECOND + +Two subscriptions attached to the SAME command topic inside ONE process, on two +distinct consumer groups, so the broker delivers the accepted command to both +and the whole reducer chain executes twice. Both executions carry the SAME +ingress correlation id, so the duplicate is invisible to any correlation-keyed +dedupe — the only observable is cardinality. + +Seam assertions (both must hold): + +1. CARDINALITY — exactly ONE command is emitted for one accepted command. +2. CORRELATION AUTHORITY — the emitted command carries the INGRESS-assigned + correlation id byte-identical. The dispatcher is not an authority on + correlation identity; it may never mint one. (Deepened in OMN-15546.) +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from unittest.mock import patch +from uuid import UUID + +import pytest + +from omnibase_core.models.errors import ModelOnexError +from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + ENV_SINGLE_OWNER_COMMAND_TOPICS, + wire_from_manifest, +) +from omnibase_infra.runtime.auto_wiring.models import ( + ModelAutoWiringManifest, + ModelContractVersion, + ModelDiscoveredContract, + ModelEventBusWiring, + ModelHandlerRef, + ModelHandlerRouting, + ModelHandlerRoutingEntry, +) +from omnibase_infra.runtime.message_dispatch_engine import MessageDispatchEngine + +pytestmark = pytest.mark.asyncio + +SEAM_FIXTURE = ( + Path(__file__).resolve().parents[2] + / "fixtures" + / "seams" + / "dispatch_correlation" + / "accepted_command.json" +) + + +# --------------------------------------------------------------------------- +# Seam doubles — faithful to the real transport contract, not to a mock's shape +# --------------------------------------------------------------------------- + + +@dataclass +class _KafkaRecord: + """Minimal record shape the wiring callback consumes (``.value`` bytes).""" + + value: bytes + + +@dataclass +class _EmittedCommand: + """One command the runtime emitted downstream of the accepted command.""" + + correlation_id: UUID | None + status: str + message_type: str + + +@dataclass +class _RecordingResultApplier: + """Stands in for ``DispatchResultApplier`` — records what would be published. + + The applier IS the emit point: every downstream command/event the runtime + produces for a consumed record goes through ``apply()``. Counting applies is + therefore counting emitted commands at the seam, without needing a broker. + """ + + emitted: list[_EmittedCommand] = field(default_factory=list) + + async def apply(self, result: object, correlation_id: UUID | None = None) -> None: + self.emitted.append( + _EmittedCommand( + correlation_id=correlation_id, + status=str(getattr(getattr(result, "status", None), "value", "?")), + message_type=str(getattr(result, "message_type", "?")), + ) + ) + + +@dataclass +class _FanOutEventBus: + """In-process bus that reproduces broker fan-out across consumer groups. + + A real broker delivers a partition record once PER consumer group. Each + auto-wired contract computes its own group id, so N contracts subscribed to + the same topic in one process each receive the record. This double models + exactly that and nothing more. + """ + + subscriptions: list[tuple[str, str, Callable[..., Awaitable[None]]]] = field( + default_factory=list + ) + + async def subscribe( + self, + *, + topic: str, + node_identity: Any, + on_message: Callable[..., Awaitable[None]], + ) -> Callable[[], Awaitable[None]]: + self.subscriptions.append( + (topic, getattr(node_identity, "node_name", "?"), on_message) + ) + + async def _unsubscribe() -> None: + return None + + return _unsubscribe + + async def publish(self, *args: object, **kwargs: object) -> None: + return None + + def subscriber_count(self, topic: str) -> int: + return sum(1 for t, _, _ in self.subscriptions if t == topic) + + async def deliver_once(self, topic: str, record: _KafkaRecord) -> None: + """Deliver ONE accepted command to every subscribed consumer group.""" + for sub_topic, _, callback in list(self.subscriptions): + if sub_topic == topic: + await callback(record) + + +# --------------------------------------------------------------------------- +# Contract fixtures — two contracts declaring the SAME command subscribe topic +# --------------------------------------------------------------------------- + + +def _handler_cls(name: str) -> type: + """A real class: ``ModelHandlerResolverContext.handler_cls`` requires ``type``.""" + + ns: dict[str, object] = {} + exec( # noqa: S102 — builds a genuinely distinct handler type per contract + "class _H:\n" + " async def handle(self, envelope):\n" + " return {'ok': True}\n", + ns, + ) + cls = ns["_H"] + assert isinstance(cls, type) + cls.__name__ = name + return cls + + +def _contract(name: str, topic: str) -> ModelDiscoveredContract: + return ModelDiscoveredContract( + name=name, + node_type="ORCHESTRATOR_GENERIC", + contract_version=ModelContractVersion(major=1, minor=0, patch=0), + contract_path=Path(f"/fake/{name}/contract.yaml"), + entry_point_name=name, + package_name="omnibase-infra", + event_bus=ModelEventBusWiring( + subscribe_topics=(topic,), + publish_topics=("onex.evt.omnibase-infra.routing-decision.v1",), + ), + handler_routing=ModelHandlerRouting( + routing_strategy="payload_type_match", + handlers=( + ModelHandlerRoutingEntry( + handler=ModelHandlerRef( + name=f"Handler{name}", + module=f"fake.{name}", + ), + event_model=None, + operation=None, + event_type="delegation-request", + topic=topic, + ), + ), + ), + ) + + +# --------------------------------------------------------------------------- +# The seam test +# --------------------------------------------------------------------------- + + +async def _boot( + contracts: tuple[ModelDiscoveredContract, ...], + bus: _FanOutEventBus, + applier: _RecordingResultApplier, + *, + strict: bool = True, +) -> tuple[MessageDispatchEngine, ModelOnexError | None]: + """Run the real wiring boot; return the engine and any fail-closed refusal.""" + engine = MessageDispatchEngine() + with ( + patch.dict( + os.environ, {ENV_SINGLE_OWNER_COMMAND_TOPICS: "1" if strict else "0"} + ), + patch( + "omnibase_infra.runtime.auto_wiring.handler_wiring._import_handler_class", + side_effect=lambda ref, *a, **kw: _handler_cls("SeamHandler"), + ), + ): + try: + await wire_from_manifest( + ModelAutoWiringManifest(contracts=contracts), + engine, + event_bus=bus, + environment="local", + result_appliers_by_contract={c.name: applier for c in contracts}, + ) + except ModelOnexError as exc: + return engine, exc + # The real boot freezes the engine before consumers may drive it; the wiring + # callback otherwise blocks waiting for freeze. + if not engine.is_frozen: + engine.freeze() + return engine, None + + +async def test_seam_dispatch_scope_emits_exactly_one_correlated_command() -> None: + """One ingress-accepted command emits exactly ONE command, ingress-correlated. + + Two halves, both required: + + A. The canonical single-owner wiring emits exactly one command for one + accepted command, and that command carries the INGRESS-assigned + correlation id byte-identical — the dispatcher minted nothing. + B. The live duplicate-owner wiring (OMN-15474's measured shape) is REFUSED + before any subscription attaches, so the double execution is impossible + rather than merely unlikely. + + Pre-fix, half B does not hold: the boot completes, two consumers attach, and + delivering the one accepted command produces TWO SUCCESS dispatches both + carrying ``a4000001-0000-4000-8000-000000000001`` (see the ticket's PR body + for the recorded RED). Half A held pre-fix and still holds — it is the + regression guard for the opposite failure (dropping the command entirely). + """ + seam = json.loads(SEAM_FIXTURE.read_text(encoding="utf-8")) + topic: str = seam["topic"] + ingress_correlation_id: str = seam["ingress_correlation_id"] + expected: int = seam["expected_emitted_command_count"] + record = _KafkaRecord(value=json.dumps(seam["envelope"]).encode("utf-8")) + + # --- A. canonical single-owner wiring --------------------------------- + bus = _FanOutEventBus() + applier = _RecordingResultApplier() + _engine, refusal = await _boot((_contract("node_delegation", topic),), bus, applier) + assert refusal is None, f"single-owner wiring must boot, got refusal: {refusal}" + assert bus.subscriber_count(topic) == 1 + + await bus.deliver_once(topic, record) + + # A.1 CARDINALITY + assert len(applier.emitted) == expected, ( + f"one ingress-accepted command on {topic} emitted " + f"{len(applier.emitted)} commands, expected {expected}; emitted " + f"correlation ids={[str(e.correlation_id) for e in applier.emitted]}" + ) + + # A.2 CORRELATION AUTHORITY IS THE INGRESS, NOT THE DISPATCHER + emitted = applier.emitted[0] + assert emitted.correlation_id is not None, ( + "emitted command carries no correlation id — the ingress-assigned id " + "was dropped at the dispatch seam" + ) + assert str(emitted.correlation_id) == ingress_correlation_id, ( + "correlation authority violated: the emitted command carries " + f"{emitted.correlation_id!s}, not the ingress-assigned " + f"{ingress_correlation_id}. The dispatcher must never mint a " + "correlation id; it may only propagate the ingress one byte-identical." + ) + + # --- B. the live duplicate-owner wiring must be refused --------------- + dup_bus = _FanOutEventBus() + dup_applier = _RecordingResultApplier() + _dup_engine, dup_refusal = await _boot( + ( + _contract("node_delegation_primary", topic), + _contract("node_delegation_effects", topic), + ), + dup_bus, + dup_applier, + ) + + assert dup_refusal is not None, ( + f"OMN-15474: two contracts own command topic {topic} in one process and " + f"the boot ACCEPTED it — {dup_bus.subscriber_count(topic)} consumers " + "attached. Each joins its own consumer group, so the accepted command " + "is delivered to both and the whole reducer chain executes twice under " + "one correlation id." + ) + assert topic in str(dup_refusal), ( + f"the refusal must name the offending command topic; got: {dup_refusal}" + ) + assert dup_bus.subscriber_count(topic) == 0, ( + "fail-closed means refusing BEFORE the side effect: " + f"{dup_bus.subscriber_count(topic)} subscriptions were already attached " + "when the boot refused, so the duplicate consumers are live anyway" + ) + + # And with nothing attached, the accepted command cannot be double-executed. + await dup_bus.deliver_once(topic, record) + assert dup_applier.emitted == [] + + # --- C. flag OFF is the documented default: warn, do not wedge the boot -- + # 8 shipped command topics violate this today (1 infra, 7 omnimarket), so a + # hard gate would refuse the next real deploy. CLAUDE.md requires the strict + # invariant to ship default-OFF and be flipped once those are compliant. + warn_bus = _FanOutEventBus() + warn_applier = _RecordingResultApplier() + _warn_engine, warn_refusal = await _boot( + ( + _contract("node_delegation_primary", topic), + _contract("node_delegation_effects", topic), + ), + warn_bus, + warn_applier, + strict=False, + ) + assert warn_refusal is None, ( + "default (flag OFF) must not refuse the boot — that is the documented " + f"strict-mode rollout sequencing; got: {warn_refusal}" + ) + assert warn_bus.subscriber_count(topic) == 2, ( + "flag OFF preserves today's behavior exactly (both consumers attach); " + "this is the measured duplicate-execution exposure the flip closes" + ) + + +async def test_seam_command_topic_single_owner_guard_does_not_touch_event_topics() -> ( + None +): + """Event-topic fan-out stays legal — the guard is command-scoped (OMN-15474). + + An over-broad guard would break every legitimate multi-consumer event topic, + which is a worse outage than the bug. This pins the boundary. + """ + event_topic = "onex.evt.omnibase-infra.routing-decision.v1" + bus = _FanOutEventBus() + applier = _RecordingResultApplier() + + _engine, refusal = await _boot( + ( + _contract("node_projection_a", event_topic), + _contract("node_projection_b", event_topic), + ), + bus, + applier, + ) + + assert refusal is None, ( + "event topics are fan-out by contract; the single-owner command guard " + f"must not reject them, but it refused: {refusal}" + ) + assert bus.subscriber_count(event_topic) == 2 diff --git a/tests/integration/runtime/test_steel_dispatch_golden_chain_live_runtime.py b/tests/integration/runtime/test_steel_dispatch_golden_chain_live_runtime.py new file mode 100644 index 0000000000..ea5b8582b7 --- /dev/null +++ b/tests/integration/runtime/test_steel_dispatch_golden_chain_live_runtime.py @@ -0,0 +1,307 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Infra-side golden chain: steel topic -> dispatch -> event_ledger row (OMN-15169). + +Scope (hostile finding #1, plan `2026-07-26-steel-node-dispatch-integration- +plan.md` §2 step 8): this test proves ONLY what `omnibase_infra` can own -- +topic -> dispatch -> a real `event_ledger` row. It does NOT drive +`steel_onslaught`'s own code (a private, separate repo) and does NOT claim +that half of the proof. The steel-side driver test +(`steel_onslaught/tests/live/test_omn15170_live_driver.py`, OMN-15170) is the +other, independently-owned half; it already passed live 2026-07-26 (real +match, real Kafka publish, correlation_id `ad230e9e-b336-4599-b870- +f6746033be47` consumed back at offset 5). + +Two checkpoints, per the plan's terminal-checkpoint requirement (never +"event published" alone -- OMN-15002/OMN-15006 already proved that class of +false-positive): + +1. NEGATIVE (allowlist-gate proof, OMN-15002 precedent). `node_ledger_ + projection_compute`'s `subscribe_topics` allowlist is the only thing that + makes this topic's events durable in `event_ledger` -- widening + `subscribe_topics` without dispatch is the documented NO_DISPATCHER class + (OMN-14594). Before the OMN-15168 paired diff was deployed to the + stability-test lane, `event_ledger` held ZERO rows for + `onex.evt.steel-onslaught.match-terminal.v1` even though the topic already + carried 14 real events (offsets 0-13, produced by steel's own OMN-15170 + live-driver runs -- a genuine, unplanned natural experiment). This is + captured as durable, structured evidence in + `tests/fixtures/golden_chains/steel_dispatch_ledger_negative_evidence.json` + (recorded 2026-07-26, verified via `docker exec ... cat contract.yaml` on + the deployed `omninode-stability-test-runtime-effects` container showing + `contract_version: 1.1.0` with no steel topic, plus a direct `psql` count + against `100.109.203.94:15436`). `test_negative_case_...` below asserts + this recorded evidence is internally consistent; it does not, and cannot, + re-derive the historical fact live (the deployed contract has since been + refreshed to include the topic -- see below -- so the negative window is + permanently closed and is recorded, not reproduced). + +2. POSITIVE (topic -> dispatch -> ledger row). `test_golden_chain_positive_ + topic_to_ledger_row` publishes ONE synthetic event with a freshly minted + `correlation_id` to the real stability-test Kafka topic, then polls + `event_ledger` directly via SQL (never `ledger.query` RPC -- direct SQL is + explicitly sanctioned by the governing plan §2 step 8) for a row carrying + that `correlation_id`. + +KNOWN LIVE BLOCKER as of 2026-07-26 (OMN-15215, filed this session, blocks +this ticket): a stability-lane warm refresh +(`omnibase_infra/scripts/runtime_build/refresh_stability_lane.sh --ref +origin/dev --execute`) was run and its own health-gate PASSED; the deployed +`node_ledger_projection_compute` contract on +`omninode-stability-test-runtime-effects` was re-verified afterward to +include the steel topic (`subscribe_topics` + paired `handler_routing` +entry, confirmed via `docker exec ... cat contract.yaml`). Despite this, two +independent synthetic publishes (one before, one after an additional +targeted restart of just `omninode-stability-test-runtime`) produced zero +`event_ledger` rows within a 60s poll window each, and `rpk group list` +shows zero live Kafka consumer groups for this contract on ANY of the 19 +topics added since OMN-15006 (not just the OMN-15168 steel topic) -- ruling +out an OMN-15168-specific cause. `test_golden_chain_positive_topic_to_ +ledger_row` is written to the correct, intended behavior and WILL currently +fail (not skip) if run live against the stability-test lane today with a +message citing OMN-15215; it is expected to pass once OMN-15215 clears. It +is skipped automatically (like every test in this module) when the lane is +unreachable, e.g. in CI, which cannot reach the private `.201`/Tailscale +network. + +Run: uv run pytest tests/integration/runtime/test_steel_dispatch_golden_chain_live_runtime.py -v -s -m integration +""" + +from __future__ import annotations + +import json +import os +import socket +import time +from pathlib import Path +from uuid import uuid4 + +import pytest + +from omnibase_infra.enums.generated.enum_steel_onslaught_topic import ( + EnumSteelOnslaughtTopic, +) + +FIXTURES_DIR = Path(__file__).parent.parent.parent / "fixtures" / "golden_chains" + +# The stability-test lane's advertised Kafka listener (matches the literal +# steel_onslaught's own OMN-15170 live-driver test dials -- verified live, +# not a hostname substitute; see that test's module docstring for why a +# hostname would break on the post-bootstrap metadata-driven reconnect). +# Overridable via STABILITY_TEST_KAFKA_BOOTSTRAP_SERVERS for any other lane. +_DEFAULT_KAFKA_BOOTSTRAP = "100.109.203.94:39092" # sanitize-ok +_ENV_KAFKA_BOOTSTRAP = "STABILITY_TEST_KAFKA_BOOTSTRAP_SERVERS" +KAFKA_BOOTSTRAP = os.environ.get(_ENV_KAFKA_BOOTSTRAP, _DEFAULT_KAFKA_BOOTSTRAP) + +# The stability-test lane's Postgres. No hardcoded default DSN (rule 8: +# fail-fast on missing env, never a silent fallback that could point at the +# wrong lane) -- a full DSN is required via env var, exactly the shape +# `tests/helpers/util_postgres.py::PostgresConfig.from_env` already +# establishes for `OMNIBASE_INFRA_DB_URL`, but under a lane-scoped name so +# this test never silently reads a DEV-lane DSN some other test configured. +_ENV_POSTGRES_DSN = "STABILITY_TEST_POSTGRES_DSN" + +TOPIC: str = EnumSteelOnslaughtTopic.EVT_MATCH_TERMINAL_V1.value + +_CONSUMER_TIMEOUT_SECONDS = 60.0 +_PRODUCER_FLUSH_TIMEOUT_SECONDS = 30.0 + + +def _can_reach(host: str, port: int, timeout: float = 3.0) -> bool: + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: + return False + + +def _kafka_bootstrap_reachable() -> bool: + parts = KAFKA_BOOTSTRAP.rsplit(":", 1) + if len(parts) != 2: + return False + host, port_str = parts + try: + port = int(port_str) + except ValueError: + return False + return _can_reach(host, port) + + +def _postgres_dsn() -> str | None: + dsn = os.environ.get(_ENV_POSTGRES_DSN) + return dsn.strip() if dsn and dsn.strip() else None + + +# Module-wide: categorize as integration. The live-infra reachability skips +# below are scoped to TestSteelDispatchGoldenChainPositiveCase ONLY -- the +# negative-case test reads a checked-in fixture and needs no live infra, so +# it must not be swept up by a module-wide skip (it would otherwise never +# run at all in an environment without stability-lane reachability, e.g. CI). +pytestmark = [pytest.mark.integration, pytest.mark.kafka] + +_live_infra_skips = [ + pytest.mark.skipif( + not _kafka_bootstrap_reachable(), + reason=( + f"stability-test Kafka on {KAFKA_BOOTSTRAP} not reachable " + f"(set {_ENV_KAFKA_BOOTSTRAP} to override)" + ), + ), + pytest.mark.skipif( + _postgres_dsn() is None, + reason=( + f"{_ENV_POSTGRES_DSN} not set -- required full DSN for the " + "stability-test lane's Postgres (postgresql://user:pass@host:port/db)" + ), + ), +] + + +def _load_json_fixture(name: str) -> dict[str, object] | list[object]: + result: dict[str, object] | list[object] = json.loads( + (FIXTURES_DIR / f"{name}.json").read_text() + ) + return result + + +class TestSteelDispatchGoldenChainNegativeCase: + """OMN-15002-precedent negative case: recorded, not re-derived live.""" + + def test_negative_case_evidence_recorded_and_self_consistent(self) -> None: + evidence = _load_json_fixture("steel_dispatch_ledger_negative_evidence") + assert isinstance(evidence, dict) + + assert evidence["ticket"] == "OMN-15169" + assert evidence["precedent"] == "OMN-15002" + assert evidence["topic"] == TOPIC + assert evidence["lane"] == "stability-test" + + # The load-bearing claim: zero ledger rows while the deployed + # contract predated the paired allowlist diff. + assert ( + evidence["steel_topic_present_in_deployed_contract_before_refresh"] is False + ) + assert evidence["event_ledger_row_count_before_refresh"] == 0 + + # The natural-experiment offsets must be non-empty and contiguous + # from 0 -- otherwise this isn't proof the allowlist (not some other + # gate) was the cause, since an empty/negative range proves nothing. + offsets = evidence["offsets_present_on_topic_with_zero_ledger_rows"] + assert offsets == sorted(offsets) + assert offsets[0] == 0 + assert len(offsets) == evidence["topic_high_watermark_at_check_time"] + + +@_live_infra_skips[0] +@_live_infra_skips[1] +class TestSteelDispatchGoldenChainPositiveCase: + """Live positive case: topic -> dispatch -> event_ledger row.""" + + def test_golden_chain_positive_topic_to_ledger_row(self) -> None: + """Publish one synthetic event; assert a real `event_ledger` row. + + Terminal checkpoint is a direct SQL readback by `correlation_id` -- + never "producer.flush() returned 0 pending" alone. A successful + Kafka delivery only proves the broker accepted the message; it does + not prove `node_ledger_projection_compute` dispatched and + `node_ledger_write_effect` persisted it (exactly the gap OMN-15002 + and now OMN-15215 both document). + """ + confluent_kafka = pytest.importorskip( + "confluent_kafka", + reason="requires confluent-kafka (uv sync --extra live, or add as a dep)", + ) + asyncpg = pytest.importorskip("asyncpg") + + correlation_id = str(uuid4()) + match_id = f"match.omn15169.golden-chain.{uuid4().hex[:12]}" + + golden = _load_json_fixture("steel_dispatch_ledger_success") + assert isinstance(golden, list) + expected_event_types = {entry["event_type"] for entry in golden} + assert "topic_event_published" in expected_event_types + assert "ledger_row_persisted" in expected_event_types + + # --- Checkpoint 0: publish to the real stability-test topic --- + producer = confluent_kafka.Producer({"bootstrap.servers": KAFKA_BOOTSTRAP}) + value = { + "event_type": "match_started", + "match_id": match_id, + "envelope": { + "correlation_id": correlation_id, + "causation_id": None, + "entity_id": match_id, + "message_id": str(uuid4()), + }, + "payload": {"probe": "OMN-15169 infra golden-chain positive case"}, + } + headers = [ + ("correlation_id", correlation_id.encode("utf-8")), + ("event_type", b"match_started"), + ("source", b"omn15169-golden-chain-test"), + ] + delivery_errors: list[str] = [] + + def _on_delivery(err: object, _msg: object) -> None: + if err is not None: + delivery_errors.append(str(err)) + + producer.produce( + topic=TOPIC, + key=match_id.encode("utf-8"), + value=json.dumps(value, default=str).encode("utf-8"), + headers=headers, + callback=_on_delivery, + ) + pending = producer.flush(_PRODUCER_FLUSH_TIMEOUT_SECONDS) + assert pending == 0, ( + f"{pending} message(s) still undelivered after flush timeout" + ) + assert not delivery_errors, ( + f"Kafka producer reported delivery failures (topic missing / " + f"partition-cap class): {delivery_errors}" + ) + + # --- Terminal checkpoint: a real event_ledger row, by correlation_id --- + import asyncio + + async def _poll() -> dict[str, object] | None: + conn = await asyncpg.connect(_postgres_dsn()) + try: + deadline = time.monotonic() + _CONSUMER_TIMEOUT_SECONDS + while time.monotonic() < deadline: + row = await conn.fetchrow( + "SELECT ledger_entry_id, topic, partition, kafka_offset, " + "correlation_id, event_type, source, ledger_written_at " + "FROM event_ledger WHERE correlation_id = $1", + correlation_id, + ) + if row is not None: + return dict(row) + await asyncio.sleep(2.0) + return None + finally: + await conn.close() + + row = asyncio.run(_poll()) + + assert row is not None, ( + f"event_ledger has no row for correlation_id={correlation_id} " + f"(topic={TOPIC}, match_id={match_id}) within " + f"{_CONSUMER_TIMEOUT_SECONDS}s of a confirmed Kafka delivery. " + "The topic->dispatch->ledger chain did not complete. Known live " + "blocker as of 2026-07-26: OMN-15215 (node_ledger_projection_" + "compute never attaches a live consumer group for this topic, " + "or any of the 18 other topics added since OMN-15006, on the " + "stability-test lane, even after a verified contract refresh)." + ) + assert row["topic"] == TOPIC + assert str(row["correlation_id"]) == correlation_id + + print( + "\nOMN-15169 infra golden chain POSITIVE case: PASS\n" + f" correlation_id={correlation_id}\n" + f" match_id={match_id}\n" + f" ledger_entry_id={row['ledger_entry_id']}\n" + f" partition={row['partition']} kafka_offset={row['kafka_offset']}\n" + ) diff --git a/tests/integration/test_application_acl_postgres16_proof.py b/tests/integration/test_application_acl_postgres16_proof.py new file mode 100644 index 0000000000..98a50eb233 --- /dev/null +++ b/tests/integration/test_application_acl_postgres16_proof.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Rebuilt PostgreSQL 16 integration proof for the generated ACL matrix.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).parents[2] +_COMPOSE = _ROOT / "docker" / "application-acl-proof" / "compose.yml" +_HAS_DOCKER = shutil.which("docker") is not None + + +@pytest.mark.slow +@pytest.mark.skipif(not _HAS_DOCKER, reason="Requires a Docker engine") +def test_rebuilt_postgres16_acl_and_rollback_proof() -> None: + project_name = f"application-acl-proof-{os.getpid()}" + command = [ + "docker", + "compose", + "-p", + project_name, + "-f", + str(_COMPOSE), + ] + result = subprocess.run( + [ + *command, + "up", + "--build", + "--abort-on-container-exit", + "--exit-code-from", + "proof", + ], + cwd=_ROOT, + capture_output=True, + text=True, + timeout=1_500, + check=False, + ) + try: + assert result.returncode == 0, result.stdout + result.stderr + output = result.stdout + result.stderr + assert "acl_phase=scaffold_wrong_database_guard status=PASS" in output + assert "acl_phase=scaffold_fresh_additive_round_trip status=PASS" in output + assert "acl_phase=rollback_atomic_failure" in output + assert "acl_status=PASS postgres_major=16" in output + finally: + subprocess.run( + [*command, "down", "--volumes", "--remove-orphans"], + cwd=_ROOT, + capture_output=True, + text=True, + timeout=120, + check=False, + ) diff --git a/tests/integration/test_application_database_domain_enforcement_postgres16.py b/tests/integration/test_application_database_domain_enforcement_postgres16.py new file mode 100644 index 0000000000..011ea2b10a --- /dev/null +++ b/tests/integration/test_application_database_domain_enforcement_postgres16.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Rebuilt PostgreSQL 16 application-domain enforcement proof.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).parents[2] +_COMPOSE = _ROOT / "docker" / "application-domain-enforcement" / "compose.yml" +_HAS_DOCKER = shutil.which("docker") is not None + + +@pytest.mark.slow +@pytest.mark.skipif(not _HAS_DOCKER, reason="Requires a Docker engine") +def test_rebuilt_postgres16_application_domain_enforcement() -> None: + project_name = f"application-domain-enforcement-{os.getpid()}" + command = ["docker", "compose", "-p", project_name, "-f", str(_COMPOSE)] + result = subprocess.run( + [ + *command, + "up", + "--build", + "--abort-on-container-exit", + "--exit-code-from", + "proof", + ], + cwd=_ROOT, + capture_output=True, + text=True, + timeout=1_500, + check=False, + ) + try: + output = result.stdout + result.stderr + assert result.returncode == 0, output + assert output.count("domain_control=") == 46 + assert "domain_control=identity-root-runtime-login status=PASS" in output + assert "domain_control=identity-root-unproven-enumeration status=PASS" in output + assert "domain_control=identity-root-runtime-membership status=PASS" in output + assert "domain_control=identity-root-runtime-set-role status=PASS" in output + assert "domain_control=canonical-policy-unrelated-role status=PASS" in output + assert "domain_control=tenant-text-key status=PASS" in output + assert "domain_control=unsafe-security-definer status=PASS" in output + assert "domain_control=security-definer-volatility-drift status=PASS" in output + assert "domain_control=old-application-database status=PASS" in output + assert "domain_control=public-catalog-leak status=PASS" in output + assert ( + "domain_control=source-tenant-partial-unique-predicate status=PASS" + in output + ) + assert ( + "domain_control=source-tenant-generated-unique-alias status=PASS" in output + ) + assert ( + "domain_control=source-tenant-transitive-whole-row-helper status=PASS" + in output + ) + assert ( + "domain_control=source-tenant-named-whole-row-helper status=PASS" in output + ) + assert "domain_control=source-tenant-check-constraint status=PASS" in output + assert "domain_control=source-tenant-trigger-body status=PASS" in output + assert "domain_control=source-tenant-dependent-view status=PASS" in output + assert ( + "application_domain_enforcement_status=PASS postgres_major=16 " + "relations=6 catalog_objects=6 pools=4 red_controls=46" + ) in output + finally: + subprocess.run( + [*command, "down", "--volumes", "--remove-orphans"], + cwd=_ROOT, + capture_output=True, + text=True, + timeout=120, + check=False, + ) diff --git a/tests/integration/test_auto_wiring_async_incompat_quarantine_integration.py b/tests/integration/test_auto_wiring_async_incompat_quarantine_integration.py index 16b41d8d22..b5b0c4f5ae 100644 --- a/tests/integration/test_auto_wiring_async_incompat_quarantine_integration.py +++ b/tests/integration/test_auto_wiring_async_incompat_quarantine_integration.py @@ -120,11 +120,28 @@ def _make_contract( def _make_dispatch_engine() -> MagicMock: + """A dispatch-engine double that answers the contract-scope question honestly. + + OMN-15474 made subscription contract-scoped: before attaching a consumer, + ``_subscribe_contract_topics`` asks the LIVE engine which dispatchers the + contract owns and refuses to attach on an empty answer, because a + zero-owner consumer would dispatch process-globally. A bare ``MagicMock`` + answers that question with an auto-created attribute whose iteration yields + nothing, so the double silently claims the contract owns no dispatcher and + the refusal fires on a contract that just registered one. Return the + dispatcher ids the caller registered — what the real + ``MessageDispatchEngine`` returns — so this test exercises the wiring + behaviour it is named for instead of the scope guard. + """ engine = MagicMock() engine._routes = {} engine._container = None engine.register_dispatcher = MagicMock() engine.register_route = MagicMock() + engine.dispatch_scoped = AsyncMock() + engine.validate_contract_dispatcher_scope = MagicMock( + side_effect=lambda _contract_name, dispatcher_ids: frozenset(dispatcher_ids) + ) engine.freeze = MagicMock() return engine diff --git a/tests/integration/test_auto_wiring_real_manifest.py b/tests/integration/test_auto_wiring_real_manifest.py index 40552d4a45..fd01946d2a 100644 --- a/tests/integration/test_auto_wiring_real_manifest.py +++ b/tests/integration/test_auto_wiring_real_manifest.py @@ -21,15 +21,26 @@ from pathlib import Path from unittest.mock import MagicMock -from uuid import UUID +from uuid import UUID, uuid4 import pytest +from pydantic import BaseModel, ConfigDict, Field from omnibase_infra.runtime.auto_wiring.discovery import discover_contracts from omnibase_infra.runtime.auto_wiring.handler_wiring import wire_from_manifest +from omnibase_infra.runtime.auto_wiring.models import ( + ModelAutoWiringManifest, + ModelContractVersion, + ModelDiscoveredContract, + ModelEventBusWiring, + ModelHandlerRef, + ModelHandlerRouting, + ModelHandlerRoutingEntry, +) from omnibase_infra.runtime.auto_wiring.models.model_discovery_error import ( ModelDiscoveryError, ) +from omnibase_infra.runtime.message_dispatch_engine import MessageDispatchEngine from omnibase_infra.runtime.service_intent_routing_loader import ( load_intent_routing_table, ) @@ -178,3 +189,185 @@ async def test_real_manifest_wiring_has_no_failures() -> None: assert not error_results, "ModelOnexError found in wiring results:\n" + "\n".join( f" {r.contract_name}: {r.reason}" for r in error_results ) + + +# --------------------------------------------------------------------------- +# OMN-16050 — registered-input-model unwrap stop, proven on the REAL manifest +# --------------------------------------------------------------------------- + +_OMN16050_TOPIC = "onex.cmd.omnibase-infra.omn16050-unwrap-probe.v1" +_OMN16050_MODULE = "tests.integration.test_auto_wiring_real_manifest" + + +class ModelOmn16050EmitRequest(BaseModel): + """Field-for-field mirror of omnimarket's ``ModelEmitRequest`` (OMN-16050). + + Declares a ``payload`` mapping plus FOUR transport marker keys + (``event_type``, ``correlation_id``, ``partition_key``, ``event_id``), which + is precisely what made it indistinguishable from a transport envelope to the + old structural heuristic. ``extra="forbid"`` mirrors the real model, so an + over-unwrap fails totally — the live DLQ signature. + + Lives here rather than in omnimarket because omnibase_infra is upstream of it + and cannot import it; the shape, not the identity, is what the defect keys on. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + event_type: str = Field(..., min_length=1) + payload: dict[str, object] = Field(default_factory=dict) + correlation_id: str | None = None + topic: str | None = None + partition_key: str | None = None + event_id: str = Field(default_factory=lambda: str(uuid4()), min_length=1) + + +class HandlerOmn16050EmitProbe: + """Canonical def-B handler: ``handle(request: ModelX) -> None``. + + Wired by the real ``wire_from_manifest`` path, so the callback under test is + the production-built one, not a hand-constructed ``_make_dispatch_callback``. + """ + + received: list[ModelOmn16050EmitRequest] = [] + + async def handle(self, request: ModelOmn16050EmitRequest) -> None: + type(self).received.append(request) + + +def _omn16050_probe_contract() -> ModelDiscoveredContract: + """An ``operation_match`` def-B EFFECT contract shaped like node_event_emit_effect.""" + return ModelDiscoveredContract( + name="node_omn16050_unwrap_probe", + node_type="EFFECT_GENERIC", + contract_version=ModelContractVersion(major=1, minor=0, patch=0), + contract_path=Path("/fake/omn16050/contract.yaml"), + entry_point_name="node_omn16050_unwrap_probe", + package_name="omnibase-infra", + event_bus=ModelEventBusWiring( + subscribe_topics=(_OMN16050_TOPIC,), + publish_topics=(), + ), + handler_routing=ModelHandlerRouting( + routing_strategy="operation_match", + handlers=( + ModelHandlerRoutingEntry( + handler=ModelHandlerRef( + name="HandlerOmn16050EmitProbe", module=_OMN16050_MODULE + ), + message_category="command", + event_type="omnibase-infra.omn16050-unwrap-probe", + operation="omn16050.probe", + ), + ), + ), + ) + + +def _omn16050_published_bytes() -> dict[str, object]: + """The live shape: one transport envelope wrapping a ModelEmitRequest. + + Mirrors the in-pod capture on onex-dev (digest sha256:35099472…) verbatim: + ``RAW KEYS: ['event_type', 'correlation_id', 'source_tool', 'payload']``. + """ + return { + "event_type": "session.started", + "correlation_id": "18a50ff5-c877-481c-b3c1-a183d8069762", + "source_tool": "defect-ab-probe", + "payload": { + "event_type": "session.started", + "correlation_id": "18a50ff5-c877-481c-b3c1-a183d8069762", + "partition_key": "session-1", + "event_id": "evt-defect-ab-probe", + "payload": { + "session_id": "18a50ff5-c877-481c-b3c1-a183d8069762", + "defect_ab_probe": True, + "emitted_at": "2026-08-13T02:46:13Z", + }, + }, + } + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_real_manifest_wiring_preserves_registered_envelope_shaped_input_model() -> ( + None +): + """Runtime-startup gate for OMN-16050: real manifest + real wiring + real dispatch. + + Satisfies the repo's Runtime Startup CI gate for a PR touching + ``auto_wiring/``: the manifest is the real one loaded from disk via + ``discover_contracts()``, ``wire_from_manifest`` runs with the kernel's + argument shape, and zero unexpected failures are asserted. + + On top of that it proves the defect is closed through the production path: + one probe contract whose def-B handler declares an envelope-SHAPED input + model (``payload`` + four transport markers, ``extra="forbid"``) is wired + alongside the real contracts, and the dispatcher the wiring registered is + invoked with the exact bytes captured in-pod. Before the fix the callback + unwrapped through the domain model to the caller's inner payload and raised + ``ValidationError`` (``event_type`` Field required + 3x extra_forbidden) — + the live ``boundary_swallow_prevented`` / DLQ signature. + """ + HandlerOmn16050EmitProbe.received.clear() + + real_manifest = discover_contracts() + combined = ModelAutoWiringManifest( + contracts=(*real_manifest.contracts, _omn16050_probe_contract()), + errors=real_manifest.errors, + ) + derived_appliers = { + contract.name: _StubResultApplier() + for contract in real_manifest.contracts + if load_intent_routing_table(Path(contract.contract_path)) + } + + engine = MessageDispatchEngine() + report = await wire_from_manifest( + manifest=combined, + dispatch_engine=engine, + event_bus=None, + subscribe_immediately=False, + result_appliers_by_contract=derived_appliers, + ) + + failed_names = { + r.contract_name for r in report.results if str(r.outcome).endswith("FAILED") + } + unexpected = failed_names - _KNOWN_UNWIRED_RAW_PROJECTIONS + assert not unexpected, ( + "wire_from_manifest() reported unexpected failure(s) against the real " + f"manifest + the OMN-16050 probe contract: {sorted(unexpected)}" + ) + + probe_result = next( + r for r in report.results if r.contract_name == "node_omn16050_unwrap_probe" + ) + assert str(probe_result.outcome).endswith("WIRED"), ( + "the OMN-16050 probe contract must WIRE — a skipped/failed probe would " + f"make this gate vacuous (outcome={probe_result.outcome}, " + f"reason={probe_result.reason})" + ) + assert len(probe_result.dispatchers_registered) == 1 + + dispatcher_id = probe_result.dispatchers_registered[0] + dispatcher = engine._dispatchers[dispatcher_id].dispatcher + + await dispatcher(_omn16050_published_bytes()) + + assert len(HandlerOmn16050EmitProbe.received) == 1, ( + "the wired dispatcher did not deliver to the handler — pre-fix this " + "raised ValidationError inside the callback and DLQ'd" + ) + request = HandlerOmn16050EmitProbe.received[0] + assert isinstance(request, ModelOmn16050EmitRequest) + assert request.event_type == "session.started" + assert request.event_id == "evt-defect-ab-probe" + assert request.partition_key == "session-1" + # The load-bearing assertion: the handler owns the CALLER's payload, and the + # unwrap stopped at the registered model instead of walking through it. + assert request.payload == { + "session_id": "18a50ff5-c877-481c-b3c1-a183d8069762", + "defect_ab_probe": True, + "emitted_at": "2026-08-13T02:46:13Z", + } diff --git a/tests/integration/test_consumer_health_pipeline.py b/tests/integration/test_consumer_health_pipeline.py index 97eec0f22c..a5954f2912 100644 --- a/tests/integration/test_consumer_health_pipeline.py +++ b/tests/integration/test_consumer_health_pipeline.py @@ -49,7 +49,11 @@ pytest.mark.kafka, ] -BOOTSTRAP_SERVERS = "localhost:19092" +# OMN-15567: read the CI-derived bootstrap address (nightly-integration.yml +# assigns a run-scoped port and, on containerized runners, a non-localhost +# host) instead of hardcoding the docker-compose.e2e.yml default -- a +# hardcoded value here silently ignores both. +BOOTSTRAP_SERVERS = os.environ.get("KAFKA_BOOTSTRAP_SERVERS", "localhost:19092") @pytest.fixture @@ -58,9 +62,11 @@ async def kafka_producer() -> AsyncGenerator[AIOKafkaProducer, None]: producer = AIOKafkaProducer( bootstrap_servers=BOOTSTRAP_SERVERS, ) - await producer.start() - yield producer - await producer.stop() + try: + await producer.start() + yield producer + finally: + await producer.stop() @pytest.fixture @@ -75,9 +81,11 @@ async def kafka_consumer() -> AsyncGenerator[AIOKafkaConsumer, None]: enable_auto_commit=True, consumer_timeout_ms=5000, ) - await consumer.start() - yield consumer - await consumer.stop() + try: + await consumer.start() + yield consumer + finally: + await consumer.stop() class TestConsumerHealthEmitterIntegration: diff --git a/tests/integration/test_kafka_contract_source_default_handler.py b/tests/integration/test_kafka_contract_source_default_handler.py index dac3804034..dc920daf68 100644 --- a/tests/integration/test_kafka_contract_source_default_handler.py +++ b/tests/integration/test_kafka_contract_source_default_handler.py @@ -32,12 +32,11 @@ def test_kafka_contract_source_materializes_default_handler_shorthand() -> None: output_model: "omnibase_core.models.dispatch.ModelHandlerOutput" metadata: handler_class: "tests.fixtures.handler_proof_noop.HandlerProofNoop" - event_bus: - subscribe_topics: - - onex.cmd.test.default-handler.v1 - publish_topics: - - onex.evt.test.default-handler-completed.v1 - consumer_purpose: consume + yaml_consumed_events: + - event_type: onex.cmd.test.default-handler.v1 + yaml_published_events: + - event_type: onex.evt.test.default-handler-completed.v1 + topic: onex.evt.test.default-handler-completed.v1 handler_routing: version: major: 1 diff --git a/tests/integration/test_omn_9741_service_kernel_health_ordering.py b/tests/integration/test_omn_9741_service_kernel_health_ordering.py index b00f8618fc..ac98f3a027 100644 --- a/tests/integration/test_omn_9741_service_kernel_health_ordering.py +++ b/tests/integration/test_omn_9741_service_kernel_health_ordering.py @@ -65,3 +65,22 @@ def test_health_server_uses_runtime_pending_mode_before_attach() -> None: assert "container=container" in start_block assert "runtime=runtime" not in start_block + + +@pytest.mark.integration +def test_failed_plugin_consumer_result_is_not_reported_as_started() -> None: + """A returned failure must branch before the consumer-started log.""" + source = _read_service_kernel_source() + pass_two_start = source.index( + "# --- Pass 2: Start consumers for ready plugins only ---" + ) + pass_two = source[ + pass_two_start : source.index("plugin_activation_duration =", pass_two_start) + ] + + result_idx = pass_two.index("consumer_result = await plugin.start_consumers") + failure_idx = pass_two.index("if not consumer_result.success:") + continue_idx = pass_two.index("continue", failure_idx) + started_idx = pass_two.index("consumers started", continue_idx) + + assert result_idx < failure_idx < continue_idx < started_idx diff --git a/tests/integration/test_projection_handler_wiring_runtime_dispatch.py b/tests/integration/test_projection_handler_wiring_runtime_dispatch.py index eb1f48852f..c500aa573f 100644 --- a/tests/integration/test_projection_handler_wiring_runtime_dispatch.py +++ b/tests/integration/test_projection_handler_wiring_runtime_dispatch.py @@ -13,9 +13,16 @@ from omnibase_infra.runtime.auto_wiring.handler_wiring import ( _make_projection_dispatch_callback, ) +from tests.helpers.application_db_topology import projection_database_target + + +@pytest.fixture(autouse=True) +def _configured_projection_dsn(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OMNIDASH_ANALYTICS_DB_URL", "postgresql://fixture") + _PATCH_BUILD_ADAPTER = ( - "omnibase_infra.runtime.auto_wiring.handler_wiring._build_sync_db_adapter" + "omnibase_infra.runtime.auto_wiring.handler_wiring._build_projection_db_adapter" ) _PATCH_ENVIRON_GET = "omnibase_infra.runtime.auto_wiring.handler_wiring.os.environ.get" @@ -41,7 +48,7 @@ def handle(self, input_data: dict[str, object]) -> dict[str, bool]: handler = DelegationProjectionRunner() callback = _make_projection_dispatch_callback( handler, - [{"name": "delegation_events", "database": "omnidash_analytics"}], + projection_database_target("delegation_events"), ("onex.evt.omniclaude.task-delegated.v1",), ) @@ -71,7 +78,7 @@ def handle(self, input_data: dict[str, object]) -> dict[str, int]: callback = _make_projection_dispatch_callback( HandlerProjectionDelegation(), - [{"name": "delegation_events", "database": "omnidash_analytics"}], + projection_database_target("delegation_events"), ("onex.evt.omniclaude.task-delegated.v1",), ) diff --git a/tests/integration/test_remote_agent_invoke_effect_contract.py b/tests/integration/test_remote_agent_invoke_effect_contract.py index dccf947b12..a27db90c7f 100644 --- a/tests/integration/test_remote_agent_invoke_effect_contract.py +++ b/tests/integration/test_remote_agent_invoke_effect_contract.py @@ -33,7 +33,7 @@ from datetime import UTC, datetime from pathlib import Path from typing import cast -from uuid import uuid4 +from uuid import UUID, uuid4 import pytest import yaml @@ -45,6 +45,7 @@ from omnibase_core.models.delegation.model_invocation_command import ( ModelInvocationCommand, ) +from omnibase_core.types import JsonType from omnibase_infra.runtime.runtime_local_ingress import ( ModelRuntimeLocalIngressRoute, validate_runtime_local_ingress_payload, @@ -91,7 +92,7 @@ def _first_enum_value(annotation: object) -> object: raise AssertionError(f"no Enum member found in annotation {annotation!r}") -def _producer_invocation_payload() -> dict[str, object]: +def _producer_invocation_payload() -> dict[str, JsonType]: """The RICH ModelInvocationCommand the producer publishes on remote-agent-invoke.v1.""" kind = _first_enum_value( ModelInvocationCommand.model_fields["invocation_kind"].annotation @@ -165,7 +166,11 @@ def test_producer_payload_validates_through_runtime_ingress(self) -> None: agent_id / forbidden extras); GREEN against the canonical contract.""" route = _ingress_route_from_contract() payload = _producer_invocation_payload() - normalized = validate_runtime_local_ingress_payload(route, payload) + normalized = validate_runtime_local_ingress_payload( + route, + payload, + correlation_id=UUID(str(payload["correlation_id"])), + ) # No field drop: the rich invocation fields survive validation. for field in ("task_id", "invocation_kind", "target_ref"): assert field in normalized, f"{field} dropped by input_model validation" diff --git a/tests/integration/test_runtime_log_bridge_pipeline.py b/tests/integration/test_runtime_log_bridge_pipeline.py index ee4a26b167..7b7917e0cd 100644 --- a/tests/integration/test_runtime_log_bridge_pipeline.py +++ b/tests/integration/test_runtime_log_bridge_pipeline.py @@ -48,7 +48,11 @@ pytest.mark.kafka, ] -BOOTSTRAP_SERVERS = "localhost:19092" +# OMN-15567: read the CI-derived bootstrap address (nightly-integration.yml +# assigns a run-scoped port and, on containerized runners, a non-localhost +# host) instead of hardcoding the docker-compose.e2e.yml default -- a +# hardcoded value here silently ignores both. +BOOTSTRAP_SERVERS = os.environ.get("KAFKA_BOOTSTRAP_SERVERS", "localhost:19092") @pytest.fixture @@ -57,9 +61,11 @@ async def kafka_producer() -> AsyncGenerator[AIOKafkaProducer, None]: producer = AIOKafkaProducer( bootstrap_servers=BOOTSTRAP_SERVERS, ) - await producer.start() - yield producer - await producer.stop() + try: + await producer.start() + yield producer + finally: + await producer.stop() @pytest.fixture @@ -74,9 +80,11 @@ async def kafka_consumer() -> AsyncGenerator[AIOKafkaConsumer, None]: enable_auto_commit=True, consumer_timeout_ms=5000, ) - await consumer.start() - yield consumer - await consumer.stop() + try: + await consumer.start() + yield consumer + finally: + await consumer.stop() class _EnableBridgeCtx: diff --git a/tests/integration/test_runtime_sub_bundle_cli.py b/tests/integration/test_runtime_sub_bundle_cli.py index 9042ccb017..bccefafa97 100644 --- a/tests/integration/test_runtime_sub_bundle_cli.py +++ b/tests/integration/test_runtime_sub_bundle_cli.py @@ -196,16 +196,21 @@ def test_generate_composed_runtime_includes_all_sub_bundles(tmp_path: Path) -> N @pytest.mark.integration -@pytest.mark.xfail( - strict=False, - reason="OMN-9345: CatalogResolver iterates a set[str] over bundle names, " - "producing non-deterministic service ordering in the generated compose. " - "Remove this marker once OMN-9345 lands.", -) def test_generate_runtime_is_deterministic_across_invocations(tmp_path: Path) -> None: """Two back-to-back `generate runtime` runs must produce byte-identical compose output. Resolver non-determinism (e.g. set iteration order) would cause operator machines to see spurious compose-file diffs. + + Expired red-authorization removed (OMN-15248): this test carried + ``xfail(strict=False, reason="OMN-9345: CatalogResolver iterates a + set[str] ...")``. OMN-9345 landed — ``resolver.py`` now accumulates + ``all_bundle_names: dict[str, None]`` (insertion-ordered) and iterates + that, so the ordering is deterministic. Because the marker was + ``strict=False`` it could never self-retire: the test xpassed silently and + the stale waiver stayed. Verified GREEN 5/5 on ``.200`` across distinct + ``PYTHONHASHSEED`` values (the axis that would expose set-iteration + randomness) before the marker was deleted. Do not re-add a non-strict + xfail here — if this regresses, fix the resolver. """ out_a = tmp_path / "runtime-a.yml" out_b = tmp_path / "runtime-b.yml" diff --git a/tests/integration/test_state_io_outbox_seam_red.py b/tests/integration/test_state_io_outbox_seam_red.py index 5a5218bdbf..06e749066d 100644 --- a/tests/integration/test_state_io_outbox_seam_red.py +++ b/tests/integration/test_state_io_outbox_seam_red.py @@ -838,9 +838,14 @@ def _bus_callback(callback: Any, applier: DispatchResultApplier) -> Any: """ class _Engine: - async def dispatch( - self, topic: str, envelope: ModelEventEnvelope[object] + async def dispatch_scoped( + self, + topic: str, + envelope: ModelEventEnvelope[object], + *, + allowed_dispatcher_ids: object, ) -> Any: + del topic, allowed_dispatcher_ids return await callback(envelope) return _make_event_bus_callback( @@ -848,6 +853,7 @@ async def dispatch( cast("Any", _Engine()), cast("Any", applier), propagate_publish_failures=True, + allowed_dispatcher_ids={"state-io-test-dispatcher"}, ) @@ -1194,14 +1200,22 @@ def test_non_outbox_boundary_still_swallows_unchanged() -> None: applier = _make_applier(bus) class _Engine: - async def dispatch( - self, topic: str, envelope: ModelEventEnvelope[object] + async def dispatch_scoped( + self, + topic: str, + envelope: ModelEventEnvelope[object], + *, + allowed_dispatcher_ids: object, ) -> Any: + del topic, allowed_dispatcher_ids return await callback(envelope) # DEFAULT flag (propagate_publish_failures NOT passed) = non-outbox behavior. on_message = _make_event_bus_callback( - TOPIC_INBOUND, cast("Any", _Engine()), cast("Any", applier) + TOPIC_INBOUND, + cast("Any", _Engine()), + cast("Any", applier), + allowed_dispatcher_ids={"state-io-test-dispatcher"}, ) raised: BaseException | None = None diff --git a/tests/integration/test_sync_db_adapter_composite_conflict_key.py b/tests/integration/test_sync_db_adapter_composite_conflict_key.py index c8393ca588..06e19279fd 100644 --- a/tests/integration/test_sync_db_adapter_composite_conflict_key.py +++ b/tests/integration/test_sync_db_adapter_composite_conflict_key.py @@ -13,13 +13,30 @@ import pytest -from omnibase_infra.runtime.auto_wiring.handler_wiring import _build_sync_db_adapter +from omnibase_infra.runtime.auto_wiring.handler_wiring import ( + _build_projection_db_adapter, +) +from tests.helpers.application_db_topology import ( + projection_database_target, + projection_database_urls, +) pytestmark = [pytest.mark.integration] +def _internal_adapter(table: str) -> object: + target = projection_database_target(table, schema="omninode_internal") + return _build_projection_db_adapter( + projection_database_urls(target, "postgresql://user:pass@host/db"), + target, + None, + None, + ) + + def test_sync_db_adapter_composite_conflict_key_sql_shape() -> None: cursor = MagicMock() + cursor.fetchone.return_value = ("omninode_runtime", "omnidash_analytics") cursor_context = MagicMock() cursor_context.__enter__.return_value = cursor conn = MagicMock() @@ -27,9 +44,9 @@ def test_sync_db_adapter_composite_conflict_key_sql_shape() -> None: conn.cursor.return_value = cursor_context with patch("psycopg2.connect", return_value=conn): - adapter = _build_sync_db_adapter("postgresql://user:pass@host/db") + adapter = _internal_adapter("cost_by_repo_snapshots") result = adapter.upsert( - "savings_estimates", + "cost_by_repo_snapshots", "session_id,event_timestamp,model_local,model_cloud_baseline", { "session_id": "sess-1", @@ -52,6 +69,7 @@ def test_sync_db_adapter_composite_conflict_key_sql_shape() -> None: def test_sync_db_adapter_single_conflict_key_unchanged() -> None: cursor = MagicMock() + cursor.fetchone.return_value = ("omninode_runtime", "omnidash_analytics") cursor_context = MagicMock() cursor_context.__enter__.return_value = cursor conn = MagicMock() @@ -59,9 +77,9 @@ def test_sync_db_adapter_single_conflict_key_unchanged() -> None: conn.cursor.return_value = cursor_context with patch("psycopg2.connect", return_value=conn): - adapter = _build_sync_db_adapter("postgresql://user:pass@host/db") + adapter = _internal_adapter("node_service_registry") result = adapter.upsert( - "node_registrations", + "node_service_registry", "node_id", {"node_id": "n1", "status": "active"}, ) diff --git a/tests/integration/test_workspace_candidate_runner_routing.py b/tests/integration/test_workspace_candidate_runner_routing.py new file mode 100644 index 0000000000..8e8ffe1875 --- /dev/null +++ b/tests/integration/test_workspace_candidate_runner_routing.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Integration coverage for the runtime candidate build runner policy.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +pytestmark = pytest.mark.integration + +REPO_ROOT = Path(__file__).parents[2] +WORKFLOW_PATH = ".github/workflows/build-workspace-candidate-runtime.yml" + + +def test_candidate_runtime_build_uses_allowlisted_clean_cloud_egress() -> None: + """The candidate and routing policy must agree on the hosted ECR lane.""" + workflow = yaml.safe_load((REPO_ROOT / WORKFLOW_PATH).read_text(encoding="utf-8")) + policy = yaml.safe_load( + (REPO_ROOT / "config/runner_routing_policy.yaml").read_text(encoding="utf-8") + ) + + assert workflow["jobs"]["build-workspace-candidate"]["runs-on"] == ("ubuntu-latest") + allowlisted_paths = {entry["path"] for entry in policy["hosted_runner_allowlist"]} + assert WORKFLOW_PATH in allowlisted_paths diff --git a/tests/integration/verification/test_registration_contract_verify.py b/tests/integration/verification/test_registration_contract_verify.py index be2dddf16b..eb26229e03 100644 --- a/tests/integration/verification/test_registration_contract_verify.py +++ b/tests/integration/verification/test_registration_contract_verify.py @@ -115,6 +115,7 @@ def watermark_fn(topic: str) -> tuple[int, int]: @pytest.mark.integration +@pytest.mark.kafka class TestRegistrationContractVerifyLive: """Integration tests against live PostgreSQL and Kafka.""" diff --git a/tests/scripts/test_check_dep_provenance_lineage_omn15604.py b/tests/scripts/test_check_dep_provenance_lineage_omn15604.py new file mode 100644 index 0000000000..6fe24bfd16 --- /dev/null +++ b/tests/scripts/test_check_dep_provenance_lineage_omn15604.py @@ -0,0 +1,648 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Hermetic tests for the dep-provenance content-lineage check (OMN-15604). + +Root cause this closes: ``find_violations`` (OMN-13873) forbids a git source +for a first-party dep, but a line carrying a well-formed +``# raw-override-ok: `` token is exempt from that rule +*unconditionally and forever* -- the token is validated only for shape, never +against whether the pinned rev's content matches the PyPI version declared +alongside it. Live incident this reproduces: ``omnibase_infra@dev`` declared +``omnibase-core==0.46.8`` while pinning git rev ``3d51b047`` (escaped via +``# raw-override-ok: OMN-15414``) whose ``src/`` tree measurably DIFFERED from +released tag ``v0.46.8``. + +This module is hermetic (no network): every case injects a fake ``resolve`` +callable into ``find_lineage_violations`` so the pure decision logic +(declared-version parsing, source/version cross-referencing, escape-token +independence, message shape) is exercised without hitting GitHub. The LIVE +half -- proving the checker against the real ``3d51b047`` vs ``v0.46.8`` pin +via the real GitHub REST API -- lives in +``tests/integration/ci/test_dep_provenance_lineage_live_omn15604.py``, for the +same reason the OMN-15538 pin-reachability gate splits the same way: the +pre-push selector always ignores ``tests/integration``, so a transient network +failure there can never make a branch locally unpushable. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "check_dep_provenance.py" + +# The two real tree SHAs from the live incident (OMN-15604 ticket evidence, +# independently reproduced against the canonical omnibase_core clone during +# this lane): `git rev-parse 3d51b047:src` and `git rev-parse v0.46.8:src`. +_PINNED_REV = "3d51b047a43ee412a7521502619d35c216dc7811" +_PINNED_SRC_TREE = "a74566fd92b0ca9bb86919df5e7f804cc4307793" +_RELEASED_TAG = "v0.46.8" +_RELEASED_SRC_TREE = "008efdba12b39cf04d90d17468523daa281fe4fd" + + +def _load_module(): + spec = importlib.util.spec_from_file_location("check_dep_provenance", _SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def mod(): + return _load_module() + + +def _write_pyproject( + tmp_path: Path, + *, + dependencies: str = '"omnibase-core==0.46.8",', + sources_block: str, +) -> Path: + content = ( + "[project]\n" + 'name = "omnibase-infra"\n' + 'version = "0.0.0"\n' + "dependencies = [\n" + f" {dependencies}\n" + "]\n" + "\n" + f"{sources_block}" + "\n" + "[tool.ruff]\n" + 'target-version = "py312"\n' + ) + path = tmp_path / "pyproject.toml" + path.write_text(content) + return path + + +def _fixed_resolver(mapping: dict[tuple[str, str], tuple[str | None, str]]): + """Build a fake `resolve` callable from a fixed {(repo, ref): (sha, detail)} map.""" + + def _resolve(repo: str, ref: str) -> tuple[str | None, str]: + try: + return mapping[(repo, ref)] + except KeyError: + return None, f"unexpected probe: repo={repo!r} ref={ref!r}" + + return _resolve + + +# --------------------------------------------------------------------------- +# RED: reproduce the exact live incident (3d51b047 vs v0.46.8) with a fake +# resolver returning the REAL tree SHAs measured against the live pin -- +# proving RED against exists-but-wrong, not a synthetic value. +# --------------------------------------------------------------------------- + + +def test_red_reproduces_the_live_incident_pin(mod, tmp_path: Path) -> None: + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + resolver = _fixed_resolver( + { + ("omnibase_core", _PINNED_REV): (_PINNED_SRC_TREE, "ok"), + ("omnibase_core", _RELEASED_TAG): (_RELEASED_SRC_TREE, "ok"), + } + ) + + violations = mod.find_lineage_violations(path.read_text(), resolve=resolver) + + assert len(violations) == 1 + assert "omnibase-core" in violations[0] + assert _PINNED_SRC_TREE in violations[0] + assert _RELEASED_SRC_TREE in violations[0] + assert "differs from" in violations[0] + + +def test_red_applies_even_with_a_valid_escape_token(mod, tmp_path: Path) -> None: + """The whole point of OMN-15604: find_violations exempts an escaped line; + find_lineage_violations must NOT -- the token only exempts the + "forbid git source" rule, never the content-matches-version rule.""" + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + text = path.read_text() + resolver = _fixed_resolver( + { + ("omnibase_core", _PINNED_REV): (_PINNED_SRC_TREE, "ok"), + ("omnibase_core", _RELEASED_TAG): (_RELEASED_SRC_TREE, "ok"), + } + ) + + # find_violations is fooled by the token (this is #2 of the two facts the + # ticket says are "stored side by side and never compared"). + assert mod.find_violations(text) == [] + # find_lineage_violations is not. + assert len(mod.find_lineage_violations(text, resolve=resolver)) == 1 + + +# --------------------------------------------------------------------------- +# GREEN: pinned rev's src/ tree matches the released tag's src/ tree. +# --------------------------------------------------------------------------- + + +def test_green_when_pinned_tree_matches_released_tree(mod, tmp_path: Path) -> None: + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + 'rev = "105f7ce0a8f4b31f6f01fc94e9b43e75984f166a" }\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + resolver = _fixed_resolver( + { + ("omnibase_core", "105f7ce0a8f4b31f6f01fc94e9b43e75984f166a"): ( + _RELEASED_SRC_TREE, + "ok", + ), + ("omnibase_core", _RELEASED_TAG): (_RELEASED_SRC_TREE, "ok"), + } + ) + + assert mod.find_lineage_violations(path.read_text(), resolve=resolver) == [] + + +def test_green_when_no_uv_sources_override_present(mod, tmp_path: Path) -> None: + """Steady state after AC1 (the git override deleted): nothing to compare, + zero resolver calls -- lineage check is a cheap no-op.""" + calls: list[tuple[str, str]] = [] + + def _resolve(repo: str, ref: str) -> tuple[str | None, str]: + calls.append((repo, ref)) + return None, "should not be called" + + block = "[tool.uv.sources]\n" + path = _write_pyproject(tmp_path, sources_block=block) + + assert mod.find_lineage_violations(path.read_text(), resolve=_resolve) == [] + assert calls == [] + + +def test_green_when_git_override_has_no_declared_version(mod, tmp_path: Path) -> None: + """A git override for a package NEVER REFERENCED by any dependency / + override-dependency entry (not even a range) has nothing to compare + against -- that shape is find_violations' problem (forbidden override), + not a lineage-mismatch problem.""" + + def _resolve(repo: str, ref: str) -> tuple[str | None, str]: + raise AssertionError("resolver must not be called with no declared version") + + path = _write_pyproject( + tmp_path, + dependencies="", + sources_block=( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + 'rev = "deadbeef" }\n' + ), + ) + assert mod.find_lineage_violations(path.read_text(), resolve=_resolve) == [] + + +# --------------------------------------------------------------------------- +# RED: the AC2 bypass an adversarial verifier found and reproduced live -- +# loosening `==0.46.8` to `>=0.46.8,<0.47.0` made the ORIGINAL implementation +# skip the package entirely (declared.get(pkg) is None for a range), so a +# git override escaped by a valid raw-override-ok token sailed through BOTH +# find_violations (token) and find_lineage_violations (no exact version to +# compare) with zero violations. Live repro command that produced CASE B: +# `check_dep_provenance.py --check-lineage` exited 0 for the identical +# divergent rev (3d51b047) + identical token with `>=0.46.8,<0.47.0` instead +# of `==0.46.8`, while the `==` form exited 1. +# --------------------------------------------------------------------------- + + +def test_red_when_declared_constraint_is_a_range_not_an_exact_pin( + mod, tmp_path: Path +) -> None: + """A range constraint next to a git override + valid escape token must + RED, not silently pass -- a range does not unambiguously name a single + released tree to compare against, so "nothing to compare" is wrong; it + is an unprovable, and therefore failing, shape.""" + path = _write_pyproject( + tmp_path, + dependencies='"omnibase-core>=0.46.8,<0.47.0",', + sources_block=( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414\n' + ), + ) + text = path.read_text() + + def _boom(repo: str, ref: str) -> tuple[str | None, str]: + raise AssertionError( + "a range constraint has no single released tree to resolve " + "against -- the resolver must not even be called" + ) + + # find_violations is fooled by the token, same as the exact-pin case. + assert mod.find_violations(text) == [] + violations = mod.find_lineage_violations(text, resolve=_boom) + assert len(violations) == 1 + assert "omnibase-core" in violations[0] + assert "non-exact" in violations[0] + + +def test_red_when_declared_constraint_is_a_bare_unversioned_name( + mod, tmp_path: Path +) -> None: + """A bare `"omnibase-core"` entry (no version operator at all) next to a + git override is likewise unprovable and must RED.""" + path = _write_pyproject( + tmp_path, + dependencies='"omnibase-core",', + sources_block=( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414\n' + ), + ) + violations = mod.find_lineage_violations( + path.read_text(), resolve=lambda repo, ref: (None, "should not be called") + ) + assert len(violations) == 1 + assert "non-exact" in violations[0] + + +# --------------------------------------------------------------------------- +# UNDETERMINED: network/resolution failure fails closed by default. +# --------------------------------------------------------------------------- + + +def test_undetermined_when_resolver_cannot_resolve(mod, tmp_path: Path) -> None: + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }}\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + + def _resolve(repo: str, ref: str) -> tuple[str | None, str]: + return None, "transport error: simulated" + + violations = mod.find_lineage_violations(path.read_text(), resolve=_resolve) + assert len(violations) == 1 + assert "UNDETERMINED lineage" in violations[0] + + +# --------------------------------------------------------------------------- +# _declared_version_specs / _repo_from_git_url unit coverage +# --------------------------------------------------------------------------- + + +def test_declared_version_specs_prefers_override_dependencies(mod) -> None: + parsed = { + "project": {"dependencies": ["omnibase-core==0.46.7"]}, + "tool": {"uv": {"override-dependencies": ["omnibase-core==0.46.8"]}}, + } + assert mod._declared_version_specs(parsed) == {"omnibase-core": "==0.46.8"} + + +def test_declared_version_specs_captures_range_constraints(mod) -> None: + """Unlike an exact-only lookup, a range is captured (not dropped) -- this + is what lets `find_lineage_violations` flag it as unprovable instead of + silently skipping the package (the AC2 range-constraint bypass).""" + parsed = {"project": {"dependencies": ["omnibase-spi>=0.23.0,<0.24.0"]}} + assert mod._declared_version_specs(parsed) == {"omnibase-spi": ">=0.23.0,<0.24.0"} + + +def test_repo_from_git_url_extracts_bare_repo_name(mod) -> None: + assert ( + mod._repo_from_git_url("https://github.com/OmniNode-ai/omnibase_core.git") + == "omnibase_core" + ) + assert ( + mod._repo_from_git_url("https://github.com/other-org/omnibase_core.git") is None + ) + + +# --------------------------------------------------------------------------- +# CLI wiring: --check-lineage is opt-in and off by default. +# --------------------------------------------------------------------------- + + +def test_check_lineage_flag_is_off_by_default( + mod, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Default no-flag invocation never imports/calls the network resolver -- + pre-commit's zero-arg invocation stays offline (unchanged from OMN-13873).""" + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + + def _boom(repo: str, ref: str) -> tuple[str | None, str]: + raise AssertionError("network resolver must not run without --check-lineage") + + monkeypatch.setattr(mod, "resolve_src_tree_sha", _boom) + # No --check-lineage: find_violations sees the escape token and exits 0, + # and find_lineage_violations is never invoked at all. + assert mod.main(["--pyproject", str(path)]) == 0 + + +def test_allow_undetermined_lineage_refused_under_ci( + mod, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CI", "true") + path = _write_pyproject(tmp_path, sources_block="[tool.uv.sources]\n") + assert ( + mod.main( + [ + "--pyproject", + str(path), + "--check-lineage", + "--allow-undetermined-lineage", + ] + ) + == 2 + ) + + +# --------------------------------------------------------------------------- +# AC3 -- escape-token reconciliation (find_escape_token_violations). +# +# Root cause: `# raw-override-ok: ` exempted `find_violations` +# unconditionally and forever -- never checked against whether `` is +# still open. Reproduced live during this ticket's own build: a well-formed +# `# raw-override-ok: OMN-15414` token against the real OMN-15414 ticket +# (confirmed Done via Linear `get_issue`) exits 0 under `--check-lineage` +# under CI=true -- the exact RED case this section closes. +# --------------------------------------------------------------------------- + + +def test_red_token_cites_a_done_ticket(mod, tmp_path: Path) -> None: + """The exact RED case the ticket names: a well-formed token citing a + ticket that resolves to Done. Carries a live (not-yet-expired) until= + date so the mandatory-until= check (below) does not itself short-circuit + before the ticket-status path is exercised.""" + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414 until=2099-01-01\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + + def _resolve_done(ticket_id: str) -> tuple[str | None, str]: + assert ticket_id == "OMN-15414" + return "Done", "ok" + + violations = mod.find_escape_token_violations( + path.read_text(), resolve_ticket=_resolve_done + ) + assert len(violations) == 1 + assert "OMN-15414" in violations[0] + assert "closed" in violations[0] + + +@pytest.mark.parametrize("status_name", ["Cancelled", "Duplicate", "done"]) +def test_red_token_cites_other_closed_statuses( + mod, tmp_path: Path, status_name: str +) -> None: + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-1 until=2099-01-01\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + violations = mod.find_escape_token_violations( + path.read_text(), resolve_ticket=lambda t: (status_name, "ok") + ) + assert len(violations) == 1 + + +def test_green_token_cites_an_open_ticket(mod, tmp_path: Path) -> None: + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-1 until=2099-01-01\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + violations = mod.find_escape_token_violations( + path.read_text(), resolve_ticket=lambda t: ("In Progress", "ok") + ) + assert violations == [] + + +def test_red_token_until_date_has_expired(mod, tmp_path: Path) -> None: + """The `until=` half needs no network at all and fires purely on date.""" + from datetime import date + + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-1 until=2026-01-01\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + + def _boom(ticket_id: str) -> tuple[str | None, str]: + raise AssertionError( + "an already-expired until= date must short-circuit before any " + "ticket-status resolution" + ) + + violations = mod.find_escape_token_violations( + path.read_text(), resolve_ticket=_boom, today=date(2026, 8, 1) + ) + assert len(violations) == 1 + assert "EXPIRED" in violations[0] + + +def test_green_token_until_date_not_yet_expired(mod, tmp_path: Path) -> None: + from datetime import date + + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-1 until=2027-01-01\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + violations = mod.find_escape_token_violations( + path.read_text(), + resolve_ticket=lambda t: ("In Progress", "ok"), + today=date(2026, 8, 1), + ) + assert violations == [] + + +def test_green_no_escape_token_present(mod, tmp_path: Path) -> None: + """No token at all is find_violations' problem, not this check's -- must + not call the resolver.""" + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }}\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + + def _boom(ticket_id: str) -> tuple[str | None, str]: + raise AssertionError("resolver must not run with no escape token") + + assert ( + mod.find_escape_token_violations(path.read_text(), resolve_ticket=_boom) == [] + ) + + +def test_red_when_token_has_no_until_and_linear_api_key_unset( + mod, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The residual AC3 gap a remediation-round adversarial verifier found: + LINEAR_API_KEY is not provisioned as a repo OR org secret anywhere in + OmniNode-ai (verified live via `gh secret list` / `gh api + orgs/.../actions/secrets`), so the ticket-status reconciliation path + never actually runs in any live enforcing environment. A token with no + until= suffix -- the EXACT shape of the live incident token, + `# raw-override-ok: OMN-15414` -- must therefore RED here rather than + fall through both conditions and pass unconditionally and forever + (which is what shipped before this fix: this fixture, run against the + original implementation, returned `[]`). + + Uses the REAL resolve_ticket_status (no injected fake) so this proves + the shipped production code path, not just a test double.""" + monkeypatch.delenv("LINEAR_API_KEY", raising=False) + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + violations = mod.find_escape_token_violations(path.read_text()) + assert len(violations) == 1 + assert "OMN-15414" in violations[0] + assert "until=" in violations[0] + + +def test_graceful_skip_when_linear_api_key_unset_but_until_date_covers_it( + mod, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Graceful degradation on a missing LINEAR_API_KEY still applies -- but + only once an until= date has already supplied a live, network-free + enforcement condition for the line. Matches check_stale_todos.py's + degrade-don't-fail posture for the SECONDARY (ticket-status) check only.""" + monkeypatch.delenv("LINEAR_API_KEY", raising=False) + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414 until=2099-01-01\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + assert mod.find_escape_token_violations(path.read_text()) == [] + + +def test_check_token_expiry_flag_wired_through_cli(mod, tmp_path: Path) -> None: + """--check-token-expiry runs find_escape_token_violations and fails the + process; omitting it does not.""" + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-1 until=2020-01-01\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + + assert mod.main(["--pyproject", str(path)]) == 0 + assert mod.main(["--pyproject", str(path), "--check-token-expiry"]) == 1 + + +# --------------------------------------------------------------------------- +# AC4 -- cascade-movability check (find_unmovable_cascade_targets, +# --check-movable). Proof this closes: a dependency-cascade run +# (`.github/workflows/dependency-cascade.yml`) that tries to move a +# [tool.uv.sources]-git-pinned package via `uv lock --upgrade-package` cannot +# -- uv always prefers the explicit source override -- and previously had no +# way to surface that as anything other than a silent "no lockfile changes" +# skip. This is the standalone CLI check that workflow now runs BEFORE +# attempting the (guaranteed no-op) lock. +# --------------------------------------------------------------------------- + + +def test_check_movable_fails_for_a_git_pinned_package(mod, tmp_path: Path) -> None: + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + violations = mod.find_unmovable_cascade_targets(path.read_text(), "omnibase-core") + assert len(violations) == 1 + assert "cannot move" in violations[0] + + +def test_check_movable_ignores_escape_token(mod, tmp_path: Path) -> None: + """A valid raw-override-ok token does NOT make the package movable by a + cascade -- the token only ever exempted find_violations.""" + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + assert mod.find_violations(path.read_text()) == [] # token exempts this check + assert ( + len(mod.find_unmovable_cascade_targets(path.read_text(), "omnibase-core")) == 1 + ) + + +def test_check_movable_underscore_and_hyphen_spelling_both_match( + mod, tmp_path: Path +) -> None: + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }}\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + assert ( + len(mod.find_unmovable_cascade_targets(path.read_text(), "omnibase_core")) == 1 + ) + + +def test_check_movable_green_when_no_override_present(mod, tmp_path: Path) -> None: + path = _write_pyproject(tmp_path, sources_block="[tool.uv.sources]\n") + assert mod.find_unmovable_cascade_targets(path.read_text(), "omnibase-core") == [] + + +def test_check_movable_flag_wired_through_cli(mod, tmp_path: Path) -> None: + """--check-movable is a standalone CLI path (used directly by + dependency-cascade.yml) independent of find_violations/--check-lineage.""" + block = ( + "[tool.uv.sources]\n" + 'omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", ' + f'rev = "{_PINNED_REV}" }} # raw-override-ok: OMN-15414\n' + ) + path = _write_pyproject(tmp_path, sources_block=block) + assert mod.main(["--pyproject", str(path), "--check-movable", "omnibase-core"]) == 1 + assert mod.main(["--pyproject", str(path), "--check-movable", "omnibase-spi"]) == 0 + + +def test_ci_yml_dep_provenance_lineage_job_also_runs_check_token_expiry() -> None: + """AC3 must be a real wired gate, not detection-only: the SAME job + already registered in ci_summary_gate.py's STRICT_GATE_JOBS + ("Dep Provenance Lineage Gate (OMN-15604)") must also invoke + --check-token-expiry, so a fail there fails the same fail-closed job the + lineage half already fails through.""" + import yaml + + repo_root = Path(__file__).resolve().parents[2] + workflow = yaml.safe_load( + (repo_root / ".github" / "workflows" / "ci.yml").read_text() + ) + job = workflow["jobs"]["dep-provenance-lineage-gate"] + assert job["name"] == "Dep Provenance Lineage Gate (OMN-15604)" + + run_scripts = "\n".join( + step.get("run", "") for step in job["steps"] if "run" in step + ) + assert "--check-lineage" in run_scripts + assert "--check-token-expiry" in run_scripts diff --git a/tests/scripts/test_check_required_env_vars.py b/tests/scripts/test_check_required_env_vars.py new file mode 100644 index 0000000000..86bdcaed08 --- /dev/null +++ b/tests/scripts/test_check_required_env_vars.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Regression tests for required compose environment validation (OMN-15009).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts import check_required_env_vars + + +@pytest.mark.unit +def test_nonempty_process_environment_satisfies_required_compose_var( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + compose = tmp_path / "compose.yml" + compose.write_text( + "services:\n effects:\n environment:\n" + " DEPLOY_AGENT_HMAC_SECRET: " + "${DEPLOY_AGENT_HMAC_SECRET:?required}\n", + encoding="utf-8", + ) + empty_env_file = tmp_path / "runtime-policy.env" + empty_env_file.write_text("", encoding="utf-8") + monkeypatch.setenv("DEPLOY_AGENT_HMAC_SECRET", "render-only-not-a-secret") + + result = check_required_env_vars.main( + [ + "--compose-file", + str(compose), + "--env-file", + str(empty_env_file), + ] + ) + + assert result == 0 + + +@pytest.mark.unit +def test_empty_process_environment_does_not_satisfy_required_compose_var( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + compose = tmp_path / "compose.yml" + compose.write_text( + "services:\n effects:\n environment:\n" + " DEPLOY_AGENT_HMAC_SECRET: " + "${DEPLOY_AGENT_HMAC_SECRET:?required}\n", + encoding="utf-8", + ) + empty_env_file = tmp_path / "runtime-policy.env" + empty_env_file.write_text("", encoding="utf-8") + monkeypatch.setenv("DEPLOY_AGENT_HMAC_SECRET", "") + + result = check_required_env_vars.main( + [ + "--compose-file", + str(compose), + "--env-file", + str(empty_env_file), + ] + ) + + assert result == 1 diff --git a/tests/scripts/test_deploy_gateway.py b/tests/scripts/test_deploy_gateway.py new file mode 100644 index 0000000000..480c9e22a7 --- /dev/null +++ b/tests/scripts/test_deploy_gateway.py @@ -0,0 +1,938 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""scripts/deploy-gateway.sh -- the OMN-15521 sanctioned .201 gateway deploy path. + +Before this script, the `omninode-gateway` compose project (the standalone +operator-edge forwarder at /opt/omninode/gateway) had NO repo-resident deploy +path: it was stood up by hand-copying files into a root-owned directory, ran +an image with an empty `org.opencontainers.image.revision` label, and had no +recorded rollback target. These tests drive the REAL script (scripts/deploy- +gateway.sh) via subprocess -- exactly the convention +tests/scripts/test_deploy_runtime_promotion_class.py already uses for +deploy-runtime.sh -- with `docker`/`sudo` replaced by inspectable fakes so the +full --execute code path runs without a live Docker daemon, systemd, or root. + +Coverage maps 1:1 to the OMN-15521 falsifiable acceptance criteria: + AC1 -- test_help_documents_runbook, + test_print_compose_cmd_targets_repo_resident_compose_project, + test_dry_run_default_performs_no_mutation + AC2 -- test_build_command_stamps_oci_provenance_build_args, + test_build_command_stamps_sibling_ref_build_args, + test_execute_deploy_produces_non_empty_image_labels + AC3 -- test_verify_deployment_red_before_files_absent, + test_verify_deployment_green_after_files_present + AC4 -- test_sync_host_files_red_before_stale_copy_diverges, + test_sync_host_files_green_after_diff_is_empty + AC6 -- test_execute_deploy_writes_registry_with_rollback_target, + test_second_deploy_records_previous_digest_as_first_deploys_active, + test_rollback_target_derived_from_running_container_not_env_file, + test_previous_image_retagged_for_retention_before_build, + test_rollback_target_not_recorded_if_previous_image_missing + (AC5 -- the OMN-12912 restart/redelivery receipt -- is proven separately by + scripts/gateway_restart_safety_proof.sh + + tests/scripts/test_gateway_restart_safety_proof.py; the receipt itself lands + on OMN-12912, not here, per that ticket's own filing instruction.) + +Remediation round (OMN-15521, 2026-08-01): a prior version of this script (a) +derived the AC6 rollback target from gateway.env's GATEWAY_IMAGE= line +instead of the container's actual running image, with no existence check -- +this produced a registry.json rollback_command that pointed at an already +pruned/dangling image; (b) omitted the OMNIBASE_COMPAT_REF / OMNIMARKET_REF / +ONEX_CHANGE_CONTROL_REF build-args deploy-runtime.sh always passes, silently +falling back to the Dockerfile's hardcoded defaults; (c) verify_deployment() +never compared the running container's actual image against the digest it +just built, so a reload that silently failed to recreate the container still +reported success. The new tests below are RED-before/GREEN-after for each. +""" + +from __future__ import annotations + +import json +import os +import re +import stat +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEPLOY_SCRIPT = REPO_ROOT / "scripts" / "deploy-gateway.sh" +RUNBOOK = REPO_ROOT / "docs" / "runbooks" / "gateway-lane-deploy.md" + + +# --------------------------------------------------------------------------- +# Fixture: fake docker + sudo + systemctl on PATH, scratch host paths +# --------------------------------------------------------------------------- + +_FAKE_DOCKER = """#!/usr/bin/env bash +set -eu +log="${GW_TEST_DOCKER_LOG:?}" +printf '%s\\n' "$*" >> "${log}" + +case "$*" in + "compose -p omninode-gateway -f docker/docker-compose.gateway.yml build"*) + exit 0 + ;; + "image inspect "*"--format={{.Id}}") + printf 'sha256:%064d\\n' "${GW_TEST_DIGEST_SEED:-1}" + exit 0 + ;; + "inspect omninode-gateway-forwarder --format {{.Image}}") + # resolve_running_container_image() / verify_deployment()'s digest + # readback. Backed by a state file so a `systemctl reload` (fake sudo, + # below) can simulate the container actually being recreated onto the + # new digest -- distinguishing "before this deploy" from "after reload". + state="${GW_TEST_RUNNING_IMAGE_STATE:?}" + if [ -f "${state}" ]; then + cat "${state}" + fi + exit 0 + ;; + "tag "*) + if [ "${GW_TEST_ROLLBACK_TARGET_MISSING:-0}" = "1" ]; then + exit 1 + fi + exit 0 + ;; + *"--format={{index .Config.Labels \\"org.opencontainers.image.revision\\"}}}"*) + printf '%s\\n' "${GW_TEST_LABEL_REVISION:-}" + exit 0 + ;; + *"image.revision"*) + printf '%s\\n' "${GW_TEST_LABEL_REVISION:-}" + exit 0 + ;; + *"build_source"*) + printf '%s\\n' "${GW_TEST_LABEL_BUILD_SOURCE:-}" + exit 0 + ;; + *"exec omninode-gateway-forwarder test -f /app/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_delivery.py") + [ "${GW_TEST_DELIVERY_PRESENT:-1}" = "1" ] + exit $? + ;; + *"exec omninode-gateway-forwarder test -f /app/src/omnibase_infra/idempotency/store_sqlite.py") + [ "${GW_TEST_SQLITE_PRESENT:-1}" = "1" ] + exit $? + ;; + *) + printf 'fake docker: unexpected invocation: %s\\n' "$*" >&2 + exit 1 + ;; +esac +""" + +_FAKE_SUDO = """#!/usr/bin/env bash +# Strip `-o -g ` (real root ownership changes are not available +# in a test sandbox) and special-case systemctl so no live unit is required. +set -eu +args=() +skip_next=0 +for a in "$@"; do + if [ "${skip_next}" = "1" ]; then skip_next=0; continue; fi + case "${a}" in + -o|-g) skip_next=1; continue ;; + esac + args+=("${a}") +done +if [ "${args[0]:-}" = "systemctl" ]; then + printf '%s\\n' "${args[*]}" >> "${GW_TEST_SYSTEMCTL_LOG:?}" + if [ "${args[1]:-}" = "reload" ] && [ "${GW_TEST_RELOAD_TAKES_EFFECT:-1}" = "1" ]; then + # Simulate the reload actually recreating the container onto whatever + # digest gateway.env holds at this point (update_gateway_env_digest() + # already ran before reload_service() is called) -- mirrors real compose + # behavior. GW_TEST_RELOAD_TAKES_EFFECT=0 simulates a reload that exits 0 + # but does not actually recreate the container (the silent-failure case + # verify_deployment()'s digest check now catches). + state="${GW_TEST_RUNNING_IMAGE_STATE:-}" + env_file="${GATEWAY_ENV_FILE:-}" + if [ -n "${state}" ] && [ -n "${env_file}" ] && [ -f "${env_file}" ]; then + new_image="$(awk -F= '/^GATEWAY_IMAGE=/{print $2; exit}' "${env_file}")" + if [ -n "${new_image}" ]; then + printf '%s' "${new_image}" > "${state}" + fi + fi + fi + exit 0 +fi +exec "${args[@]}" +""" + + +def _write_fake_bin(bin_dir: Path) -> None: + bin_dir.mkdir(parents=True, exist_ok=True) + docker = bin_dir / "docker" + docker.write_text(_FAKE_DOCKER, encoding="utf-8") + docker.chmod(docker.stat().st_mode | stat.S_IEXEC) + + sudo = bin_dir / "sudo" + sudo.write_text(_FAKE_SUDO, encoding="utf-8") + sudo.chmod(sudo.stat().st_mode | stat.S_IEXEC) + + +class _Harness: + """One scratch environment for a subprocess run of deploy-gateway.sh.""" + + def __init__(self, tmp_path: Path) -> None: + self.tmp_path = tmp_path + self.bin_dir = tmp_path / "bin" + self.host_dir = tmp_path / "opt-omninode-gateway" + self.env_file = tmp_path / "gateway.env" + self.registry_dir = tmp_path / "home" / ".omnibase" / "gateway" + self.docker_log = tmp_path / "docker.log" + self.systemctl_log = tmp_path / "systemctl.log" + self.running_image_state = tmp_path / "running-image.state" + _write_fake_bin(self.bin_dir) + self.env_file.write_text( + "GATEWAY_IMAGE=sha256:" + ("0" * 64) + "\n" + "GATEWAY_AWS_PROFILE=gateway\n" + "GATEWAY_AWS_CONFIG_FILE=/dev/null\n" + "GATEWAY_AWS_CERTIFICATE_FILE=/dev/null\n" + "GATEWAY_AWS_PRIVATE_KEY_FILE=/dev/null\n" + "GATEWAY_AWS_SIGNING_HELPER_FILE=/dev/null\n" + "GATEWAY_TPM_DEVICE=/dev/null\n" + "GATEWAY_TPM_GROUP_ID=105\n" + "GATEWAY_CONTAINER_UID=1000\n" + "GATEWAY_CONTAINER_GID=1000\n", + encoding="utf-8", + ) + # Simulates a container already running this digest before the + # deploy under test -- the rollback-target source of truth now that + # it is read from `docker inspect`, not gateway.env's own line. + self.running_image_state.write_text("sha256:" + ("0" * 64), encoding="utf-8") + + def env(self, **overrides: str) -> dict[str, str]: + e = os.environ.copy() + e["PATH"] = f"{self.bin_dir}{os.pathsep}{e['PATH']}" + e["HOME"] = str(self.tmp_path / "home") + e["GATEWAY_HOST_DIR"] = str(self.host_dir) + e["GATEWAY_ENV_FILE"] = str(self.env_file) + e["GATEWAY_REGISTRY_DIR"] = str(self.registry_dir) + e["GW_TEST_DOCKER_LOG"] = str(self.docker_log) + e["GW_TEST_SYSTEMCTL_LOG"] = str(self.systemctl_log) + e["GW_TEST_RUNNING_IMAGE_STATE"] = str(self.running_image_state) + e.update(overrides) + return e + + def run(self, *args: str, **env_overrides: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", str(DEPLOY_SCRIPT), *args], + cwd=REPO_ROOT, + env=self.env(**env_overrides), + capture_output=True, + text=True, + check=False, + ) + + def registry(self) -> dict[str, Any]: + result: dict[str, Any] = json.loads( + (self.registry_dir / "registry.json").read_text(encoding="utf-8") + ) + return result + + +@pytest.fixture +def harness(tmp_path: Path) -> _Harness: + return _Harness(tmp_path) + + +# --------------------------------------------------------------------------- +# AC1 -- committed, repo-resident deploy path +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_script_exists_and_is_executable() -> None: + assert DEPLOY_SCRIPT.is_file(), "scripts/deploy-gateway.sh must exist (AC1)" + mode = DEPLOY_SCRIPT.stat().st_mode + assert mode & stat.S_IXUSR, "scripts/deploy-gateway.sh must be executable" + + +@pytest.mark.unit +def test_runbook_exists_alongside_cold_lane_bringup() -> None: + assert RUNBOOK.is_file(), ( + "docs/runbooks/gateway-lane-deploy.md must exist alongside " + "docs/runbooks/cold-lane-full-bringup.md (AC1)" + ) + + +@pytest.mark.unit +def test_help_documents_runbook() -> None: + result = subprocess.run( + ["bash", str(DEPLOY_SCRIPT), "--help"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + assert "OMN-15521" in result.stdout + + +@pytest.mark.unit +def test_help_rollback_reads_registry_command_not_hand_reconstructed() -> None: + """AC1 (round-4 remediation): --help's ROLLBACK section must tell the + operator to read `rollback_command` out of registry.json and run it + verbatim -- NOT to hand-reconstruct a sed substitution from + `previous_digest`. + + The hand-reconstruction recipe this replaces was the exact hazard + docs/runbooks/gateway-lane-deploy.md now warns against: `rollback_command` + (and `previous_digest`) are JSON `null` when there is no rollback target, + and a `null` printed through `jq -r` renders as the literal 4-character + string "null", which a hand-written `sed` substitution writes straight + into gateway.env's GATEWAY_IMAGE= line. The systemd unit's ExecStartPre + digest-format assertion then refuses to start the container on the next + restart/reboot. The script's own --help was still shipping that recipe + after the runbook was fixed -- the operator surface most likely to be + read at 3am contradicted the doc. + """ + result = subprocess.run( + ["bash", str(DEPLOY_SCRIPT), "--help"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + help_text = result.stdout + + assert "ROLLBACK" in help_text, "--help must document rollback at all" + assert "rollback_command" in help_text, ( + "--help must name registry.json's rollback_command field as the " + "rollback source of truth" + ) + assert "jq -r .rollback_command" in help_text, ( + "--help must show the operator how to READ the pre-filled command " + "out of the registry, not how to rebuild it" + ) + + # The hazard itself: no hand-fill template may survive anywhere in --help. + assert "" not in help_text, ( + "--help must not print a fill-in-the-blank rollback template -- " + "hand-substituting previous_digest is precisely what corrupts " + "gateway.env when the field is null" + ) + assert "sed -i" not in help_text, ( + "--help must not carry a sed-reconstruction recipe; the only sed " + "the operator should ever run for rollback is the one registry.json " + "already pre-filled in rollback_command" + ) + + # And it must say what to do when there IS no rollback target. + assert "null" in help_text, ( + "--help must state that rollback_command is null when there is no " + "rollback target (first deploy / previous image pruned)" + ) + + +@pytest.mark.unit +def test_help_rollback_matches_runbook_guidance() -> None: + """The script's --help and the runbook must not drift apart on rollback: + both must point at `jq -r .rollback_command`, and neither may carry the + `` hand-fill template. The round-3 defect was exactly + this drift -- the runbook was corrected, the script's usage() was not. + """ + result = subprocess.run( + ["bash", str(DEPLOY_SCRIPT), "--help"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + runbook_text = RUNBOOK.read_text(encoding="utf-8") + + assert "jq -r .rollback_command" in runbook_text, ( + "runbook must document reading rollback_command from the registry" + ) + assert "" not in runbook_text, ( + "runbook must not carry the hand-fill rollback template" + ) + assert "jq -r .rollback_command" in result.stdout, ( + "usage() must mirror the runbook's rollback instruction" + ) + + +@pytest.mark.unit +def test_print_compose_cmd_targets_repo_resident_compose_project( + harness: _Harness, +) -> None: + """AC1: the deploy path must target compose project omninode-gateway from + the REPO copy of docker-compose.gateway.yml -- never a hand-copied path. + """ + result = harness.run("--print-compose-cmd") + assert result.returncode == 0, result.stderr + assert "-p omninode-gateway" in result.stdout + assert "-f docker/docker-compose.gateway.yml" in result.stdout + assert "/opt/omninode/gateway" not in result.stdout + + +@pytest.mark.unit +def test_dry_run_default_performs_no_mutation(harness: _Harness) -> None: + """Bare invocation (no --execute) must not touch the host dir, env file, + or registry -- mirrors deploy-runtime.sh's dry-run-by-default contract. + """ + before_env = harness.env_file.read_text(encoding="utf-8") + result = harness.run() + assert result.returncode == 0, result.stderr + assert "Dry Run" in result.stdout + assert not harness.host_dir.exists() + assert harness.env_file.read_text(encoding="utf-8") == before_env + assert not (harness.registry_dir / "registry.json").exists() + assert not harness.docker_log.exists() or harness.docker_log.read_text() == "" + + +# --------------------------------------------------------------------------- +# AC2 -- OCI provenance labels +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_build_command_stamps_oci_provenance_build_args(harness: _Harness) -> None: + """AC2 mechanism: the constructed build invocation must carry the same + provenance build-args every omnibase-infra runtime container gets. This is + the root-cause fix for the ticket's own finding (`rev=(empty) + src=release` on the hand-built container) -- the original hand build never + passed these flags at all. + """ + result = harness.run("--print-compose-cmd") + assert result.returncode == 0, result.stderr + assert re.search(r"--build-arg VCS_REF=[0-9a-f]{12}", result.stdout) + assert re.search(r"--build-arg RUNTIME_VERSION=\d+\.\d+\.\d+", result.stdout) + assert "--build-arg COMPOSE_PROJECT=omninode-gateway" in result.stdout + assert re.search(r"--build-arg RUNTIME_SOURCE_HASH=[0-9a-f]{12}", result.stdout) + assert "--build-arg BUILD_SOURCE=release" in result.stdout + assert "--build-arg PROMOTION_CLASS=clean-main" in result.stdout + assert "--build-arg NON_MAIN_LINEAGE=false" in result.stdout + + +@pytest.mark.unit +def test_build_command_stamps_sibling_ref_build_args(harness: _Harness) -> None: + """AC2 remediation (OMN-15521): scripts/deploy-runtime.sh's build_images() + passes OMNIBASE_COMPAT_REF / OMNIMARKET_REF / ONEX_CHANGE_CONTROL_REF + unconditionally on every build -- a prior version of this script silently + dropped all three, so the gateway image fell back to the Dockerfile's + hardcoded ARG defaults (OMNIBASE_COMPAT_REF=v0.5.5, + ONEX_CHANGE_CONTROL_REF=v0.5.3, OMNIMARKET_REF=dev). That is exactly how + the deployed gateway container's onex-change-control pin (0.5.3) drifted + from the omnibase-infra runtime container's pin (0.5.1) on the same + `.201` box. OMNI_HOME is explicitly cleared so the fallback strings are + deterministic regardless of the host running this test. + """ + result = harness.run("--print-compose-cmd", OMNI_HOME="") + assert result.returncode == 0, result.stderr + assert "--build-arg OMNIBASE_COMPAT_REF=main" in result.stdout + assert "--build-arg OMNIMARKET_REF=dev" in result.stdout + assert "--build-arg ONEX_CHANGE_CONTROL_REF=main" in result.stdout + + +@pytest.mark.unit +def test_current_compose_file_declares_no_provenance_build_args() -> None: + """RED-before-the-fix control: docker-compose.gateway.yml's OWN declared + build.args block (what a bare `docker compose build` would use with no + extra flags -- i.e. exactly how the lane was hand-built on 2026-07-29) + carries only BUILD_SOURCE/EXPECTED_BUILD_SOURCE. This is why the running + container had an empty org.opencontainers.image.revision label; the fix + lives in the deploy script's explicit --build-arg list, not in the + compose file itself (matching how docker-compose.infra.yml + deploy- + runtime.sh's build_images() are already split the same way). + """ + compose_text = (REPO_ROOT / "docker" / "docker-compose.gateway.yml").read_text( + encoding="utf-8" + ) + build_block_match = re.search( + r"build:\n(.*?)\n container_name:", compose_text, re.DOTALL + ) + assert build_block_match is not None + build_block = build_block_match.group(1) + assert "VCS_REF" not in build_block + assert "COMPOSE_PROJECT" not in build_block + + +@pytest.mark.unit +def test_execute_deploy_produces_non_empty_image_labels(harness: _Harness) -> None: + """GREEN-after (mechanism level): after --execute, verify_deployment() + reads back a non-empty org.opencontainers.image.revision from the fake + docker inspect -- the exact probe cited in the ticket's AC2. + """ + result = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="3541ac805b86", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + ) + assert result.returncode == 0, result.stderr + result.stdout + assert "AC2 OK: org.opencontainers.image.revision=3541ac805b86" in result.stdout + + +# --------------------------------------------------------------------------- +# AC3 -- the two OMN-12912 files present in the running container +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_verify_deployment_red_before_files_absent(harness: _Harness) -> None: + """RED: the ticket's own §2 probe state (pre-#2556 image) -- both files + absent -- must be surfaced as a hard failure, not silently accepted. + """ + result = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="deadbeef0000", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="0", + GW_TEST_SQLITE_PRESENT="0", + ) + assert result.returncode != 0 + assert "AC3 FAILED" in result.stdout + result.stderr + + +@pytest.mark.unit +def test_verify_deployment_green_after_files_present(harness: _Harness) -> None: + """GREEN: once the deployed image carries #2556, both files resolve and + the deploy reports success -- the exact AC3 probe flipping. + """ + result = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="3541ac805b86", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + ) + assert result.returncode == 0, result.stderr + result.stdout + assert "AC3 OK" in result.stdout + + +@pytest.mark.unit +def test_verify_deployment_red_before_reload_silently_fails(harness: _Harness) -> None: + """RED (OMN-15521 remediation, exists-but-wrong): a `systemctl reload` + that exits 0 without actually recreating the container must NOT be + reported as a successful deploy. A prior version of verify_deployment() + only checked image labels and file presence -- both of which the STALE + (still-running, pre-deploy) container also satisfies once it has already + been deployed once -- so a silently-failed recreate on any deploy after + the first read as success on every subsequent run. + """ + result = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="3541ac805b86", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + GW_TEST_RELOAD_TAKES_EFFECT="0", + ) + assert result.returncode != 0 + assert "AC-VERIFY FAILED" in result.stdout + result.stderr + assert "did not take effect" in result.stdout + result.stderr + + +# --------------------------------------------------------------------------- +# AC4 -- host compose file matches the merged-dev repo copy +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_sync_host_files_red_before_stale_copy_diverges(harness: _Harness) -> None: + """RED (exists-but-wrong, not merely absent): pre-populate the host dir + with a stale compose file -- exactly the 2026-07-29 hand-copy scenario, + where /opt/omninode/gateway/docker-compose.gateway.yml lacked #2556's + gateway-delivery-state volume. Before any sync, diff is non-empty. + """ + harness.host_dir.mkdir(parents=True) + (harness.host_dir / "docker-compose.gateway.yml").write_text( + "services: {}\n# stale pre-#2556 copy\n", encoding="utf-8" + ) + real_compose = (REPO_ROOT / "docker" / "docker-compose.gateway.yml").read_text( + encoding="utf-8" + ) + assert ( + harness.host_dir / "docker-compose.gateway.yml" + ).read_text() != real_compose, "fixture must actually diverge from the repo copy" + + +@pytest.mark.unit +def test_sync_host_files_green_after_diff_is_empty(harness: _Harness) -> None: + """GREEN: after --execute, the host copy is byte-identical to the repo + copy that produced the running container -- the exact AC4 probe. + """ + harness.host_dir.mkdir(parents=True) + (harness.host_dir / "docker-compose.gateway.yml").write_text( + "services: {}\n# stale pre-#2556 copy\n", encoding="utf-8" + ) + + result = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="3541ac805b86", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + ) + assert result.returncode == 0, result.stderr + result.stdout + assert "AC4 OK" in result.stdout + + repo_compose = (REPO_ROOT / "docker" / "docker-compose.gateway.yml").read_text( + encoding="utf-8" + ) + host_compose = (harness.host_dir / "docker-compose.gateway.yml").read_text( + encoding="utf-8" + ) + assert host_compose == repo_compose + + repo_canary = ( + REPO_ROOT / "docker" / "gateway" / "beta-gateway-canary.yaml" + ).read_text(encoding="utf-8") + host_canary = (harness.host_dir / "gateway" / "beta-gateway-canary.yaml").read_text( + encoding="utf-8" + ) + assert host_canary == repo_canary + + +# --------------------------------------------------------------------------- +# AC6 -- rollback target recorded via registry.json +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_execute_deploy_writes_registry_with_rollback_target( + harness: _Harness, +) -> None: + result = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="3541ac805b86", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + GW_TEST_DIGEST_SEED="1", + ) + assert result.returncode == 0, result.stderr + result.stdout + + registry = harness.registry() + assert registry["compose_project"] == "omninode-gateway" + assert registry["active_digest"] == "sha256:" + ("0" * 63) + "1" + assert registry["previous_digest"] == "sha256:" + ("0" * 64) + assert registry["git_sha"] + assert registry["rollback_command"] + rollback_command = str(registry["rollback_command"]) + assert "GATEWAY_IMAGE=sha256:" + ("0" * 64) in rollback_command + + # A reload must actually have been requested. + assert harness.systemctl_log.exists() + assert ( + "systemctl reload onex-gateway-forwarder" in harness.systemctl_log.read_text() + ) + + +@pytest.mark.unit +def test_second_deploy_records_previous_digest_as_first_deploys_active( + harness: _Harness, +) -> None: + """The rollback target must chain: deploy #2's `previous_digest` must + equal deploy #1's `active_digest`, not the pre-existing gateway.env value + from before either deploy ran. + """ + first = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="aaaaaaaaaaaa", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + GW_TEST_DIGEST_SEED="1", + ) + assert first.returncode == 0, first.stderr + first.stdout + first_registry = harness.registry() + + second = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="bbbbbbbbbbbb", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + GW_TEST_DIGEST_SEED="2", + ) + assert second.returncode == 0, second.stderr + second.stdout + second_registry = harness.registry() + + assert second_registry["previous_digest"] == first_registry["active_digest"] + assert second_registry["active_digest"] != first_registry["active_digest"] + + +@pytest.mark.unit +def test_rollback_target_derived_from_running_container_not_env_file( + harness: _Harness, +) -> None: + """RED-before-the-fix control / GREEN-after (OMN-15521 remediation): a + prior version of this script awk'd the rollback target out of + gateway.env's GATEWAY_IMAGE= line. That line can drift from what the + container is actually running (a previous deploy that wrote the file but + was killed before reload; a manual edit) -- exactly what this fixture + reproduces: gateway.env claims one digest, the running container (the + fake docker inspect state file) reports a different one. The recorded + previous_digest must be the ACTUAL running digest, never the stale + env-file value. + """ + stale_env_digest = "sha256:" + ("e" * 64) + harness.env_file.write_text( + harness.env_file.read_text(encoding="utf-8").replace( + "GATEWAY_IMAGE=sha256:" + ("0" * 64), + f"GATEWAY_IMAGE={stale_env_digest}", + ), + encoding="utf-8", + ) + actually_running_digest = "sha256:" + ("a" * 64) + harness.running_image_state.write_text(actually_running_digest, encoding="utf-8") + + result = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="3541ac805b86", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + GW_TEST_DIGEST_SEED="1", + ) + assert result.returncode == 0, result.stderr + result.stdout + + registry = harness.registry() + assert registry["previous_digest"] == actually_running_digest + assert registry["previous_digest"] != stale_env_digest + + +@pytest.mark.unit +def test_previous_image_retagged_for_retention_before_build( + harness: _Harness, +) -> None: + """OMN-15521 remediation: the previous running image must be retagged + under a durable name (`docker tag + docker-gateway-forwarder:previous`) BEFORE the build moves + BUILD_IMAGE_TAG onto the new image -- otherwise the old image becomes + untagged/dangling the instant the build succeeds and is eligible for + collection by a routine `docker image prune` before anyone needs it for + rollback. Order matters, not just occurrence. + """ + previous_digest = "sha256:" + ("a" * 64) + harness.running_image_state.write_text(previous_digest, encoding="utf-8") + + result = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="3541ac805b86", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + ) + assert result.returncode == 0, result.stderr + result.stdout + + lines = harness.docker_log.read_text(encoding="utf-8").splitlines() + tag_idx = next((i for i, line in enumerate(lines) if line.startswith("tag ")), None) + build_idx = next( + ( + i + for i, line in enumerate(lines) + if line.startswith("compose -p omninode-gateway") + ), + None, + ) + assert tag_idx is not None, f"expected a `docker tag ...` call, got: {lines}" + assert build_idx is not None, ( + f"expected a `docker compose ... build` call, got: {lines}" + ) + assert f"tag {previous_digest} docker-gateway-forwarder:previous" in lines[tag_idx] + assert tag_idx < build_idx, ( + "retention tag must be applied BEFORE the build moves " + "BUILD_IMAGE_TAG off the previous image" + ) + + +@pytest.mark.unit +def test_rollback_target_not_recorded_if_previous_image_missing( + harness: _Harness, +) -> None: + """Fail-closed (OMN-15521 remediation): if the previous running image no + longer resolves locally (already pruned), the script must not record it + as a rollback target -- recording an unvalidated digest verbatim was + exactly how the previous version produced a registry.json + rollback_command pointing at an image `docker image inspect` could not + find (confirmed live on `.201`: registry previous_digest + sha256:b51b380d... resolved to "No such image"). + """ + previous_digest = "sha256:" + ("a" * 64) + harness.running_image_state.write_text(previous_digest, encoding="utf-8") + + result = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="3541ac805b86", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + GW_TEST_ROLLBACK_TARGET_MISSING="1", + ) + assert result.returncode == 0, result.stderr + result.stdout + + registry = harness.registry() + assert registry["previous_digest"] is None + assert registry["rollback_command"] is None, ( + "OMN-15521 remediation round 3: a null previous_digest must produce a " + "null rollback_command, never a sed command built from an empty " + "digest -- confirmed live on .201: a prior version emitted " + '"GATEWAY_IMAGE=" (nothing after the =) here, which would have ' + "corrupted gateway.env's GATEWAY_IMAGE= line to an unparseable value " + "on the next accidental run, wedging the systemd unit's " + "ExecStartPre digest-format assertion on the following restart." + ) + + +@pytest.mark.unit +def test_first_deploy_records_no_rollback_target(harness: _Harness) -> None: + """First-ever deploy: no container is running yet (the fake docker + inspect state file is absent), so there is nothing to roll back to -- + registry.json must record previous_digest as null, not a fabricated or + stale value. + """ + harness.running_image_state.unlink() + + result = harness.run( + "--execute", + GW_TEST_LABEL_REVISION="3541ac805b86", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + ) + assert result.returncode == 0, result.stderr + result.stdout + + registry = harness.registry() + assert registry["previous_digest"] is None + assert registry["rollback_command"] is None, ( + "first deploy has no rollback target -- rollback_command must be " + "null, not a sed command with an empty GATEWAY_IMAGE= value" + ) + docker_log = harness.docker_log.read_text(encoding="utf-8") + assert not any(line.startswith("tag ") for line in docker_log.splitlines()), ( + "must not attempt to retag an empty previous digest" + ) + + +@pytest.mark.unit +def test_skip_reload_leaves_container_on_previous_digest(harness: _Harness) -> None: + """--skip-reload must still write gateway.env + registry but must NOT + invoke systemctl -- a deliberate escape hatch, not a silent no-op. + """ + result = harness.run( + "--execute", + "--skip-reload", + GW_TEST_LABEL_REVISION="3541ac805b86", + GW_TEST_LABEL_BUILD_SOURCE="release", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + ) + assert result.returncode == 0, result.stderr + result.stdout + assert ( + "gateway.env is updated but the running container still has the OLD digest" + in (result.stdout + result.stderr) + ) + assert not harness.systemctl_log.exists() or harness.systemctl_log.read_text() == "" + + +@pytest.mark.unit +def test_execute_without_env_file_fails_closed(harness: _Harness) -> None: + """AC1 fail-closed case: --execute must refuse (not silently proceed) + when GATEWAY_ENV_FILE is missing, rather than building/deploying with + unresolved compose interpolation variables. + """ + harness.env_file.unlink() + result = harness.run("--execute") + assert result.returncode != 0 + assert "GATEWAY_ENV_FILE not found" in result.stderr + assert not (harness.registry_dir / "registry.json").exists() + + +# --------------------------------------------------------------------------- +# BUILD_SOURCE=workspace staging (OMN-15521 remediation round 3) +# +# A prior version of this script honoured BUILD_SOURCE=workspace for the +# stamped labels (promotion_class/non_main_lineage) but never actually staged +# workspace/sibling-repos/ -- unlike scripts/deploy-runtime.sh's +# build_images(), which always calls stage_workspace_if_needed() first. +# docker/Dockerfile.runtime unconditionally COPYs workspace/sibling-repos/, +# so a workspace-mode build silently used the committed placeholder (or +# whatever stale staging happened to already be sitting in the checkout) +# while still stamping workspace-provenance labels the prod-promotion gate +# and lineage guard consume. A full live staging run needs real OMNI_HOME +# sibling git clones, so -- matching the established convention +# tests/scripts/test_deploy_runtime_build_context.py already uses for the +# identical deploy-runtime.sh wiring (test_deploy_runtime_stages_workspace_ +# and_passes_omni_home_arg / test_deploy_runtime_runs_sibling_lock_pin_ +# preflight) -- these are static assertions on the wiring, not a live +# staging run. +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_deploy_gateway_stages_workspace_before_build() -> None: + """AC2 remediation: BUILD_SOURCE=workspace must stage sibling repos + before build_image() runs, or the build silently uses stale/placeholder + workspace/sibling-repos/ content while still claiming workspace + provenance in its stamped labels. + """ + deploy_script = DEPLOY_SCRIPT.read_text(encoding="utf-8") + + assert 'stage_workspace_if_needed "${repo_root}"' in deploy_script + stage_call_idx = deploy_script.index('stage_workspace_if_needed "${repo_root}"\n') + build_call_idx = deploy_script.index('build_image "${repo_root}" "${git_sha}"') + assert stage_call_idx < build_call_idx, ( + "stage_workspace_if_needed must run BEFORE build_image in main(), or " + "the build reads workspace/sibling-repos/ before it is populated" + ) + + # The staging function itself must invoke the SAME script deploy-runtime.sh + # uses -- reused machinery, not a parallel reimplementation. + assert ( + 'stage_script="${repo_root}/scripts/runtime_build/stage_workspace.sh"' + in deploy_script + ) + assert 'bash "${stage_script}"' in deploy_script + + # Release mode (the default) must not attempt to stage anything. + assert ( + 'if [[ "${build_source}" != "workspace" ]]; then\n return 0' + in (deploy_script.split("stage_workspace_if_needed() {", 1)[1]) + ) + + +@pytest.mark.unit +def test_deploy_gateway_requires_omni_home_for_workspace_build_source( + harness: _Harness, +) -> None: + """BUILD_SOURCE=workspace with no OMNI_HOME must fail closed before any + build/mutation -- mirrors deploy-runtime.sh's validate_build_source_config. + """ + result = harness.run( + "--execute", + BUILD_SOURCE="workspace", + OMNI_HOME="", + GW_TEST_LABEL_REVISION="3541ac805b86", + GW_TEST_LABEL_BUILD_SOURCE="workspace", + GW_TEST_DELIVERY_PRESENT="1", + GW_TEST_SQLITE_PRESENT="1", + ) + assert result.returncode != 0 + assert "BUILD_SOURCE=workspace requires OMNI_HOME" in ( + result.stdout + result.stderr + ) + assert not harness.docker_log.exists() or "compose -p" not in ( + harness.docker_log.read_text(encoding="utf-8") + ), "must fail before attempting a build, not mid-build" + + +@pytest.mark.unit +def test_deploy_gateway_runs_sibling_lock_pin_preflight_in_workspace_mode() -> None: + """Workspace staging must run the OMN-12987 lock-pin preflight before + build, same as deploy-runtime.sh -- the recurrence guard for a stale + vendored sibling silently shipping (the 2026-06-11 stability crash). + """ + deploy_script = DEPLOY_SCRIPT.read_text(encoding="utf-8") + + assert 'check_sibling_lock_pins "${repo_root}" "${omni_home}"' in deploy_script + assert "scripts/runtime_build/check_sibling_lock_pins.py" in deploy_script + assert "Refusing to build a stale image." in deploy_script + # Current CLI (OMN-12977/12987): --lock / repeatable --repo / --output, + # never the removed --provenance-out flag. + assert "--lock" in deploy_script + assert "--output" in deploy_script + assert "--provenance-out" not in deploy_script diff --git a/tests/scripts/test_deploy_runners_sync_paths.py b/tests/scripts/test_deploy_runners_sync_paths.py index aa8e1b946b..616ee924d9 100644 --- a/tests/scripts/test_deploy_runners_sync_paths.py +++ b/tests/scripts/test_deploy_runners_sync_paths.py @@ -30,9 +30,9 @@ import importlib.util import re -import shutil import subprocess import sys +import tempfile from pathlib import Path from typing import Any @@ -128,39 +128,28 @@ def _run_rsync_artifacts() -> list[str]: "rsync_artifacts", ] ) - result = subprocess.run( - ["bash", "-c", harness], - capture_output=True, - text=True, - check=False, - env={ - "PATH": f"{Path(__file__).resolve().parent / '_stub_bin'}:/usr/bin:/bin", - }, - ) + with tempfile.TemporaryDirectory(prefix="rsync-stub-") as stub_dir_name: + stub_dir = Path(stub_dir_name) + stub_rsync = stub_dir / "rsync" + stub_rsync.write_text( + "#!/usr/bin/env bash\nprintf '%s\\n' \"$@\"\n", encoding="utf-8" + ) + stub_rsync.chmod(0o755) + result = subprocess.run( + ["bash", "-c", harness], + capture_output=True, + text=True, + check=False, + env={ + "PATH": f"{stub_dir}:/usr/bin:/bin", + }, + ) assert result.returncode == 0, ( f"rsync_artifacts() harness failed:\nstdout={result.stdout}\nstderr={result.stderr}" ) return result.stdout.splitlines() -@pytest.fixture(scope="module", autouse=True) -def _stub_rsync_binary() -> Any: - """Install a stub ``rsync`` on a fixed PATH dir that just echoes its argv. - - Fixed path (not tmp_path per-test) because the harness's PATH is built as - a literal string inside the bash -c invocation above. - """ - stub_dir = Path(__file__).resolve().parent / "_stub_bin" - stub_dir.mkdir(exist_ok=True) - stub_rsync = stub_dir / "rsync" - stub_rsync.write_text( - "#!/usr/bin/env bash\nprintf '%s\\n' \"$@\"\n", encoding="utf-8" - ) - stub_rsync.chmod(0o755) - yield - shutil.rmtree(stub_dir, ignore_errors=True) - - @pytest.mark.parametrize("required_path", _REQUIRED_OMNI_CURL_PATHS) def test_rsync_artifacts_actually_syncs_omni_curl(required_path: str) -> None: """The real rsync_artifacts() function must pass both omni-curl source diff --git a/tests/scripts/test_deploy_runtime_build_context.py b/tests/scripts/test_deploy_runtime_build_context.py index f17b48a2be..13bc515f8b 100644 --- a/tests/scripts/test_deploy_runtime_build_context.py +++ b/tests/scripts/test_deploy_runtime_build_context.py @@ -46,6 +46,25 @@ def _dockerfile_workspace_copy_sources() -> list[str]: return sources +def _dockerfile_config_copy_sources() -> list[str]: + """Every Dockerfile.runtime COPY source that pulls from the config/ tree. + + Same rationale as `_dockerfile_workspace_copy_sources` (OMN-12987), applied + to config/ (OMN-15696): sync_files() must rsync config/ into the deployed + build context or a COPY config/ fails workspace-mode `docker build` with + "failed to calculate checksum ...: not found". + """ + dockerfile = DOCKERFILE.read_text(encoding="utf-8") + sources: list[str] = [] + for match in _COPY_LINE_RE.finditer(dockerfile): + tokens = match.group("args").split() + if any(tok.startswith("--from=") for tok in tokens): + continue + operands = [tok for tok in tokens if not tok.startswith("--")] + sources.extend(src for src in operands[:-1] if src.startswith("config/")) + return sources + + @pytest.mark.unit def test_deploy_runtime_syncs_runtime_dockerfile_copy_sources() -> None: """deploy-runtime.sh must ship paths copied by Dockerfile.runtime.""" @@ -104,6 +123,47 @@ def test_deploy_runtime_stages_every_workspace_copy_source() -> None: ) +@pytest.mark.unit +def test_deploy_runtime_stages_every_config_copy_source() -> None: + """Every `COPY config/` in Dockerfile.runtime must be staged. + + Regression guard for OMN-15696: Dockerfile.runtime COPYs + config/runner_fleet.yaml (added by OMN-15676), but sync_files() never + rsynced config/ into the deployed build context, so any --force redeploy + or cold bring-up that recreates deployed// failed the image build + with "failed to calculate checksum of ref ...:/config/runner_fleet.yaml: + not found" -- the same COPY-without-matching-rsync class OMN-12987 fixed + for workspace/. + + This test derives the config/ COPY sources from the live Dockerfile and + asserts deploy-runtime.sh stages each -- either via an exact-file rsync + argument, or by rsyncing the containing config/ directory -- so a future + Dockerfile COPY without a matching rsync fails CI. + """ + deploy_script = DEPLOY_SCRIPT.read_text(encoding="utf-8") + + config_sources = _dockerfile_config_copy_sources() + # The Dockerfile must at minimum COPY the known runner_fleet.yaml path; a + # regex that silently matched nothing would make this guard vacuously pass. + assert "config/runner_fleet.yaml" in config_sources + + directory_staged = '"${repo_root}/config/"' in deploy_script + + missing: list[str] = [] + for source in config_sources: + staged = directory_staged or f'"${{repo_root}}/{source}"' in deploy_script + if not staged: + missing.append(source) + + assert not missing, ( + "Dockerfile.runtime COPYs these config/ paths but deploy-runtime.sh " + f"does not stage them into the deployed build context: {missing}. Add an " + "rsync of each (or of config/ as a whole) into sync_files() or " + "workspace-mode `docker build` will fail with 'failed to calculate " + "checksum ...: not found' (OMN-15696)." + ) + + def _init_git_repo(path: Path, marker: str) -> str: path.mkdir(parents=True) (path / "marker.txt").write_text(marker, encoding="utf-8") diff --git a/tests/scripts/test_deploy_runtime_core_contracts_resolution.py b/tests/scripts/test_deploy_runtime_core_contracts_resolution.py index b0642f0155..1304333766 100644 --- a/tests/scripts/test_deploy_runtime_core_contracts_resolution.py +++ b/tests/scripts/test_deploy_runtime_core_contracts_resolution.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT """deploy-runtime.sh step 3b must resolve omnibase_core runtime contracts from the diff --git a/tests/scripts/test_deploy_runtime_lane_attribution.py b/tests/scripts/test_deploy_runtime_lane_attribution.py new file mode 100644 index 0000000000..08592d15ef --- /dev/null +++ b/tests/scripts/test_deploy_runtime_lane_attribution.py @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""deploy-runtime.sh must enforce lane-deploy attribution + the grant interlock (OMN-15218). + +Before OMN-15218 the sanctioned deploy path recorded WHAT was deployed +(registry.json) but nothing recorded WHO deployed it or WHY, and nothing checked +whether live prod-promotion grants were pinned to the proof the deploy was about +to replace. Two stability-lane rebuilds in two days (2026-07-26T21:45Z, +2026-07-27T10:05-10:09Z) were consequently unattributable. + +Two kinds of test here: + + * wiring assertions over the script text (the repo's existing idiom for + deploy-runtime.sh gates), and + * an EXECUTED harness that extracts the guard function and runs it in bash + against a stub preflight, proving the seam actually hard-fails and actually + captures the record — not merely that the tokens appear in the file. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import stat +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEPLOY_SCRIPT = REPO_ROOT / "scripts" / "deploy-runtime.sh" +REFRESH_SCRIPT = REPO_ROOT / "scripts" / "runtime_build" / "refresh_stability_lane.sh" +PREFLIGHT = REPO_ROOT / "scripts" / "preflight_lane_deploy_attribution.py" + + +def _script_text() -> str: + return DEPLOY_SCRIPT.read_text(encoding="utf-8") + + +def _extract_function(text: str, name: str) -> str: + match = re.search(rf"^{name}\s*\(\)\s*\{{.*?^\}}", text, re.DOTALL | re.MULTILINE) + assert match is not None, f"{name}() not found in deploy-runtime.sh" + return match.group(0) + + +# --- wiring ------------------------------------------------------------------ + + +@pytest.mark.unit +def test_preflight_script_exists_and_is_the_single_source_of_truth() -> None: + assert PREFLIGHT.is_file(), "the attribution/interlock preflight must exist" + assert "scripts/preflight_lane_deploy_attribution.py" in _script_text() + + +@pytest.mark.unit +def test_defines_and_calls_the_attribution_guard() -> None: + text = _script_text() + assert re.search(r"^guard_lane_deploy_attribution\s*\(\)", text, re.MULTILINE) + # Defined AND called (definition + main() invocation). + assert text.count("guard_lane_deploy_attribution") >= 2 + + +@pytest.mark.unit +def test_guard_hard_fails_rather_than_warning() -> None: + body = _extract_function(_script_text(), "guard_lane_deploy_attribution") + assert "exit 1" in body, "the attribution guard must hard-fail, never warn" + assert "log_warn" not in body, ( + "a warning-only attribution guard is the old behavior" + ) + + +@pytest.mark.unit +def test_guard_runs_before_any_mutation() -> None: + """The guard must precede build/sync/restart, not follow them.""" + text = _script_text() + main_body = text[text.index("\nmain() {") :] + guard_at = main_body.index("guard_lane_deploy_attribution ") + for later in ( + "sync_files ", + "build_images ", + "restart_services ", + "bringup_full_stack ", + "write_registry ", + ): + assert guard_at < main_body.index(later), ( + f"{later.strip()} must run AFTER the attribution guard" + ) + + +@pytest.mark.unit +def test_registry_carries_the_attribution_record() -> None: + body = _extract_function(_script_text(), "write_registry") + assert "attribution: $attribution" in body, ( + "registry.json must carry who/why, not just what" + ) + + +@pytest.mark.unit +def test_refresh_stability_lane_runs_the_preflight_before_it_mutates_anything() -> None: + text = REFRESH_SCRIPT.read_text(encoding="utf-8") + assert "preflight_lane_deploy_attribution.py" in text + preflight_at = text.index("ATTRIBUTION_PREFLIGHT=") + # docker tag (rollback anchor) and the ambient-clone checkout are this + # script's own mutations; both happen after deploy-runtime.sh is chosen but + # BEFORE it is invoked, so the preflight has to precede them here too. + assert preflight_at < text.index(' docker tag "') + assert preflight_at < text.index("checkout --force --detach") + assert "attribution: $attribution" in text, ( + "the refresh receipt must carry the attribution record" + ) + + +@pytest.mark.unit +def test_lane_derivation_is_shared_not_duplicated() -> None: + text = _script_text() + assert re.search(r"^resolve_lane_name\s*\(\)", text, re.MULTILINE) + hotpatch = _extract_function(text, "guard_hotpatch_ledger") + assert "resolve_lane_name" in hotpatch, ( + "hot-patch guard must reuse the shared lane derivation" + ) + + +# --- executed harness -------------------------------------------------------- + + +HARNESS_PRELUDE = """ +set -uo pipefail +log_step() { printf '[step] %s\\n' "$*" >&2; } +log_info() { printf '[info] %s\\n' "$*" >&2; } +log_warn() { printf '[warn] %s\\n' "$*" >&2; } +log_error() { printf '[error] %s\\n' "$*" >&2; } +log_cmd() { printf '[cmd] %s\\n' "$*" >&2; } +SCRIPT_NAME="deploy-runtime.sh" +DEPLOY_INVOCATION_ARGS=(--execute --restart) +LANE_ATTRIBUTION_RECORD_JSON="" +ONEX_DEPLOY_REASON_VAR="ONEX_DEPLOY_REASON" +ONEX_DEPLOY_GRANT_ACK_VAR="ONEX_DEPLOY_GRANT_ACK" +MODE="execute" +""" + +STUB_PREFLIGHT = """#!/usr/bin/env python3 +import json, os, sys +print(json.dumps({"result": "REFUSE" if os.environ.get("STUB_REFUSE") else "ALLOW", + "lane": "stability-test", "argv": sys.argv[1:]})) +sys.exit(1 if os.environ.get("STUB_REFUSE") else 0) +""" + + +def _harness( + tmp_path: Path, *, refuse: bool, drop_preflight: bool = False +) -> subprocess.CompletedProcess[str]: + """Run the real guard function from deploy-runtime.sh against a stub preflight.""" + fake_repo = tmp_path / "repo" + (fake_repo / "scripts").mkdir(parents=True) + (fake_repo / ".venv" / "bin").mkdir(parents=True) + # Pin python_bin resolution to this interpreter so the harness never depends + # on uv/system python being present or on a pyproject in the fake repo. + venv_python = fake_repo / ".venv" / "bin" / "python" + venv_python.symlink_to(sys.executable) + + if not drop_preflight: + stub = fake_repo / "scripts" / "preflight_lane_deploy_attribution.py" + stub.write_text(STUB_PREFLIGHT, encoding="utf-8") + stub.chmod(stub.stat().st_mode | stat.S_IEXEC) + + text = _script_text() + script = "\n".join( + [ + HARNESS_PRELUDE, + _extract_function(text, "resolve_lane_name"), + _extract_function(text, "guard_lane_deploy_attribution"), + 'guard_lane_deploy_attribution "$1" "$2"', + 'printf "RECORD:%s\\n" "${LANE_ATTRIBUTION_RECORD_JSON}"', + ] + ) + harness = tmp_path / "harness.sh" + harness.write_text(script, encoding="utf-8") + + env = dict(os.environ) + if refuse: + env["STUB_REFUSE"] = "1" + else: + env.pop("STUB_REFUSE", None) + return subprocess.run( + ["bash", str(harness), str(fake_repo), "omnibase-infra-stability-test"], + capture_output=True, + text=True, + env=env, + check=False, + timeout=120, + ) + + +@pytest.mark.unit +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash not available") +def test_guard_aborts_the_deploy_when_the_preflight_refuses(tmp_path: Path) -> None: + """RED on the old behavior: the deploy continued regardless.""" + result = _harness(tmp_path, refuse=True) + assert result.returncode == 1, result.stdout + result.stderr + assert "RECORD:" not in result.stdout, ( + "execution must stop at the refusal, not continue" + ) + assert "REFUSED this deploy" in result.stderr + + +@pytest.mark.unit +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash not available") +def test_guard_captures_the_record_and_passes_lane_identity(tmp_path: Path) -> None: + result = _harness(tmp_path, refuse=False) + assert result.returncode == 0, result.stdout + result.stderr + captured = result.stdout.split("RECORD:", 1)[1].strip() + record = json.loads(captured) + argv = record["argv"] + assert "--lane" in argv and argv[argv.index("--lane") + 1] == "stability-test" + assert "--compose-project" in argv + assert argv[argv.index("--compose-project") + 1] == "omnibase-infra-stability-test" + assert ( + "--source" in argv and argv[argv.index("--source") + 1] == "deploy-runtime.sh" + ) + assert "--check-only" not in argv, "execute mode must write the durable record" + + +@pytest.mark.unit +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash not available") +def test_guard_refuses_when_the_preflight_is_missing(tmp_path: Path) -> None: + """Deleting the mechanism must not silently restore the old behavior.""" + result = _harness(tmp_path, refuse=False, drop_preflight=True) + assert result.returncode == 1 + assert "attribution preflight not found" in result.stderr diff --git a/tests/scripts/test_deploy_runtime_lane_overlay.py b/tests/scripts/test_deploy_runtime_lane_overlay.py index f498a7eea3..639852b0ad 100644 --- a/tests/scripts/test_deploy_runtime_lane_overlay.py +++ b/tests/scripts/test_deploy_runtime_lane_overlay.py @@ -99,16 +99,65 @@ def test_defines_lane_overlay_resolver_functions() -> None: @pytest.mark.unit -def test_dev_project_gets_no_overlay() -> None: - """The bare dev project runs from infra.yml alone (fixed dev names are correct).""" +def test_dev_project_layers_only_the_dev_lane_overlay() -> None: + """The bare dev project gets infra.yml + the dev-lane overlay, nothing else. + + Changed by OMN-15379 (operator ruling 15). It used to be infra.yml ALONE — + the dev lane's fixed container names are correct in the base, so it needed + no overlay. It now layers ``docker-compose.dev-lane.yml``, whose entire + content is ``ONEX_MIGRATION_LANE=dev`` on forward-migration: the lane + indicator that releases the node_projection_registration trio (0000 CREATE / + 0001 heartbeat / 0002 ENABLE + FORCE ROW LEVEL SECURITY) from the operator + fence, making the lab lane the FORCE proving ground. + + Why a separate file rather than a line in the base: every non-dev lane + overlay MERGES infra.yml, and stability-test's forward-migration override is + a single ``container_name:`` line, so it inherits the base ``environment:`` + block wholesale. The indicator in the base would therefore be inherited by + stability-test, prod, judge and any lane added later — fail-OPEN. Inverted + this way, a lane that does not load the dev overlay carries no indicator and + the runner applies the FULL fence. + """ result = _run_overlay_resolver("omnibase-infra") assert result.returncode == 0, result.stderr out = result.stdout.strip() - assert out == "-f /DEPLOY/docker/docker-compose.infra.yml", out + assert out == ( + "-f /DEPLOY/docker/docker-compose.infra.yml " + "-f /DEPLOY/docker/docker-compose.dev-lane.yml" + ), out + # infra.yml first, so the dev overlay's environment merges on top of it. + assert out.index("docker-compose.infra.yml") < out.index( + "docker-compose.dev-lane.yml" + ) assert "stability-test" not in out assert "prod" not in out +@pytest.mark.unit +@pytest.mark.parametrize( + "compose_project", + [ + "omnibase-infra-stability-test", + "omnibase-infra-prod", + "omnibase-infra-judge", + ], +) +def test_no_non_dev_lane_ever_loads_the_dev_lane_overlay(compose_project: str) -> None: + """OMN-15379: the lane indicator must be unreachable from every other lane. + + The negative half of the pair above, and the load-bearing one: the dev + overlay is the ONLY thing that releases the fenced registration migrations, + so a lane that loaded it would apply FORCE ROW LEVEL SECURITY to + node_service_registry unattended — precisely the class of unattended posture + change the operator fence exists to prevent. + """ + result = _run_overlay_resolver(compose_project) + assert result.returncode == 0, result.stderr + assert "dev-lane" not in result.stdout, ( + f"{compose_project} loads the dev-lane overlay: {result.stdout.strip()}" + ) + + @pytest.mark.unit @pytest.mark.parametrize( ("compose_project", "lane"), diff --git a/tests/scripts/test_deploy_runtime_registry_commit_on_success.py b/tests/scripts/test_deploy_runtime_registry_commit_on_success.py new file mode 100644 index 0000000000..0200c79a82 --- /dev/null +++ b/tests/scripts/test_deploy_runtime_registry_commit_on_success.py @@ -0,0 +1,635 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""deploy-runtime.sh must not write-ahead deployment state (OMN-15352). + +Defect: `write_registry()` ran at Phase 9, BEFORE Phase 10 (`build_images`) and +every phase downstream of it (migration preflight, restart, RT-6 readback) that +can actually fail. `DEPLOY_DIR_TO_CLEANUP=""` immediately after the write also +disarmed the orphan-directory cleanup at the same moment the false claim was +made. The `--force` backup-restore branch in `cleanup_on_exit()` restored the +deployed directory + migration tree but explicitly declined to touch +`registry.json`, leaving it asserting the failed run's git_sha/deployed_at. A +failed deploy therefore left `registry.json` claiming a version that was never +actually running -- observed live 2026-07-29T00:26-00:41Z on the `.201` dev lane +(workflow `wf_55998f90`; OMN-15352 description + comment `1564e60c`). +F3 (companion defect): the implicit `docker compose build` retag of +`-:latest` was never protected -- a failed deploy +left `:latest` resolving to an untested image, so a later +`docker compose up -d` without `--build` would silently swap it in. + +Fix: `write_registry()` moves to commit-on-success -- it is called once, right +before `DEPLOYMENT_COMPLETE=true`, after every phase that can fail has passed. +`DEPLOY_DIR_TO_CLEANUP` stays armed for the whole deploy instead of being +disabled right after the (now-removed) early write, so a failure at any later +phase lets `cleanup_on_exit()` remove the orphaned directory. A new +`snapshot_latest_image_tags()` / `restore_latest_image_tags()` pair records +each RUNTIME_BUILD_SERVICES image's pre-build `:latest` id and restores it (or +removes an unverified tag that had no prior state) on any non-success exit. + +These tests drive the ACTUAL script seam per `feedback_test_the_artifact_that_ +runs`: `main()` is extracted verbatim (unmodified control flow, unmodified +`write_registry()` / `cleanup_on_exit()` / `snapshot_latest_image_tags()` / +`restore_latest_image_tags()` / `guard_existing_deployment()` / +`restore_migration_tree_after_revert()` / `snapshot_migration_tree()` / +`assert_deployed_migration_tree_synced()`) and executed under bash with only +the true I/O boundaries stubbed: `docker` is a local file-backed fake (no +daemon required), and the heavy phases with no bearing on this defect +(rsync-driven sync, compose validation, network/Kafka/Postgres readiness, +attribution/lineage preflights) are replaced with no-op or controllable fakes. +This is not a surrogate reimplementation of the ordering -- the real bash +`main()` text runs, and a real `jq`-backed `write_registry()` either does or +does not touch the filesystem depending on where in that real control flow the +injected failure lands. +""" + +from __future__ import annotations + +import json +import os +import re +import stat +import subprocess +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEPLOY_SCRIPT = REPO_ROOT / "scripts" / "deploy-runtime.sh" + +FAKE_SERVICES = ["fake-svc-a", "fake-svc-b"] + + +def _script_text() -> str: + return DEPLOY_SCRIPT.read_text(encoding="utf-8") + + +def _script_noncomment() -> str: + """deploy-runtime.sh with comment-only lines stripped (see sibling tests).""" + lines = [ + line + for line in _script_text().splitlines() + if not line.lstrip().startswith("#") + ] + return "\n".join(lines) + + +def _extract_function(name: str) -> str: + text = _script_text() + match = re.search( + rf"^{re.escape(name)}\s*\(\)\s*\{{.*?\n\}}", + text, + re.DOTALL | re.MULTILINE, + ) + assert match is not None, ( + f"could not extract function {name}() from deploy-runtime.sh" + ) + return match.group(0) + + +# --------------------------------------------------------------------------- +# Static wiring assertions (repo idiom): prove the ordering in the source text +# without executing anything. +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_write_registry_call_moved_after_readback_in_main() -> None: + """write_registry() must be called AFTER readback_deployed_ref(), not before build.""" + text = _script_text() + main_body = text[text.index("\nmain() {") :] + build_idx = main_body.index('build_images "${deploy_target}"') + readback_idx = main_body.index('readback_deployed_ref "${git_sha}"') + write_registry_idx = main_body.index('write_registry "${version}"') + complete_idx = main_body.index("DEPLOYMENT_COMPLETE=true") + + assert build_idx < readback_idx, "test fixture assumption: build precedes readback" + assert readback_idx < write_registry_idx, ( + "write_registry() must run AFTER readback_deployed_ref() (commit-on-" + "success, OMN-15352) -- it must not still run before the phases that " + "can fail." + ) + assert write_registry_idx < complete_idx, ( + "write_registry() must run BEFORE DEPLOYMENT_COMPLETE=true is set, so " + "the registry commit and the completion flag land together." + ) + + +@pytest.mark.unit +def test_deploy_dir_cleanup_not_disarmed_before_build() -> None: + """DEPLOY_DIR_TO_CLEANUP must stay armed through build/restart/readback. + + The old code cleared DEPLOY_DIR_TO_CLEANUP="" immediately after the + (write-ahead) registry write, before build_images() ran -- disarming the + orphan-directory cleanup at the exact moment the false registry claim was + made. Only one `DEPLOY_DIR_TO_CLEANUP=""` assignment may exist in main(), + and it must be co-located with the (now deferred) registry write, after + build/restart/readback. + """ + text = _script_noncomment() + main_body = text[text.index("\nmain() {") :] + clear_positions = [ + m.start() for m in re.finditer(r'DEPLOY_DIR_TO_CLEANUP=""', main_body) + ] + assert len(clear_positions) == 1, ( + f'expected exactly one DEPLOY_DIR_TO_CLEANUP="" in main(), found ' + f"{len(clear_positions)} -- an early clear before build/restart/readback " + "would re-open the write-ahead window." + ) + build_idx = main_body.index('build_images "${deploy_target}"') + readback_idx = main_body.index('readback_deployed_ref "${git_sha}"') + assert clear_positions[0] > build_idx, ( + "DEPLOY_DIR_TO_CLEANUP must not be disarmed before build_images() runs." + ) + assert clear_positions[0] > readback_idx, ( + "DEPLOY_DIR_TO_CLEANUP must not be disarmed before readback_deployed_ref() runs." + ) + + +@pytest.mark.unit +def test_snapshot_latest_image_tags_runs_before_build() -> None: + text = _script_text() + main_body = text[text.index("\nmain() {") :] + snap_idx = main_body.index('snapshot_latest_image_tags "${compose_project}"') + build_idx = main_body.index('build_images "${deploy_target}"') + assert snap_idx < build_idx, ( + "snapshot_latest_image_tags() must run BEFORE build_images() so it " + "captures the pre-build :latest state (OMN-15352 F3)." + ) + + +@pytest.mark.unit +def test_cleanup_on_exit_restores_latest_tags_only_on_non_success() -> None: + body = _extract_function("cleanup_on_exit") + match = re.search( + r'if \[\[ "\$\{DEPLOYMENT_COMPLETE\}" != "true" \]\]; then\s*\n\s*restore_latest_image_tags', + body, + ) + assert match is not None, ( + "restore_latest_image_tags() must be called from cleanup_on_exit() " + "guarded on DEPLOYMENT_COMPLETE != true, so a successful deploy never " + "reverts its own freshly-built :latest tag." + ) + + +@pytest.mark.unit +def test_force_restore_branch_no_longer_claims_registry_is_stale() -> None: + """The backup-restore branch must not warn about stale registry metadata. + + Now that write_registry() is commit-on-success, the restore branch (which + only runs when DEPLOYMENT_COMPLETE != true, i.e. write_registry() never ran + this invocation) can never leave registry.json stale -- the old log_warn + was prose describing a problem the write-ahead ordering created; it must + not survive un-mechanized. + """ + text = _script_text() + assert "may contain stale metadata" not in text, ( + "the stale-registry warning is obsolete: write_registry() is now " + "commit-on-success, so a restore branch never runs after it wrote " + "this invocation's registry entry (OMN-15352)." + ) + + +# --------------------------------------------------------------------------- +# Executed harness -- drives the real main() control flow end to end. +# --------------------------------------------------------------------------- + +_LOG_FUNCS = """ +log_step() { printf 'STEP: %s\\n' "$*" >&2; } +log_info() { printf 'INFO: %s\\n' "$*" >&2; } +log_warn() { printf 'WARN: %s\\n' "$*" >&2; } +log_error() { printf 'ERR: %s\\n' "$*" >&2; } +log_cmd() { printf 'CMD: %s\\n' "$*" >&2; } +""" + +# Every function main() calls that is NOT central to the OMN-15352 fix is +# replaced with a small controllable fake. The functions central to the fix +# (write_registry, cleanup_on_exit, snapshot_latest_image_tags, +# restore_latest_image_tags, guard_existing_deployment, +# restore_migration_tree_after_revert, snapshot_migration_tree, +# assert_deployed_migration_tree_synced) are extracted VERBATIM from the real +# script below and are never stubbed. +_STUB_FUNCS = """ +parse_args() { :; } +validate_prerequisites() { :; } +resolve_repo_root() { printf '%s\\n' "${FAKE_REPO_ROOT}"; } +validate_repo_structure() { :; } +read_version() { printf '%s\\n' "${FAKE_VERSION}"; } +read_git_sha() { printf '%s\\n' "${FAKE_GIT_SHA}"; } +check_git_dirty() { :; } +validate_build_source_config() { :; } +guard_prod_promotion_lineage() { :; } +resolve_compose_project() { printf '%s\\n' "${FAKE_COMPOSE_PROJECT}"; } +guard_lane_deploy_attribution() { :; } +guard_hotpatch_ledger() { :; } +check_compose_project_collision() { :; } +show_preview() { :; } +acquire_lock() { :; } +sync_files() { + local dst="$2" + mkdir -p "${dst}" + printf '%s' "${FAKE_MARKER_CONTENT}" > "${dst}/marker.txt" +} +sanity_check() { :; } +build_images() { + if [[ "${FAKE_BUILD_FAIL:-0}" == "1" ]]; then + log_error "fake build failure" + return 1 + fi + local project="$2" + local svc + for svc in "${RUNTIME_BUILD_SERVICES[@]}"; do + docker tag "${FAKE_NEW_IMAGE_ID}-${svc}" "${project}-${svc}:latest" + done + return 0 +} +ensure_core_infra_ready() { :; } +warm_broker_topic_provisioning() { :; } +run_runtime_migration_preflight() { + if [[ "${FAKE_MIGRATION_FAIL:-0}" == "1" ]]; then + log_error "fake migration preflight failure" + return 1 + fi + return 0 +} +bringup_full_stack() { :; } +restart_services() { :; } +verify_deployment() { :; } +readback_deployed_ref() { + if [[ "${FAKE_READBACK_FAIL:-0}" == "1" ]]; then + log_error "fake readback failure" + return 1 + fi + return 0 +} +show_summary() { :; } +prune_old_deployments() { :; } +""" + + +def _write_docker_stub(bin_dir: Path) -> None: + """Fake `docker` answering `image inspect`, `tag`, `rmi` from a file-backed + store under $DOCKER_STUB_DIR/images, logging every call to calls.log.""" + stub = bin_dir / "docker" + stub.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + 'printf "%s\\n" "$*" >> "${DOCKER_STUB_DIR}/calls.log"\n' + "\n" + 'if [[ "$1" == "image" && "$2" == "inspect" ]]; then\n' + ' ref="$3"\n' + ' safe="$(printf "%s" "${ref}" | tr "/:" "__")"\n' + ' file="${DOCKER_STUB_DIR}/images/${safe}"\n' + ' if [[ -f "${file}" ]]; then\n' + ' cat "${file}"\n' + " exit 0\n" + " fi\n" + " exit 1\n" + "fi\n" + "\n" + 'if [[ "$1" == "tag" ]]; then\n' + ' src="$2"\n' + ' dest="$3"\n' + ' safe="$(printf "%s" "${dest}" | tr "/:" "__")"\n' + ' printf "%s" "${src}" > "${DOCKER_STUB_DIR}/images/${safe}"\n' + " exit 0\n" + "fi\n" + "\n" + 'if [[ "$1" == "rmi" ]]; then\n' + ' ref="$2"\n' + ' safe="$(printf "%s" "${ref}" | tr "/:" "__")"\n' + ' rm -f "${DOCKER_STUB_DIR}/images/${safe}"\n' + " exit 0\n" + "fi\n" + "\n" + "exit 1\n", + encoding="utf-8", + ) + stub.chmod(stub.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +def _read_latest_tag(stub_dir: Path, compose_project: str, service: str) -> str | None: + safe = f"{compose_project}-{service}:latest".replace("/", "_").replace(":", "_") + f = stub_dir / "images" / safe + return f.read_text(encoding="utf-8") if f.is_file() else None + + +def _build_harness(tmp_path: Path) -> tuple[str, dict[str, str]]: + """Assemble the full harness script text: log fns, stubs, real extracted + functions (verbatim), the real main(), then `trap cleanup_on_exit EXIT; + main "$@"`. Returns (script_text, base_env).""" + deploy_root = tmp_path / "deploy_root" + deploy_root.mkdir(exist_ok=True) + stub_dir = tmp_path / "stubs" + (stub_dir / "images").mkdir(parents=True, exist_ok=True) + if not (stub_dir / "calls.log").exists(): + (stub_dir / "calls.log").write_text("", encoding="utf-8") + _write_docker_stub(stub_dir) + + fake_repo_root = tmp_path / "fake_repo" + fake_repo_root.mkdir(exist_ok=True) + + services_literal = " ".join(f'"{s}"' for s in FAKE_SERVICES) + + globals_prelude = "\n".join( + [ + "set -euo pipefail", + f'DEPLOY_ROOT="{deploy_root}"', + 'REGISTRY_FILE="${DEPLOY_ROOT}/registry.json"', + 'LOCK_DIR="${DEPLOY_ROOT}/.deploy.lock"', + 'SCRIPT_NAME="deploy-runtime.sh"', + f"RUNTIME_BUILD_SERVICES=({services_literal})", + 'MIGRATION_TREE_REL_PATH="docker/migrations/forward"', + 'DEPLOY_DIR_TO_CLEANUP=""', + 'FORCE_BACKUP_DIR=""', + 'MIGRATION_TREE_SNAPSHOT_DIR=""', + 'LATEST_TAG_SNAPSHOT_FILE=""', + 'DEPLOY_COMPOSE_PROJECT=""', + "DEPLOYMENT_COMPLETE=false", + 'LANE_ATTRIBUTION_RECORD_JSON=""', + 'COMPOSE_PROFILE="runtime"', + 'MODE="execute"', + "PRINT_COMPOSE_CMD=false", + "PROD_LANE=false", + "COLD_FULL_BRINGUP=false", + "COLD_START_KAFKA_TIMEOUT_SECONDS=180", + 'FORCE="${FAKE_FORCE:-false}"', + 'RESTART="${FAKE_RESTART:-false}"', + "DEPLOY_INVOCATION_ARGS=()", + ] + ) + + script = "\n".join( + [ + globals_prelude, + _LOG_FUNCS, + _STUB_FUNCS, + _extract_function("guard_existing_deployment"), + _extract_function("assert_deployed_migration_tree_synced"), + _extract_function("snapshot_migration_tree"), + _extract_function("restore_migration_tree_after_revert"), + _extract_function("snapshot_latest_image_tags"), + _extract_function("restore_latest_image_tags"), + _extract_function("write_registry"), + _extract_function("cleanup_on_exit"), + _extract_function("main"), + "trap 'cleanup_on_exit' EXIT", + 'main "$@"', + ] + ) + + env = dict(os.environ) + env["PATH"] = f"{stub_dir}{os.pathsep}{env['PATH']}" + env["DOCKER_STUB_DIR"] = str(stub_dir) + env["FAKE_REPO_ROOT"] = str(fake_repo_root) + env["FAKE_COMPOSE_PROJECT"] = "fake-project" + env["FAKE_MARKER_CONTENT"] = "fresh-sync-content" + env["FAKE_NEW_IMAGE_ID"] = "built-new" + + return script, env + + +def _run( + tmp_path: Path, + *, + restart: bool, + force: bool, + version: str, + git_sha: str, + migration_fail: bool = False, + readback_fail: bool = False, + build_fail: bool = False, +) -> tuple[subprocess.CompletedProcess[str], Path, Path]: + script, env = _build_harness(tmp_path) + env["FAKE_VERSION"] = version + env["FAKE_GIT_SHA"] = git_sha + env["FAKE_MIGRATION_FAIL"] = "1" if migration_fail else "0" + env["FAKE_READBACK_FAIL"] = "1" if readback_fail else "0" + env["FAKE_BUILD_FAIL"] = "1" if build_fail else "0" + env["FAKE_RESTART"] = "true" if restart else "false" + env["FAKE_FORCE"] = "true" if force else "false" + + harness = tmp_path / "harness.sh" + harness.write_text(script, encoding="utf-8") + + args = ["--execute"] + if restart: + args.append("--restart") + if force: + args.append("--force") + + result = subprocess.run( + ["bash", str(harness), *args], + capture_output=True, + text=True, + check=False, + env=env, + timeout=60, + ) + deploy_root = tmp_path / "deploy_root" + deploy_target = deploy_root / "deployed" / version + return result, deploy_root, deploy_target + + +@pytest.mark.unit +def test_fresh_deploy_failed_migration_preflight_never_writes_registry( + tmp_path: Path, +) -> None: + """RED on the old behavior: a fresh (non--force) deploy that fails at the + migration preflight (after build) must leave NO registry.json at all -- + write_registry() must never have been reached.""" + result, deploy_root, deploy_target = _run( + tmp_path, + restart=True, + force=False, + version="9.9.9", + git_sha="abc123def456", + migration_fail=True, + ) + assert result.returncode != 0, result.stdout + result.stderr + + registry_file = deploy_root / "registry.json" + assert not registry_file.exists(), ( + "registry.json must not exist after a failed fresh deploy -- " + "write_registry() must run only after the migration preflight (and " + "everything before it) has succeeded. stderr:\n" + result.stderr + ) + # Orphan cleanup: DEPLOY_DIR_TO_CLEANUP stayed armed through build/preflight + # (no early clear), so cleanup_on_exit() must have removed the freshly + # synced (but never-committed) deploy_target directory. + assert not deploy_target.exists(), ( + "the orphaned deploy_target directory must be removed by " + "cleanup_on_exit() when it was never committed to the registry.\n" + + result.stderr + ) + + +@pytest.mark.unit +def test_fresh_deploy_failed_migration_preflight_untags_unverified_latest( + tmp_path: Path, +) -> None: + """F3: a fresh deploy's :latest tag (no prior state) must be removed, not + left resolving to the untested image built just before the failure.""" + tmp_path_stubs = tmp_path + result, _deploy_root, _ = _run( + tmp_path_stubs, + restart=True, + force=False, + version="9.9.9", + git_sha="abc123def456", + migration_fail=True, + ) + assert result.returncode != 0, result.stdout + result.stderr + + stub_dir = tmp_path / "stubs" + for svc in FAKE_SERVICES: + tag = _read_latest_tag(stub_dir, "fake-project", svc) + assert tag is None, ( + f"fake-project-{svc}:latest must be removed after a failed deploy " + f"with no prior :latest state, got {tag!r}.\n{result.stderr}" + ) + + +@pytest.mark.unit +def test_force_redeploy_failed_migration_preflight_leaves_registry_byte_identical( + tmp_path: Path, +) -> None: + """AC1/AC2: a --force redeploy over an existing (registered, running) + deployment that fails at the migration preflight must leave registry.json + BYTE-IDENTICAL to its pre-run content, and the deploy_target directory and + :latest tags restored to their pre-run state.""" + version = "9.9.9" + old_git_sha = "111111111111" + new_git_sha = "222222222222" + + # --- Seed a pre-existing deployment: registry.json (written by the REAL + # write_registry(), so byte-for-byte format matches) + a deploy_target + # directory with OLD content + a prior :latest tag per service. + deploy_root = tmp_path / "deploy_root" + deploy_root.mkdir() + deploy_target = deploy_root / "deployed" / version + deploy_target.mkdir(parents=True) + (deploy_target / "marker.txt").write_text("old-content", encoding="utf-8") + + stub_dir = tmp_path / "stubs" + (stub_dir / "images").mkdir(parents=True) + (stub_dir / "calls.log").write_text("", encoding="utf-8") + _write_docker_stub(stub_dir) + for svc in FAKE_SERVICES: + safe = f"fake-project-{svc}:latest".replace("/", "_").replace(":", "_") + (stub_dir / "images" / safe).write_text(f"prior-{svc}", encoding="utf-8") + + seed_script = "\n".join( + [ + "set -euo pipefail", + f'DEPLOY_ROOT="{deploy_root}"', + 'REGISTRY_FILE="${DEPLOY_ROOT}/registry.json"', + 'LANE_ATTRIBUTION_RECORD_JSON=""', + 'COMPOSE_PROFILE="runtime"', + _LOG_FUNCS, + _extract_function("write_registry"), + ( + f'write_registry "{version}" "{old_git_sha}" ' + f'"{deploy_target}" "/fake/repo" "fake-project"' + ), + ] + ) + seed_env = dict(os.environ) + seed_result = subprocess.run( + ["bash", "-c", seed_script], + capture_output=True, + text=True, + check=False, + env=seed_env, + timeout=30, + ) + assert seed_result.returncode == 0, seed_result.stdout + seed_result.stderr + registry_file = deploy_root / "registry.json" + pre_run_registry_bytes = registry_file.read_bytes() + assert b"111111111111" in pre_run_registry_bytes + + # --- Run the real harness: --force redeploy, fails at migration preflight. + script, env = _build_harness(tmp_path) + env["FAKE_VERSION"] = version + env["FAKE_GIT_SHA"] = new_git_sha + env["FAKE_MIGRATION_FAIL"] = "1" + env["FAKE_READBACK_FAIL"] = "0" + env["FAKE_BUILD_FAIL"] = "0" + env["FAKE_MARKER_CONTENT"] = "new-content" + env["FAKE_RESTART"] = "true" + env["FAKE_FORCE"] = "true" + + harness = tmp_path / "harness.sh" + harness.write_text(script, encoding="utf-8") + result = subprocess.run( + ["bash", str(harness), "--execute", "--restart", "--force"], + capture_output=True, + text=True, + check=False, + env=env, + timeout=60, + ) + assert result.returncode != 0, result.stdout + result.stderr + + # AC1: registry.json byte-identical to its pre-run content. + post_run_registry_bytes = registry_file.read_bytes() + assert post_run_registry_bytes == pre_run_registry_bytes, ( + "registry.json must be byte-identical after a failed --force redeploy " + "-- write_registry() must never have run this invocation.\n" + f"pre: {pre_run_registry_bytes!r}\n" + f"post: {post_run_registry_bytes!r}\n" + f"stderr: {result.stderr}" + ) + + # The directory must be restored to its pre-run (OLD) content, not left on + # the freshly-synced (new, failed) content. + assert (deploy_target / "marker.txt").read_text( + encoding="utf-8" + ) == "old-content", ( + "deploy_target must be restored to its pre-run content on a failed " + f"--force redeploy.\n{result.stderr}" + ) + + # F3: :latest must resolve to the same image id as pre-run for every + # service (restored from the snapshot taken before the build). + for svc in FAKE_SERVICES: + tag = _read_latest_tag(stub_dir, "fake-project", svc) + assert tag == f"prior-{svc}", ( + f"fake-project-{svc}:latest must be restored to its pre-run id " + f"'prior-{svc}', got {tag!r}.\n{result.stderr}" + ) + + +@pytest.mark.unit +def test_success_path_writes_registry_and_keeps_new_latest_tags( + tmp_path: Path, +) -> None: + """AC3: an unregressed success path still ends with registry.json + reflecting the newly deployed SHA and :latest pointing at the newly built + image (no restore fires on a completed deploy).""" + version = "9.9.9" + git_sha = "abc123def456" + + result, deploy_root, deploy_target = _run( + tmp_path, + restart=True, + force=False, + version=version, + git_sha=git_sha, + ) + assert result.returncode == 0, result.stdout + result.stderr + + registry_file = deploy_root / "registry.json" + assert registry_file.is_file(), "a successful deploy must write registry.json" + registry = json.loads(registry_file.read_text(encoding="utf-8")) + assert registry["git_sha"] == git_sha + assert registry["active_version"] == version + assert registry["deploy_path"] == str(deploy_target) + + stub_dir = tmp_path / "stubs" + for svc in FAKE_SERVICES: + tag = _read_latest_tag(stub_dir, "fake-project", svc) + assert tag == f"built-new-{svc}", ( + f"a successful deploy must keep its newly built :latest tag for " + f"{svc}, got {tag!r} (a restore must not have fired).\n{result.stderr}" + ) diff --git a/tests/scripts/test_deploy_runtime_rt6_scoped_readback.py b/tests/scripts/test_deploy_runtime_rt6_scoped_readback.py new file mode 100644 index 0000000000..74c53e1cdf --- /dev/null +++ b/tests/scripts/test_deploy_runtime_rt6_scoped_readback.py @@ -0,0 +1,271 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""RT-6 deploy readback must verify exactly the rebuilt service scope [OMN-15348]. + +Defect: `readback_deployed_ref()` in `scripts/deploy-runtime.sh` was hardcoded to +probe only the `omninode-runtime` container's image-revision label, regardless of +`RUNTIME_BUILD_SERVICES_OVERRIDE` (the OMN-14873 scoped-rebuild override). Observed +live 2026-07-28T23:08-23:17Z on the .201 dev lane: a correctly scoped rebuild +(`RUNTIME_BUILD_SERVICES_OVERRIDE=runtime-effects`) rebuilt + recreated only +`omninode-runtime-effects`. RT-6 then compared the UNTOUCHED `omninode-runtime` +container's stale label against the new build's `VCS_REF`, false-FAILed, and +auto-triggered `restore-previous-deployment` -- reverting `docker-compose.infra.yml` +and `registry.json` on disk while the freshly-recreated `runtime-effects` container +stayed live. + +The fix loops the readback over `RUNTIME_BUILD_SERVICES` (the array +`deploy-runtime.sh` already resolves from `RUNTIME_BUILD_SERVICES_OVERRIDE`, or the +full `RUNTIME_SERVICES` set when unset) instead of a single hardcoded container, so +an out-of-scope container's stale label is never probed -- it can neither fail the +deploy nor trigger restore. + +These tests drive the ACTUAL script seam: `readback_deployed_ref()` is extracted +(with its real dependencies `resolve_lane_runtime_container_name`, +`resolve_lane_overlay_filename`, `resolve_compose_file_args`) and executed under +bash against the REAL `scripts/verify_deployed_versions.py`, with only `docker` +stubbed on PATH (no daemon required). This is not a surrogate: the exact bash +control flow that decided which container(s) to probe, and the real Python readback +script's pass/fail logic, both run for real. +""" + +from __future__ import annotations + +import os +import re +import stat +import subprocess +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEPLOY_SCRIPT = REPO_ROOT / "scripts" / "deploy-runtime.sh" + +GIT_SHA = "abc123def456" +STALE_SHA = "111111111111" +VERSION = "9.9.9" + + +def _script_text() -> str: + return DEPLOY_SCRIPT.read_text(encoding="utf-8") + + +def _extract_function(name: str) -> str: + """Return the source text of a single top-level bash function ``name()``.""" + text = _script_text() + match = re.search( + rf"^{re.escape(name)}\s*\(\)\s*\{{.*?\n\}}", + text, + re.DOTALL | re.MULTILINE, + ) + assert match is not None, ( + f"could not extract function {name}() from deploy-runtime.sh" + ) + return match.group(0) + + +def _write_docker_stub(bin_dir: Path) -> Path: + """Write a fake `docker` on PATH that answers `compose ps -q`, `inspect`, + and `exec ... uv pip show` from files under $DOCKER_STUB_DIR, and appends + every invocation to $DOCKER_STUB_DIR/calls.log for call-scope assertions. + """ + stub = bin_dir / "docker" + stub.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + 'printf "%s\\n" "$*" >> "${DOCKER_STUB_DIR}/calls.log"\n' + "\n" + 'if [[ "$1" == "compose" ]]; then\n' + " shift\n" + ' args=("$@")\n' + " n=${#args[@]}\n" + " for ((i = 0; i < n; i++)); do\n" + ' if [[ "${args[$i]}" == "ps" ]]; then\n' + ' service="${args[$((n - 1))]}"\n' + ' map_file="${DOCKER_STUB_DIR}/ps/${service}"\n' + ' if [[ -f "${map_file}" ]]; then\n' + ' cat "${map_file}"\n' + " exit 0\n" + " fi\n" + " exit 1\n" + " fi\n" + " done\n" + " exit 1\n" + "fi\n" + "\n" + 'if [[ "$1" == "inspect" ]]; then\n' + ' container="$2"\n' + ' rev_file="${DOCKER_STUB_DIR}/revision/${container}"\n' + ' if [[ -f "${rev_file}" ]]; then\n' + ' cat "${rev_file}"\n' + " fi\n" + " exit 0\n" + "fi\n" + "\n" + 'if [[ "$1" == "exec" ]]; then\n' + ' container="$2"\n' + ' package="${*: -1}"\n' + ' ver_file="${DOCKER_STUB_DIR}/version/${container}"\n' + ' if [[ -f "${ver_file}" ]]; then\n' + ' printf "Name: %s\\nVersion: %s\\n" "${package}" "$(cat "${ver_file}")"\n' + " exit 0\n" + " fi\n" + " exit 1\n" + "fi\n" + "\n" + "exit 1\n", + encoding="utf-8", + ) + stub.chmod(stub.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + return stub + + +def _run_readback( + tmp_path: Path, + *, + runtime_build_services: list[str], + ps_map: dict[str, str], + revision_map: dict[str, str], + version_map: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + """Extract + execute readback_deployed_ref() with real dependencies, stubbed docker.""" + stub_dir = tmp_path / "stubs" + stub_dir.mkdir() + (stub_dir / "ps").mkdir() + (stub_dir / "revision").mkdir() + (stub_dir / "version").mkdir() + (stub_dir / "calls.log").write_text("", encoding="utf-8") + _write_docker_stub(stub_dir) + + for service, container in ps_map.items(): + (stub_dir / "ps" / service).write_text(container + "\n", encoding="utf-8") + for container, revision in revision_map.items(): + (stub_dir / "revision" / container).write_text(revision, encoding="utf-8") + for container, version in (version_map or {}).items(): + (stub_dir / "version" / container).write_text(version, encoding="utf-8") + + services_literal = " ".join(f'"{s}"' for s in runtime_build_services) + + harness = "\n".join( + [ + "set -euo pipefail", + "log_step() { printf 'STEP: %s\\n' \"$*\" >&2; }", + "log_info() { printf 'INFO: %s\\n' \"$*\" >&2; }", + "log_warn() { printf 'WARN: %s\\n' \"$*\" >&2; }", + "log_error() { printf 'ERR: %s\\n' \"$*\" >&2; }", + "log_cmd() { printf 'CMD: %s\\n' \"$*\" >&2; }", + _extract_function("resolve_lane_overlay_filename"), + _extract_function("resolve_compose_file_args"), + _extract_function("resolve_lane_runtime_container_name"), + _extract_function("readback_deployed_ref"), + f"RUNTIME_BUILD_SERVICES=({services_literal})", + ( + f'readback_deployed_ref "{GIT_SHA}" "{VERSION}" ' + f'"omnibase-infra" "{REPO_ROOT}" "/tmp/fake-deploy-target"' + ), + ] + ) + + env = dict(os.environ) + env["PATH"] = f"{stub_dir}{os.pathsep}{env['PATH']}" + env["DOCKER_STUB_DIR"] = str(stub_dir) + + return subprocess.run( + ["bash", "-c", harness], + capture_output=True, + text=True, + check=False, + env=env, + ) + + +@pytest.mark.unit +def test_scoped_run_ignores_out_of_scope_stale_container(tmp_path: Path) -> None: + """(i) Scoped run: out-of-scope omninode-runtime has a stale label -> RT-6 + passes and never even probes the out-of-scope container (no restore).""" + result = _run_readback( + tmp_path, + runtime_build_services=["runtime-effects"], + ps_map={"runtime-effects": "omninode-runtime-effects"}, + revision_map={ + "omninode-runtime-effects": GIT_SHA, + # Deliberately stale/never-referenced: omninode-runtime is NOT in + # RUNTIME_BUILD_SERVICES for this scoped run. + "omninode-runtime": STALE_SHA, + }, + ) + assert result.returncode == 0, result.stderr + calls_log = (tmp_path / "stubs" / "calls.log").read_text(encoding="utf-8") + # Word-boundary match: "omninode-runtime" as its own token must not appear + # (it would only appear as a prefix of "omninode-runtime-effects" today). + assert re.search(r"(? None: + """(ii) Scoped run: the in-scope container genuinely fails readback -> RT-6 + fails (exit 1), which is the same exit code that fires deploy-runtime.sh's + auto-restore trap.""" + result = _run_readback( + tmp_path, + runtime_build_services=["runtime-effects"], + ps_map={"runtime-effects": "omninode-runtime-effects"}, + revision_map={"omninode-runtime-effects": STALE_SHA}, + ) + assert result.returncode == 1, ( + f"expected RT-6 to fail-closed on a genuine in-scope mismatch; " + f"stdout={result.stdout!r} stderr={result.stderr!r}" + ) + assert "runtime-effects" in result.stderr + + +@pytest.mark.unit +def test_unscoped_run_verifies_full_default_service_set(tmp_path: Path) -> None: + """(iii) Unscoped run (no override): every RUNTIME_SERVICES member is + verified, unchanged from the pre-fix single-container behavior for + omninode-runtime, plus every sibling service now also covered.""" + full_services = [ + "omninode-runtime", + "runtime-effects", + "runtime-worker", + "projection-api", + "agent-actions-consumer", + "skill-lifecycle-consumer", + "intelligence-api", + "omninode-contract-resolver", + ] + ps_map = { + "runtime-effects": "omninode-runtime-effects", + "runtime-worker": "omninode-runtime-worker", + "projection-api": "omnimarket-projection-api", + "agent-actions-consumer": "omninode-agent-actions-consumer", + "skill-lifecycle-consumer": "omninode-skill-lifecycle-consumer", + "intelligence-api": "omnibase-intelligence-api", + "omninode-contract-resolver": "omninode-contract-resolver", + } + revision_map = dict.fromkeys(ps_map.values(), GIT_SHA) + # omninode-runtime is resolved via resolve_lane_runtime_container_name, not + # the ps map, for the dev project used here ("omnibase-infra"). + revision_map["omninode-runtime"] = GIT_SHA + version_map = {"omninode-runtime": VERSION} + + result = _run_readback( + tmp_path, + runtime_build_services=full_services, + ps_map=ps_map, + revision_map=revision_map, + version_map=version_map, + ) + assert result.returncode == 0, result.stderr + calls_log = (tmp_path / "stubs" / "calls.log").read_text(encoding="utf-8") + for service in ps_map: + assert service in calls_log, ( + f"unscoped run must probe every default RUNTIME_SERVICES member, " + f"missing '{service}' in docker calls:\n{calls_log}" + ) + # The package-version check is still asserted for the primary container. + assert "uv pip show omnibase-infra" in calls_log diff --git a/tests/scripts/test_forward_migration_advisory_lock.py b/tests/scripts/test_forward_migration_advisory_lock.py new file mode 100644 index 0000000000..2febf04a78 --- /dev/null +++ b/tests/scripts/test_forward_migration_advisory_lock.py @@ -0,0 +1,723 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""OMN-15291 — the forward-migration runner must serialize concurrent runs. + +Defect (``docs/deep-dives/JULY_27_2026_DEEP_DIVE.md``, "Remaining friction"): +``scripts/run-forward-migrations.sh`` applied every migration through an +unsynchronized check-then-act — ``SELECT`` from ``schema_migrations``, then +``psql -f``, then ``INSERT ... ON CONFLICT DO NOTHING`` — with **no advisory +lock of any kind**. Two concurrent runners both read "not applied" and both +executed the same file; non-idempotent DDL then errored in the loser, and the +``ON CONFLICT`` hid the double-apply so the tracking table still looked clean. + +OMN-15254 fixed the sibling defect in ``omninode_infra``'s k8s Job runners. +This is the same canonical single-session lock ported to this runner in POSIX +``sh``. + +These tests drive **the artifact that actually runs** — the shipped +``scripts/run-forward-migrations.sh``, executed twice concurrently against one +real Postgres with a deliberately slow, non-idempotent migration. +``test_lock_free_runner_does_not_serialize`` is the RED control: the identical +harness against a copy of the runner with the lock block mechanically stripped, +asserting the critical sections DO overlap. Without that control the +serialization assertion would be unfalsifiable — it would also pass against a +harness that never runs anything concurrently. + +Database selection, in order: + 1. ``MIGRATION_LOCK_TEST_HOST``/``_PORT``/``_USER``/``_PASSWORD``/``_DB``. + 2. An ephemeral local cluster via ``initdb``/``pg_ctl`` if those binaries exist. + 3. Skip — unless ``REQUIRE_MIGRATION_LOCK_DB`` is set, in which case fail. + A check that silently skips is a check that does not exist. +""" + +from __future__ import annotations + +import os +import re +import shutil +import signal +import socket +import subprocess +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from collections.abc import Iterator + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNNER = REPO_ROOT / "scripts" / "run-forward-migrations.sh" + +BEGIN_MARKER = "# ---- BEGIN canonical forward-migration advisory lock (OMN-15291) ----" +END_MARKER = "# ---- END canonical forward-migration advisory lock (OMN-15291) ----" + +# The migration applied by the live proofs. The 3s pg_sleep is the critical +# section: long enough that a genuinely concurrent second runner is +# observably inside it too. It is deliberately NOT idempotent: +# a second application raises "relation already exists", which is exactly the +# production failure the lock prevents. The probe rows bracket the window in +# which a runner is inside the apply. +RACE_MIGRATION_SQL = """\ +INSERT INTO public.apply_probe (label, phase) + VALUES (current_setting('application_name'), 'start'); +SELECT pg_sleep(3); +CREATE TABLE public.t_race (id INT PRIMARY KEY); +INSERT INTO public.apply_probe (label, phase) + VALUES (current_setting('application_name'), 'end'); +""" + +SETUP_SQL = """\ +CREATE TABLE public.db_metadata ( + id BOOLEAN PRIMARY KEY DEFAULT TRUE, + migrations_complete BOOLEAN NOT NULL DEFAULT FALSE, + runner_completed_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +INSERT INTO public.db_metadata (id) VALUES (TRUE); +CREATE TABLE public.apply_probe ( + label TEXT NOT NULL, + phase TEXT NOT NULL, + at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); +-- Pre-created so the runner's own `CREATE TABLE IF NOT EXISTS` bootstrap is a +-- no-op. Two lock-free runners executing that statement concurrently lose to a +-- duplicate-key error on pg_type_typname_nsp_index before either reaches the +-- apply loop -- a real symptom of the same defect, but it would pre-empt the +-- apply race these proofs are about and make the RED control host-load +-- dependent. +CREATE TABLE public.schema_migrations ( + migration_id TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + checksum TEXT NOT NULL, + source_set TEXT NOT NULL +); +""" + + +def _runner_text() -> str: + return RUNNER.read_text() + + +def extract_lock_block(text: str | None = None) -> str: + """Return the canonical lock block, markers stripped. + + Raises with a specific message if the markers are missing or duplicated — + that is how a silent removal of the lock surfaces as a test failure rather + than as a vacuous pass. + """ + lines = (text if text is not None else _runner_text()).splitlines() + starts = [i for i, ln in enumerate(lines) if ln.strip() == BEGIN_MARKER] + ends = [i for i, ln in enumerate(lines) if ln.strip() == END_MARKER] + if len(starts) != 1 or len(ends) != 1 or ends[0] <= starts[0]: + msg = ( + "scripts/run-forward-migrations.sh: expected exactly one canonical " + "advisory-lock block delimited by the OMN-15291 markers (found " + f"{len(starts)} begin / {len(ends)} end markers)" + ) + raise AssertionError(msg) + return "\n".join(lines[starts[0] + 1 : ends[0]]) + "\n" + + +def strip_lock_block(text: str) -> str: + """The pre-OMN-15291 runner: byte-identical minus the lock block. + + Derived from the shipped artifact rather than pinned as a copy, so the RED + control can never drift away from the thing it is the control for. + """ + lines = text.splitlines(keepends=True) + start = next(i for i, ln in enumerate(lines) if ln.strip() == BEGIN_MARKER) + end = next(i for i, ln in enumerate(lines) if ln.strip() == END_MARKER) + remaining = lines[:start] + lines[end + 1 :] + # The lock-free runner has no lock to assert on either. + return "".join( + ln for ln in remaining if ln.strip() != "assert_migration_lock_still_held" + ) + + +# -------------------------------------------------------------------------- +# Static assertions — no database required, so they gate every PR. +# -------------------------------------------------------------------------- + + +def test_runner_carries_the_canonical_lock_block() -> None: + block = extract_lock_block() + assert "pg_advisory_lock(" in block, ( + "the OMN-15291 block must actually acquire an advisory lock" + ) + + +def test_runner_never_uses_a_cross_session_unlock() -> None: + """The original defect's signature: an unlock from a session that never held it.""" + offenders = [ + ln + for ln in _runner_text().splitlines() + if "pg_advisory_unlock" in ln and not ln.strip().startswith("#") + ] + assert not offenders, ( + "pg_advisory_unlock() releases only locks held by the CALLING session; " + "this runner holds its lock in a dedicated session that is released by " + f"disconnecting (OMN-15291). Offending lines: {offenders}" + ) + + +def test_lock_is_never_acquired_through_a_one_shot_psql() -> None: + """A ``psql -c`` acquisition drops the lock as soon as that psql exits.""" + for line in _runner_text().splitlines(): + stripped = line.strip() + if stripped.startswith("#") or "pg_advisory_lock" not in stripped: + continue + assert not stripped.startswith("psql "), ( + f"`{stripped}` acquires the advisory lock in a one-shot psql session " + "that exits immediately, releasing it (OMN-15291)" + ) + + +def test_runner_asserts_the_lock_survived_the_whole_run() -> None: + calls = [ + ln + for ln in _runner_text().splitlines() + if ln.strip() == "assert_migration_lock_still_held" + ] + assert len(calls) == 1, ( + "expected exactly one final `assert_migration_lock_still_held` call so a " + "holder session that died mid-run fails the runner instead of flipping " + f"the migration-gate sentinel HEALTHY (found {len(calls)})" + ) + + +def test_held_ness_check_is_bound_to_our_own_session() -> None: + """ "Somebody holds it" is not proof that WE hold it.""" + block = extract_lock_block() + assert "application_name = '${MIGRATION_LOCK_TAG}'" in block, ( + "the held-ness probe must match our own application_name tag, otherwise " + "another runner's lock reads as ours (OMN-15291)" + ) + + +def test_acquisition_failure_is_fatal_not_advisory() -> None: + block = extract_lock_block() + assert block.count("exit 1") >= 2, ( + "both acquisition failure paths (holder died, deadline exceeded) must " + "exit nonzero — proceeding unserialized is the defect" + ) + + +def test_runner_is_posix_sh() -> None: + """The container shell is busybox ash; bashisms would fail only in prod.""" + assert _runner_text().splitlines()[0] == "#!/bin/sh" + block = extract_lock_block() + assert not re.search(r"^\s*local\s", block, re.MULTILINE), ( + "`local` is not POSIX and is unavailable in every /bin/sh this runner " + "may execute under" + ) + + +# -------------------------------------------------------------------------- +# Live concurrency proof. +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PgTarget: + """Connection coordinates for the scratch Postgres under test.""" + + host: str + port: int + user: str + password: str + dbname: str + + def env(self) -> dict[str, str]: + return {**os.environ, "PGPASSWORD": self.password} + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _find_pg_binary(name: str) -> str | None: + found = shutil.which(name) + if found: + return found + # (root, glob) split deliberately: the wildcard sits in a PARENT component, + # so Path().parent would itself contain a literal "*" and + # never exist -- the fallback would silently never fire and the live suite + # would skip on any host without psql already on PATH. + for root, pattern in ( + ("/opt/homebrew/opt", "postgresql@*/bin"), + ("/usr/lib/postgresql", "*/bin"), + ): + base = Path(root) + if not base.exists(): + continue + for candidate in sorted(base.glob(pattern)): + binary = candidate / name + if binary.exists(): + return str(binary) + return None + + +def _unavailable(reason: str) -> None: + if os.environ.get("REQUIRE_MIGRATION_LOCK_DB"): + pytest.fail( + "REQUIRE_MIGRATION_LOCK_DB is set but the advisory-lock concurrency " + f"proof cannot run: {reason}" + ) + pytest.skip(reason) + + +def _psql(target: PgTarget, sql: str) -> str: + result = subprocess.run( + [ + _find_pg_binary("psql") or "psql", + "-h", + target.host, + "-p", + str(target.port), + "-U", + target.user, + "-d", + target.dbname, + "-v", + "ON_ERROR_STOP=1", + "-tAc", + sql, + ], + check=True, + capture_output=True, + text=True, + env=target.env(), + ) + return result.stdout.strip() + + +@pytest.fixture +def pg_target() -> Iterator[PgTarget]: + """A scratch Postgres: external if configured, else an ephemeral cluster. + + Function-scoped on purpose — each proof needs a virgin database (the racing + migration creates a table that must not already exist). + """ + if not _find_pg_binary("psql"): + _unavailable("psql client not available") + + host = os.environ.get("MIGRATION_LOCK_TEST_HOST") + if host: + target = PgTarget( + host=host, + port=int(os.environ.get("MIGRATION_LOCK_TEST_PORT", "5432")), + user=os.environ.get("MIGRATION_LOCK_TEST_USER", "postgres"), + password=os.environ.get("MIGRATION_LOCK_TEST_PASSWORD", "postgres"), + dbname=os.environ.get("MIGRATION_LOCK_TEST_DB", "postgres"), + ) + scratch = f"omn15291_{int(time.time() * 1000) % 100_000_000}" + _psql(target, f'CREATE DATABASE "{scratch}"') + scoped = PgTarget( + host=target.host, + port=target.port, + user=target.user, + password=target.password, + dbname=scratch, + ) + try: + _psql(scoped, SETUP_SQL) + yield scoped + finally: + _psql(target, f'DROP DATABASE IF EXISTS "{scratch}" WITH (FORCE)') + return + + initdb = _find_pg_binary("initdb") + pg_ctl = _find_pg_binary("pg_ctl") + if not initdb or not pg_ctl: + _unavailable( + "no MIGRATION_LOCK_TEST_HOST and no local initdb/pg_ctl to start an " + "ephemeral cluster" + ) + return + + # Short base dir: the unix socket path has a ~100 char limit. + with tempfile.TemporaryDirectory(dir="/tmp", prefix="omn15291-") as base: + datadir = Path(base) / "pgdata" + subprocess.run( + [initdb, "-D", str(datadir), "-U", "postgres", "--auth=trust", "--no-sync"], + check=True, + capture_output=True, + ) + port = _free_port() + subprocess.run( + [ + pg_ctl, + "-D", + str(datadir), + "-l", + str(Path(base) / "pg.log"), + "-o", + f"-p {port} -c listen_addresses=127.0.0.1 " + f"-c unix_socket_directories={base}", + "-w", + "start", + ], + check=True, + capture_output=True, + ) + target = PgTarget( + host="127.0.0.1", + port=port, + user="postgres", + password="postgres", + dbname="postgres", + ) + try: + _psql(target, SETUP_SQL) + yield target + finally: + subprocess.run( + [pg_ctl, "-D", str(datadir), "-m", "immediate", "-w", "stop"], + check=False, + capture_output=True, + ) + + +@pytest.fixture +def migrations_dir(tmp_path: Path) -> Path: + forward = tmp_path / "migrations" / "forward" + forward.mkdir(parents=True) + (forward / "001_race_target.sql").write_text(RACE_MIGRATION_SQL) + # OMN-15349: the runner unconditionally requires the fence manifest to + # exist under MIGRATIONS_DIR; none of this fixture's ids need gating. + (forward / "fenced-node-migrations.yaml").write_text( + "fenced_node_migrations: []\n", encoding="utf-8" + ) + # OMN-15336 item 4 repair: the runner also unconditionally requires the + # FORCE-RLS grandfather snapshot to exist under MIGRATIONS_DIR now; this + # fixture has no node migrations at all, so an empty list is correct. + (forward / "grandfathered-force-rls-migrations.yaml").write_text( + "grandfathered_force_rls_migrations: []\n", encoding="utf-8" + ) + return forward + + +def _runner_env( + target: PgTarget, migrations: Path, label: str, wait: int +) -> dict[str, str]: + psql_dir = str(Path(_find_pg_binary("psql") or "psql").parent) + return { + **os.environ, + "PATH": f"{psql_dir}{os.pathsep}{os.environ.get('PATH', '')}", + "PGAPPNAME": label, + "POSTGRES_USER": target.user, + "POSTGRES_PASSWORD": target.password, + "POSTGRES_HOST": target.host, + "POSTGRES_PORT": str(target.port), + "POSTGRES_DB": target.dbname, + "MIGRATIONS_DIR": str(migrations), + "NODE_MIGRATIONS_DIR": str(migrations / "nodes"), + "MIGRATION_LOCK_WAIT_SECONDS": str(wait), + } + + +def _spawn( + runner: Path, target: PgTarget, migrations: Path, label: str, wait: int = 60 +) -> subprocess.Popen[str]: + return subprocess.Popen( + ["/bin/sh", str(runner)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=_runner_env(target, migrations, label, wait), + ) + + +def _intervals(target: PgTarget) -> dict[str, tuple[float, float | None]]: + rows = _psql( + target, + "SELECT label, phase, extract(epoch from at) FROM public.apply_probe " + "ORDER BY at", + ) + start: dict[str, float] = {} + end: dict[str, float] = {} + for line in filter(None, rows.splitlines()): + label, phase, at = line.split("|") + (start if phase == "start" else end)[label] = float(at) + return {label: (at, end.get(label)) for label, at in start.items()} + + +def _overlaps(intervals: dict[str, tuple[float, float | None]]) -> bool: + labels = sorted(intervals) + for i, first in enumerate(labels): + for second in labels[i + 1 :]: + a_start, a_end = intervals[first] + b_start, b_end = intervals[second] + if a_start < (b_end if b_end is not None else float("inf")) and b_start < ( + a_end if a_end is not None else float("inf") + ): + return True + return False + + +@pytest.mark.integration +def test_lock_free_runner_does_not_serialize( + pg_target: PgTarget, migrations_dir: Path, tmp_path: Path +) -> None: + """RED control: strip the lock block and concurrent runs stop being safe. + + The safe signature the fixed runner produces is exact and narrow: both runs + exit 0, one applies, one observes the migration as already applied, and the + critical sections never overlap. This asserts the lock-free runner does NOT + produce it. Asserting only "the sections overlap" would be host-load + dependent -- which unserialized step the loser dies on varies -- and a RED + control that flakes is worse than none. + """ + legacy = tmp_path / "run-forward-migrations.legacy.sh" + legacy.write_text(strip_lock_block(_runner_text())) + + procs = [_spawn(legacy, pg_target, migrations_dir, label) for label in ("a", "b")] + outputs = [proc.communicate(timeout=120) for proc in procs] + codes = [proc.returncode for proc in procs] + intervals = _intervals(pg_target) + + overlapped = _overlaps(intervals) + failed = any(code != 0 for code in codes) + assert overlapped or failed, ( + "the lock-free runner was expected to be unsafe under concurrency (the " + f"OMN-15291 defect) but both runs completed cleanly and serialized: " + f"intervals={intervals} codes={codes}\n{outputs}" + ) + + applied = [out for out, _ in outputs if "apply 001_race_target.sql" in out] + skipped = [out for out, _ in outputs if "already applied" in out] + safe_signature = ( + codes == [0, 0] and len(applied) == 1 and len(skipped) == 1 and not overlapped + ) + assert not safe_signature, ( + "the lock-free runner produced the SAME safe outcome as the locked one, " + "so the GREEN proof below cannot distinguish them and is vacuous: " + f"intervals={intervals} codes={codes}" + ) + + +@pytest.mark.integration +def test_shipped_runner_serializes_concurrent_runs( + pg_target: PgTarget, migrations_dir: Path +) -> None: + """GREEN: the shipped runner admits exactly one run at a time.""" + procs = [_spawn(RUNNER, pg_target, migrations_dir, label) for label in ("a", "b")] + outputs = [proc.communicate(timeout=180) for proc in procs] + + for proc, (out, _) in zip(procs, outputs, strict=True): + assert proc.returncode == 0, out + + intervals = _intervals(pg_target) + assert not _overlaps(intervals), ( + "concurrent forward-migration runners entered the apply at the same time " + f"— the advisory lock is not serializing: {intervals}" + ) + # Exactly one runner applied it; the other observed it as already applied. + assert len(intervals) == 1, ( + f"expected one applying runner, got {sorted(intervals)} — the migration " + "was applied twice" + ) + applied = [out for out, _ in outputs if "apply 001_race_target.sql" in out] + skipped = [out for out, _ in outputs if "already applied" in out] + assert len(applied) == 1 and len(skipped) == 1, ( + f"expected one apply and one skip across the two runs: {outputs}" + ) + + +@pytest.mark.integration +def test_contended_lock_fails_loud_instead_of_proceeding( + pg_target: PgTarget, migrations_dir: Path +) -> None: + """A lock we cannot get must abort with a message, not hang and not proceed.""" + lock_id = os.environ.get("FORWARD_MIGRATION_LOCK_ID", "100010") + holder = subprocess.Popen( + [ + _find_pg_binary("psql") or "psql", + "-h", + pg_target.host, + "-p", + str(pg_target.port), + "-U", + pg_target.user, + "-d", + pg_target.dbname, + "-v", + "ON_ERROR_STOP=1", + "-q", + ], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + text=True, + env=pg_target.env(), + ) + assert holder.stdin is not None + holder.stdin.write(f"SELECT pg_advisory_lock({lock_id});\n") + holder.stdin.flush() + # Wait until the foreign holder is actually granted the lock. + deadline = time.time() + 30 + while time.time() < deadline: + granted = _psql( + pg_target, + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND granted " # noqa: S608 - lock_id is a numeric runner constant, not user input + f"AND classid::bigint * 4294967296 + objid::bigint = {lock_id}", + ) + if granted != "0": + break + time.sleep(0.2) + else: # pragma: no cover - environment failure, not a product defect + holder.kill() + pytest.fail("could not establish the contending advisory-lock holder") + + try: + proc = _spawn(RUNNER, pg_target, migrations_dir, "waiter", wait=2) + out, _ = proc.communicate(timeout=120) + finally: + holder.stdin.close() + holder.kill() + holder.wait(timeout=30) + + assert proc.returncode != 0, ( + f"a runner that could not acquire the lock exited 0 and proceeded: {out}" + ) + assert "could not acquire advisory lock" in out, ( + f"expected a specific fail-loud message naming the lock: {out}" + ) + assert "apply 001_race_target.sql" not in out, ( + f"the runner applied migrations without holding the lock: {out}" + ) + + +# -------------------------------------------------------------------------- +# Signal handling: a terminated runner must STOP, not resume unserialized. +# -------------------------------------------------------------------------- + +# The shipped form. In POSIX sh only the EXIT trap is terminal, so a combined +# `trap ... EXIT HUP INT TERM` handler that returns normally RESUMES the script +# -- releasing the lock and then continuing to apply migrations unserialized. +SHIPPED_SIGNAL_TRAP = "trap 'release_migration_lock' EXIT\n" +COMBINED_SIGNAL_TRAP = "trap 'release_migration_lock' EXIT HUP INT TERM\n" + + +def _resuming_signal_variant(text: str) -> str: + """The pre-fix trap shape: one combined handler that resumes on signal.""" + lines = text.splitlines(keepends=True) + out = [] + for line in lines: + if line == SHIPPED_SIGNAL_TRAP: + out.append(COMBINED_SIGNAL_TRAP) + elif line.startswith("trap 'release_migration_lock; echo"): + continue # the terminating signal handler is what we are removing + else: + out.append(line) + return "".join(out) + + +def test_signal_and_exit_traps_are_separate() -> None: + """Static ratchet for the shape CodeRabbit flagged.""" + text = _runner_text() + assert SHIPPED_SIGNAL_TRAP in text, ( + "EXIT must have its own trap; a combined EXIT+signal trap resumes the " + "script after a signal in POSIX sh" + ) + assert COMBINED_SIGNAL_TRAP not in text + signal_traps = [ + ln + for ln in text.splitlines() + if ln.startswith("trap ") and "HUP INT TERM" in ln + ] + assert len(signal_traps) == 1 and "exit 1" in signal_traps[0], ( + "the HUP/INT/TERM handler must exit non-zero itself, otherwise the " + f"runner resumes with its lock released: {signal_traps}" + ) + + +@pytest.fixture +def two_migrations_dir(tmp_path: Path) -> Path: + forward = tmp_path / "migrations" / "forward" + forward.mkdir(parents=True) + (forward / "001_race_target.sql").write_text(RACE_MIGRATION_SQL) + (forward / "002_after_signal.sql").write_text( + "INSERT INTO public.apply_probe (label, phase) VALUES ('second', 'applied');\n" + ) + # OMN-15349: the runner unconditionally requires the fence manifest to + # exist under MIGRATIONS_DIR; none of this fixture's ids need gating. + (forward / "fenced-node-migrations.yaml").write_text( + "fenced_node_migrations: []\n", encoding="utf-8" + ) + # OMN-15336 item 4 repair: the runner also unconditionally requires the + # FORCE-RLS grandfather snapshot to exist under MIGRATIONS_DIR now; this + # fixture has no node migrations at all, so an empty list is correct. + (forward / "grandfathered-force-rls-migrations.yaml").write_text( + "grandfathered_force_rls_migrations: []\n", encoding="utf-8" + ) + return forward + + +def _run_until_signalled( + runner: Path, target: PgTarget, migrations: Path +) -> subprocess.Popen[str]: + proc = _spawn(runner, target, migrations, "signalled") + # Signal only once the runner is demonstrably inside migration 001. + deadline = time.time() + 60 + while time.time() < deadline: + if _psql(target, "SELECT count(*) FROM public.apply_probe") != "0": + break + time.sleep(0.1) + else: # pragma: no cover - environment failure + proc.kill() + pytest.fail("runner never entered the first migration") + proc.send_signal(signal.SIGTERM) + return proc + + +@pytest.mark.integration +def test_resuming_signal_trap_keeps_applying_migrations( + pg_target: PgTarget, two_migrations_dir: Path, tmp_path: Path +) -> None: + """RED control for the signal fix: the combined trap resumes and applies 002.""" + legacy = tmp_path / "run-forward-migrations.combined-trap.sh" + legacy.write_text(_resuming_signal_variant(_runner_text())) + + proc = _run_until_signalled(legacy, pg_target, two_migrations_dir) + out, _ = proc.communicate(timeout=120) + + applied_after = _psql( + pg_target, + "SELECT count(*) FROM public.apply_probe WHERE label = 'second'", + ) + assert applied_after == "1", ( + "the combined-trap runner was expected to resume after SIGTERM and apply " + f"migration 002 with its lock released, but did not: {out}" + ) + + +@pytest.mark.integration +def test_shipped_signal_trap_stops_the_run( + pg_target: PgTarget, two_migrations_dir: Path +) -> None: + """GREEN: SIGTERM aborts the run instead of continuing unserialized.""" + proc = _run_until_signalled(RUNNER, pg_target, two_migrations_dir) + out, _ = proc.communicate(timeout=120) + + assert proc.returncode != 0, f"a signalled runner exited 0: {out}" + applied_after = _psql( + pg_target, + "SELECT count(*) FROM public.apply_probe WHERE label = 'second'", + ) + assert applied_after == "0", ( + "migration 002 was applied AFTER the runner was signalled and its lock " + f"released -- the signal handler resumed instead of terminating: {out}" + ) + sentinel = _psql( + pg_target, "SELECT migrations_complete FROM public.db_metadata WHERE id" + ) + assert sentinel == "f", ( + f"a signalled run left the migration gate sentinel TRUE: {out}" + ) diff --git a/tests/scripts/test_gateway_restart_safety_proof.py b/tests/scripts/test_gateway_restart_safety_proof.py new file mode 100644 index 0000000000..ab0f1000e8 --- /dev/null +++ b/tests/scripts/test_gateway_restart_safety_proof.py @@ -0,0 +1,308 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""scripts/gateway_restart_safety_proof.sh -- OMN-15521 AC5 remediation. + +A prior remediation-round finding (OMN-15521) was that AC5 -- "the OMN-12912 +restart/redelivery proof (source-offset-ack / dedupe receipt) is re-run +against the newly deployed forwarder" -- had NO implementing work at all: the +PR body and runbook converted the ticket's own "that receipt lands on +OMN-12912, not this ticket" filing instruction into a blanket "out of scope," +skipping the proof even though the container was force-recreated during the +same session (the one moment it was cheap to capture). + +This script is the implementing work: a real, executable restart-durability +smoke proof driven via subprocess against the REAL script, exactly the +convention tests/scripts/test_deploy_gateway.py already uses, with +`docker`/`sudo` replaced by inspectable fakes. +""" + +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +PROOF_SCRIPT = REPO_ROOT / "scripts" / "gateway_restart_safety_proof.sh" + +_FAKE_DOCKER = """#!/usr/bin/env bash +set -eu +case "$*" in + "exec omninode-gateway-forwarder true") + # Reachability probe (require_reachable()). GW_TEST_UNREACHABLE_BEFORE + # simulates a container that was never reachable in the first place; + # GW_TEST_UNREACHABLE_AFTER simulates one that stopped responding + # sometime during the restart (discriminated by whether the reload has + # already been logged -- the same ordering the real script uses). + if [ "${GW_TEST_UNREACHABLE_BEFORE:-0}" = "1" ]; then + exit 1 + fi + if [ "${GW_TEST_UNREACHABLE_AFTER:-0}" = "1" ] && [ -s "${GW_TEST_SYSTEMCTL_LOG:-/dev/null}" ]; then + exit 1 + fi + exit 0 + ;; + "exec omninode-gateway-forwarder python3 -c "*) + state="${GW_TEST_SNAPSHOT_STATE:?}" + if [ -f "${state}" ]; then + cat "${state}" + else + printf '0\\t0\\n' + fi + exit 0 + ;; + "inspect omninode-gateway-forwarder --format {{.State.Health.Status}}") + printf '%s\\n' "${GW_TEST_HEALTH_STATUS:-healthy}" + exit 0 + ;; + "inspect omninode-gateway-forwarder --format {{.Id}}|{{.State.StartedAt}}") + cat "${GW_TEST_IDENTITY_STATE:?}" + exit 0 + ;; + *) + printf 'fake docker: unexpected invocation: %s\\n' "$*" >&2 + exit 1 + ;; +esac +""" + +_FAKE_SUDO = """#!/usr/bin/env bash +set -eu +if [ "${1:-}" = "systemctl" ] && [ "${2:-}" = "reload" ]; then + printf '%s\\n' "$*" >> "${GW_TEST_SYSTEMCTL_LOG:?}" + # Simulate the restart producing GW_TEST_AFTER_SNAPSHOT (if set) by + # overwriting the snapshot state file the fake docker exec reads. + if [ -n "${GW_TEST_AFTER_SNAPSHOT:-}" ] && [ -n "${GW_TEST_SNAPSHOT_STATE:-}" ]; then + printf '%s' "${GW_TEST_AFTER_SNAPSHOT}" > "${GW_TEST_SNAPSHOT_STATE}" + fi + # Simulate the reload actually RECREATING the container (default: yes -- + # matches real `systemctl reload` per deploy-gateway.sh's own doc). Set + # GW_TEST_RELOAD_RECREATES=0 to simulate a reload that exits 0 and reports + # healthy without recreating anything -- the false-green this script's + # container_identity() check exists to catch. + if [ "${GW_TEST_RELOAD_RECREATES:-1}" = "1" ] && [ -n "${GW_TEST_IDENTITY_STATE:-}" ]; then + printf 'id-after-reload|2026-08-01T00:05:00.000000000Z' > "${GW_TEST_IDENTITY_STATE}" + fi + exit 0 +fi +exec "$@" +""" + + +def _write_fake_bin(bin_dir: Path) -> None: + bin_dir.mkdir(parents=True, exist_ok=True) + docker = bin_dir / "docker" + docker.write_text(_FAKE_DOCKER, encoding="utf-8") + docker.chmod(docker.stat().st_mode | stat.S_IEXEC) + + sudo = bin_dir / "sudo" + sudo.write_text(_FAKE_SUDO, encoding="utf-8") + sudo.chmod(sudo.stat().st_mode | stat.S_IEXEC) + + +class _Harness: + def __init__(self, tmp_path: Path) -> None: + self.tmp_path = tmp_path + self.bin_dir = tmp_path / "bin" + self.systemctl_log = tmp_path / "systemctl.log" + self.snapshot_state = tmp_path / "snapshot.state" + self.identity_state = tmp_path / "identity.state" + _write_fake_bin(self.bin_dir) + self.snapshot_state.write_text("292\t1785611578.75", encoding="utf-8") + self.identity_state.write_text( + "id-before-reload|2026-08-01T00:00:00.000000000Z", encoding="utf-8" + ) + + def env(self, **overrides: str) -> dict[str, str]: + e = os.environ.copy() + e["PATH"] = f"{self.bin_dir}{os.pathsep}{e['PATH']}" + e["GW_TEST_SYSTEMCTL_LOG"] = str(self.systemctl_log) + e["GW_TEST_SNAPSHOT_STATE"] = str(self.snapshot_state) + e["GW_TEST_IDENTITY_STATE"] = str(self.identity_state) + e["GATEWAY_RESTART_PROOF_HEALTHY_TIMEOUT_SECONDS"] = "3" + e.update(overrides) + return e + + def run(self, *args: str, **env_overrides: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", str(PROOF_SCRIPT), *args], + cwd=REPO_ROOT, + env=self.env(**env_overrides), + capture_output=True, + text=True, + check=False, + ) + + +@pytest.fixture +def harness(tmp_path: Path) -> _Harness: + return _Harness(tmp_path) + + +@pytest.mark.unit +def test_script_exists_and_is_executable() -> None: + assert PROOF_SCRIPT.is_file(), ( + "scripts/gateway_restart_safety_proof.sh must exist (AC5)" + ) + mode = PROOF_SCRIPT.stat().st_mode + assert mode & stat.S_IXUSR, ( + "scripts/gateway_restart_safety_proof.sh must be executable" + ) + + +@pytest.mark.unit +def test_help_documents_ac5_scope() -> None: + result = subprocess.run( + ["bash", str(PROOF_SCRIPT), "--help"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + assert "OMN-15521" in result.stdout + assert "OMN-12912" in result.stdout + + +@pytest.mark.unit +def test_proof_green_when_records_survive_restart(harness: _Harness) -> None: + """GREEN: rows before (292) <= rows after (293, i.e. traffic kept + flowing and nothing was lost) and the container reports healthy -- the + exact restart-durability probe this script exists to run. + """ + result = harness.run( + GW_TEST_HEALTH_STATUS="healthy", + GW_TEST_AFTER_SNAPSHOT="293\t1785611600.0", + ) + assert result.returncode == 0, result.stderr + result.stdout + assert "AC5-PROOF OK" in result.stdout + assert "rows 292 -> 293" in result.stdout + assert "Container identity changed across restart" in result.stdout + assert ( + "systemctl reload onex-gateway-forwarder" in harness.systemctl_log.read_text() + ) + + +@pytest.mark.unit +def test_proof_red_when_records_are_lost_across_restart(harness: _Harness) -> None: + """RED (exists-but-wrong): the durable marker store must not lose rows + across a restart. Simulates exactly the failure OMN-12912's WAL + + synchronous=FULL design is meant to prevent -- the store came back with + FEWER rows than before the restart. + """ + result = harness.run( + GW_TEST_HEALTH_STATUS="healthy", + GW_TEST_AFTER_SNAPSHOT="0\t0", + ) + assert result.returncode != 0 + assert "AC5-PROOF FAILED" in result.stdout + result.stderr + assert "did NOT survive" in result.stdout + result.stderr + + +@pytest.mark.unit +def test_proof_red_when_container_never_reports_healthy(harness: _Harness) -> None: + """RED: a restart that never returns the container to Docker-healthy + must fail loudly, not report success -- this is the "false-green + gateway" OMN-12912's own PR description names as the failure mode its + readiness gating exists to prevent. + """ + result = harness.run( + GW_TEST_HEALTH_STATUS="unhealthy", + GW_TEST_AFTER_SNAPSHOT="293\t1785611600.0", + ) + assert result.returncode != 0 + assert "did not report Docker-healthy" in result.stdout + result.stderr + + +@pytest.mark.unit +def test_proof_warns_on_empty_store_before_restart(harness: _Harness) -> None: + """An empty store before the restart makes the proof trivially true (an + empty set survives anything) -- must warn, not silently claim strong + evidence. + """ + harness.snapshot_state.write_text("0\t0", encoding="utf-8") + result = harness.run( + GW_TEST_HEALTH_STATUS="healthy", + GW_TEST_AFTER_SNAPSHOT="0\t0", + ) + assert result.returncode == 0, result.stderr + result.stdout + assert "trivially true" in result.stdout + result.stderr + + +# --------------------------------------------------------------------------- +# Remediation round 3 (OMN-15521): a fake `sudo` that exits 0 and does +# nothing must not read as a successful restart-safety proof. The prior +# version only checked Docker-healthy + row-count-not-decreasing, both of +# which a stale, never-recreated container trivially satisfies. +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_proof_red_when_reload_does_not_actually_recreate_container( + harness: _Harness, +) -> None: + """RED (exists-but-wrong, not merely absent): a `sudo systemctl reload` + that exits 0, reports the container Docker-healthy (because the STALE + container was already healthy), and leaves row counts unchanged (no + GW_TEST_AFTER_SNAPSHOT override, so before == after) must NOT be reported + as AC5-PROOF OK. Proved by execution with a fake `sudo` that never + recreates the container (GW_TEST_RELOAD_RECREATES=0): before this fix, + this exact fixture printed 'AC5-PROOF OK ... rows 292 -> 292' with + returncode 0. The container's own identity (Id + StartedAt) is the only + signal that distinguishes this from a genuine restart. + """ + result = harness.run( + GW_TEST_HEALTH_STATUS="healthy", + GW_TEST_RELOAD_RECREATES="0", + ) + assert result.returncode != 0 + assert "AC5-PROOF FAILED" in result.stdout + result.stderr + assert "was NOT actually recreated" in result.stdout + result.stderr + assert "AC5-PROOF OK" not in result.stdout + + +@pytest.mark.unit +def test_proof_red_when_container_unreachable_after_restart( + harness: _Harness, +) -> None: + """RED (OMN-15521 remediation round 3): if the container stops + responding to `docker exec` sometime during the restart (reachable + before, not after), the proof must fail loudly instead of silently + reading the idempotency store as empty. Before this fix, snapshot()'s own + `except Exception: print('0\\t0')` (plus the `|| printf '0\\t0\\n'` + fallback) collapsed an unreachable container to the same "0\\t0" reading + as a genuinely empty store, so after_count(0) < before_count(292) was the + only signal -- and if before_count also happened to read 0 (e.g. a + concurrently-unreachable pre-check), the comparison went vacuously green. + """ + result = harness.run( + GW_TEST_HEALTH_STATUS="healthy", + GW_TEST_UNREACHABLE_AFTER="1", + ) + assert result.returncode != 0 + assert "AC5-PROOF FAILED" in result.stdout + result.stderr + assert "not reachable via 'docker exec'" in result.stdout + result.stderr + assert "after restart" in result.stdout + result.stderr + assert "AC5-PROOF OK" not in result.stdout + + +@pytest.mark.unit +def test_proof_red_when_container_unreachable_before_restart( + harness: _Harness, +) -> None: + """Fail-closed precondition: the proof must refuse to even start against + a container it cannot reach before mutating anything (reloading a + container it never proved was reachable would make the "before" snapshot + meaningless). + """ + result = harness.run(GW_TEST_UNREACHABLE_BEFORE="1") + assert result.returncode != 0 + assert "AC5-PROOF FAILED" in result.stdout + result.stderr + assert "not reachable via 'docker exec'" in result.stdout + result.stderr + assert "before restart" in result.stdout + result.stderr + assert not harness.systemctl_log.exists(), ( + "must not attempt the restart at all if the pre-restart reachability " + "check fails" + ) diff --git a/tests/scripts/test_git_hook_environment_hermeticity.py b/tests/scripts/test_git_hook_environment_hermeticity.py new file mode 100644 index 0000000000..77453a8a5a --- /dev/null +++ b/tests/scripts/test_git_hook_environment_hermeticity.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Regression coverage for Git-hook environment isolation (OMN-15555).""" + +from __future__ import annotations + +import os + +import pytest + +_GIT_KEYS_AT_COLLECTION = tuple( + sorted(key for key in os.environ if key.startswith("GIT_")) +) + + +@pytest.mark.unit +def test_git_hook_environment_is_scrubbed_before_collection() -> None: + """Caller repository authority must be gone before test modules import.""" + assert _GIT_KEYS_AT_COLLECTION == () + + +@pytest.mark.unit +def test_git_environment_is_scrubbed_at_test_boundary( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The shared helper removes Git state introduced during a test session.""" + monkeypatch.setenv("GIT_DIR", "/must/not/be/used") + monkeypatch.setenv("GIT_CONFIG_PARAMETERS", "'user.name'='must-not-leak'") + + from tests.conftest import _strip_inherited_git_environment + + _strip_inherited_git_environment() + + assert not any(key.startswith("GIT_") for key in os.environ) diff --git a/tests/scripts/test_node_migration_fence_parity.py b/tests/scripts/test_node_migration_fence_parity.py new file mode 100644 index 0000000000..4ef6b92391 --- /dev/null +++ b/tests/scripts/test_node_migration_fence_parity.py @@ -0,0 +1,2322 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""OMN-15336 — the compose runner must honour the same node-migration fence as +the k8s runner. + +Defect, found by the 2026-07-28 operator-ordered migration audit and confirmed +live: the operator fence gating the tenant-RLS node migrations existed ONLY in +``omninode_infra/k8s/migrations/omnibase-infra-migrate.yaml``. +``scripts/run-forward-migrations.sh`` — the runner every *compose* lane executes +(dev, stability-test, judge, prod all mount it; see +``docker/catalog/services/forward-migration.yaml``) — had no fence and no skip +branch of any kind in its node loop, so it applied every ``nodes/*/*.sql`` it +discovered. + +Live readback 2026-07-28/29 on ``.201`` (read-only, four lanes, db +``omnidash_analytics``): + +* dev (compose) — all 6 gated ids in the ledger (+ registration 0002); 6 + tables with ``relforcerowsecurity`` +* stability-test — all 6 gated ids in the ledger (+ registration 0002); 6 + tables with ``relforcerowsecurity`` +* prod — only registration 0000/0001; ZERO forced tables +* judge — only registration 0000/0001; ZERO forced tables + +So the breach is wider than the ticket recorded (stability-test, the designated +proof lane, is also over the fence), AND the prod + judge compose lanes are +clean but *pending*: without this fence the next forward-migration run on either +would apply the gated tenant-RLS migrations unattended. Rolling back the two +breached lanes is deliberately NOT part of this change — that disposition is an +outstanding operator decision. + +THE SEAM (cross-repo, and drift-prone by construction) +------------------------------------------------------ +Both runners walk the SAME vendored SQL tree +(``docker/migrations/forward/nodes//*.sql``, kept in sync by +``scripts/sync-node-migrations.sh``) and both mint the SAME id +``node::``. The fence is therefore a shared list over a shared +id space, matched field-by-field: + +Each field is stated as ``: k8s Job -> compose runner``. + +* id grammar: ``node:${node_name}:$(basename "$sql_file")`` -> + ``node:${node_name}:${filename}`` — identical strings. +* list contents AND order: ``FENCED_NODE_MIGRATION_IDS=( ... )`` (bash array) -> + ``FENCED_NODE_MIGRATION_IDS="..."`` (newline-delimited string); same seven + ids, same order. +* match: exact string equality per element -> ``grep -Fxq`` (exact whole-line). +* position in loop: before the already-applied probe -> before the + already-applied probe. +* on match: log + ``node_skipped++`` + ``continue`` -> log + ``NODE_SKIPPED++`` + + ``continue``. +* ledger row written: NO -> NO. +* env-overridable: NO (literal array) -> NO (unconditional assignment). + +Syntax differs and must: the k8s Job runs ``bash`` (arrays, ``set -euo +pipefail``); this runner is ``#!/bin/sh`` under busybox ash in the migration +container, where arrays do not exist. The SEAM is the id list and the +semantics, not the shell dialect. + +"Skip + record" means record THE SKIP — counted into ``NODE_SKIPPED`` and named +on stdout. A fenced id is deliberately NOT inserted into ``schema_migrations``, +matching the k8s runner: a ledger row would make the eventual un-fencing a +silent no-op (the runner would read the row, call it already applied, and never +run the migration). + +DRIFT HAZARD / follow-up — RESOLVED for the baseline by OMN-15349 +------------------------------------------------------------------ +Originally there was no single source of truth for this list: it was +duplicated in two repos and only prose and these tests kept them equal. +OMN-15349 (Option 1, operator ruling R-f 2026-08-05) removed that duplication +for the BASELINE fence: ``docker/migrations/forward/fenced-node-migrations.yaml`` +is now the sole committed list, and ``scripts/run-forward-migrations.sh`` +parses it at runtime instead of carrying its own literal copy (see the +"single-sourced" comment in the OMN-15336 fence block there). The k8s Job +(``omninode_infra``) parses the same manifest out of the same +``omnibase-infra-migrate`` image it already pulls. + +What single-sourcing the baseline does NOT do: make the two runners' EFFECTIVE +fences equal. Each runner layers its own, independently operator-ruled, +lane-scoped RELEASE on top of the shared baseline (see the OMN-15379 section +below and ``manifest_ids()``/``K8S_RULING_21_RELEASE`` here) — a release is an +environment-specific operator decision, not fence data, so it deliberately +stays out of the manifest. ``test_fence_matches_omninode_infra_k8s_runner`` +below therefore asserts baseline-superset + known-release-subtraction, not +raw equality. + +* Always on, gating every PR: ``test_manifest_pins_the_known_baseline_fence`` + pins the manifest's content, exact and IN ORDER — the same change-control + friction the old pinned-tuple test gave the shell script, now pointed at + the actual single source. ``test_manifest_shell_parse_matches_yaml_parse`` + guards the OTHER hazard single-sourcing introduces: both runners parse this + YAML with a plain ``sed`` one-liner (no YAML library in ``/bin/sh`` or the + k8s Job's minimal bash), so a manifest reformatted in a way ``yaml.safe_load`` + still accepts but the sed grammar cannot (e.g. single-quoted ids, an + unindented list) would silently ship a truncated or empty fence in + production while every YAML-aware tool kept reading it fine. +* Opt-in: ``test_fence_matches_omninode_infra_k8s_runner`` diffs the manifest + baseline against the live k8s manifest's effective (post-ruling-21) list. It + is opt-in on purpose — it depends on an ``omninode_infra`` checkout whose + freshness this repo cannot guarantee, and an always-on version went RED on + the ``.200`` build host purely because that clone sat two commits behind + dev. False REDs from another repo's local staleness are worse than the gap + they close. + +Residual gap, unchanged by OMN-15349: a NEW baseline id added to the k8s side +alone (as opposed to a release-policy change) would still land unnoticed here +until someone runs the opt-in check — single-sourcing the data removes the +"two literals to keep in sync" hazard, it does not make one repo's CI aware of +the other repo's uncommitted intentions. + +Live proofs drive THE ARTIFACT THAT RUNS — the shipped +``scripts/run-forward-migrations.sh``, executed against a real Postgres. +``test_fence_free_runner_applies_the_fenced_migration`` is the RED control: the +same harness against a copy of the runner with the fence mechanically stripped, +asserting it reproduces the ``.201`` breach (fenced SQL applied, ledger row +written). Without that control the GREEN proof would be unfalsifiable — it would +also pass against a harness that never reached the node loop. + +Database selection matches the sibling OMN-15291 module: external +``MIGRATION_LOCK_TEST_HOST``/``_PORT``/``_USER``/``_PASSWORD``/``_DB``, else an +ephemeral ``initdb``/``pg_ctl`` cluster, else skip — unless +``REQUIRE_MIGRATION_LOCK_DB`` is set, in which case fail. A check that silently +skips is a check that does not exist. + +OMN-15379 — LANE-SCOPED RELEASE (operator ruling 15, 2026-07-29) +---------------------------------------------------------------- +Two changes land on top of the above. + +1. ``node:node_projection_registration:0002_node_service_registry_tenant_rls.sql`` + joins the fence. It was in the k8s list from the start and missing from this + one. That was not cosmetic: with 0000 (the CREATE) fenced and 0002 (the + dependent ALTER) not, every COLD compose lane ran 0002 against a table that + had never been created and died — + ``0002_node_service_registry_tenant_rls.sql:41 ERROR: relation + "node_service_registry" does not exist`` — taking the forward-migration + one-shot to exit 3 and the whole lane with it (measured on ``.201``, + 2026-07-29). A fence that gates a CREATE but not its dependent ALTER is worse + than no fence. This also makes the two lists identical again at SEVEN ids. + +2. Operator ruling 15 extends node_service_registry FORCE ROW LEVEL SECURITY to + the LAB LANE ONLY, so the compose runner gains a lane-scoped release: with + ``ONEX_MIGRATION_LANE=dev`` the registration TRIO (0000/0001/0002) applies in + full — CREATE, heartbeat columns, ENABLE + FORCE RLS — making the lab the + proving ground that generates the evidence the staging un-fence is waiting + on. The omninode_infra k8s fence is UNCHANGED. + +The release is fail-closed on three independent axes, one test each: + +* ``ONEX_MIGRATION_LANE`` unset -> release nothing. An UNKNOWN value -> release + nothing, and say so on stderr. +* The release SET is committed in the runner. The env var selects among literal + policies; it never carries ids, and the release is only consulted for ids the + fence already covers, so a lane can only ever un-gate a SUBSET of the fence. +* The indicator is NOT in ``docker-compose.infra.yml``. Every lane overlay + MERGES that base (stability-test's forward-migration override is a lone + ``container_name:`` line — it inherits the base ``environment:`` wholesale), + so a value there would be inherited by stability-test, prod, judge and any + future lane. It lives in ``docker/docker-compose.dev-lane.yml``, which only + the dev/lab project loads. + +The live half drives the shipped runner over the REAL vendored registration SQL +BOTH WAYS against one Postgres — dev lane applies the trio and +``pg_class.relforcerowsecurity`` reads true; default lane skips it and +``node_service_registry`` does not exist. Neither leg can pass vacuously: a +runner that ignored the indicator would fail one of them whichever way it +defaulted. A fenced delegation id rides along in the same run as the negative +control, since ruling 15 released the registration trio and nothing else. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import shutil +import subprocess +import tempfile +import time +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +import yaml + +# Bound by assignment rather than `from ... import`: the OMN-15291 module owns +# the scratch-Postgres harness, and re-exporting its `pg_target` fixture as an +# import makes every test that takes it as a parameter read as a redefinition. +from tests.scripts import test_forward_migration_advisory_lock as _advisory_lock + +RUNNER = _advisory_lock.RUNNER +PgTarget = _advisory_lock.PgTarget +_find_pg_binary = _advisory_lock._find_pg_binary +_psql = _advisory_lock._psql +_free_port = _advisory_lock._free_port +_unavailable = _advisory_lock._unavailable +pg_target = _advisory_lock.pg_target + +if TYPE_CHECKING: + from collections.abc import Iterator + +REPO_ROOT = Path(__file__).resolve().parents[2] + +FENCE_BEGIN = "# ---- BEGIN operator fence — node migration ids (OMN-15336) ----" +FENCE_END = "# ---- END operator fence — node migration ids (OMN-15336) ----" +SKIP_BEGIN = "# ---- BEGIN fenced-id skip (OMN-15336) ----" +SKIP_END = "# ---- END fenced-id skip (OMN-15336) ----" +# OMN-15336 item 4: the unclassified-FORCE-RLS guard. Two blocks — the +# predicate's own definition (sits beside is_fenced_node_migration) and the +# call site (sits in the node loop, after the already-applied probe). +FORCE_RLS_GUARD_DEF_BEGIN = ( + "# ---- BEGIN unclassified FORCE ROW LEVEL SECURITY guard (OMN-15336 item 4) ----" +) +FORCE_RLS_GUARD_DEF_END = ( + "# ---- END unclassified FORCE ROW LEVEL SECURITY guard (OMN-15336 item 4) ----" +) +FORCE_RLS_GUARD_CALL_BEGIN = ( + "# ---- BEGIN unclassified FORCE ROW LEVEL SECURITY guard call " + "(OMN-15336 item 4) ----" +) +FORCE_RLS_GUARD_CALL_END = ( + "# ---- END unclassified FORCE ROW LEVEL SECURITY guard call " + "(OMN-15336 item 4) ----" +) +# OMN-15336 item 4 repair (D1, 2026-08-05): the grandfather-snapshot block — +# the fix for the guard's over-fire against the established, pre-guard tree. +GRANDFATHER_BLOCK_BEGIN = ( + "# ---- BEGIN FORCE ROW LEVEL SECURITY grandfather snapshot " + "(OMN-15336 item 4 repair) ----" +) +GRANDFATHER_BLOCK_END = ( + "# ---- END FORCE ROW LEVEL SECURITY grandfather snapshot " + "(OMN-15336 item 4 repair) ----" +) + +# --- OMN-15349 single-sourced manifest --------------------------------------- +MANIFEST_RELPATH = "docker/migrations/forward/fenced-node-migrations.yaml" +MANIFEST_PATH = REPO_ROOT / MANIFEST_RELPATH + +# The regex both runners actually execute against the manifest at runtime +# (POSIX `sed`; neither /bin/sh nor the k8s Job's bash has a YAML library). +# This is the REAL parse path in production. +_MANIFEST_ID_LINE = re.compile(r'^\s*-\s*id:\s*"([^"]*)"', re.MULTILINE) + + +def parse_shell_manifest_ids(text: str) -> tuple[str, ...]: + """Reproduce the shell `sed` extraction of `- id: "..."` lines verbatim.""" + return tuple(_MANIFEST_ID_LINE.findall(text)) + + +def _load_manifest() -> list[dict[str, str]]: + """Independent oracle: a real YAML parse of the manifest, schema-checked.""" + doc = yaml.safe_load(MANIFEST_PATH.read_text(encoding="utf-8")) + assert isinstance(doc, dict) and "fenced_node_migrations" in doc, ( + f"{MANIFEST_RELPATH} must be a mapping with a top-level " + "'fenced_node_migrations' key" + ) + entries = doc["fenced_node_migrations"] + assert isinstance(entries, list) and entries, ( + f"{MANIFEST_RELPATH}'s fenced_node_migrations must be a non-empty list" + ) + for i, entry in enumerate(entries): + assert isinstance(entry, dict) and "id" in entry, ( + f"{MANIFEST_RELPATH} entry {i} is not a mapping with an 'id' key: {entry!r}" + ) + return entries + + +def manifest_ids() -> tuple[str, ...]: + """Ordered ids from the manifest, via the YAML-parse oracle.""" + return tuple(entry["id"] for entry in _load_manifest()) + + +# The four OMN-14974/OMN-15313 delegation ids. Kept as their own tuple so a +# failure says WHICH half of the fence moved. +FENCED_DELEGATION_IDS = ( + "node:node_projection_delegation:0023_delegation_rls_tenant_isolation.sql", + "node:node_projection_delegation:0024_drop_unwired_routing_columns.sql", + "node:node_projection_delegation:0025_delegation_judge_verdict_events_tenant_id.sql", + "node:node_projection_delegation:" + "0026_delegation_judge_verdict_events_rls_tenant_isolation.sql", +) +# The OMN-15335/OMN-15343 registration TRIO. 0002 was in the k8s list from the +# start and missing here until OMN-15379 — a fence that gated the CREATE (0000) +# but not the dependent ALTER (0002) took every cold compose lane to exit 3 with +# `relation "node_service_registry" does not exist`. +FENCED_REGISTRATION_IDS = ( + "node:node_projection_registration:0000_create_node_service_registry.sql", + "node:node_projection_registration:0001_add_heartbeat_columns.sql", + "node:node_projection_registration:0002_node_service_registry_tenant_rls.sql", +) +# The OMN-15717/OMN-15376 node_pr_review_bot id. Fenced (not SQL-edited) to +# satisfy the OMN-15376 shape-reconciliation gate without changing the file's +# content sha256, which is bound to an already-applied production row (see +# fenced-node-migrations.yaml's own rationale comment for the full argument). +# Not releasable on any lane today — no ONEX_MIGRATION_LANE value un-gates it. +FENCED_PR_REVIEW_BOT_IDS = ( + "node:node_pr_review_bot:001_create_review_bot_bypass_log.sql", +) +# OMN-15336 item 4 / OMN-15656: contract-declared TENANT domain (unlike +# node_service_registry, this is not a misclassification), held for the same +# OMN-15301 writer-tenant-context reason as the delegation quartet. Was never +# in this manifest on any runner before this entry — see the manifest's own +# docstring for the incident. +FENCED_INFERENCE_RESPONSE_IDS = ( + "node:node_projection_delegation_inference_response:" + "0003_inference_response_text_rls_tenant_isolation.sql", +) +# Pinned expectation for the manifest content (OMN-15349): the baseline fence, +# exact and in order. A manifest edit that moves this must update the pin in +# the same PR — same change-control friction the pre-OMN-15349 shell-literal +# pin gave, now pointed at the actual single source instead of a copy of it. +EXPECTED_FENCE = ( + FENCED_DELEGATION_IDS + + FENCED_REGISTRATION_IDS + + FENCED_PR_REVIEW_BOT_IDS + + FENCED_INFERENCE_RESPONSE_IDS +) + +# --- OMN-15336 item 4 repair (D1, 2026-08-05): FORCE-RLS grandfather snapshot +# -------------------------------------------------------------------------- +# See docker/migrations/forward/grandfathered-force-rls-migrations.yaml's own +# header for what this is and why it is a snapshot, not an allowlist. Pinned +# here with the SAME change-control friction as EXPECTED_FENCE: growing this +# tuple without a corresponding manifest edit (or vice versa) fails +# test_grandfather_pins_the_snapshot_baseline closed. +GRANDFATHER_MANIFEST_RELPATH = ( + "docker/migrations/forward/grandfathered-force-rls-migrations.yaml" +) +GRANDFATHER_MANIFEST_PATH = REPO_ROOT / GRANDFATHER_MANIFEST_RELPATH + +EXPECTED_GRANDFATHER = ( + "node:node_canary_score_reducer:0002_capability_scores_tenant_id_and_rls.sql", + "node:node_projection_context_roi:003_context_roi_scores_tenant_id_and_rls.sql", + "node:node_projection_cost_summary:0002_llm_cost_aggregates_tenant_id_and_rls.sql", + "node:node_projection_dep_health:002_dep_health_findings_tenant_id_and_rls.sql", + "node:node_projection_instruction_eval:" + "0002_instruction_eval_aggregate_snapshots_tenant_id_and_rls.sql", + "node:node_projection_pattern_learning:" + "0001_pattern_learning_artifacts_tenant_id_and_rls.sql", + "node:node_projection_routing_decision:" + "0022_agent_routing_decisions_tenant_id_and_rls.sql", + "node:node_projection_savings:081_savings_estimates_rls_tenant_isolation.sql", + "node:node_projection_skill_executions:" + "0002_skill_execution_snapshots_tenant_id_and_rls.sql", +) + + +def _load_grandfather_manifest() -> list[dict[str, str]]: + """Independent oracle: a real YAML parse of the grandfather manifest.""" + doc = yaml.safe_load(GRANDFATHER_MANIFEST_PATH.read_text(encoding="utf-8")) + assert isinstance(doc, dict) and "grandfathered_force_rls_migrations" in doc, ( + f"{GRANDFATHER_MANIFEST_RELPATH} must be a mapping with a top-level " + "'grandfathered_force_rls_migrations' key" + ) + entries = doc["grandfathered_force_rls_migrations"] + assert isinstance(entries, list) and entries, ( + f"{GRANDFATHER_MANIFEST_RELPATH}'s grandfathered_force_rls_migrations " + "must be a non-empty list" + ) + for i, entry in enumerate(entries): + assert isinstance(entry, dict) and "id" in entry, ( + f"{GRANDFATHER_MANIFEST_RELPATH} entry {i} is not a mapping with " + f"an 'id' key: {entry!r}" + ) + return entries + + +def grandfather_manifest_ids() -> tuple[str, ...]: + """Ordered ids from the grandfather manifest, via the YAML-parse oracle.""" + return tuple(entry["id"] for entry in _load_grandfather_manifest()) + + +# --- OMN-15349 k8s-side release (operator ruling 21, OMN-15332 comment +# 1a067542, 2026-07-31T14:05Z GO) -------------------------------------------- +# Unlike the lab-lane release below (env-gated, ruling 15), ruling 21 +# authorized a DURABLE release of the registration trio on the k8s Job, which +# serves exactly one environment (staging/onex-dev). This is a k8s-side +# runner policy, not manifest data (see the manifest file's own docstring for +# why) — pinned here only so the opt-in cross-repo test +# (`test_fence_matches_omninode_infra_k8s_runner`) can state the expected +# baseline-minus-release relationship instead of asserting raw equality. +K8S_RULING_21_RELEASE = ( + "node:node_projection_registration:0000_create_node_service_registry.sql", + "node:node_projection_registration:0001_add_heartbeat_columns.sql", + "node:node_projection_registration:0002_node_service_registry_tenant_rls.sql", +) + +# --- OMN-15379 lane-scoped release (operator ruling 15, 2026-07-29) ---------- +# Ruling 15: node_service_registry FORCE ROW LEVEL SECURITY extends to the LAB +# LANE ONLY. The lab (compose dev lane, project `omnibase-infra`) applies the +# registration trio in full as the proving ground. The omninode_infra k8s +# fence is a SEPARATE release (ruling 21, K8S_RULING_21_RELEASE above) — not +# "unchanged," and not the same mechanism (durable vs env-gated). See the +# CORRECTION comment in run-forward-migrations.sh's LANE-SCOPED FENCE RELEASE +# block for the same fix applied at the runner. +LANE_INDICATOR_ENV = "ONEX_MIGRATION_LANE" +DEV_LANE_VALUE = "dev" +# Spelled out rather than aliased to FENCED_REGISTRATION_IDS: if the two are +# meant to be equal, that equality is an assertion, not a definition. Aliasing +# would let a change to one silently move the other. +LANE_RELEASED_IDS = ( + "node:node_projection_registration:0000_create_node_service_registry.sql", + "node:node_projection_registration:0001_add_heartbeat_columns.sql", + "node:node_projection_registration:0002_node_service_registry_tenant_rls.sql", +) + +BASE_COMPOSE_RELPATH = "docker/docker-compose.infra.yml" +DEV_LANE_OVERLAY_RELPATH = "docker/docker-compose.dev-lane.yml" +CATALOG_SERVICE_RELPATH = "docker/catalog/services/forward-migration.yaml" +# Every lane that MERGES the base compose file. The lane indicator must not be +# reachable from any of them. +NON_DEV_OVERLAY_RELPATHS = ( + "docker/docker-compose.stability-test.yml", + "docker/docker-compose.prod.yml", + "docker/docker-compose.judge.yml", +) + +# An id that is NOT fenced, used by the live proofs as the discriminator: if the +# node loop applied nothing at all, the fenced-ids-absent assertion would pass +# vacuously. +UNFENCED_CONTROL_ID = "node:node_projection_delegation:0099_unfenced_control.sql" + + +def _runner_text() -> str: + return RUNNER.read_text() + + +def extract_fence_block(text: str | None = None) -> str: + """Return the fence definition block, markers stripped. + + Raises with a specific message if the markers are missing or duplicated — + that is how a silent removal of the fence surfaces as a test failure rather + than as a vacuous pass. + """ + return _extract_marked( + text if text is not None else _runner_text(), FENCE_BEGIN, FENCE_END + ) + + +def extract_skip_branch(text: str | None = None) -> str: + """Return the in-loop fenced-id skip branch, markers stripped.""" + return _extract_marked( + text if text is not None else _runner_text(), SKIP_BEGIN, SKIP_END + ) + + +def _extract_marked(text: str, begin: str, end: str) -> str: + lines = text.splitlines() + starts = [i for i, ln in enumerate(lines) if ln.strip() == begin] + ends = [i for i, ln in enumerate(lines) if ln.strip() == end] + if len(starts) != 1 or len(ends) != 1 or ends[0] <= starts[0]: + msg = ( + f"scripts/run-forward-migrations.sh: expected exactly one block " + f"delimited by {begin!r} / {end!r} (found {len(starts)} begin / " + f"{len(ends)} end markers)" + ) + raise AssertionError(msg) + return "\n".join(lines[starts[0] + 1 : ends[0]]) + "\n" + + +def extract_force_rls_guard_def(text: str | None = None) -> str: + """Return the unclassified-FORCE-RLS predicate's own definition block.""" + return _extract_marked( + text if text is not None else _runner_text(), + FORCE_RLS_GUARD_DEF_BEGIN, + FORCE_RLS_GUARD_DEF_END, + ) + + +def extract_force_rls_guard_call(text: str | None = None) -> str: + """Return the in-loop call site of the unclassified-FORCE-RLS guard.""" + return _extract_marked( + text if text is not None else _runner_text(), + FORCE_RLS_GUARD_CALL_BEGIN, + FORCE_RLS_GUARD_CALL_END, + ) + + +def extract_grandfather_block(text: str | None = None) -> str: + """Return the FORCE-RLS grandfather-snapshot block, markers stripped.""" + return _extract_marked( + text if text is not None else _runner_text(), + GRANDFATHER_BLOCK_BEGIN, + GRANDFATHER_BLOCK_END, + ) + + +def strip_force_rls_guard(text: str) -> str: + """The pre-OMN-15336-item-4 runner: byte-identical minus the guard. + + Derived from the shipped artifact, same discipline as ``strip_fence`` + above, so the RED-control-for-the-RED-control can never drift from the + thing it is the control for. + """ + out = text + for begin, end in ( + (FORCE_RLS_GUARD_DEF_BEGIN, FORCE_RLS_GUARD_DEF_END), + (FORCE_RLS_GUARD_CALL_BEGIN, FORCE_RLS_GUARD_CALL_END), + ): + lines = out.splitlines(keepends=True) + start = next(i for i, ln in enumerate(lines) if ln.strip() == begin) + stop = next(i for i, ln in enumerate(lines) if ln.strip() == end) + out = "".join(lines[:start] + lines[stop + 1 :]) + return out + + +def strip_fence(text: str) -> str: + """The pre-OMN-15336 runner: byte-identical minus both fence blocks. + + Derived from the shipped artifact rather than pinned as a copy, so the RED + control can never drift away from the thing it is the control for. + """ + out = text + for begin, end in ((FENCE_BEGIN, FENCE_END), (SKIP_BEGIN, SKIP_END)): + lines = out.splitlines(keepends=True) + start = next(i for i, ln in enumerate(lines) if ln.strip() == begin) + stop = next(i for i, ln in enumerate(lines) if ln.strip() == end) + out = "".join(lines[:start] + lines[stop + 1 :]) + return out + + +def parse_lane_release_policies(block: str) -> dict[str, tuple[str, ...]]: + """Parse the OMN-15379 ``case "${ONEX_MIGRATION_LANE}" in ... esac`` policy. + + Returns ``{case-label: released ids}``. A label whose arm assigns the empty + string maps to ``()`` — i.e. FULLY FENCED. A label whose arm makes no + assignment at all maps to ``None``, which every caller treats as a defect: + an un-assigned arm would inherit whatever the previous arm left behind. + """ + case_match = re.search( + r'case\s+"\$\{ONEX_MIGRATION_LANE\}"\s+in\n(?P.*?)\nesac', + block, + re.DOTALL, + ) + assert case_match is not None, ( + 'the lane-release policy must be a `case "${ONEX_MIGRATION_LANE}" in ' + "... esac` over COMMITTED arms. If the shape moved, fix this parser — " + "do not restate the policy here." + ) + policies: dict[str, tuple[str, ...] | None] = {} + for arm in case_match.group("body").split(";;"): + label_match = re.match(r"\s*(?P