diff --git a/.github/workflows/node-migration-sync.yml b/.github/workflows/node-migration-sync.yml new file mode 100644 index 0000000000..4a9965d4c6 --- /dev/null +++ b/.github/workflows/node-migration-sync.yml @@ -0,0 +1,85 @@ +# Node Migration Vendor Sync Gate (OMN-13124) +# +# WHY THIS EXISTS +# omnimarket projection nodes ship SQL under +# src/omnimarket/nodes//migrations/*.sql +# which must be vendored into omnibase_infra under +# docker/migrations/forward/nodes// (by scripts/sync-node-migrations.sh) +# so the forward-migration runner recreates the projection tables on a clean +# redeploy. The local pre-commit hook (onex-check-node-migration-sync) only +# runs in the infra repo and only when an infra commit is made; an omnimarket +# PR that adds a node migration cannot fire it. OMN-13124's +# node_projection_pattern_learning merged WITHOUT its vendored copy, so +# pattern_learning_artifacts was absent from omnidash_analytics on a clean +# redeploy and the golden-chain pattern_learning chain only passed via a live +# hot-fix (golden-chain sweep run12 Finding). This gate is the enforcement +# layer: every infra PR (and merge_group) re-runs the vendor --check against +# the current omnimarket dev tip, so an un-vendored or drifted node migration +# fails CI before merge. + +name: node-migration-sync + +on: + pull_request: + branches: [main, dev] + push: + branches: [main] + merge_group: + +permissions: + contents: read + +jobs: + node-migration-sync: + name: node-migration-sync + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 5 + + steps: + - name: Checkout omnibase_infra + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Checkout omnimarket (sibling — node migration source of truth) + uses: actions/checkout@v6 + with: + repository: OmniNode-ai/omnimarket + ref: dev + path: omnimarket-src + token: ${{ secrets.OMNI_GITHUB_TOKEN || github.token }} + # Sparse checkout: only the node tree carrying migrations/*.sql. + sparse-checkout: | + src/omnimarket/nodes + sparse-checkout-cone-mode: false + + - name: Run node migration vendor sync --check + shell: bash + env: + # The script's resolution order prefers OMNIMARKET_SRC (a repo root that + # contains src/omnimarket/nodes). Point it at the sibling checkout so the + # gate compares the vendored tree against the omnimarket dev tip and never + # silently passes on an unresolvable source. + OMNIMARKET_SRC: ${{ github.workspace }}/omnimarket-src + run: | + set -euo pipefail + echo "::group::node migration vendor sync --check" + if bash scripts/sync-node-migrations.sh --check; then + echo "::endgroup::" + echo "::notice::node migration vendor tree is in sync with omnimarket dev tip." + else + rc=$? + echo "::endgroup::" + if [[ "${rc}" -eq 2 ]]; then + echo "::error::Could not resolve the omnimarket source tree (exit 2). The sibling checkout is missing or empty." + else + echo "::error::Vendored node migrations under docker/migrations/forward/nodes/ are OUT OF SYNC with omnimarket. Run 'scripts/sync-node-migrations.sh' locally and commit the diff. An un-vendored node migration means its projection table will be MISSING after a clean redeploy (golden-chain run12 Finding)." + fi + exit "${rc}" + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1919d55d3a..c9f4833521 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -149,14 +149,22 @@ repos: pass_filenames: false always_run: false stages: [pre-commit] - # OMN-12559: vendored omnimarket node migrations must stay in sync with - # source. Skips silently when omnimarket source is not checked out. + # OMN-12559 / OMN-13124: vendored omnimarket node migrations must stay in + # sync with source. MUST always_run — the prior files:-scoped trigger only + # fired when a file under docker/migrations/forward/nodes/ (or the sync + # script) was staged. A node migration added in the *omnimarket* repo + # (e.g. node_projection_pattern_learning, OMN-13124) never touches those + # infra paths, so the hook silently passed and the un-vendored migration + # merged — the table was missing in omnidash_analytics on a clean redeploy + # (golden-chain run12 Finding). always_run closes that gap. The script + # itself returns exit 2 (gate fires) when the omnimarket source is + # unresolvable unless SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE=1. - id: onex-check-node-migration-sync name: ONEX Node Migration Vendor Sync Check entry: bash scripts/sync-node-migrations.sh --check language: system pass_filenames: false - files: ^(docker/migrations/forward/nodes/|scripts/sync-node-migrations\.sh) + always_run: true stages: [pre-commit] # Reject any staged .env file at any path depth (OMN-2476) # Uses (^|/)\.env([._-].+)?$ to block root/.env, config/.env, .env.production, .env.staging, diff --git a/docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql b/docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql new file mode 100644 index 0000000000..4009439cc6 --- /dev/null +++ b/docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql @@ -0,0 +1,52 @@ +-- Migration: 0000_create_gate_projection_tables.sql +-- Node: node_omnigate_projection +-- Ticket: OMN-13067 +-- Creates gate_activity and gate_metrics tables for the OmniGate projection API. +-- These tables back the projection API endpoints for +-- onex.snapshot.projection.gate.activity.v1 and +-- onex.snapshot.projection.gate.metrics.v1. + +CREATE TABLE IF NOT EXISTS gate_activity ( + id BIGSERIAL PRIMARY KEY, + repository_id TEXT NOT NULL, + project_name TEXT NOT NULL DEFAULT '', + branch TEXT NOT NULL DEFAULT '', + base_sha TEXT NOT NULL DEFAULT '', + head_sha TEXT NOT NULL DEFAULT '', + diff_hash TEXT, + config_hash TEXT, + status TEXT NOT NULL, + action TEXT, + reason TEXT NOT NULL DEFAULT '', + total_checks INTEGER NOT NULL DEFAULT 0, + failed_checks INTEGER NOT NULL DEFAULT 0, + advisory_checks INTEGER NOT NULL DEFAULT 0, + pending_checks INTEGER NOT NULL DEFAULT 0, + observed_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS gate_activity_observed_at_idx + ON gate_activity (observed_at DESC); + +CREATE INDEX IF NOT EXISTS gate_activity_repository_id_idx + ON gate_activity (repository_id); + +CREATE INDEX IF NOT EXISTS gate_activity_status_idx + ON gate_activity (status); + +-- Aggregate metrics snapshot — single row upserted on each event. +-- id=1 is the canonical singleton row. +CREATE TABLE IF NOT EXISTS gate_metrics ( + id INTEGER PRIMARY KEY DEFAULT 1, + total_events INTEGER NOT NULL DEFAULT 0, + passed INTEGER NOT NULL DEFAULT 0, + failed INTEGER NOT NULL DEFAULT 0, + advisory INTEGER NOT NULL DEFAULT 0, + pending INTEGER NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Seed the singleton metrics row so the projection API returns 1 row immediately. +INSERT INTO gate_metrics (id, total_events, passed, failed, advisory, pending, updated_at) +VALUES (1, 0, 0, 0, 0, 0, NOW()) +ON CONFLICT (id) DO NOTHING; diff --git a/docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql b/docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql new file mode 100644 index 0000000000..976b9d0b17 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql @@ -0,0 +1,68 @@ +-- Migration: 0000_create_pattern_learning_artifacts +-- Node: node_projection_pattern_learning +-- Target DB: omnidash_analytics (omnibase_infra postgres on .201:5436) +-- +-- Purpose: Projection table for onex.evt.omniintelligence.pattern-stored.v1. +-- Consumed by node_projection_pattern_learning. Closes the golden chain +-- pattern_learning chain (OMN-13124, clears OMN-13102): the tail table had no +-- consumer for pattern-stored.v1, so the sweep timed out (0/N golden chains). +-- +-- UPSERT key: pattern_id (latest-state-wins). Idempotent: IF NOT EXISTS. +-- Schema derived from omnibase_infra migration 064 + omnidash read-model, with +-- correlation_id added (the golden-chain expected field) and NOT-NULL defaults +-- relaxed so sparse pattern-stored events project without violating constraints. + +CREATE TABLE IF NOT EXISTS pattern_learning_artifacts ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + pattern_id UUID NOT NULL, + pattern_name VARCHAR(255) NOT NULL DEFAULT '', + pattern_type VARCHAR(100) NOT NULL DEFAULT '', + language VARCHAR(50), + lifecycle_state TEXT NOT NULL DEFAULT 'candidate', + state_changed_at TIMESTAMPTZ, + composite_score NUMERIC(10, 6) NOT NULL DEFAULT 0, + scoring_evidence JSONB NOT NULL DEFAULT '{}', + signature JSONB NOT NULL DEFAULT '{}', + metrics JSONB DEFAULT '{}', + metadata JSONB DEFAULT '{}', + correlation_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT pk_pattern_learning_artifacts PRIMARY KEY (id), + CONSTRAINT uq_pattern_learning_pattern_id UNIQUE (pattern_id) +); + +-- Backfill correlation_id on pre-existing deployments of this table that were +-- created by the legacy omnibase_infra 064 migration (which lacked the column). +ALTER TABLE pattern_learning_artifacts + ADD COLUMN IF NOT EXISTS correlation_id TEXT; + +CREATE INDEX IF NOT EXISTS idx_patlearn_lifecycle_state + ON pattern_learning_artifacts (lifecycle_state); + +CREATE INDEX IF NOT EXISTS idx_patlearn_composite_score + ON pattern_learning_artifacts (composite_score DESC); + +CREATE INDEX IF NOT EXISTS idx_patlearn_state_changed_at + ON pattern_learning_artifacts (state_changed_at DESC); + +CREATE INDEX IF NOT EXISTS idx_patlearn_created_at + ON pattern_learning_artifacts (created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_patlearn_updated_at + ON pattern_learning_artifacts (updated_at DESC); + +CREATE INDEX IF NOT EXISTS idx_patlearn_pattern_name + ON pattern_learning_artifacts (pattern_name); + +COMMENT ON TABLE pattern_learning_artifacts IS + 'Pattern learning projection from onex.evt.omniintelligence.pattern-stored.v1. ' + 'UPSERT key: pattern_id. Golden chain pattern_learning tail table (OMN-13124).'; + +COMMENT ON COLUMN pattern_learning_artifacts.pattern_id IS + 'Unique pattern identifier. Used as UPSERT conflict target.'; + +COMMENT ON COLUMN pattern_learning_artifacts.correlation_id IS + 'Correlation ID for distributed tracing. Golden-chain expected field.';