fix(OMN-13062): migration-gate vacuity fix — sentinel discipline, wait-for-postgres, skip-manifest, sync-check nonzero (retro A-10)

docker/migrations/forward/085_add_runner_completed_at_to_db_metadata.sql

@@ -0,0 +1,38 @@
+-- =============================================================================
+-- MIGRATION: Add runner_completed_at column to db_metadata (OMN-13062)
+-- =============================================================================
+-- Ticket: OMN-13062 (migration-gate vacuity fix — retro A-10)
+-- Recurrences: OMN-12885, OMN-12934
+-- Version: 1.0.0
+--
+-- PURPOSE:
+--   Extends db_metadata with runner_completed_at TIMESTAMPTZ.  The forward
+--   migration runner (run-forward-migrations.sh) stamps this column as its
+--   FINAL act after every migration in the infra and node sets succeeds.
+--
+--   This timestamp is the durable evidence that the runner reached
+--   successful completion.  Combined with the migrations_complete=TRUE
+--   sentinel (OMN-3737), the migration-gate healthcheck now has two
+--   independent signals:
+--     1. migrations_complete=TRUE  — the flag was set without error
+--     2. runner_completed_at IS NOT NULL — the runner reached its final act
+--
+--   The runner clears migrations_complete to FALSE at the start of every run
+--   and sets it (together with runner_completed_at) only after all migrations
+--   succeed.  Any nonzero exit from any migration leaves the gate UNHEALTHY.
+--
+-- IDEMPOTENCY:
+--   ALTER TABLE ... ADD COLUMN IF NOT EXISTS is safe to re-run.
+--
+-- ROLLBACK:
+--   See rollback/rollback_085_add_runner_completed_at_to_db_metadata.sql
+-- =============================================================================
+
+ALTER TABLE public.db_metadata
+    ADD COLUMN IF NOT EXISTS runner_completed_at TIMESTAMPTZ;
+
+COMMENT ON COLUMN public.db_metadata.runner_completed_at IS
+    'Stamped by run-forward-migrations.sh as its final act after all '
+    'infra and node migrations succeed.  NULL means the runner never '
+    'completed successfully (partial run, first boot, or mid-run failure). '
+    'OMN-13062 (retro A-10).';

docker/migrations/rollback/rollback_085_add_runner_completed_at_to_db_metadata.sql

@@ -0,0 +1,9 @@
+-- =============================================================================
+-- ROLLBACK: Remove runner_completed_at column from db_metadata (OMN-13062)
+-- =============================================================================
+-- Ticket: OMN-13062 (migration-gate vacuity fix — retro A-10)
+-- Version: 1.0.0
+-- =============================================================================
+
+ALTER TABLE public.db_metadata
+    DROP COLUMN IF EXISTS runner_completed_at;

docker/migrations/schema_fingerprint.sha256

@@ -1,6 +1,6 @@
 # Schema migration fingerprint for omnibase_infra (auto-generated)
 # Regenerate: python scripts/check_schema_fingerprint.py stamp
 # Verify:     python scripts/check_schema_fingerprint.py verify
-sha256:dc4b265ba91d2d28ca02b75546cf73f4c05be55e8a3ecd27585653959a19c734
-generated_at: 2026-06-09T17:45:31Z
-migration_file_count: 70
+sha256:b24f163de9b716a0db87d15af329ddeb008f3d6531c6404867b844aed304451a
+generated_at: 2026-06-12T16:41:04Z
+migration_file_count: 71

docker/migrations/skip-manifest.yaml

@@ -0,0 +1,34 @@
+# skip-manifest.yaml — Intentionally-skipped migration IDs
+#
+# Ticket: OMN-13062 (migration-gate vacuity fix — retro A-10)
+#
+# PURPOSE
+#   The forward migration runner (run-forward-migrations.sh) checks this
+#   file before applying each migration.  A migration listed here is treated
+#   as already-applied (a skip record is written to schema_migrations with
+#   checksum "skip-manifest") without executing the SQL.
+#
+# WHEN TO USE
+#   - A migration was discovered to be a no-op on warm volumes (already
+#     applied via docker-entrypoint-initdb.d on first boot).
+#   - A migration contains a known-safe but environment-specific statement
+#     that must not re-run on warm volumes.
+#   - Never use this to hide a broken migration — fix the SQL instead.
+#
+# HOW TO ADD AN ENTRY
+#   1. Confirm the migration is safe to skip (e.g., idempotent but already
+#      applied, or intentionally empty).
+#   2. Add the entry to skipped_migrations below.
+#   3. Commit the manifest change in the SAME PR that deems the migration
+#      unrunnable.  Operator env cannot inject skips.
+#
+# ENTRY FORMAT
+#   id:     "docker/<NNN_name.sql>"   — migration_id as recorded in schema_migrations
+#   reason: "human-readable reason"
+#   ticket: "OMN-XXXX"               — Linear ticket authorising the skip
+#
+# EXIT CODES for run-forward-migrations.sh
+#   0 — all migrations applied (or skipped by this manifest)
+#   1 — migration failure (nonzero psql exit)
+
+skipped_migrations: []

scripts/run-forward-migrations.sh

@@ -14,6 +14,27 @@
 # mechanism for keeping warm Postgres volumes up-to-date with new migrations.
 #
 # Ticket: OMN-4175 (Forward migration runner for warm Postgres volumes)
+# Ticket: OMN-13062 (migration-gate vacuity fix — retro A-10)
+#
+# ---------------------------------------------------------------------------
+# Sentinel discipline (OMN-13062)
+# ---------------------------------------------------------------------------
+# migrations_complete is cleared to FALSE at the start of every runner
+# invocation and set to TRUE only as the FINAL act after all infra and
+# node migrations apply without error. Any nonzero exit from any migration
+# leaves the flag FALSE, making the migration-gate healthcheck UNHEALTHY.
+#
+# The committed per-migration skip-manifest is the SOLE escape for migrations
+# that must be intentionally skipped:
+#   docker/migrations/skip-manifest.yaml
+# Format:
+#   skipped_migrations:
+#     - id: "docker/NNN_name.sql"
+#       reason: "..."
+#       ticket: "OMN-XXXX"
+# The runner reads this at startup; a listed migration_id is treated as
+# already-applied without executing the SQL. New entries must be committed
+# in the same PR that deems the migration unrunnable.
 #
 # ---------------------------------------------------------------------------
 # Node-owned migration auto-discovery (OMN-12559)
@@ -40,6 +61,7 @@
 #   MIGRATIONS_DIR    (default: /migrations/forward)
 #   NODE_MIGRATIONS_DIR (default: ${MIGRATIONS_DIR}/nodes)
 #   NODE_POSTGRES_DB  (default: POSTGRES_DB; compose sets omnidash_analytics)
+#   PG_WAIT_RETRIES   (default: 30 — number of 2s waits for postgres ready)
 
 set -e
 
@@ -50,9 +72,50 @@ PGDB="${POSTGRES_DB:-omnibase_infra}"
 MIGRATIONS_DIR="${MIGRATIONS_DIR:-/migrations/forward}"
 NODE_MIGRATIONS_DIR="${NODE_MIGRATIONS_DIR:-${MIGRATIONS_DIR}/nodes}"
 NODE_PGDB="${NODE_POSTGRES_DB:-${PGDB}}"
+PG_WAIT_RETRIES="${PG_WAIT_RETRIES:-30}"
 
 export PGPASSWORD="${POSTGRES_PASSWORD}"
 
+# ---------------------------------------------------------------------------
+# Skip-manifest: load intentionally-skipped migration ids (OMN-13062)
+# ---------------------------------------------------------------------------
+# Format: YAML file with a top-level list "skipped_migrations" each entry has
+# "id" (e.g. "docker/038_placeholder.sql") and optionally "reason" / "ticket".
+# Only a committed manifest is honoured — operator env cannot inject skips.
+SKIP_MANIFEST="${MIGRATIONS_DIR%/forward}/skip-manifest.yaml"
+SKIPPED_IDS=""
+if [ -f "${SKIP_MANIFEST}" ]; then
+  echo "[forward-migration] Loading skip-manifest: ${SKIP_MANIFEST}"
+  # Extract quoted id: values from YAML using portable sed (no yq/python/gawk).
+  # Handles lines of the form:  - id: "docker/NNN_name.sql"
+  SKIPPED_IDS="$(sed -n 's/^[[:space:]]*-[[:space:]]*id:[[:space:]]*"\([^"]*\)".*/\1/p' \
+    "${SKIP_MANIFEST}" 2>/dev/null || true)"
+fi
+
+is_skipped_by_manifest() {
+  migration_id="$1"
+  if [ -z "${SKIPPED_IDS}" ]; then
+    return 1
+  fi
+  echo "${SKIPPED_IDS}" | grep -Fxq "${migration_id}"
+}
+
+# ---------------------------------------------------------------------------
+# 0. Wait for Postgres to be ready (first-boot initdb race guard, OMN-13062)
+# ---------------------------------------------------------------------------
+echo "[forward-migration] Waiting for Postgres to accept connections..."
+retries=0
+until psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" -c "SELECT 1" >/dev/null 2>&1; do
+  retries=$((retries + 1))
+  if [ "$retries" -ge "$PG_WAIT_RETRIES" ]; then
+    echo "[forward-migration] ERROR: Postgres not ready after ${PG_WAIT_RETRIES} retries. Aborting." >&2
+    exit 1
+  fi
+  echo "[forward-migration]   postgres not ready (attempt ${retries}/${PG_WAIT_RETRIES}), retrying in 2s..."
+  sleep 2
+done
+echo "[forward-migration] Postgres is ready."
+
 validate_database_identifier() {
   database="$1"
   if ! printf '%s' "$database" | grep -Eq '^[A-Za-z_][A-Za-z0-9_-]*$'; then
@@ -96,6 +159,33 @@ CREATE TABLE IF NOT EXISTS public.schema_migrations (
 );
 "
 
+# ---------------------------------------------------------------------------
+# 1a. Clear the sentinel at the start of every run (OMN-13062)
+# ---------------------------------------------------------------------------
+# This ensures that any mid-run failure leaves migrations_complete=FALSE so
+# the migration-gate healthcheck stays UNHEALTHY. The sentinel is only set
+# TRUE as the very last act of this script (after all migrations succeed).
+# We use a conditional UPDATE so this is a no-op on volumes that have not
+# yet applied migration 037 (migrations_complete column may not exist yet).
+echo "[forward-migration] Clearing migration sentinel (will be re-set on successful completion)..."
+psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" -c "
+DO \$\$
+BEGIN
+  IF EXISTS (
+    SELECT 1 FROM information_schema.columns
+    WHERE table_schema = 'public'
+      AND table_name = 'db_metadata'
+      AND column_name = 'migrations_complete'
+  ) THEN
+    UPDATE public.db_metadata
+    SET migrations_complete = FALSE,
+        updated_at = NOW()
+    WHERE id = TRUE;
+  END IF;
+END;
+\$\$;
+" 2>/dev/null || true
+
 # ---------------------------------------------------------------------------
 # 2. Apply pending migrations in sorted order
 # ---------------------------------------------------------------------------
@@ -108,6 +198,18 @@ for migration_file in $(ls "${MIGRATIONS_DIR}"/*.sql | sort); do
   filename=$(basename "$migration_file")
   migration_id="docker/${filename}"
 
+  # Honour skip-manifest: treat manifest-listed migrations as already applied
+  if is_skipped_by_manifest "${migration_id}"; then
+    echo "[forward-migration]   skip  ${filename} (skip-manifest)"
+    SKIPPED=$((SKIPPED + 1))
+    # Record in schema_migrations so the table stays consistent.
+    psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" \
+      -c "INSERT INTO public.schema_migrations (migration_id, checksum, source_set)
+          VALUES ('${migration_id}', 'skip-manifest', 'docker')
+          ON CONFLICT (migration_id) DO NOTHING;"
+    continue
+  fi
+
   # Check if already applied
   already_applied=$(psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" \
     -tAc "SELECT 1 FROM public.schema_migrations WHERE migration_id = '${migration_id}'" 2>/dev/null || true)
@@ -199,3 +301,18 @@ else
 fi
 
 echo "[forward-migration] Complete: ${APPLIED} infra applied, ${SKIPPED} infra skipped; ${NODE_APPLIED} node applied, ${NODE_SKIPPED} node skipped."
+
+# ---------------------------------------------------------------------------
+# 4. Set the sentinel TRUE only after ALL migrations succeed (OMN-13062)
+# ---------------------------------------------------------------------------
+# This is the FINAL act. Any earlier failure leaves migrations_complete=FALSE.
+# runner_completed_at records the timestamp of this successful completion.
+echo "[forward-migration] All migrations complete. Setting sentinel TRUE..."
+psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" -v ON_ERROR_STOP=1 -c "
+UPDATE public.db_metadata
+SET migrations_complete = TRUE,
+    runner_completed_at = NOW(),
+    updated_at = NOW()
+WHERE id = TRUE;
+"
+echo "[forward-migration] Sentinel set. Migration gate will report HEALTHY."

scripts/sync-node-migrations.sh

@@ -95,17 +95,20 @@ PY
 
 OMK_RESOLVED="$(resolve_omnimarket_src || true)"
 if [ -z "${OMK_RESOLVED}" ]; then
-  # In --check mode (pre-commit / CI), the omnimarket source tree is frequently
-  # not checked out. The vendored files are the durable source of truth and are
-  # validated independently by the migration tests; a drift check is only
-  # meaningful when the source is present. Skip (success) rather than fail so the
-  # gate never produces false negatives on runners without omnimarket.
-  if [ "${CHECK_MODE}" -eq 1 ]; then
-    echo "[sync-node-migrations] omnimarket source not resolvable — skipping drift check." >&2
-    exit 0
-  fi
   echo "[sync-node-migrations] ERROR: could not resolve omnimarket source tree." >&2
   echo "  Set OMNIMARKET_SRC=<omnimarket repo root>, or OMNI_HOME=<omni_home>, or pip install omnimarket." >&2
+  if [ "${CHECK_MODE}" -eq 1 ]; then
+    # OMN-13062 (retro A-10): --check with unresolvable source is a vacuous gate.
+    # Silently passing when the source is absent means drift is never detected.
+    # Exit 2 (source-unresolvable) so the CI gate fires and the operator is
+    # required to either supply OMNIMARKET_SRC / OMNI_HOME or explicitly skip
+    # this check via SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE=1.
+    if [ "${SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE:-0}" = "1" ]; then
+      echo "[sync-node-migrations] SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE=1 — skipping unresolvable-source error." >&2
+      exit 0
+    fi
+    exit 2
+  fi
   exit 2
 fi