fix(OMN-13666): runtime entrypoint — secondary-DB schema-fingerprint stamp failure is non-fatal

contracts/OMN-13666.yaml

@@ -0,0 +1,49 @@
+schema_version: '1.0.0'
+ticket_id: OMN-13666
+title: 'Runtime entrypoint: secondary-DB schema-fingerprint stamp failure is non-fatal (primary stays fatal)'
+summary: >-
+  The runtime image entrypoint (docker/entrypoint-runtime.sh) stamps schema fingerprints into db_metadata per-DB before starting the kernel. In prod, the runtime DB user lacks write permission on the omniintelligence (secondary, non-owned) database's db_metadata table, so its stamp failed with "permission denied for table db_metadata"; after 5 attempts the entrypoint exited 1 and the container crash-looped, holding all 7 onex-prod runtime deployments at 0/1. The runtime's OWN database (omnibase_infra) stamps fine. This fix introduces a principled required-vs-best-effort stamp policy in the entrypoint: the PRIMARY/owned DB (omnibase_infra) stamp stays REQUIRED — a failure after retries aborts boot loudly (exit 1) so a NULL/stale-fingerprint kernel never starts — while a SECONDARY/non-owned DB (omniintelligence) stamp is BEST-EFFORT — a failure logs a clear WARNING and boot proceeds (exit 0). No DB GRANT is added (operator chose the entrypoint-policy route). No gate is weakened.
+is_seam_ticket: false
+interface_change: false
+interfaces_touched: []
+evidence_requirements:
+  - kind: tests
+    description: >-
+      Behavioral tests execute the real entrypoint with a stubbed python: a primary-DB stamp failure aborts boot (exit 1), a secondary-DB stamp failure warns and boot proceeds (exit 0); plus static source guards and shellcheck-clean.
+    command: 'uv run pytest tests/unit/docker/test_runtime_entrypoint_stamp_tolerance.py -q'
+  - kind: ci
+    description: CI pipeline green on PR
+    command: 'gh pr checks <PR> --repo OmniNode-ai/omnibase_infra'
+emergency_bypass:
+  enabled: false
+  justification: ''
+  follow_up_ticket_id: ''
+dod_evidence:
+  - id: dod-001
+    description: >-
+      Entrypoint marks the primary DB (omnibase_infra) stamp REQUIRED and the secondary DB (omniintelligence) stamp BEST-EFFORT.
+    source: generated
+    checks:
+      - check_type: command
+        check_value: 'grep -q ''stamp_fingerprint "omnibase_infra" "${OMNIBASE_INFRA_DB_URL}" "required"'' docker/entrypoint-runtime.sh && grep -q ''stamp_fingerprint "omniintelligence" "${OMNIINTELLIGENCE_DB_URL}" "optional"'' docker/entrypoint-runtime.sh'
+  - id: dod-002
+    description: >-
+      A required-stamp failure exits non-zero (fatal); the optional branch only warns (non-fatal continue).
+    source: generated
+    checks:
+      - check_type: command
+        check_value: 'grep -q ''if \[ "${REQUIRED}" = "required" \]; then'' docker/entrypoint-runtime.sh && grep -q "aborting boot" docker/entrypoint-runtime.sh && grep -q "continuing best-effort" docker/entrypoint-runtime.sh'
+  - id: dod-003
+    description: >-
+      Behavioral + static tests for the required/optional stamp policy pass.
+    source: generated
+    checks:
+      - check_type: command
+        check_value: 'uv run pytest tests/unit/docker/test_runtime_entrypoint_stamp_tolerance.py -q'
+  - id: dod-004
+    description: >-
+      Entrypoint script is shellcheck-clean.
+    source: generated
+    checks:
+      - check_type: command
+        check_value: 'shellcheck docker/entrypoint-runtime.sh'

docker/entrypoint-runtime.sh

@@ -66,26 +66,37 @@ echo "========================"
 # omniintelligence (e.g. code_entities, code_relationships), the stored
 # fingerprint was stale and the service failed health checks.
 #
+# OMN-13666: Required vs best-effort stamp policy.
+#   - The runtime's OWN database (omnibase_infra) is REQUIRED. Its db_metadata
+#     row drives the kernel's startup fingerprint assertion; if the stamp cannot
+#     succeed the kernel would start with a NULL/stale fingerprint and crash-loop
+#     anyway, so we fail FAST and loud here with a clear cause.
+#   - Secondary / non-owned databases (e.g. omniintelligence) are BEST-EFFORT.
+#     The runtime DB user legitimately lacks write permission on another
+#     service's db_metadata table ("permission denied for table db_metadata").
+#     A failure there must NOT take the whole runtime down -- it is logged as a
+#     WARNING and boot proceeds. The owning service stamps its own fingerprint.
+#
 # Retry logic: up to 5 attempts with 1s sleep between failures handles
 # transient DB-not-ready conditions at container startup.
 #
 # Exit code handling from util_schema_fingerprint:
 #   0 = success (fingerprint stamped)
 #   2 = schema mismatch (no point retrying -- bail immediately)
 #   1 = connection or general error (retry)
-#
-# Fail-open: kernel starts regardless of stamp outcome. The kernel's own
-# fingerprint assertion will catch any real problems.
 
 stamp_fingerprint() {
   # Stamp schema fingerprint for a single database.
-  # Usage: stamp_fingerprint <manifest_name> <db_url>
+  # Usage: stamp_fingerprint <manifest_name> <db_url> <required>
+  #   required="required" -> a failed stamp aborts boot (exit 1)
+  #   required="optional" -> a failed stamp warns and boot continues
   MANIFEST_NAME="$1"
   DB_URL="$2"
+  REQUIRED="$3"
 
   # Safe log: strip scheme and userinfo, show only host:port/db
   SAFE_DSN=$(echo "${DB_URL}" | sed 's|^[^/]*//[^@]*@||')
-  echo "[entrypoint] Stamping schema fingerprint for ${MANIFEST_NAME} (db: ${SAFE_DSN})..."
+  echo "[entrypoint] Stamping schema fingerprint for ${MANIFEST_NAME} (db: ${SAFE_DSN}, ${REQUIRED})..."
 
   STAMP_OK=0
   ATTEMPT=1
@@ -112,20 +123,24 @@ stamp_fingerprint() {
   done
 
   if [ "${STAMP_OK}" -eq 0 ]; then
-    echo "[entrypoint] WARNING: ${MANIFEST_NAME} fingerprint stamp did not succeed -- continuing"
+    if [ "${REQUIRED}" = "required" ]; then
+      echo "[entrypoint] ERROR: ${MANIFEST_NAME} (PRIMARY/owned DB) fingerprint stamp failed -- aborting boot" >&2
+      exit 1
+    fi
+    echo "[entrypoint] WARNING: ${MANIFEST_NAME} (secondary/non-owned DB) fingerprint stamp did not succeed -- continuing best-effort"
   fi
 }
 
-# Stamp omnibase_infra (primary, always required)
+# Stamp omnibase_infra (PRIMARY/owned DB -- REQUIRED: failure aborts boot)
 if [ -n "${OMNIBASE_INFRA_DB_URL:-}" ]; then
-  stamp_fingerprint "omnibase_infra" "${OMNIBASE_INFRA_DB_URL}"
+  stamp_fingerprint "omnibase_infra" "${OMNIBASE_INFRA_DB_URL}" "required"
 else
   echo "[entrypoint] OMNIBASE_INFRA_DB_URL not set -- skipping fingerprint stamp"
 fi
 
-# Stamp omniintelligence (optional, activates only when plugin DB is configured)
+# Stamp omniintelligence (SECONDARY/non-owned DB -- BEST-EFFORT: failure warns)
 if [ -n "${OMNIINTELLIGENCE_DB_URL:-}" ]; then
-  stamp_fingerprint "omniintelligence" "${OMNIINTELLIGENCE_DB_URL}"
+  stamp_fingerprint "omniintelligence" "${OMNIINTELLIGENCE_DB_URL}" "optional"
 else
   echo "[entrypoint] OMNIINTELLIGENCE_DB_URL not set -- skipping omniintelligence fingerprint stamp"
 fi

tests/unit/docker/test_runtime_entrypoint_stamp_tolerance.py

@@ -0,0 +1,186 @@
+# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc.
+# SPDX-License-Identifier: MIT
+"""Regression tests for entrypoint schema-fingerprint stamp tolerance (OMN-13666).
+
+Policy under test:
+    * The runtime's OWN database (omnibase_infra) stamp is REQUIRED -- a failure
+      after all retries aborts boot (exit 1) so the crash cause is loud and the
+      kernel never starts with a NULL/stale fingerprint.
+    * A SECONDARY / non-owned database (omniintelligence) stamp is BEST-EFFORT --
+      a failure (e.g. "permission denied for table db_metadata") warns and boot
+      proceeds (exit 0).
+
+The behavioral tests execute the real ``docker/entrypoint-runtime.sh`` with a
+stubbed ``python`` on PATH whose exit code is driven per-manifest by env vars,
+so no Docker, Postgres, or privilege drop is involved. The CMD that the
+entrypoint exec's at the end is ``true`` (exit 0), so a clean run exits 0.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import stat
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from tests.unit.docker.conftest import DOCKER_DIR
+
+pytestmark = [pytest.mark.unit]
+
+ENTRYPOINT = DOCKER_DIR / "entrypoint-runtime.sh"
+
+# Stub "python" that emulates util_schema_fingerprint stamp exit codes.
+# It parses the --manifest value out of its args and looks up an exit code from
+# an env var (STUB_RC_<MANIFEST>), defaulting to 0 (success). Any non-stamp
+# invocation (e.g. render modules, which the entrypoint guards behind unset env
+# vars and therefore never calls here) also exits 0.
+_PYTHON_STUB = """#!/bin/sh
+manifest=""
+prev=""
+for arg in "$@"; do
+  if [ "$prev" = "--manifest" ]; then
+    manifest="$arg"
+  fi
+  prev="$arg"
+done
+case "$manifest" in
+  omnibase_infra) exit "${STUB_RC_OMNIBASE_INFRA:-0}" ;;
+  omniintelligence) exit "${STUB_RC_OMNIINTELLIGENCE:-0}" ;;
+  *) exit 0 ;;
+esac
+"""
+
+
+def _run_entrypoint(
+    tmp_path: Path,
+    *,
+    infra_rc: int = 0,
+    intel_rc: int = 0,
+    with_intel_db: bool = True,
+) -> subprocess.CompletedProcess[str]:
+    """Run the real entrypoint with a stubbed python and controlled stamp RCs."""
+    bindir = tmp_path / "bin"
+    bindir.mkdir()
+    stub = bindir / "python"
+    stub.write_text(_PYTHON_STUB)
+    stub.chmod(stub.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+
+    env = {
+        # Minimal PATH: stub python first, then real sh/sed/echo/sleep tools.
+        "PATH": f"{bindir}:/usr/bin:/bin",
+        "OMNIBASE_INFRA_DB_URL": "postgresql://u:p@db:5432/omnibase_infra",
+        "STUB_RC_OMNIBASE_INFRA": str(infra_rc),
+        "STUB_RC_OMNIINTELLIGENCE": str(intel_rc),
+    }
+    if with_intel_db:
+        env["OMNIINTELLIGENCE_DB_URL"] = "postgresql://u:p@db:5432/omniintelligence"
+
+    # CMD the entrypoint exec's at the end; "true" exits 0 on a clean boot path.
+    return subprocess.run(
+        ["sh", str(ENTRYPOINT), "true"],
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=60,
+        check=False,
+    )
+
+
+def test_clean_stamp_both_dbs_boots() -> None:
+    """Both stamps succeed -> entrypoint reaches exec and exits 0."""
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as td:
+        result = _run_entrypoint(Path(td), infra_rc=0, intel_rc=0)
+    assert result.returncode == 0, result.stderr
+    assert "Schema fingerprint stamped for omnibase_infra." in result.stdout
+    assert "Schema fingerprint stamped for omniintelligence." in result.stdout
+    assert "Starting runtime kernel..." in result.stdout
+
+
+def test_secondary_db_stamp_failure_is_nonfatal() -> None:
+    """omniintelligence stamp fails (perm denied) -> WARNING, boot continues (exit 0)."""
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as td:
+        # rc=1 emulates "permission denied for table db_metadata" general error.
+        result = _run_entrypoint(Path(td), infra_rc=0, intel_rc=1)
+    assert result.returncode == 0, result.stderr
+    assert "Schema fingerprint stamped for omnibase_infra." in result.stdout
+    assert (
+        "WARNING: omniintelligence (secondary/non-owned DB) fingerprint stamp "
+        "did not succeed -- continuing best-effort" in result.stdout
+    )
+    # Boot proceeds past the stamp section.
+    assert "Starting runtime kernel..." in result.stdout
+
+
+def test_primary_db_stamp_failure_is_fatal() -> None:
+    """omnibase_infra stamp fails -> entrypoint aborts boot (exit 1), no kernel start."""
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as td:
+        result = _run_entrypoint(Path(td), infra_rc=1, intel_rc=0)
+    assert result.returncode == 1
+    assert (
+        "ERROR: omnibase_infra (PRIMARY/owned DB) fingerprint stamp failed "
+        "-- aborting boot" in result.stderr
+    )
+    # Must NOT reach kernel start when the primary DB stamp fails.
+    assert "Starting runtime kernel..." not in result.stdout
+
+
+def test_secondary_db_optional_when_db_url_unset() -> None:
+    """No OMNIINTELLIGENCE_DB_URL -> stamp skipped, boot proceeds (exit 0)."""
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as td:
+        result = _run_entrypoint(Path(td), infra_rc=0, with_intel_db=False)
+    assert result.returncode == 0, result.stderr
+    assert (
+        "OMNIINTELLIGENCE_DB_URL not set -- skipping omniintelligence "
+        "fingerprint stamp" in result.stdout
+    )
+    assert "Starting runtime kernel..." in result.stdout
+
+
+# ---------------------------------------------------------------------------
+# Static guards on the script source -- keep the required/optional contract.
+# ---------------------------------------------------------------------------
+
+
+def test_entrypoint_marks_primary_required_and_secondary_optional() -> None:
+    source = ENTRYPOINT.read_text()
+    assert (
+        'stamp_fingerprint "omnibase_infra" "${OMNIBASE_INFRA_DB_URL}" "required"'
+        in source
+    )
+    assert (
+        'stamp_fingerprint "omniintelligence" "${OMNIINTELLIGENCE_DB_URL}" "optional"'
+        in source
+    )
+
+
+def test_required_stamp_failure_exits_nonzero_in_source() -> None:
+    source = ENTRYPOINT.read_text()
+    # The required branch must exit non-zero; the optional branch must only warn.
+    assert 'if [ "${REQUIRED}" = "required" ]; then' in source
+    assert "exit 1" in source.split('if [ "${REQUIRED}" = "required" ]; then', 1)[1]
+
+
+def test_shellcheck_clean_if_available() -> None:
+    shellcheck = shutil.which("shellcheck")
+    if shellcheck is None:
+        pytest.skip("shellcheck not installed")
+    result = subprocess.run(
+        [shellcheck, str(ENTRYPOINT)],
+        capture_output=True,
+        text=True,
+        timeout=60,
+        env={**os.environ},
+        check=False,
+    )
+    assert result.returncode == 0, result.stdout + result.stderr