From 4fd654ffe577e90c2c90c0aebf1c87e21f40014f Mon Sep 17 00:00:00 2001 From: jonahgabriel Date: Sat, 27 Jun 2026 09:20:09 -0400 Subject: [PATCH] fix(OMN-13666): secondary-DB schema-fingerprint stamp failure is non-fatal in runtime entrypoint The runtime image entrypoint stamps schema fingerprints into db_metadata per-DB before starting the kernel. In prod the runtime DB user lacks write permission on the omniintelligence (secondary, non-owned) database's db_metadata table, so its stamp failed with 'permission denied for table db_metadata'; after 5 attempts the entrypoint exited 1 and the container crash-looped, holding all 7 onex-prod runtime deployments at 0/1. Introduce a principled required-vs-best-effort stamp policy: - PRIMARY/owned DB (omnibase_infra) stays REQUIRED: a failed stamp aborts boot (exit 1) loudly so a NULL/stale-fingerprint kernel never starts. - SECONDARY/non-owned DB (omniintelligence) is BEST-EFFORT: a failed stamp logs a clear WARNING and boot proceeds (exit 0). No DB GRANT added (operator chose the entrypoint-policy route). No gate weakened. Adds behavioral tests that run the real entrypoint with a stubbed python (primary failure -> exit 1; secondary failure -> exit 0) plus static source guards and shellcheck. Evidence-Ticket: OMN-13666 --- contracts/OMN-13666.yaml | 49 +++++ docker/entrypoint-runtime.sh | 35 +++- ...test_runtime_entrypoint_stamp_tolerance.py | 186 ++++++++++++++++++ 3 files changed, 260 insertions(+), 10 deletions(-) create mode 100644 contracts/OMN-13666.yaml create mode 100644 tests/unit/docker/test_runtime_entrypoint_stamp_tolerance.py diff --git a/contracts/OMN-13666.yaml b/contracts/OMN-13666.yaml new file mode 100644 index 0000000000..dda9e964ff --- /dev/null +++ b/contracts/OMN-13666.yaml @@ -0,0 +1,49 @@ +schema_version: '1.0.0' +ticket_id: OMN-13666 +title: 'Runtime entrypoint: secondary-DB schema-fingerprint stamp failure is non-fatal (primary stays fatal)' +summary: >- + The runtime image entrypoint (docker/entrypoint-runtime.sh) stamps schema fingerprints into db_metadata per-DB before starting the kernel. In prod, the runtime DB user lacks write permission on the omniintelligence (secondary, non-owned) database's db_metadata table, so its stamp failed with "permission denied for table db_metadata"; after 5 attempts the entrypoint exited 1 and the container crash-looped, holding all 7 onex-prod runtime deployments at 0/1. The runtime's OWN database (omnibase_infra) stamps fine. This fix introduces a principled required-vs-best-effort stamp policy in the entrypoint: the PRIMARY/owned DB (omnibase_infra) stamp stays REQUIRED — a failure after retries aborts boot loudly (exit 1) so a NULL/stale-fingerprint kernel never starts — while a SECONDARY/non-owned DB (omniintelligence) stamp is BEST-EFFORT — a failure logs a clear WARNING and boot proceeds (exit 0). No DB GRANT is added (operator chose the entrypoint-policy route). No gate is weakened. +is_seam_ticket: false +interface_change: false +interfaces_touched: [] +evidence_requirements: + - kind: tests + description: >- + Behavioral tests execute the real entrypoint with a stubbed python: a primary-DB stamp failure aborts boot (exit 1), a secondary-DB stamp failure warns and boot proceeds (exit 0); plus static source guards and shellcheck-clean. + command: 'uv run pytest tests/unit/docker/test_runtime_entrypoint_stamp_tolerance.py -q' + - kind: ci + description: CI pipeline green on PR + command: 'gh pr checks --repo OmniNode-ai/omnibase_infra' +emergency_bypass: + enabled: false + justification: '' + follow_up_ticket_id: '' +dod_evidence: + - id: dod-001 + description: >- + Entrypoint marks the primary DB (omnibase_infra) stamp REQUIRED and the secondary DB (omniintelligence) stamp BEST-EFFORT. + source: generated + checks: + - check_type: command + check_value: 'grep -q ''stamp_fingerprint "omnibase_infra" "${OMNIBASE_INFRA_DB_URL}" "required"'' docker/entrypoint-runtime.sh && grep -q ''stamp_fingerprint "omniintelligence" "${OMNIINTELLIGENCE_DB_URL}" "optional"'' docker/entrypoint-runtime.sh' + - id: dod-002 + description: >- + A required-stamp failure exits non-zero (fatal); the optional branch only warns (non-fatal continue). + source: generated + checks: + - check_type: command + check_value: 'grep -q ''if \[ "${REQUIRED}" = "required" \]; then'' docker/entrypoint-runtime.sh && grep -q "aborting boot" docker/entrypoint-runtime.sh && grep -q "continuing best-effort" docker/entrypoint-runtime.sh' + - id: dod-003 + description: >- + Behavioral + static tests for the required/optional stamp policy pass. + source: generated + checks: + - check_type: command + check_value: 'uv run pytest tests/unit/docker/test_runtime_entrypoint_stamp_tolerance.py -q' + - id: dod-004 + description: >- + Entrypoint script is shellcheck-clean. + source: generated + checks: + - check_type: command + check_value: 'shellcheck docker/entrypoint-runtime.sh' diff --git a/docker/entrypoint-runtime.sh b/docker/entrypoint-runtime.sh index 7fa1467626..0fc6291027 100755 --- a/docker/entrypoint-runtime.sh +++ b/docker/entrypoint-runtime.sh @@ -66,6 +66,17 @@ echo "========================" # omniintelligence (e.g. code_entities, code_relationships), the stored # fingerprint was stale and the service failed health checks. # +# OMN-13666: Required vs best-effort stamp policy. +# - The runtime's OWN database (omnibase_infra) is REQUIRED. Its db_metadata +# row drives the kernel's startup fingerprint assertion; if the stamp cannot +# succeed the kernel would start with a NULL/stale fingerprint and crash-loop +# anyway, so we fail FAST and loud here with a clear cause. +# - Secondary / non-owned databases (e.g. omniintelligence) are BEST-EFFORT. +# The runtime DB user legitimately lacks write permission on another +# service's db_metadata table ("permission denied for table db_metadata"). +# A failure there must NOT take the whole runtime down -- it is logged as a +# WARNING and boot proceeds. The owning service stamps its own fingerprint. +# # Retry logic: up to 5 attempts with 1s sleep between failures handles # transient DB-not-ready conditions at container startup. # @@ -73,19 +84,19 @@ echo "========================" # 0 = success (fingerprint stamped) # 2 = schema mismatch (no point retrying -- bail immediately) # 1 = connection or general error (retry) -# -# Fail-open: kernel starts regardless of stamp outcome. The kernel's own -# fingerprint assertion will catch any real problems. stamp_fingerprint() { # Stamp schema fingerprint for a single database. - # Usage: stamp_fingerprint + # Usage: stamp_fingerprint + # required="required" -> a failed stamp aborts boot (exit 1) + # required="optional" -> a failed stamp warns and boot continues MANIFEST_NAME="$1" DB_URL="$2" + REQUIRED="$3" # Safe log: strip scheme and userinfo, show only host:port/db SAFE_DSN=$(echo "${DB_URL}" | sed 's|^[^/]*//[^@]*@||') - echo "[entrypoint] Stamping schema fingerprint for ${MANIFEST_NAME} (db: ${SAFE_DSN})..." + echo "[entrypoint] Stamping schema fingerprint for ${MANIFEST_NAME} (db: ${SAFE_DSN}, ${REQUIRED})..." STAMP_OK=0 ATTEMPT=1 @@ -112,20 +123,24 @@ stamp_fingerprint() { done if [ "${STAMP_OK}" -eq 0 ]; then - echo "[entrypoint] WARNING: ${MANIFEST_NAME} fingerprint stamp did not succeed -- continuing" + if [ "${REQUIRED}" = "required" ]; then + echo "[entrypoint] ERROR: ${MANIFEST_NAME} (PRIMARY/owned DB) fingerprint stamp failed -- aborting boot" >&2 + exit 1 + fi + echo "[entrypoint] WARNING: ${MANIFEST_NAME} (secondary/non-owned DB) fingerprint stamp did not succeed -- continuing best-effort" fi } -# Stamp omnibase_infra (primary, always required) +# Stamp omnibase_infra (PRIMARY/owned DB -- REQUIRED: failure aborts boot) if [ -n "${OMNIBASE_INFRA_DB_URL:-}" ]; then - stamp_fingerprint "omnibase_infra" "${OMNIBASE_INFRA_DB_URL}" + stamp_fingerprint "omnibase_infra" "${OMNIBASE_INFRA_DB_URL}" "required" else echo "[entrypoint] OMNIBASE_INFRA_DB_URL not set -- skipping fingerprint stamp" fi -# Stamp omniintelligence (optional, activates only when plugin DB is configured) +# Stamp omniintelligence (SECONDARY/non-owned DB -- BEST-EFFORT: failure warns) if [ -n "${OMNIINTELLIGENCE_DB_URL:-}" ]; then - stamp_fingerprint "omniintelligence" "${OMNIINTELLIGENCE_DB_URL}" + stamp_fingerprint "omniintelligence" "${OMNIINTELLIGENCE_DB_URL}" "optional" else echo "[entrypoint] OMNIINTELLIGENCE_DB_URL not set -- skipping omniintelligence fingerprint stamp" fi diff --git a/tests/unit/docker/test_runtime_entrypoint_stamp_tolerance.py b/tests/unit/docker/test_runtime_entrypoint_stamp_tolerance.py new file mode 100644 index 0000000000..8a75fc23f0 --- /dev/null +++ b/tests/unit/docker/test_runtime_entrypoint_stamp_tolerance.py @@ -0,0 +1,186 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Regression tests for entrypoint schema-fingerprint stamp tolerance (OMN-13666). + +Policy under test: + * The runtime's OWN database (omnibase_infra) stamp is REQUIRED -- a failure + after all retries aborts boot (exit 1) so the crash cause is loud and the + kernel never starts with a NULL/stale fingerprint. + * A SECONDARY / non-owned database (omniintelligence) stamp is BEST-EFFORT -- + a failure (e.g. "permission denied for table db_metadata") warns and boot + proceeds (exit 0). + +The behavioral tests execute the real ``docker/entrypoint-runtime.sh`` with a +stubbed ``python`` on PATH whose exit code is driven per-manifest by env vars, +so no Docker, Postgres, or privilege drop is involved. The CMD that the +entrypoint exec's at the end is ``true`` (exit 0), so a clean run exits 0. +""" + +from __future__ import annotations + +import os +import shutil +import stat +import subprocess +from pathlib import Path + +import pytest + +from tests.unit.docker.conftest import DOCKER_DIR + +pytestmark = [pytest.mark.unit] + +ENTRYPOINT = DOCKER_DIR / "entrypoint-runtime.sh" + +# Stub "python" that emulates util_schema_fingerprint stamp exit codes. +# It parses the --manifest value out of its args and looks up an exit code from +# an env var (STUB_RC_), defaulting to 0 (success). Any non-stamp +# invocation (e.g. render modules, which the entrypoint guards behind unset env +# vars and therefore never calls here) also exits 0. +_PYTHON_STUB = """#!/bin/sh +manifest="" +prev="" +for arg in "$@"; do + if [ "$prev" = "--manifest" ]; then + manifest="$arg" + fi + prev="$arg" +done +case "$manifest" in + omnibase_infra) exit "${STUB_RC_OMNIBASE_INFRA:-0}" ;; + omniintelligence) exit "${STUB_RC_OMNIINTELLIGENCE:-0}" ;; + *) exit 0 ;; +esac +""" + + +def _run_entrypoint( + tmp_path: Path, + *, + infra_rc: int = 0, + intel_rc: int = 0, + with_intel_db: bool = True, +) -> subprocess.CompletedProcess[str]: + """Run the real entrypoint with a stubbed python and controlled stamp RCs.""" + bindir = tmp_path / "bin" + bindir.mkdir() + stub = bindir / "python" + stub.write_text(_PYTHON_STUB) + stub.chmod(stub.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + env = { + # Minimal PATH: stub python first, then real sh/sed/echo/sleep tools. + "PATH": f"{bindir}:/usr/bin:/bin", + "OMNIBASE_INFRA_DB_URL": "postgresql://u:p@db:5432/omnibase_infra", + "STUB_RC_OMNIBASE_INFRA": str(infra_rc), + "STUB_RC_OMNIINTELLIGENCE": str(intel_rc), + } + if with_intel_db: + env["OMNIINTELLIGENCE_DB_URL"] = "postgresql://u:p@db:5432/omniintelligence" + + # CMD the entrypoint exec's at the end; "true" exits 0 on a clean boot path. + return subprocess.run( + ["sh", str(ENTRYPOINT), "true"], + env=env, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + +def test_clean_stamp_both_dbs_boots() -> None: + """Both stamps succeed -> entrypoint reaches exec and exits 0.""" + import tempfile + + with tempfile.TemporaryDirectory() as td: + result = _run_entrypoint(Path(td), infra_rc=0, intel_rc=0) + assert result.returncode == 0, result.stderr + assert "Schema fingerprint stamped for omnibase_infra." in result.stdout + assert "Schema fingerprint stamped for omniintelligence." in result.stdout + assert "Starting runtime kernel..." in result.stdout + + +def test_secondary_db_stamp_failure_is_nonfatal() -> None: + """omniintelligence stamp fails (perm denied) -> WARNING, boot continues (exit 0).""" + import tempfile + + with tempfile.TemporaryDirectory() as td: + # rc=1 emulates "permission denied for table db_metadata" general error. + result = _run_entrypoint(Path(td), infra_rc=0, intel_rc=1) + assert result.returncode == 0, result.stderr + assert "Schema fingerprint stamped for omnibase_infra." in result.stdout + assert ( + "WARNING: omniintelligence (secondary/non-owned DB) fingerprint stamp " + "did not succeed -- continuing best-effort" in result.stdout + ) + # Boot proceeds past the stamp section. + assert "Starting runtime kernel..." in result.stdout + + +def test_primary_db_stamp_failure_is_fatal() -> None: + """omnibase_infra stamp fails -> entrypoint aborts boot (exit 1), no kernel start.""" + import tempfile + + with tempfile.TemporaryDirectory() as td: + result = _run_entrypoint(Path(td), infra_rc=1, intel_rc=0) + assert result.returncode == 1 + assert ( + "ERROR: omnibase_infra (PRIMARY/owned DB) fingerprint stamp failed " + "-- aborting boot" in result.stderr + ) + # Must NOT reach kernel start when the primary DB stamp fails. + assert "Starting runtime kernel..." not in result.stdout + + +def test_secondary_db_optional_when_db_url_unset() -> None: + """No OMNIINTELLIGENCE_DB_URL -> stamp skipped, boot proceeds (exit 0).""" + import tempfile + + with tempfile.TemporaryDirectory() as td: + result = _run_entrypoint(Path(td), infra_rc=0, with_intel_db=False) + assert result.returncode == 0, result.stderr + assert ( + "OMNIINTELLIGENCE_DB_URL not set -- skipping omniintelligence " + "fingerprint stamp" in result.stdout + ) + assert "Starting runtime kernel..." in result.stdout + + +# --------------------------------------------------------------------------- +# Static guards on the script source -- keep the required/optional contract. +# --------------------------------------------------------------------------- + + +def test_entrypoint_marks_primary_required_and_secondary_optional() -> None: + source = ENTRYPOINT.read_text() + assert ( + 'stamp_fingerprint "omnibase_infra" "${OMNIBASE_INFRA_DB_URL}" "required"' + in source + ) + assert ( + 'stamp_fingerprint "omniintelligence" "${OMNIINTELLIGENCE_DB_URL}" "optional"' + in source + ) + + +def test_required_stamp_failure_exits_nonzero_in_source() -> None: + source = ENTRYPOINT.read_text() + # The required branch must exit non-zero; the optional branch must only warn. + assert 'if [ "${REQUIRED}" = "required" ]; then' in source + assert "exit 1" in source.split('if [ "${REQUIRED}" = "required" ]; then', 1)[1] + + +def test_shellcheck_clean_if_available() -> None: + shellcheck = shutil.which("shellcheck") + if shellcheck is None: + pytest.skip("shellcheck not installed") + result = subprocess.run( + [shellcheck, str(ENTRYPOINT)], + capture_output=True, + text=True, + timeout=60, + env={**os.environ}, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr