diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d202caca3..d5ca0ac42c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1922,6 +1922,170 @@ jobs: PY echo "All 24 tables present." + # ============================================================================ + # Integration Silent-Skip Guard (OMN-14172) + # An @pytest.mark.integration real-DB test that self-skips because Postgres + # looks absent is a FALSE-GREEN: the required Tests job goes green while the + # real-DB assertion never ran (the exact class of gap that let the omnimarket + # OMN-14058 missing-column defect and the OMN-9041 UUID-cast crash reach dev + # via a silent skip). Detection-only sweeps get ignored, so this ships as a + # mechanical CI gate + pre-commit hook in the same PR (Operating Rule #5). + # + # This job PROVISIONS Postgres (mirroring migration-integration), applies all + # migrations, exports the connection env, and runs a CURATED set of + # Postgres-only integration proofs, then asserts via check_integration_skips.py + # that (1) none skipped citing a *provisioned* service's absence and (2) at + # least one integration test actually executed (zero-collection is itself a + # false-green). Legitimately-optional skips (Kafka/Redpanda broker not + # provisioned by design, live-LLM opt-in flags, catalog-data conditions) remain + # allowed — the gate is keyed to the services this job actually provisions + # (scripts/ci/integration_skip_guard.yaml). + # + # Wired as BLOCKING via the CI Summary umbrella context: this job's display + # name is in ci_summary_gate.py SKIPPABLE_GATE_JOBS (present + success/skipped + # required, same tier as migration-integration) AND the default-deny sweep + # fails CI Summary on any non-success/skipped conclusion. No NEW + # branch-protection required context is registered here — that is deferred to + # the operator/Codex once the job proves green on real PRs. + # SYNC: scripts/ci/check_integration_skips.py + pre-commit hook + # 'integration-skip-guard'. Update both when the config changes. + # ============================================================================ + integration-guard: + name: Integration Silent-Skip Guard (OMN-14172) + needs: [occ-preflight, zone-filter] + if: needs.occ-preflight.result == 'success' && (needs.zone-filter.outputs.docs_only != 'true') + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 15 + # Same Postgres service block as migration-integration: a real DB so the + # @pytest.mark.integration real-DB proofs EXECUTE instead of self-skipping. + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_PASSWORD: test_password + POSTGRES_DB: omnibase_infra + ports: + - 5432/tcp + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Setup Python and uv + uses: ./.github/actions/setup-python-uv + with: + python-version: ${{ env.PYTHON_VERSION }} + uv-version: ${{ env.UV_VERSION }} + cache-version: ${{ env.CACHE_VERSION }} + cache-enabled: "false" + install-args: '--reinstall-package omnibase-core --reinstall-package omnibase-spi --frozen' + + - name: Resolve Postgres service host + id: postgres_host + env: + POSTGRES_PORT: ${{ job.services.postgres.ports['5432'] }} + run: | + python - <<'PY' >> "$GITHUB_OUTPUT" + import os + import socket + + port = int(os.environ["POSTGRES_PORT"]) + + def can_connect(host: str) -> bool: + try: + with socket.create_connection((host, port), timeout=1.0): + return True + except OSError: + return False + + if can_connect("127.0.0.1"): + print("host=127.0.0.1") + raise SystemExit(0) + + with open("/proc/net/route", encoding="utf-8") as routes: + for line in routes.readlines()[1:]: + fields = line.split() + if len(fields) >= 3 and fields[1] == "00000000": + gateway_hex = fields[2] + gateway = ".".join( + str(int(gateway_hex[i : i + 2], 16)) + for i in (6, 4, 2, 0) + ) + if can_connect(gateway): + print(f"host={gateway}") + raise SystemExit(0) + + raise SystemExit( + f"could not reach Postgres service on localhost or default gateway port {port}" + ) + PY + + - name: Apply all migrations + env: + OMNIBASE_INFRA_DB_URL: postgresql://postgres:test_password@${{ steps.postgres_host.outputs.host }}:${{ job.services.postgres.ports['5432'] }}/omnibase_infra + run: | + uv run python scripts/run-migrations.py + + # Export the connection env the curated integration proofs read. The + # canonical PostgresConfig.from_env() reads ONLY OMNIBASE_INFRA_DB_URL (a + # full DSN, no POSTGRES_* fallback), so that var is what flips the curated + # proofs from "skipped" to "executed"; the POSTGRES_* set is exported too so + # the guard stays correct if the curated set broadens to tests that read + # discrete vars. + - name: Export integration Postgres env + env: + RESOLVED_HOST: ${{ steps.postgres_host.outputs.host }} + RESOLVED_PORT: ${{ job.services.postgres.ports['5432'] }} + run: | + { + echo "OMNIBASE_INFRA_DB_URL=postgresql://postgres:test_password@${RESOLVED_HOST}:${RESOLVED_PORT}/omnibase_infra" + echo "POSTGRES_HOST=${RESOLVED_HOST}" + echo "POSTGRES_PORT=${RESOLVED_PORT}" + echo "POSTGRES_USER=postgres" + echo "POSTGRES_PASSWORD=test_password" + echo "POSTGRES_DB=omnibase_infra" + echo "POSTGRES_DATABASE=omnibase_infra" + } >> "$GITHUB_ENV" + echo "Exported integration Postgres env (host=${RESOLVED_HOST} port=${RESOLVED_PORT})" + + # Run ONLY the curated Postgres-only integration proofs (kept in sync with + # integration_skip_guard.yaml curated_test_paths). The selected proof is + # Postgres-only and self-contained against the freshly-migrated DB. Real integration + # FAILURES fail this step directly; the silent-skip enforcement below runs + # regardless (if: always()). + - name: Run curated Postgres integration proofs (Postgres provisioned) + run: | + uv run pytest \ + tests/integration/handlers/test_registration_storage_postgres_uuid_cast.py::TestRegistrationStoragePostgresUuidCast::test_query_round_trip_produces_stdlib_uuid \ + -rs -v --tb=short \ + --junitxml=integration-guard.xml + + # Fail-closed silent-skip enforcement. RED if any integration test skipped + # citing a provisioned service's absence, or if zero integration tests ran. + - name: Enforce no missing-service silent-skips (OMN-14172) + if: always() + run: | + uv run python scripts/ci/check_integration_skips.py \ + --junit integration-guard.xml + + - name: Upload integration-guard results + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-guard-results + path: integration-guard.xml + retention-days: 7 + ai-slop-check: needs: occ-preflight timeout-minutes: 30 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2fdf54a81a..3992e1057a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -807,6 +807,25 @@ repos: files: ^contracts/llm_endpoints\.yaml$ pass_filenames: false stages: [pre-commit] + # OMN-14172: integration silent-skip false-green gate. An + # @pytest.mark.integration real-DB test that self-skips because a + # PROVISIONED service (Postgres) is absent is a false-green — the required + # Tests job goes green while the real-DB assertion never ran (the exact gap + # that let OMN-9041/OMN-14058-class defects reach dev). The --selftest runs + # the gate's own logic against synthetic pass/fail JUnit reports (case (a) + # provisioned -> PASS, case (b) reintroduced missing-service silent skip -> + # RED) so the gate cannot silently regress, and validates the shipped + # integration_skip_guard.yaml parses. No DB needed locally — the CI job + # 'integration-guard' is the full runtime gate. SYNC with ci.yml job + # 'integration-guard' (Operating Rule #5). + - id: integration-skip-guard + name: Integration silent-skip guard selftest (OMN-14172) + language: system + entry: uv run python scripts/ci/check_integration_skips.py --selftest + files: >- + ^(scripts/ci/check_integration_skips\.py|scripts/ci/integration_skip_guard\.yaml|tests/ci/test_check_integration_skips\.py|\.github/workflows/ci\.yml|tests/integration/runtime/db/test_postgres_repository_runtime_integration\.py|tests/integration/handlers/test_registration_storage_postgres_uuid_cast\.py)$ + pass_filenames: false + stages: [pre-commit] # OMN-13291 (W0): validator-requirements fleet gate. Reports ALL missing # required gates for omnibase_infra against omnibase_core's # architecture-handshakes/validator-requirements.yaml, ratcheted by this diff --git a/scripts/ci/check_integration_skips.py b/scripts/ci/check_integration_skips.py new file mode 100644 index 0000000000..5fc04ad935 --- /dev/null +++ b/scripts/ci/check_integration_skips.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""OMN-14172 — silent-skip false-green prevention gate (omnibase_infra). + +An `@pytest.mark.integration` real-service test that self-skips because its +DB/service/env is absent produces a *false-green*: the required Tests job +reports success while the real-DB assertion never ran. On a merge-gating job +that ACTUALLY provisions that service (Postgres, per the `integration-guard` +job), such a skip is a defect the gate must turn RED. + +This script parses the pytest JUnit-XML report produced by the guard job (which +provisions Postgres, applies migrations, and runs the curated Postgres-only +integration proofs) and fails when: + + 1. any integration test was SKIPPED citing a *provisioned* service's absence + (`required_services[*].missing_skip_patterns`), or + 2. fewer than `require_executed_min` integration tests actually EXECUTED + (guards the zero-collection / all-skip false-green — a marker typo or a + broken selector collecting nothing). + +Legitimately-optional skips — Kafka/Redpanda broker (not provisioned by design), +Consul/Vault/Qdrant, live-LLM / live-e2e opt-in flags — are never treated as +false-greens, because those resources are deliberately not provisioned on the +merge-gating job. + +Usage +----- + check_integration_skips.py --junit FILE [FILE ...] [--config CFG] [--strict] + check_integration_skips.py --selftest [--config CFG] + +`--selftest` runs the gate's own logic against synthetic pass/fail JUnit reports +(the case-(b) regression: a reintroduced missing-service silent skip MUST turn +the gate red). It needs no DB and is what the pre-commit hook runs locally. + +Exit codes: 0 = gate passed; 1 = false-green detected / gate failed; 2 = usage +or input error (fail-closed — a missing/unparsable report is a failure). + +SYNC: ci.yml job `integration-guard` + pre-commit hook `integration-skip-guard`. +""" + +from __future__ import annotations + +import argparse +import re +import tempfile +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +_DEFAULT_CONFIG = Path(__file__).resolve().parent / "integration_skip_guard.yaml" + + +@dataclass(frozen=True) +class SkipRecord: + """A single skipped test case extracted from a JUnit report.""" + + testid: str + reason: str + + +@dataclass +class GuardConfig: + silent_skip_allowed: bool + require_executed_min: int + required_service_patterns: dict[str, list[re.Pattern[str]]] + allowed_optional_patterns: list[re.Pattern[str]] = field(default_factory=list) + + @classmethod + def load(cls, path: Path) -> GuardConfig: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + req: dict[str, list[re.Pattern[str]]] = {} + for svc, spec in (raw.get("required_services") or {}).items(): + pats = [ + re.compile(p, re.IGNORECASE) + for p in (spec or {}).get("missing_skip_patterns", []) + ] + req[svc] = pats + allowed = [ + re.compile(p, re.IGNORECASE) + for p in (raw.get("allowed_optional_skip_patterns") or []) + ] + return cls( + silent_skip_allowed=bool(raw.get("silent_skip_allowed", False)), + require_executed_min=int(raw.get("require_executed_min", 1)), + required_service_patterns=req, + allowed_optional_patterns=allowed, + ) + + +@dataclass +class ReportStats: + executed: int = 0 + skipped: list[SkipRecord] = field(default_factory=list) + total_cases: int = 0 + + +def _testid(case: ET.Element) -> str: + classname = case.get("classname", "") + name = case.get("name", "") + return f"{classname}::{name}" if classname else name + + +def parse_junit(paths: list[Path]) -> ReportStats: + """Aggregate executed/skipped counts across one or more JUnit-XML files. + + A test case is "executed" when it has no child (passed, failed, or + errored all count as ran — the point is that the assertion body executed). + """ + stats = ReportStats() + for path in paths: + if not path.is_file(): + raise FileNotFoundError(f"JUnit report not found: {path}") + try: + root = ET.parse(path).getroot() # noqa: S314 — JUnit XML is CI-generated, not untrusted + except ET.ParseError as exc: # fail-closed: an unparsable report is a failure + raise ValueError(f"could not parse JUnit report {path}: {exc}") from exc + for case in root.iter("testcase"): + stats.total_cases += 1 + skipped_el = case.find("skipped") + if skipped_el is not None: + reason = skipped_el.get("message", "") or (skipped_el.text or "") + stats.skipped.append( + SkipRecord(testid=_testid(case), reason=reason.strip()) + ) + else: + stats.executed += 1 + return stats + + +def classify_skip(reason: str, cfg: GuardConfig) -> tuple[str | None, bool]: + """Return (offending_service | None, is_allowed_optional). + + offending_service is set when the skip reason names a PROVISIONED service's + absence (a false-green). is_allowed_optional is True when the reason matches + a known legitimately-optional pattern. + """ + offending: str | None = None + for svc, pats in cfg.required_service_patterns.items(): + if any(p.search(reason) for p in pats): + offending = svc + break + is_allowed = any(p.search(reason) for p in cfg.allowed_optional_patterns) + return offending, is_allowed + + +def evaluate(stats: ReportStats, cfg: GuardConfig, strict: bool) -> list[str]: + """Return a list of violation messages (empty == gate passes).""" + violations: list[str] = [] + + if cfg.silent_skip_allowed: + # Explicit, documented opt-out. Never used in practice (config pins False). + return violations + + for rec in stats.skipped: + offending, is_allowed = classify_skip(rec.reason, cfg) + if offending is not None: + violations.append( + f"FALSE-GREEN: integration test {rec.testid!r} SKIPPED because " + f"provisioned service {offending!r} looked absent — reason: " + f"{rec.reason!r}. The merge-gating job provisions {offending}; this " + f"test SHOULD have run. Wire the service env or fix the skip guard." + ) + elif strict and not is_allowed: + violations.append( + f"UNCLASSIFIED-SKIP (strict): integration test {rec.testid!r} skipped " + f"with an unrecognised reason: {rec.reason!r}. Classify it in " + f"integration_skip_guard.yaml (required_services vs " + f"allowed_optional_skip_patterns)." + ) + + if stats.executed < cfg.require_executed_min: + violations.append( + f"ZERO/UNDER-COLLECTION: only {stats.executed} integration test(s) " + f"executed (require >= {cfg.require_executed_min}). A marker typo or a " + f"broken selector collecting nothing is itself a false-green. " + f"(total cases seen: {stats.total_cases})" + ) + + return violations + + +def run_gate(paths: list[Path], cfg: GuardConfig, strict: bool) -> int: + stats = parse_junit(paths) + violations = evaluate(stats, cfg, strict) + print("=== integration silent-skip guard (OMN-14172) ===") + print( + f"cases={stats.total_cases} executed={stats.executed} " + f"skipped={len(stats.skipped)} require_executed_min={cfg.require_executed_min}" + ) + for rec in stats.skipped: + offending, is_allowed = classify_skip(rec.reason, cfg) + tag = ( + f"BAD[{offending}]" + if offending + else ("ok-optional" if is_allowed else "ok-unclassified") + ) + print(f" skip {tag}: {rec.testid} :: {rec.reason}") + if violations: + print("\nGATE FAILED — silent-skip false-green(s) detected:") + for v in violations: + print(f"::error::{v}") + return 1 + print("\nGATE PASSED — no missing-service silent-skips; integration tests ran.") + return 0 + + +# --------------------------------------------------------------------------- +# Self-test: encodes case (a) PASS and case (b) RED with synthetic JUnit reports +# that mirror real pytest output for the curated omnibase_infra Postgres proofs. +# This is what the pre-commit hook runs locally (no DB required) and is the +# regression proof that the gate cannot silently stop catching the reintroduced +# silent-skip. The skip messages below are the VERBATIM strings the two curated +# tests emit when Postgres is absent (grepped from tests/). +# --------------------------------------------------------------------------- + +_JUNIT_PASS = """ + + + + + + +""" + +_JUNIT_SILENT_SKIP = """ + + + + + + + +""" + + +def selftest(cfg: GuardConfig) -> int: + ok = True + with tempfile.TemporaryDirectory() as td: + pass_xml = Path(td) / "pass.xml" + skip_xml = Path(td) / "skip.xml" + pass_xml.write_text(_JUNIT_PASS, encoding="utf-8") + skip_xml.write_text(_JUNIT_SILENT_SKIP, encoding="utf-8") + + # Case (a): Postgres provisioned, integration tests ran -> gate PASSES. + v_pass = evaluate(parse_junit([pass_xml]), cfg, strict=False) + if v_pass: + ok = False + print(f"SELFTEST FAIL (case a should pass): {v_pass}") + else: + print("SELFTEST ok: case (a) provisioned -> PASS") + + # Case (b): missing-service silent skip reintroduced -> gate goes RED. + v_skip = evaluate(parse_junit([skip_xml]), cfg, strict=False) + if not v_skip: + ok = False + print("SELFTEST FAIL (case b should be RED): gate did not flag silent skip") + else: + print( + f"SELFTEST ok: case (b) silent-skip -> RED ({len(v_skip)} violation(s))" + ) + + if not cfg.required_service_patterns: + ok = False + print("SELFTEST FAIL: config declares no required_services") + if cfg.silent_skip_allowed: + ok = False + print("SELFTEST FAIL: silent_skip_allowed must be false") + + print("SELFTEST PASSED" if ok else "SELFTEST FAILED") + return 0 if ok else 1 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--junit", nargs="+", type=Path, help="JUnit-XML report(s)") + parser.add_argument("--config", type=Path, default=_DEFAULT_CONFIG) + parser.add_argument( + "--strict", + action="store_true", + help="also fail on integration skips with unclassified reasons", + ) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args(argv) + + try: + cfg = GuardConfig.load(args.config) + except (OSError, yaml.YAMLError, re.error) as exc: + print(f"::error::could not load guard config {args.config}: {exc}") + return 2 + + if args.selftest: + return selftest(cfg) + + if not args.junit: + print("::error::--junit is required (or use --selftest)") + return 2 + + try: + return run_gate(args.junit, cfg, args.strict) + except (FileNotFoundError, ValueError) as exc: + print(f"::error::{exc}") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/ci_summary_gate.py b/scripts/ci/ci_summary_gate.py index f561bde9c7..a00d251cee 100644 --- a/scripts/ci/ci_summary_gate.py +++ b/scripts/ci/ci_summary_gate.py @@ -94,6 +94,7 @@ # a legitimate skip path in ci.yml (docs-only diff, or event-scoped ``if:``). SKIPPABLE_GATE_JOBS: tuple[str, ...] = ( "Migration Integration Test", # migration-integration (skips on docs-only) + "Integration Silent-Skip Guard (OMN-14172)", # integration-guard (skips on docs-only) "Contract Compliance", # compliance "Contract Compliance Check", # contract-compliance "Contract Sync Gate (Wave C) [OMN-8915]", # contract-sync-gate (skips on push) diff --git a/scripts/ci/integration_skip_guard.yaml b/scripts/ci/integration_skip_guard.yaml new file mode 100644 index 0000000000..600c99f741 --- /dev/null +++ b/scripts/ci/integration_skip_guard.yaml @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# OMN-14172 — silent-skip false-green prevention gate config (omnibase_infra). +# +# WHY THIS EXISTS +# --------------- +# `@pytest.mark.integration` real-service tests self-skip (pytest.skip / skipif) +# when their DB/service/env is absent. On a merge-gating job that ACTUALLY +# provisions that service, such a skip is a *false-green*: the required Tests job +# reports success while the real-DB assertion never ran. This recurred across +# repos (OMN-10330 omninode_infra, OMN-14167 omnimarket) and each time it hid a +# genuine latent defect. In omnibase_infra the analogous real-DB proof is the +# OMN-9041 UUID-cast regression guard +# (tests/integration/handlers/test_registration_storage_postgres_uuid_cast.py) — +# a production crash that a silent Postgres-absent skip would have hidden. +# +# `silent_skip_allowed: false` is the OMN-13575 anchor — a required-integration +# test that self-skips because a PROVISIONED service is missing is NOT allowed to +# pass silently. This gate makes that class mechanically RED. +# +# HOW IT STAYS LOW-FALSE-POSITIVE +# ------------------------------- +# The gate is keyed to the services the merge-gating guard job actually +# PROVISIONS (see `required_services`). The `integration-guard` job in ci.yml +# provisions Postgres (postgres:16-alpine), applies all migrations, exports the +# connection env, and runs a CURATED set of Postgres-only integration proofs +# (see `curated_test_paths`). A skip only fails the gate when its reason names a +# *provisioned* service's absence — i.e. Postgres is right there and the test +# SHOULD have run. Legitimately-optional skips (Kafka/Redpanda broker, Consul, +# Vault, Qdrant, live-LLM / live-e2e opt-in flags, catalog-data conditions) are +# NOT provisioned by this job and are never treated as false-greens. +silent_skip_allowed: false +# Curated Postgres-only integration proofs the `integration-guard` job runs. +# INFORMATIONAL (the gate keys on junit skip reasons, not this list) — kept in +# sync with the ci.yml `integration-guard` "Run curated Postgres integration +# proofs" step. Chosen because it is Postgres-only and self-contained against a +# freshly-migrated DB: the registration test auto-creates its required +# `node_registrations` schema. +curated_test_paths: + - "tests/integration/handlers/test_registration_storage_postgres_uuid_cast.py::TestRegistrationStoragePostgresUuidCast::test_query_round_trip_produces_stdlib_uuid" +# At least this many integration tests must actually EXECUTE (not skip) on the +# provisioning job. Guards the zero-collection / all-skip false-green (a marker +# typo, a broken collection selecting nothing, or every curated proof silently +# skipping because Postgres looked absent). +require_executed_min: 1 +# Services the merge-gating guard job PROVISIONS. A skip whose reason matches one +# of these absence patterns is a false-green -> FAIL. Patterns are Python regex, +# matched case-insensitively against the pytest skip reason (message + location). +# The patterns cover the full omnibase_infra Postgres-absence skip vocabulary +# (grepped from tests/) so the gate stays correct if the curated set broadens. +required_services: + postgres: + provisioned_by: "ci.yml `integration-guard` job (postgres:16-alpine service, migrations applied, OMNIBASE_INFRA_DB_URL + POSTGRES_* exported)" + missing_skip_patterns: + # test_postgres_repository_runtime_integration.py db_pool fixture + - "PostgreSQL integration tests skipped" + - "Missing OMNIBASE_INFRA_DB_URL" + - "PostgreSQL configured but not reachable" + # test_registration_storage_postgres_uuid_cast.py / dlq / snapshot skipif + - "PostgreSQL not available" + # ledger / injection_effectiveness conftests + - "PostgreSQL not configured" + - "PostgreSQL not reachable" + # snapshot db_pool fixture + - "PostgreSQL DSN not available" + # projector_shell / snapshot generic-DB skips + - "Database not configured" + - 'Database not reachable' + # registration_contract_verify + - "No database URL configured" +# Skip reasons that are legitimately optional even for an integration-marked test. +# These name resources the merge-gating job deliberately does NOT provision, so a +# skip on them is expected, not a false-green. Used by `--strict` to assert every +# observed skip is classified (so a NEW, unrecognised skip reason gets a human +# look instead of silently passing). The default (non-strict) gate never fails on +# these — it only fails on `required_services` patterns above. +allowed_optional_skip_patterns: + - "Redpanda broker not reachable" # Kafka/Redpanda broker not provisioned + - "Kafka not reachable" # Kafka/Redpanda broker not provisioned + - "Kafka broker" # Kafka/Redpanda broker not provisioned + - "Consul" # Consul not provisioned in the guard job + - "Vault" # Vault not provisioned in the guard job + - "Qdrant" # Qdrant not provisioned in the guard job + - "Valkey" # Valkey not provisioned in the guard job + - "manifest not found in catalog" # catalog-data condition, not a service absence + - "not reachable at http" # live LLM endpoint SLO probes (opt-in) + - "OMNI_ALLOW" # generic live opt-in flag phrasing + - "set OMN_ALLOW" # generic live opt-in flag phrasing + - "requires @pytest.mark.integration" # conftest guard for opt-in bus fixtures + - "Docker" # docker-in-docker integration tests (opt-in) diff --git a/tests/ci/test_check_integration_skips.py b/tests/ci/test_check_integration_skips.py new file mode 100644 index 0000000000..101d361843 --- /dev/null +++ b/tests/ci/test_check_integration_skips.py @@ -0,0 +1,214 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Tests for the integration silent-skip false-green gate (OMN-14172). + +Encodes the regression the gate exists to prevent: an `@pytest.mark.integration` +real-service test that self-skips because a PROVISIONED service (Postgres) looks +absent must turn the gate RED (case b), while a job where the same tests actually +ran — plus a legitimately-optional Kafka-broker skip — must PASS (case a). + +The synthetic JUnit fixtures below mirror the exact schema pytest emits for the +two curated omnibase_infra Postgres proofs +(``test_postgres_repository_runtime_integration`` — creates its own table — and +``test_registration_storage_postgres_uuid_cast`` — the OMN-9041 UUID-cast +regression guard). The ```` strings are the verbatim +Postgres-absence reasons those tests emit, grepped from ``tests/``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.ci.check_integration_skips import ( + GuardConfig, + evaluate, + main, + parse_junit, + selftest, +) + +_CONFIG = ( + Path(__file__).resolve().parents[2] + / "scripts" + / "ci" + / "integration_skip_guard.yaml" +) + +# Postgres provisioned: the two real-DB proofs RAN; only a Kafka-broker skip. +_JUNIT_PROVISIONED = """ + + + + + + +""" + +# Postgres removed / OMNIBASE_INFRA_DB_URL unset: both real-DB proofs SILENTLY +# SKIP. This is the exact false-green that would let an OMN-9041-class defect +# reach dev. +_JUNIT_SILENT_SKIP = """ + + + + + + + +""" + + +@pytest.fixture +def cfg() -> GuardConfig: + return GuardConfig.load(_CONFIG) + + +def _write(tmp_path: Path, name: str, body: str) -> Path: + p = tmp_path / name + p.write_text(body, encoding="utf-8") + return p + + +@pytest.mark.unit +def test_shipped_config_is_fail_closed(cfg: GuardConfig) -> None: + assert cfg.silent_skip_allowed is False + assert "postgres" in cfg.required_service_patterns + assert cfg.required_service_patterns["postgres"], "postgres needs skip patterns" + assert cfg.require_executed_min >= 1 + + +@pytest.mark.unit +def test_case_a_provisioned_service_passes(cfg: GuardConfig, tmp_path: Path) -> None: + junit = _write(tmp_path, "a.xml", _JUNIT_PROVISIONED) + stats = parse_junit([junit]) + assert stats.executed == 2 # both real-DB proofs ran + assert evaluate(stats, cfg, strict=False) == [] + + +@pytest.mark.unit +def test_case_b_silent_skip_goes_red(cfg: GuardConfig, tmp_path: Path) -> None: + junit = _write(tmp_path, "b.xml", _JUNIT_SILENT_SKIP) + stats = parse_junit([junit]) + assert stats.executed == 0 + violations = evaluate(stats, cfg, strict=False) + assert violations, "reintroduced missing-service silent skip must be caught" + joined = " ".join(violations) + assert "FALSE-GREEN" in joined + assert "postgres" in joined + + +@pytest.mark.unit +def test_case_b_main_exit_code_is_nonzero(cfg: GuardConfig, tmp_path: Path) -> None: + junit = _write(tmp_path, "b.xml", _JUNIT_SILENT_SKIP) + assert main(["--junit", str(junit), "--config", str(_CONFIG)]) == 1 + + +@pytest.mark.unit +def test_case_a_main_exit_code_is_zero(cfg: GuardConfig, tmp_path: Path) -> None: + junit = _write(tmp_path, "a.xml", _JUNIT_PROVISIONED) + assert main(["--junit", str(junit), "--config", str(_CONFIG)]) == 0 + + +@pytest.mark.unit +def test_kafka_broker_skip_is_not_a_false_green( + cfg: GuardConfig, tmp_path: Path +) -> None: + body = """ + + + +""" + junit = _write(tmp_path, "kafka.xml", body) + assert evaluate(parse_junit([junit]), cfg, strict=False) == [] + + +@pytest.mark.unit +def test_catalog_manifest_skip_is_not_a_false_green( + cfg: GuardConfig, tmp_path: Path +) -> None: + # tests/integration/test_catalog_extra_networks.py emits this catalog-data + # skip; it names "postgres" but is NOT a service-absence false-green. + body = """ + + + +""" + junit = _write(tmp_path, "catalog.xml", body) + assert evaluate(parse_junit([junit]), cfg, strict=False) == [] + + +@pytest.mark.unit +def test_all_postgres_absence_strings_are_caught( + cfg: GuardConfig, tmp_path: Path +) -> None: + # Every real Postgres-absence skip string grepped from tests/ must classify + # as a false-green (offending service == postgres), so the gate stays correct + # if the curated set broadens. + reasons = [ + "PostgreSQL integration tests skipped: PostgreSQL configured but not reachable", + "PostgreSQL not available (set OMNIBASE_INFRA_DB_URL)", + "PostgreSQL not configured or not reachable", + "PostgreSQL not reachable", + "PostgreSQL DSN not available", + "Database not configured (set OMNIBASE_INFRA_DB_URL or POSTGRES_PASSWORD)", + "Database not reachable: Connection refused", + "No database URL configured (set OMNIBASE_INFRA_DB_URL)", + ] + cases = "".join( + f'' + for i, reason in enumerate(reasons) + ) + body = ( + '' + f'' + '' + f"{cases}" + ) + junit = _write(tmp_path, "all.xml", body) + violations = evaluate(parse_junit([junit]), cfg, strict=False) + assert len(violations) == len(reasons), violations + assert all("FALSE-GREEN" in v for v in violations) + + +@pytest.mark.unit +def test_zero_collection_is_a_false_green(cfg: GuardConfig, tmp_path: Path) -> None: + body = '' + junit = _write(tmp_path, "empty.xml", body) + violations = evaluate(parse_junit([junit]), cfg, strict=False) + assert any("UNDER-COLLECTION" in v for v in violations) + + +@pytest.mark.unit +def test_missing_report_fails_closed(cfg: GuardConfig) -> None: + assert main(["--junit", "/nonexistent/nope.xml", "--config", str(_CONFIG)]) == 2 + + +@pytest.mark.unit +def test_strict_flags_unclassified_skip(cfg: GuardConfig, tmp_path: Path) -> None: + body = """ + + + +""" + junit = _write(tmp_path, "weird.xml", body) + assert evaluate(parse_junit([junit]), cfg, strict=False) == [] + assert any( + "UNCLASSIFIED-SKIP" in v + for v in evaluate(parse_junit([junit]), cfg, strict=True) + ) + + +@pytest.mark.unit +def test_embedded_selftest_passes(cfg: GuardConfig) -> None: + assert selftest(cfg) == 0 diff --git a/tests/integration/handlers/test_registration_storage_postgres_uuid_cast.py b/tests/integration/handlers/test_registration_storage_postgres_uuid_cast.py index bf64f8fd43..29becc0534 100644 --- a/tests/integration/handlers/test_registration_storage_postgres_uuid_cast.py +++ b/tests/integration/handlers/test_registration_storage_postgres_uuid_cast.py @@ -90,6 +90,7 @@ async def handler( database=pg["database"], user=pg["user"], password=pg["password"], + auto_create_schema=True, ) yield handler await handler.shutdown()