diff --git a/contracts/OMN-13011.yaml b/contracts/OMN-13011.yaml new file mode 100644 index 0000000000..cdf155077e --- /dev/null +++ b/contracts/OMN-13011.yaml @@ -0,0 +1,89 @@ +# OMN-13011 contract file for omnibase_infra deploy-gate. +# The canonical ticket contract and PASS receipts live in onex_change_control. +schema_version: "1.0.0" +ticket_id: "OMN-13011" +title: "LANE CENSUS RECONCILIATION ratchet — declared desired-state per lane, scheduled + on-demand, drift = auto-ticket" +summary: >- + The class fix for the recurring lane-drift regression. Nothing reconciled the + declared desired state of a runtime lane against what is actually running, so + the same failure kept recurring with zero signal: volume config drift + (OMN-12945), WORKER_REPLICAS silent zero (OMN-12988/12990), and on 2026-06-11 + prod runtime containers plus the broker network were silently absent for hours + during demo prep. This ships a per-lane DESIRED-STATE census: (a) DECLARED in a + versioned lane manifest (deploy/lane-census/lane-manifest.yaml) — container set, + network, replicas, image-tag pattern per lane (stability-test/prod/judge/dev), + derived from the canonical compose lane files; (b) RECONCILED on a schedule on + .201 by sharing the OMN-13008 systemd timer (a drop-in 4th ExecStart on + onex-disk-gc.service, never a second timer) and on-demand via runtime_sweep; + (c) drift = a typed bus event + Linear auto-ticket naming exactly what is + missing/extra (container_absent, network_detached, replicas_zero, + unexpected_container, oneshot_failed/stuck, image_tag_mismatch). Fail-fast, + no warn-only mode (gates-block policy). Builds on the OMN-12988 deploy-agent + RUNTIME census (deploy-time) as the complementary steady-state reconciler. +is_seam_ticket: false +interface_change: false +interfaces_touched: [] +evidence_requirements: + - kind: "tests" + description: "Pure-planner drift tests incl. the 2026-06-11 prod red fixture pass" + command: >- + uv run pytest tests/unit/scripts/test_lane_census_plan.py -q + - kind: "tests" + description: "Event-builder, manifest-parity, dry-run, and timer-dropin tests pass" + command: >- + uv run pytest + tests/unit/scripts/test_lane_census_event.py + tests/unit/scripts/test_lane_census_manifest_parity.py + tests/unit/scripts/test_lane_census_dry_run.py + tests/unit/scripts/test_lane_census_timer_dropin.py -q + - kind: "ci" + description: "New source passes ruff and mypy --strict" + command: >- + uv run ruff check scripts/lane_census_plan.py scripts/lane_census_event.py + && uv run mypy scripts/lane_census_plan.py scripts/lane_census_event.py --strict + - kind: "manual" + description: >- + On .201, after OMN-13008's install-disk-gc.sh, run + deploy/lane-census/install-lane-census.sh and confirm via + `systemctl --user cat onex-disk-gc.service` that the lane-census ExecStart + is the 4th pass on the shared unit (no second timer) +emergency_bypass: + enabled: false + justification: "" + follow_up_ticket_id: "" +dod_evidence: + - id: "dod-planner-red-fixture" + description: "Pure planner reports network_detached + container_absent for the 2026-06-11 prod outage" + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/scripts/test_lane_census_plan.py -q + - id: "dod-event-and-ticket" + description: "Typed drift event builds stable alert_key + ticket title/body naming exactly what is wrong" + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/scripts/test_lane_census_event.py -q + - id: "dod-manifest-parity" + description: "Lane manifest is locked in step with the compose lane files; no service declares replicas 0" + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/scripts/test_lane_census_manifest_parity.py -q + - id: "dod-dry-run-failfast" + description: "Dry-run never publishes; drift exits 30; no broker => no localhost-default publish" + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/scripts/test_lane_census_dry_run.py -q + - id: "dod-shared-timer" + description: "Lane-census shares the OMN-13008 onex-disk-gc.service via drop-in; no second timer" + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/scripts/test_lane_census_timer_dropin.py -q diff --git a/deploy/lane-census/install-lane-census.sh b/deploy/lane-census/install-lane-census.sh new file mode 100755 index 0000000000..5caabd40d9 --- /dev/null +++ b/deploy/lane-census/install-lane-census.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# install-lane-census.sh — Register the lane-census reconcile on the SHARED +# onex-disk-gc.service systemd USER timer (OMN-13011). +# +# COORDINATION (OMN-13008 / PR #1952): the disk-GC PR owns onex-disk-gc.service + +# onex-disk-gc.timer. This installer does NOT create a second timer — it drops a +# systemd override (onex-disk-gc.service.d/20-lane-census.conf) that appends the +# lane-census ExecStart to that one unit, then reloads. The reconcile runs on the +# same hourly tick as the GC/watermark passes. +# +# Prerequisite: OMN-13008's `bash deploy/disk-gc/install-disk-gc.sh` must have run +# first (it installs the base onex-disk-gc.service/.timer). If the base unit is +# absent this installer fails fast and tells you to run the disk-gc installer. +# +# Usage (on 192.168.86.201, user scope, no sudo): +# bash deploy/lane-census/install-lane-census.sh # install drop-in + reload +# bash deploy/lane-census/install-lane-census.sh --uninstall +# bash deploy/lane-census/install-lane-census.sh --status + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +DROPIN_SRC="${SCRIPT_DIR}/onex-disk-gc.service.d/20-lane-census.conf" +USER_UNIT_DIR="${HOME}/.config/systemd/user" +BASE_SERVICE="${USER_UNIT_DIR}/onex-disk-gc.service" +DROPIN_DST_DIR="${USER_UNIT_DIR}/onex-disk-gc.service.d" +DROPIN_DST="${DROPIN_DST_DIR}/20-lane-census.conf" + +if [[ "${1:-}" == "--uninstall" ]]; then + echo "Removing lane-census drop-in..." + rm -f "$DROPIN_DST" + systemctl --user daemon-reload 2>/dev/null || true + echo "Done. lane-census drop-in removed (base onex-disk-gc unit untouched)." + exit 0 +fi + +if [[ "${1:-}" == "--status" ]]; then + echo "Effective onex-disk-gc.service (base + drop-ins):" + systemctl --user cat onex-disk-gc.service --no-pager 2>/dev/null || \ + echo " (base unit not installed — run deploy/disk-gc/install-disk-gc.sh first)" + exit 0 +fi + +if [[ ! -f "$BASE_SERVICE" ]]; then + echo "ERROR: base onex-disk-gc.service not installed at $BASE_SERVICE" >&2 + echo "Run OMN-13008's installer first: bash deploy/disk-gc/install-disk-gc.sh" >&2 + exit 1 +fi + +echo "Installing lane-census drop-in onto the shared onex-disk-gc.service..." +chmod +x "${SCRIPT_DIR}/../../scripts/lane-census-check.sh" 2>/dev/null || true +mkdir -p "$DROPIN_DST_DIR" +cp "$DROPIN_SRC" "$DROPIN_DST" +systemctl --user daemon-reload + +echo "" +echo "Done. lane-census reconcile registered on the shared onex-disk-gc timer." +echo " Drop-in: $DROPIN_DST" +echo "" +echo "Verify: systemctl --user cat onex-disk-gc.service # base + lane-census ExecStart" +echo "Run now: systemctl --user start onex-disk-gc.service" +echo "Logs: journalctl --user -u onex-disk-gc.service -f" +echo "On-demand: bash scripts/lane-census-check.sh --json" diff --git a/deploy/lane-census/lane-manifest.yaml b/deploy/lane-census/lane-manifest.yaml new file mode 100644 index 0000000000..5919e34ec6 --- /dev/null +++ b/deploy/lane-census/lane-manifest.yaml @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# lane-manifest.yaml — Versioned DESIRED-STATE census for the .201 runtime lanes (OMN-13011). +# +# THE CLASS FIX. We kept regressing the same way — nothing reconciled desired vs +# actual lane state: +# - volume config drift (OMN-12945 family) +# - WORKER_REPLICAS silent zero (OMN-12988 / OMN-12990) +# - 2026-06-11: prod runtime containers + broker network silently absent for +# hours during demo prep, with ZERO alert. +# +# This file is the single source of truth for what each lane MUST look like in +# steady state. scripts/lane_census_plan.py reads it and diffs it against the +# live `docker ps` / `docker network ls` inventory; scripts/lane-census-check.sh +# drives the reconcile, emits a typed bus event, and fails fast (non-zero) on any +# drift. Nothing about lane shape is hardcoded in the scripts — it all lives here. +# +# Maintenance: the container_name and network values below are derived from the +# canonical compose lane files (docker/docker-compose..yml `container_name` +# fields and the `networks:` block). When a lane gains/loses a container or +# renames its network, update BOTH the compose file and this manifest in the same +# PR. tests/unit/scripts/test_lane_census_manifest_parity.py asserts the manifest +# stays in lock-step with the compose files so the manifest can never silently +# drift away from the lane it is supposed to police. +# +# Container kinds: +# service — long-running; MUST be in `docker ps` Running state. Absence = drift. +# oneshot — run-to-completion migration/init container; expected to have Exited +# (0). It is NOT drift for a oneshot to be absent (compose removes it) +# or Exited; it IS drift for a oneshot to be Running-stuck or Exited +# non-zero. Tracked, not required-running. +# +# replicas: the REQUIRED running count for a service container. 1 unless a lane +# explicitly scales out. A service whose replicas resolve to 0 is itself drift +# (the WORKER_REPLICAS silent-zero regression): declaring replicas: 0 here is +# forbidden by the parity test. +# +# image_tag_pattern: a regex every running container's image tag must match for +# the lane. Guards against a lane silently running a stale/wrong generation. The +# special token "${RUNTIME_TAG}" is resolved at check time from the deploy-agent +# resolved runtime version; absence of a resolved tag relaxes the check to the +# default pattern (never a hard failure on an unresolvable tag — that is a +# separate signal, not a census drift). + +schema_version: "1.0.0" + +# Default image-tag pattern applied to a lane that does not override it. +default_image_tag_pattern: '.+' + +lanes: + stability-test: + compose_file: docker/docker-compose.stability-test.yml + compose_project: omnibase-infra-stability-test + network: omnibase-infra-stability-test-network + image_tag_pattern: '.+' + services: + - name: omnibase-infra-stability-test-postgres + kind: service + replicas: 1 + - name: omnibase-infra-stability-test-redpanda + kind: service + replicas: 1 + - name: omnibase-infra-stability-test-redpanda-partition-cap + kind: oneshot + - name: omnibase-infra-stability-test-valkey + kind: service + replicas: 1 + - name: omnibase-infra-stability-test-keycloak + kind: service + replicas: 1 + - name: omnibase-infra-stability-test-forward-migration + kind: oneshot + - name: omnibase-infra-stability-test-migration-gate + kind: oneshot + - name: omnibase-infra-stability-test-intelligence-migration + kind: oneshot + - name: omninode-stability-test-runtime + kind: service + replicas: 1 + - name: omninode-stability-test-runtime-effects + kind: service + replicas: 1 + - name: omnimarket-stability-test-projection-api + kind: service + replicas: 1 + # The worker that OMN-12988 / OMN-12990 silently scaled to 0. Declared + # replicas: 1 here so its absence (replicas 0 => no container) is loud drift. + - name: omninode-stability-test-runtime-worker + kind: service + replicas: 1 + + prod: + compose_file: docker/docker-compose.prod.yml + compose_project: omnibase-infra-prod + network: omnibase-infra-prod-network + image_tag_pattern: '.+' + services: + - name: omnibase-infra-prod-postgres + kind: service + replicas: 1 + - name: omnibase-infra-prod-redpanda + kind: service + replicas: 1 + - name: omnibase-infra-prod-redpanda-partition-cap + kind: oneshot + - name: omnibase-infra-prod-valkey + kind: service + replicas: 1 + - name: omnibase-infra-prod-forward-migration + kind: oneshot + - name: omnibase-infra-prod-migration-gate + kind: oneshot + - name: omnibase-infra-prod-intelligence-migration + kind: oneshot + # The four runtime services that were silently absent on 2026-06-11. + - name: omninode-prod-runtime + kind: service + replicas: 1 + - name: omninode-prod-runtime-effects + kind: service + replicas: 1 + - name: omnimarket-prod-projection-api + kind: service + replicas: 1 + - name: omninode-prod-runtime-worker + kind: service + replicas: 1 + + judge: + compose_file: docker/docker-compose.judge.yml + compose_project: omnibase-infra-judge + network: omnibase-infra-judge-network + image_tag_pattern: '.+' + services: + - name: omnibase-infra-judge-postgres + kind: service + replicas: 1 + - name: omnibase-infra-judge-redpanda + kind: service + replicas: 1 + - name: omnibase-infra-judge-redpanda-partition-cap + kind: oneshot + - name: omnibase-infra-judge-valkey + kind: service + replicas: 1 + - name: omnibase-infra-judge-forward-migration + kind: oneshot + - name: omnibase-infra-judge-migration-gate + kind: oneshot + - name: omnibase-infra-judge-intelligence-migration + kind: oneshot + - name: omninode-judge-runtime + kind: service + replicas: 1 + - name: omninode-judge-runtime-effects + kind: service + replicas: 1 + - name: omnimarket-judge-projection-api + kind: service + replicas: 1 + + # The dev lane (compose project `omnibase-infra`, the default infra stack) uses + # generated compose with non-prefixed container names; its census is the + # deploy-agent RUNTIME-scope verifier (OMN-12988). The lane manifest treats dev + # as OPTIONAL (reconcile-when-present): dev is a developer lane that is legitimately + # down most of the time, so its absence is not a steady-state regression. When dev + # IS up, the listed services must be running. This avoids a noisy false-positive + # ticket every time a developer stops their local stack. + dev: + compose_file: docker/docker-compose.generated.yml + compose_project: omnibase-infra + network: omnibase-infra_default + image_tag_pattern: '.+' + optional: true + services: + - name: omninode-runtime + kind: service + replicas: 1 + - name: omninode-runtime-effects + kind: service + replicas: 1 diff --git a/deploy/lane-census/onex-disk-gc.service.d/20-lane-census.conf b/deploy/lane-census/onex-disk-gc.service.d/20-lane-census.conf new file mode 100644 index 0000000000..fb0494d813 --- /dev/null +++ b/deploy/lane-census/onex-disk-gc.service.d/20-lane-census.conf @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# 20-lane-census.conf — systemd drop-in extending the SHARED onex-disk-gc.service +# with the lane-census reconcile pass (OMN-13011). +# +# COORDINATION (OMN-13008 / PR #1952): the disk-GC PR owns onex-disk-gc.service + +# onex-disk-gc.timer. Rather than install a SECOND timer (per the ticket: "share +# the timer unit rather than adding a second"), this drop-in appends a 4th +# ExecStart to that ONE unit. systemd runs drop-ins after the base unit's own +# ExecStart lines, so the lane-census reconcile runs after the GC passes on the +# same hourly tick. +# +# Why a drop-in instead of editing the base .service: the base unit lives in a +# separate in-flight PR. A drop-in lets both PRs land independently with zero +# merge collision on the unit file. When OMN-13008 merges, install-lane-census.sh +# copies this drop-in into ~/.config/systemd/user/onex-disk-gc.service.d/ and +# reloads — no second timer, no duplicate schedule. +# +# Install (on .201, user scope, after OMN-13008's install-disk-gc.sh has run): +# bash deploy/lane-census/install-lane-census.sh +# +# Check: +# systemctl --user cat onex-disk-gc.service # shows base + this drop-in +# journalctl --user -u onex-disk-gc.service -f + +[Service] +# Reconcile every declared lane against actual. Non-zero exit on drift is the +# expected maintenance signal (the watermark pass uses the same '-' fail-soft +# convention so one drift does not fail the whole maintenance unit). The drift +# event is published to the bus inside the script when KAFKA_BOOTSTRAP_SERVERS +# is present. +ExecStart=-/bin/bash %h/Code/omni_home/omnibase_infra/scripts/lane-census-check.sh diff --git a/scripts/lane-census-check.sh b/scripts/lane-census-check.sh new file mode 100755 index 0000000000..25df8e09c7 --- /dev/null +++ b/scripts/lane-census-check.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# lane-census-check.sh — Reconcile declared desired-state lanes vs actual (OMN-13011). +# +# THE CLASS FIX. Nothing reconciled desired vs actual lane state, so the same +# drift kept recurring with zero signal: +# - volume config drift (OMN-12945 family) +# - WORKER_REPLICAS silent zero (OMN-12988 / OMN-12990) +# - 2026-06-11: prod runtime containers + broker network silently absent, no alert +# +# This script gathers the live docker inventory, diffs it against the versioned +# lane manifest (deploy/lane-census/lane-manifest.yaml) via the pure planner +# (scripts/lane_census_plan.py), and on drift publishes a typed bus event +# (scripts/lane_census_event.py) so the sweep auto-ticket path opens a Linear +# ticket naming exactly what is missing/extra. +# +# Fail-fast, NO warn-only mode (gates-block policy): a drift on a non-optional +# lane exits non-zero. The systemd unit treats that as a maintenance signal. +# +# Usage: +# ./scripts/lane-census-check.sh # reconcile ALL non-optional lanes +# ./scripts/lane-census-check.sh --lane prod # one lane +# ./scripts/lane-census-check.sh --dry-run # print event/plan, do NOT publish +# ./scripts/lane-census-check.sh --json # emit the plan JSON to stdout +# +# Exit codes: 0 no drift, 30 drift detected (event emitted), 2 bad args, 3 missing deps. +# +# Runs on .201 via the SHARED onex-disk-gc.timer (4th ExecStart — coordinated with +# OMN-13008 rather than a second timer). Log: ~/.local/log/onex/lane-census.log + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +LANE="" +DRY_RUN=false +EMIT_JSON=false +LOG_FILE="${HOME}/.local/log/onex/lane-census.log" +DRIFT_TOPIC="onex.evt.infra.lane-census-drift.v1" + +while [[ $# -gt 0 ]]; do + case "$1" in + --lane) LANE="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + --json) EMIT_JSON=true; shift ;; + --help|-h) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +mkdir -p "$(dirname "$LOG_FILE")" +log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [lane-census] $*" | tee -a "$LOG_FILE" >&2; } + +command -v docker >/dev/null 2>&1 || { echo "ERROR: docker not found" >&2; exit 3; } +command -v python3 >/dev/null 2>&1 || { echo "ERROR: python3 not found" >&2; exit 3; } + +HOST="${LANE_CENSUS_HOST:-$(hostname)}" +log "Starting (lane=${LANE:-ALL}, host=$HOST, $( [[ "$DRY_RUN" == true ]] && echo DRY-RUN || echo LIVE ))" + +# --------------------------------------------------------------------------- +# Gather actual state. Inventory goes to per-run scratch files (never /tmp) and +# is handed to the pure planner on stdin as a JSON envelope. No env-var size +# limit, no decision logic in bash. +# --------------------------------------------------------------------------- +SCRATCH="$(mktemp -d "$(dirname "$LOG_FILE")/lane-census.XXXXXX")" +trap 'rm -rf "$SCRATCH"' EXIT + +docker ps -a --no-trunc --format '{{json .}}' >"$SCRATCH/ps.ndjson" 2>/dev/null || : >"$SCRATCH/ps.ndjson" +docker network ls --format '{{.Name}}' >"$SCRATCH/networks.txt" 2>/dev/null || : >"$SCRATCH/networks.txt" + +# Resolve the runtime tag from the deploy-agent runtime version when available +# (relaxes to the default pattern if unresolvable — see the planner). +RUNTIME_TAG="${RUNTIME_TAG:-}" + +ENVELOPE_JSON="$( + SCRATCH_DIR="$SCRATCH" LANE="$LANE" RUNTIME_TAG="$RUNTIME_TAG" python3 -c ' +import json, os +d = os.environ["SCRATCH_DIR"] +lane = os.environ.get("LANE") or None +containers = [] +with open(os.path.join(d, "ps.ndjson")) as fh: + for line in fh: + line = line.strip() + if line: + containers.append(json.loads(line)) +networks = [n.strip() for n in open(os.path.join(d, "networks.txt")) if n.strip()] +print(json.dumps({ + "lane": lane, + "containers": containers, + "networks": networks, + "runtime_tag": os.environ.get("RUNTIME_TAG") or None, +})) +' +)" + +PLAN_JSON="$(echo "$ENVELOPE_JSON" | python3 "${SCRIPT_DIR}/lane_census_plan.py")" + +if [[ "$EMIT_JSON" == true ]]; then + echo "$PLAN_JSON" +fi + +HAS_DRIFT="$(echo "$PLAN_JSON" | python3 -c 'import json,sys;print(json.load(sys.stdin)["has_drift"])')" + +if [[ "$HAS_DRIFT" != "True" ]]; then + log "No lane drift. Desired == actual." + exit 0 +fi + +# Build the typed drift event from the plan. +EVENT_JSON="$(echo "$PLAN_JSON" | LANE_CENSUS_HOST="$HOST" python3 "${SCRIPT_DIR}/lane_census_event.py")" +log "DRIFT detected:" +# Render each finding to stderr. The event JSON is passed via env (EVENT_JSON) so +# the single-quoted heredoc body needs no shell escaping of inner Python quotes. +EVENT_JSON="$EVENT_JSON" python3 <<'PYEOF' >&2 || true +import json, os +event = json.loads(os.environ["EVENT_JSON"]) +for finding in event["findings"]: + print(" [{severity}] {kind} {container} ({lane}): {detail}".format(**finding)) +PYEOF + +if [[ "$DRY_RUN" == true ]]; then + echo "$EVENT_JSON" + log "DRY-RUN — not publishing drift event." + exit 30 +fi + +# Publish to the bus. The broker address MUST come from KAFKA_BOOTSTRAP_SERVERS — +# fail-fast, no localhost/default fallback (Rule 8). The systemd unit injects it; +# an operator running by hand must export it. We never hardcode a broker / LAN IP. +if [[ -z "${KAFKA_BOOTSTRAP_SERVERS:-}" ]]; then + log "KAFKA_BOOTSTRAP_SERVERS unset — cannot publish. Drift event logged above for manual replay." + exit 30 +fi +BOOTSTRAP="$KAFKA_BOOTSTRAP_SERVERS" +if command -v rpk >/dev/null 2>&1; then + if echo "$EVENT_JSON" | rpk topic produce "$DRIFT_TOPIC" --brokers "$BOOTSTRAP" >>"$LOG_FILE" 2>&1; then + log "published lane-census-drift event to $DRIFT_TOPIC via rpk" + else + log "FAILED to publish via rpk (broker=$BOOTSTRAP) — event logged above for manual replay" + fi +else + log "rpk not found — event logged above; install rpk or wire a producer to publish" +fi + +# Fail-fast on drift (gates-block policy, no warn-only mode). +exit 30 diff --git a/scripts/lane_census_event.py b/scripts/lane_census_event.py new file mode 100644 index 0000000000..489ca5c7aa --- /dev/null +++ b/scripts/lane_census_event.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""lane_census_event.py — Build the typed lane-census-drift bus event (OMN-13011). + +Pure builder so the drift-event schema is deterministic and unit-testable. Takes +the drift plan emitted by lane_census_plan.py and produces one JSON object on +stdout for `rpk topic produce` to publish to onex.evt.infra.lane-census-drift.v1. + +The event is the alert authority: a downstream consumer (the runtime_sweep / +sweep auto-ticket path) creates the Linear ticket naming exactly what is missing +or extra. This keeps a single ticket-creation authority rather than letting every +cron script talk to Linear directly — the same pattern as the disk-watermark +event (OMN-13008). + +The `alert_key` deduplicates: one open ticket per (host, lane, kind-set) so a +persistent outage does not spam a new ticket every reconcile tick. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +from datetime import UTC, datetime +from typing import Any + +DRIFT_TOPIC = "onex.evt.infra.lane-census-drift.v1" + +# A drift finding is a flat string mapping (lane, kind, container, detail, severity) +# as emitted by lane_census_plan.py. The plan envelope is loosely typed (str/Any) +# because it round-trips through JSON; the helpers below narrow it locally. +Finding = dict[str, str] +Plan = dict[str, Any] + + +def _findings(plan: Plan) -> list[Finding]: + """Narrow the loosely-typed plan envelope to its findings list.""" + raw = plan.get("findings", []) + return list(raw) + + +def _alert_key(host: str, plan: Plan) -> str: + """Stable dedupe key over the (host, lane, kind, container) finding set.""" + signature = sorted( + f"{f['lane']}:{f['kind']}:{f['container']}" for f in _findings(plan) + ) + digest = hashlib.sha256("|".join(signature).encode("utf-8")).hexdigest()[:16] + return f"lane-census-drift:{host}:{digest}" + + +def _ticket_title(plan: Plan) -> str: + """One-line title naming exactly what is wrong.""" + findings = _findings(plan) + if not findings: + return "lane census: no drift" + lanes = sorted({f["lane"] for f in findings}) + # Summarize by kind for a precise, greppable title. + kinds: dict[str, int] = {} + for f in findings: + kinds[f["kind"]] = kinds.get(f["kind"], 0) + 1 + kind_summary = ", ".join( + f"{count}x {kind}" for kind, count in sorted(kinds.items()) + ) + return f"fix(infra): lane drift [{', '.join(lanes)}] — {kind_summary}" + + +def _ticket_body(host: str, plan: Plan) -> str: + findings = _findings(plan) + lanes_checked: list[str] = list(plan.get("lanes_checked", [])) + lines = [ + "## Lane census drift detected", + "", + f"Host: `{host}`", + f"Lanes checked: {', '.join(lanes_checked)}", + "", + "The desired-state lane census (deploy/lane-census/lane-manifest.yaml)", + "does not match the live runtime. Drift items below name exactly what is", + "missing or extra. This is the regression class the lane-census ratchet", + "(OMN-13011) exists to catch — it would have fired on 2026-06-11 when prod", + "runtime containers and the broker network were silently absent.", + "", + "## Findings", + "", + ] + for f in findings: + lines.append( + f"- **[{f['severity']}] {f['kind']}** `{f['container']}` ({f['lane']}): {f['detail']}" + ) + lines.append("") + lines.append("## Action") + lines.append( + "Bring the lane back to declared state (compose up the absent containers / " + "reattach the network), or update the lane manifest in the same PR if the " + "desired state legitimately changed. Do not silence the check." + ) + return "\n".join(lines) + + +def build_event( + *, + host: str, + plan: Plan, + topic: str = DRIFT_TOPIC, + now: datetime | None = None, +) -> dict[str, Any]: + """Construct the typed lane-census-drift event payload.""" + now = now or datetime.now(UTC) + findings = _findings(plan) + severity = ( + "critical" if any(f["severity"] == "critical" for f in findings) else "warning" + ) + return { + "schema_version": "1.0.0", + "event_type": "lane-census-drift", + "topic": topic, + "host": host, + "emitted_at": now.isoformat(), + "severity": severity, + "lanes_checked": plan.get("lanes_checked", []), + "drift_count": len(findings), + "findings": findings, + "alert_key": _alert_key(host, plan), + "ticket_title": _ticket_title(plan), + "ticket_body": _ticket_body(host, plan), + } + + +def main() -> int: + + host = os.environ.get("LANE_CENSUS_HOST", "") + if not host: + # Fail-fast: never silently fabricate a host into the alert_key. + print("ERROR: LANE_CENSUS_HOST must be set", file=sys.stderr) + return 2 + plan = json.load(sys.stdin) + event = build_event(host=host, plan=plan) + json.dump(event, sys.stdout) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/lane_census_plan.py b/scripts/lane_census_plan.py new file mode 100644 index 0000000000..c2c859bea8 --- /dev/null +++ b/scripts/lane_census_plan.py @@ -0,0 +1,346 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""lane_census_plan.py — Pure desired-vs-actual lane reconciler (OMN-13011). + +THE CLASS FIX for the recurring lane-drift regression. Nothing reconciled the +declared desired state of a runtime lane against what is actually running, so the +same failures kept recurring with zero signal: + + - volume config drift (OMN-12945 family) + - WORKER_REPLICAS silent zero — worker scaled to 0, no alert (OMN-12988/12990) + - 2026-06-11: prod runtime containers + broker network silently absent for hours + +This module is a *pure* planner: it takes the versioned lane manifest +(deploy/lane-census/lane-manifest.yaml) as the DESIRED state and a JSON envelope +of the live docker inventory as the ACTUAL state, and emits a deterministic list +of typed drift findings. It performs NO I/O — the shell driver +(scripts/lane-census-check.sh) gathers the docker inventory and publishes the +bus event / ticket. Keeping the decision logic here makes drift detection fully +unit-testable, including the 2026-06-11 prod red fixture. + +Drift kinds (named exactly so the auto-ticket says precisely what is wrong): + container_absent — a required `service` container is not running + replicas_zero — a required service resolved to 0 running replicas + network_detached — the lane's declared network does not exist + oneshot_failed — a run-to-completion container Exited non-zero + oneshot_stuck — a run-to-completion container is still Running + image_tag_mismatch — a running container's image tag fails the lane pattern + unexpected_container — a container labeled for the lane that the manifest does + not declare + +Fail-fast policy: any drift on a non-optional lane is a hard signal. There is no +warn-only mode (gates-block policy). The driver exits non-zero on drift. + +Input envelope (stdin JSON): + { + "lane": "" | null, # null => reconcile all non-optional lanes + "containers": [ # `docker ps -a --format '{{json .}}'` rows, + {"Names": "...", "State": "running"|"exited"|..., + "Image": "repo:tag", "Labels": "k=v,k2=v2", + "Status": "Up 3 hours" | "Exited (0) ..."}, + ... + ], + "networks": ["omnibase-infra-prod-network", ...], # `docker network ls` + "runtime_tag": "0.37.0" | null # resolved runtime tag (optional) + } + +Output (stdout JSON): + { + "schema_version": "1.0.0", + "lanes_checked": ["prod", ...], + "findings": [ {lane, kind, container, detail, severity}, ... ], + "has_drift": true|false + } +""" + +from __future__ import annotations + +import json +import os +import re +import sys +from pathlib import Path +from typing import Any + +import yaml + +SCHEMA_VERSION = "1.0.0" + +# Drift severities. critical => a required service is down / network gone (the +# 2026-06-11 class of outage). warning => degraded but not lane-down. +_SEVERITY_CRITICAL = "critical" +_SEVERITY_WARNING = "warning" + +_DEFAULT_MANIFEST = ( + Path(__file__).resolve().parent.parent + / "deploy" + / "lane-census" + / "lane-manifest.yaml" +) + + +def load_manifest(path: Path | None = None) -> dict[str, Any]: + """Load the versioned lane manifest (desired state).""" + manifest_path = path or _DEFAULT_MANIFEST + with open(manifest_path, encoding="utf-8") as fh: + data = yaml.safe_load(fh) + if not isinstance(data, dict) or "lanes" not in data: + raise ValueError(f"lane manifest missing 'lanes': {manifest_path}") + return data + + +def _running(state: str, status: str) -> bool: + """A container counts as running only when docker reports it running.""" + return state.lower() == "running" or status.lower().startswith("up") + + +def _exit_code(status: str) -> int | None: + """Parse the exit code out of a docker status string, e.g. 'Exited (0) ...'.""" + match = re.search(r"Exited \((\d+)\)", status) + return int(match.group(1)) if match else None + + +def _tag_of(image: str) -> str: + """Return the tag portion of a 'repo:tag' image reference (default 'latest').""" + # Strip a registry host:port (the first colon may belong to the host). + # docker image refs: [registry[:port]/]repo[:tag][@digest] + ref = image.split("@", 1)[0] + last_segment = ref.rsplit("/", 1)[-1] + if ":" in last_segment: + return last_segment.rsplit(":", 1)[1] + return "latest" + + +def _finding( + lane: str, kind: str, container: str, detail: str, severity: str +) -> dict[str, str]: + return { + "lane": lane, + "kind": kind, + "container": container, + "detail": detail, + "severity": severity, + } + + +def reconcile_lane( + lane_name: str, + lane_spec: dict[str, Any], + *, + actual_by_name: dict[str, dict[str, Any]], + networks: set[str], + lane_labeled_names: set[str], + default_tag_pattern: str, + runtime_tag: str | None, +) -> list[dict[str, str]]: + """Diff one lane's desired state against the actual inventory.""" + findings: list[dict[str, str]] = [] + optional = bool(lane_spec.get("optional", False)) + + declared = lane_spec.get("services", []) + declared_names = {svc["name"] for svc in declared} + + # An optional lane that is entirely down is NOT drift (developer lane). We + # detect "entirely down" as: none of its declared service containers are + # running. If ANY are running, it is partially up and we reconcile it. + if optional: + any_running = any( + _running( + actual_by_name.get(svc["name"], {}).get("State", ""), + actual_by_name.get(svc["name"], {}).get("Status", ""), + ) + for svc in declared + if svc.get("kind", "service") == "service" + ) + if not any_running: + return findings # lane legitimately down; no ticket + + # 1. Network presence — the broker/lane network must exist. + declared_network = lane_spec.get("network") + if declared_network and declared_network not in networks: + findings.append( + _finding( + lane_name, + "network_detached", + declared_network, + f"lane network {declared_network!r} is not present in " + f"`docker network ls` — containers cannot reach the broker", + _SEVERITY_CRITICAL, + ) + ) + + tag_pattern = lane_spec.get("image_tag_pattern") or default_tag_pattern + # ${RUNTIME_TAG} is resolved from the deploy-agent runtime version when + # available; an unresolvable token relaxes to the default pattern (a missing + # runtime tag is a separate signal, not a census drift). + if "${RUNTIME_TAG}" in tag_pattern: + tag_pattern = re.escape(runtime_tag) if runtime_tag else default_tag_pattern + compiled_pattern = re.compile(tag_pattern) + + # 2. Per-declared-service diff. + for svc in declared: + name = svc["name"] + kind = svc.get("kind", "service") + actual = actual_by_name.get(name) + + if kind == "oneshot": + # Run-to-completion container. Absence (compose removed it) is fine. + if actual is None: + continue + status = actual.get("Status", "") + if _running(actual.get("State", ""), status): + findings.append( + _finding( + lane_name, + "oneshot_stuck", + name, + f"migration/init container {name!r} is still Running " + f"(expected run-to-completion): {status}", + _SEVERITY_WARNING, + ) + ) + continue + code = _exit_code(status) + if code is not None and code != 0: + findings.append( + _finding( + lane_name, + "oneshot_failed", + name, + f"migration/init container {name!r} Exited non-zero " + f"(code {code}): {status}", + _SEVERITY_CRITICAL, + ) + ) + continue + + # kind == service: MUST be running with the required replica count. + required_replicas = int(svc.get("replicas", 1)) + if actual is None or not _running( + actual.get("State", ""), actual.get("Status", "") + ): + # Distinguish replicas_zero (declared scaled-out but down) only for + # clarity; for a single declared container, absent == container_absent. + kind_name = ( + "replicas_zero" if required_replicas == 0 else "container_absent" + ) + findings.append( + _finding( + lane_name, + kind_name, + name, + f"required service container {name!r} is not running " + f"(desired replicas={required_replicas}); lane " + f"{lane_name!r} is degraded", + _SEVERITY_CRITICAL, + ) + ) + continue + + # Running — verify image tag matches the lane pattern. + tag = _tag_of(actual.get("Image", "")) + if not compiled_pattern.fullmatch(tag): + findings.append( + _finding( + lane_name, + "image_tag_mismatch", + name, + f"container {name!r} runs image tag {tag!r} which does not " + f"match lane pattern {tag_pattern!r}", + _SEVERITY_WARNING, + ) + ) + + # 3. Unexpected containers — anything labeled for this lane but not declared. + for actual_name in lane_labeled_names: + if actual_name not in declared_names: + findings.append( + _finding( + lane_name, + "unexpected_container", + actual_name, + f"container {actual_name!r} carries com.omninode.lane=" + f"{lane_name!r} but is not declared in the lane manifest", + _SEVERITY_WARNING, + ) + ) + + return findings + + +def _labels_to_dict(labels: str) -> dict[str, str]: + out: dict[str, str] = {} + for pair in labels.split(","): + if "=" in pair: + key, value = pair.split("=", 1) + out[key.strip()] = value.strip() + return out + + +def build_plan(envelope: dict[str, Any], manifest: dict[str, Any]) -> dict[str, Any]: + """Build the deterministic drift plan from the desired manifest + actual state.""" + requested_lane = envelope.get("lane") + containers = envelope.get("containers", []) + networks = set(envelope.get("networks", [])) + runtime_tag = envelope.get("runtime_tag") + default_pattern = manifest.get("default_image_tag_pattern", ".+") + + actual_by_name: dict[str, dict[str, Any]] = {} + lane_labels: dict[str, set[str]] = {} + for row in containers: + name = (row.get("Names") or row.get("Name") or "").lstrip("/").strip() + if not name: + continue + actual_by_name[name] = row + labels = _labels_to_dict(row.get("Labels", "")) + lane_label = labels.get("com.omninode.lane") + if lane_label: + lane_labels.setdefault(lane_label, set()).add(name) + + lanes = manifest["lanes"] + if requested_lane: + if requested_lane not in lanes: + raise ValueError(f"unknown lane {requested_lane!r}") + target_lanes = {requested_lane: lanes[requested_lane]} + else: + target_lanes = lanes + + findings: list[dict[str, str]] = [] + checked: list[str] = [] + for lane_name, lane_spec in target_lanes.items(): + checked.append(lane_name) + findings.extend( + reconcile_lane( + lane_name, + lane_spec, + actual_by_name=actual_by_name, + networks=networks, + lane_labeled_names=lane_labels.get(lane_name, set()), + default_tag_pattern=default_pattern, + runtime_tag=runtime_tag, + ) + ) + + return { + "schema_version": SCHEMA_VERSION, + "lanes_checked": checked, + "findings": findings, + "has_drift": len(findings) > 0, + } + + +def main() -> int: + manifest_path = os.environ.get("LANE_MANIFEST") + manifest = load_manifest(Path(manifest_path) if manifest_path else None) + envelope = json.load(sys.stdin) + plan = build_plan(envelope, manifest) + json.dump(plan, sys.stdout) + sys.stdout.write("\n") + # Exit 0 always — the planner reports; the shell driver decides exit policy + # so the JSON plan is always emittable for dry-run/inspection. + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/scripts/test_lane_census_dry_run.py b/tests/unit/scripts/test_lane_census_dry_run.py new file mode 100644 index 0000000000..10ead82ff1 --- /dev/null +++ b/tests/unit/scripts/test_lane_census_dry_run.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Dry-run + fail-fast safety tests for lane-census-check.sh (OMN-13011). + +Properties proven here: + 1. --dry-run NEVER publishes to the bus (no `rpk produce`). + 2. On drift, the script exits 30 (fail-fast, no warn-only mode). + 3. With no broker configured, a live run does NOT publish (fail-fast, no + localhost default) — it logs for manual replay and still exits 30 on drift. + 4. A clean lane (no drift) exits 0. + +We shim `docker`, `rpk`, and `hostname` on PATH with a recorder so the test is +hermetic and asserts on the exact subcommands the script issued. +""" + +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[3] +_SCRIPT = _REPO / "scripts" / "lane-census-check.sh" + + +def _shim(bin_dir: Path, name: str, body: str, calllog: Path) -> None: + shim = bin_dir / name + shim.write_text(f'#!/usr/bin/env bash\necho "{name} $*" >> "{calllog}"\n{body}\n') + shim.chmod(shim.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +def _run( + args: list[str], + tmp_path: Path, + *, + ps_rows: str, + networks: str, + broker: str | None, +) -> tuple[subprocess.CompletedProcess[str], str]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + calllog = tmp_path / "calls.log" + calllog.write_text("") + + # Write the canned inventory to files the shim cats — avoids any inline + # quoting/printf interpretation of the JSON rows. + ps_file = tmp_path / "ps.ndjson" + ps_file.write_text(ps_rows) + net_file = tmp_path / "networks.txt" + net_file.write_text(networks) + + # docker shim: ps -a -> rows; network ls -> networks; everything else empty. + docker_body = ( + 'case "$*" in\n' + f' *"ps -a"*) cat "{ps_file}" ;;\n' + f' *"network ls"*) cat "{net_file}" ;;\n' + " *) : ;;\n" + "esac\n" + "exit 0" + ) + _shim(bin_dir, "docker", docker_body, calllog) + _shim(bin_dir, "rpk", "exit 0", calllog) + _shim(bin_dir, "hostname", 'echo "omninode-pc"', calllog) + + env = dict(os.environ) + env["PATH"] = f"{bin_dir}:{env['PATH']}" + env["HOME"] = str(tmp_path) # log dir under tmp, never ~ + env.pop("KAFKA_BOOTSTRAP_SERVERS", None) + if broker: + env["KAFKA_BOOTSTRAP_SERVERS"] = broker + + proc = subprocess.run( + ["bash", str(_SCRIPT), *args], + capture_output=True, + text=True, + env=env, + timeout=60, + check=False, + ) + return proc, calllog.read_text() + + +# A prod lane with the runtime containers absent + network detached (tonight's +# outage), so every live run here MUST detect drift. +_PROD_OUTAGE_PS = ( + '{"Names":"omnibase-infra-prod-postgres","State":"running",' + '"Status":"Up 2 hours","Image":"postgres:16","Labels":"com.omninode.lane=prod"}\n' +) +_NO_NETWORKS = "bridge\nhost\n" + + +def test_dry_run_does_not_publish(tmp_path: Path) -> None: + proc, calls = _run( + ["--lane", "prod", "--dry-run"], + tmp_path, + ps_rows=_PROD_OUTAGE_PS, + networks=_NO_NETWORKS, + broker="redpanda:9092", + ) + assert proc.returncode == 30, proc.stderr + assert "produce" not in calls, f"dry-run published to bus:\n{calls}" + + +def test_drift_exits_30(tmp_path: Path) -> None: + proc, _ = _run( + ["--lane", "prod"], + tmp_path, + ps_rows=_PROD_OUTAGE_PS, + networks=_NO_NETWORKS, + broker="redpanda:9092", + ) + assert proc.returncode == 30, proc.stderr + + +def test_live_without_broker_does_not_publish(tmp_path: Path) -> None: + """No KAFKA_BOOTSTRAP_SERVERS => no publish (fail-fast, no localhost default).""" + proc, calls = _run( + ["--lane", "prod"], + tmp_path, + ps_rows=_PROD_OUTAGE_PS, + networks=_NO_NETWORKS, + broker=None, + ) + assert "produce" not in calls, f"published with no broker configured:\n{calls}" + assert proc.returncode == 30 + + +def test_clean_lane_exits_zero(tmp_path: Path) -> None: + """A fully-running prod lane with its network present exits 0 (no drift).""" + # Build a healthy prod inventory from the manifest. + import yaml + + manifest = yaml.safe_load( + (_REPO / "deploy" / "lane-census" / "lane-manifest.yaml").read_text() + ) + rows = [] + for svc in manifest["lanes"]["prod"]["services"]: + name = svc["name"] + if svc.get("kind") == "oneshot": + rows.append( + f'{{"Names":"{name}","State":"exited",' + '"Status":"Exited (0) 1 hour ago",' + '"Image":"x:1","Labels":"com.omninode.lane=prod"}' + ) + else: + rows.append( + f'{{"Names":"{name}","State":"running","Status":"Up 1 hour",' + '"Image":"x:1","Labels":"com.omninode.lane=prod"}' + ) + ps = "\n".join(rows) + "\n" + networks = "omnibase-infra-prod-network\nbridge\n" + proc, calls = _run( + ["--lane", "prod"], + tmp_path, + ps_rows=ps, + networks=networks, + broker="redpanda:9092", + ) + assert proc.returncode == 0, proc.stderr + assert "produce" not in calls, "clean lane must not publish" diff --git a/tests/unit/scripts/test_lane_census_event.py b/tests/unit/scripts/test_lane_census_event.py new file mode 100644 index 0000000000..2ceb4fe0fe --- /dev/null +++ b/tests/unit/scripts/test_lane_census_event.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Tests for the lane-census-drift bus event builder (OMN-13011).""" + +from __future__ import annotations + +import importlib.util +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[3] +_EVENT_PATH = _REPO / "scripts" / "lane_census_event.py" + + +def _load() -> Any: + spec = importlib.util.spec_from_file_location("lane_census_event", _EVENT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +EV = _load() + +_FIXED_NOW = datetime(2026, 6, 11, 22, 0, 0, tzinfo=UTC) + + +def _prod_outage_plan() -> dict[str, object]: + return { + "schema_version": "1.0.0", + "lanes_checked": ["prod"], + "has_drift": True, + "findings": [ + { + "lane": "prod", + "kind": "network_detached", + "container": "omnibase-infra-prod-network", + "detail": "lane network is not present", + "severity": "critical", + }, + { + "lane": "prod", + "kind": "container_absent", + "container": "omninode-prod-runtime", + "detail": "required service not running", + "severity": "critical", + }, + ], + } + + +def test_event_is_critical_when_any_finding_critical() -> None: + event = EV.build_event(host="omninode-pc", plan=_prod_outage_plan(), now=_FIXED_NOW) + assert event["severity"] == "critical" + assert event["event_type"] == "lane-census-drift" + assert event["topic"] == "onex.evt.infra.lane-census-drift.v1" + assert event["drift_count"] == 2 + assert event["emitted_at"] == "2026-06-11T22:00:00+00:00" + + +def test_event_is_warning_when_no_critical() -> None: + plan = { + "lanes_checked": ["judge"], + "findings": [ + { + "lane": "judge", + "kind": "image_tag_mismatch", + "container": "omninode-judge-runtime", + "detail": "stale tag", + "severity": "warning", + } + ], + } + event = EV.build_event(host="h", plan=plan, now=_FIXED_NOW) + assert event["severity"] == "warning" + + +def test_alert_key_is_stable_and_dedupes() -> None: + """Same finding set => same alert_key (one ticket per outage, not per tick).""" + plan = _prod_outage_plan() + k1 = EV.build_event(host="omninode-pc", plan=plan, now=_FIXED_NOW)["alert_key"] + k2 = EV.build_event(host="omninode-pc", plan=plan, now=datetime.now(UTC))[ + "alert_key" + ] + assert k1 == k2 + assert k1.startswith("lane-census-drift:omninode-pc:") + + +def test_alert_key_changes_with_finding_set() -> None: + plan_a = _prod_outage_plan() + plan_b = _prod_outage_plan() + plan_b["findings"] = plan_b["findings"][:1] # only the network finding + ka = EV.build_event(host="h", plan=plan_a, now=_FIXED_NOW)["alert_key"] + kb = EV.build_event(host="h", plan=plan_b, now=_FIXED_NOW)["alert_key"] + assert ka != kb + + +def test_ticket_title_names_lane_and_kinds() -> None: + title = EV.build_event(host="h", plan=_prod_outage_plan(), now=_FIXED_NOW)[ + "ticket_title" + ] + assert "prod" in title + assert "network_detached" in title + assert "container_absent" in title + + +def test_ticket_body_lists_every_finding() -> None: + body = EV.build_event(host="h", plan=_prod_outage_plan(), now=_FIXED_NOW)[ + "ticket_body" + ] + assert "omnibase-infra-prod-network" in body + assert "omninode-prod-runtime" in body + assert "OMN-13011" in body diff --git a/tests/unit/scripts/test_lane_census_manifest_parity.py b/tests/unit/scripts/test_lane_census_manifest_parity.py new file mode 100644 index 0000000000..73339d6ac8 --- /dev/null +++ b/tests/unit/scripts/test_lane_census_manifest_parity.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Manifest <-> compose parity ratchet for the lane census (OMN-13011). + +The lane manifest is the desired-state authority, but it can only police a lane +if it stays in lock-step with that lane's compose file. These tests assert that +every container_name declared in a lane's compose file appears in the manifest +(and vice-versa) so the manifest can never silently drift away from the lane it +is supposed to reconcile. A lane that gains/renames a container without updating +the manifest in the same PR fails CI here — that is the whole point. + +This also closes the OMN-12988 census gap where +docker/catalog/services/runtime-worker.yaml carried container_name: null: the +manifest sources concrete names from the compose lane files, not the catalog +stub. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[3] +_MANIFEST_PATH = _REPO / "deploy" / "lane-census" / "lane-manifest.yaml" + +# Lanes whose compose files have concrete container_name values we can diff. +# dev uses generated/non-prefixed names and is intentionally optional/loose, so +# it is parity-checked against its own service list only, not a compose scrape. +_COMPOSE_LANES = ("stability-test", "prod", "judge") + + +def _load_manifest() -> dict: + with open(_MANIFEST_PATH, encoding="utf-8") as fh: + return yaml.safe_load(fh) + + +def _compose_container_names(lane: str) -> set[str]: + compose_path = _REPO / "docker" / f"docker-compose.{lane}.yml" + raw = compose_path.read_text(encoding="utf-8") + # The compose files use custom !override / !!merge tags that safe_load cannot + # parse, so scrape container_name lines directly. + return set(re.findall(r"^\s*container_name:\s*(\S+)\s*$", raw, re.MULTILINE)) + + +@pytest.mark.parametrize("lane", _COMPOSE_LANES) +def test_every_compose_container_is_declared(lane: str) -> None: + """No compose container_name may be missing from the lane manifest.""" + manifest = _load_manifest() + declared = {s["name"] for s in manifest["lanes"][lane]["services"]} + compose_names = _compose_container_names(lane) + missing = compose_names - declared + assert not missing, ( + f"lane {lane!r}: compose declares containers absent from the lane " + f"manifest (update deploy/lane-census/lane-manifest.yaml in the same PR): " + f"{sorted(missing)}" + ) + + +@pytest.mark.parametrize("lane", _COMPOSE_LANES) +def test_no_manifest_phantom_containers(lane: str) -> None: + """No manifest service may reference a container the compose file lacks.""" + manifest = _load_manifest() + declared = {s["name"] for s in manifest["lanes"][lane]["services"]} + compose_names = _compose_container_names(lane) + phantom = declared - compose_names + assert not phantom, ( + f"lane {lane!r}: lane manifest declares containers the compose file does " + f"not define (stale manifest entry): {sorted(phantom)}" + ) + + +@pytest.mark.parametrize("lane", _COMPOSE_LANES) +def test_lane_network_matches_compose(lane: str) -> None: + """The manifest's declared network must exist in the compose networks block.""" + manifest = _load_manifest() + declared_network = manifest["lanes"][lane]["network"] + compose_path = _REPO / "docker" / f"docker-compose.{lane}.yml" + raw = compose_path.read_text(encoding="utf-8") + network_names = set( + re.findall(r"^\s*name:\s*(omnibase-infra-\S+-network)\s*$", raw, re.MULTILINE) + ) + assert declared_network in network_names, ( + f"lane {lane!r}: manifest network {declared_network!r} not found among " + f"compose networks {sorted(network_names)}" + ) + + +def test_no_service_declares_replicas_zero() -> None: + """A service may never declare replicas: 0 — that is the silent-drop surface.""" + manifest = _load_manifest() + for lane, spec in manifest["lanes"].items(): + for svc in spec["services"]: + if svc.get("kind", "service") == "service": + assert svc.get("replicas", 1) >= 1, ( + f"lane {lane!r} service {svc['name']!r} declares replicas 0 — " + f"forbidden (the WORKER_REPLICAS silent-zero regression)" + ) + + +def test_runtime_worker_declared_in_every_runtime_lane() -> None: + """The worker that silently dropped (OMN-12988) must be a required service.""" + manifest = _load_manifest() + for lane in ("stability-test", "prod"): + names = {s["name"] for s in manifest["lanes"][lane]["services"]} + worker = next((n for n in names if n.endswith("runtime-worker")), None) + assert worker is not None, f"lane {lane!r} missing a runtime-worker service" diff --git a/tests/unit/scripts/test_lane_census_plan.py b/tests/unit/scripts/test_lane_census_plan.py new file mode 100644 index 0000000000..d8a82a98a4 --- /dev/null +++ b/tests/unit/scripts/test_lane_census_plan.py @@ -0,0 +1,287 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Pure-planner tests for the lane-census reconciler (OMN-13011). + +The planner diffs the versioned desired-state lane manifest against a live docker +inventory and emits typed drift findings. These tests pin every drift kind plus +the headline red fixture: the 2026-06-11 prod outage (runtime containers absent + +broker network detached) MUST produce drift findings that would have ticketed +hours before a human noticed. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + +import pytest + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[3] +_PLAN_PATH = _REPO / "scripts" / "lane_census_plan.py" +_MANIFEST_PATH = _REPO / "deploy" / "lane-census" / "lane-manifest.yaml" + + +def _load_planner() -> Any: + spec = importlib.util.spec_from_file_location("lane_census_plan", _PLAN_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +PLAN = _load_planner() +MANIFEST = PLAN.load_manifest(_MANIFEST_PATH) + + +def _container( + name: str, + *, + lane: str, + state: str = "running", + status: str = "Up 3 hours", + image: str = "omninode-runtime:0.37.0", +) -> dict[str, str]: + return { + "Names": name, + "State": state, + "Status": status, + "Image": image, + "Labels": f"com.omninode.lane={lane},com.omninode.layer=runtime", + } + + +def _healthy_prod_containers() -> list[dict[str, str]]: + """All declared prod services running, migrations exited 0.""" + services = list(MANIFEST["lanes"]["prod"]["services"]) + rows: list[dict[str, str]] = [] + for svc in services: + if svc.get("kind") == "oneshot": + rows.append( + _container( + svc["name"], + lane="prod", + state="exited", + status="Exited (0) 2 hours ago", + ) + ) + else: + rows.append(_container(svc["name"], lane="prod")) + return rows + + +def _kinds(findings: list[dict[str, str]]) -> set[str]: + return {f["kind"] for f in findings} + + +def test_healthy_prod_lane_no_drift() -> None: + """A fully-running prod lane with its network present yields zero drift.""" + envelope = { + "lane": "prod", + "containers": _healthy_prod_containers(), + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + assert plan["has_drift"] is False, plan["findings"] + assert plan["lanes_checked"] == ["prod"] + + +def test_tonight_prod_red_fixture_runtime_absent_and_network_detached() -> None: + """RED FIXTURE — the 2026-06-11 outage. + + prod runtime containers (main/effects/worker/projection-api) absent AND the + broker network detached. The reconciler MUST emit a network_detached finding + plus a container_absent finding for every missing runtime service — exactly + the ticket that should have fired hours before a human noticed. + """ + # Only the infra/migration layer is up; the four runtime services and the + # network are gone. + surviving = [ + _container("omnibase-infra-prod-postgres", lane="prod"), + _container("omnibase-infra-prod-redpanda", lane="prod"), + _container("omnibase-infra-prod-valkey", lane="prod"), + _container( + "omnibase-infra-prod-migration-gate", + lane="prod", + state="exited", + status="Exited (0) 4 hours ago", + ), + ] + envelope = { + "lane": "prod", + "containers": surviving, + "networks": [], # broker network detached + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + + assert plan["has_drift"] is True + kinds = _kinds(plan["findings"]) + assert "network_detached" in kinds, plan["findings"] + assert "container_absent" in kinds, plan["findings"] + + absent = { + f["container"] for f in plan["findings"] if f["kind"] == "container_absent" + } + # Every runtime service must be named as absent. + for runtime_svc in ( + "omninode-prod-runtime", + "omninode-prod-runtime-effects", + "omnimarket-prod-projection-api", + "omninode-prod-runtime-worker", + ): + assert runtime_svc in absent, f"{runtime_svc} not reported absent: {absent}" + + # network_detached + container_absent are all critical severity. + crit = [f for f in plan["findings"] if f["severity"] == "critical"] + assert crit, "tonight's outage must be critical severity" + + +def test_worker_replicas_zero_silent_drop_is_drift() -> None: + """The WORKER_REPLICAS silent-zero regression (OMN-12988/12990). + + A worker scaled to 0 produces no container; the planner must report it as + drift on the declared replicas:1 service. + """ + containers = _healthy_prod_containers() + containers = [c for c in containers if c["Names"] != "omninode-prod-runtime-worker"] + envelope = { + "lane": "prod", + "containers": containers, + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + absent = { + f["container"] for f in plan["findings"] if f["kind"] == "container_absent" + } + assert "omninode-prod-runtime-worker" in absent + + +def test_oneshot_failed_exit_nonzero_is_critical_drift() -> None: + """A migration container that Exited non-zero is critical drift.""" + containers = _healthy_prod_containers() + for c in containers: + if c["Names"] == "omnibase-infra-prod-migration-gate": + c["state"] = "exited" + c["State"] = "exited" + c["Status"] = "Exited (1) 5 minutes ago" + envelope = { + "lane": "prod", + "containers": containers, + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + failed = [f for f in plan["findings"] if f["kind"] == "oneshot_failed"] + assert failed and failed[0]["severity"] == "critical", plan["findings"] + + +def test_oneshot_stuck_running_is_warning_drift() -> None: + """A migration container still Running (never completed) is warning drift.""" + containers = _healthy_prod_containers() + for c in containers: + if c["Names"] == "omnibase-infra-prod-forward-migration": + c["State"] = "running" + c["Status"] = "Up 6 hours" + envelope = { + "lane": "prod", + "containers": containers, + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + stuck = [f for f in plan["findings"] if f["kind"] == "oneshot_stuck"] + assert stuck and stuck[0]["severity"] == "warning" + + +def test_unexpected_lane_labeled_container_is_drift() -> None: + """A container labeled for the lane but not declared is unexpected_container.""" + containers = _healthy_prod_containers() + containers.append(_container("omninode-prod-rogue-shadow", lane="prod")) + envelope = { + "lane": "prod", + "containers": containers, + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + unexpected = { + f["container"] for f in plan["findings"] if f["kind"] == "unexpected_container" + } + assert "omninode-prod-rogue-shadow" in unexpected + + +def test_image_tag_mismatch_is_drift() -> None: + """A running container whose tag fails the lane pattern is drift.""" + # Pin a strict pattern lane manifest in-memory. + manifest = PLAN.load_manifest(_MANIFEST_PATH) + manifest["lanes"]["prod"]["image_tag_pattern"] = r"0\.37\..+" + containers = _healthy_prod_containers() + for c in containers: + if c["Names"] == "omninode-prod-runtime": + c["Image"] = "omninode-runtime:0.30.0-stale" + envelope = { + "lane": "prod", + "containers": containers, + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, manifest) + mism = [f for f in plan["findings"] if f["kind"] == "image_tag_mismatch"] + assert mism and mism[0]["container"] == "omninode-prod-runtime" + + +def test_optional_dev_lane_entirely_down_is_not_drift() -> None: + """The optional dev lane being fully down must NOT ticket (developer lane).""" + envelope = { + "lane": "dev", + "containers": [], + "networks": [], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + assert plan["has_drift"] is False, plan["findings"] + + +def test_optional_dev_lane_partially_up_is_drift() -> None: + """A partially-up optional lane IS reconciled (one service up, one missing).""" + envelope = { + "lane": "dev", + "containers": [ + _container("omninode-runtime", lane="dev"), + # omninode-runtime-effects missing + ], + "networks": ["omnibase-infra_default"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + absent = { + f["container"] for f in plan["findings"] if f["kind"] == "container_absent" + } + assert "omninode-runtime-effects" in absent + + +def test_unknown_lane_raises() -> None: + with pytest.raises(ValueError): + PLAN.build_plan( + {"lane": "does-not-exist", "containers": [], "networks": []}, MANIFEST + ) + + +def test_all_lanes_default_excludes_nothing_required() -> None: + """With lane=None all manifest lanes are checked.""" + envelope = {"lane": None, "containers": [], "networks": [], "runtime_tag": None} + plan = PLAN.build_plan(envelope, MANIFEST) + assert set(plan["lanes_checked"]) == set(MANIFEST["lanes"].keys()) + + +def test_tag_parsing_handles_registry_host_port() -> None: + """`_tag_of` must not mistake a registry host:port for the tag.""" + assert PLAN._tag_of("registry.local:5000/omninode-runtime:0.37.0") == "0.37.0" + assert PLAN._tag_of("omninode-runtime") == "latest" + assert PLAN._tag_of("omninode-runtime@sha256:abc") == "latest" diff --git a/tests/unit/scripts/test_lane_census_timer_dropin.py b/tests/unit/scripts/test_lane_census_timer_dropin.py new file mode 100644 index 0000000000..18f4571a2e --- /dev/null +++ b/tests/unit/scripts/test_lane_census_timer_dropin.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Systemd coordination tests for the lane-census timer drop-in (OMN-13011). + +The ticket requires sharing the OMN-13008 timer unit rather than adding a second +one. These tests assert the coordination contract: + 1. The lane-census ExecStart is delivered as a drop-in for the SHARED + onex-disk-gc.service (not a new .timer / .service). + 2. The drop-in invokes scripts/lane-census-check.sh. + 3. The installer refuses to install a second timer and depends on the base unit. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[3] +_LANE_CENSUS_DIR = _REPO / "deploy" / "lane-census" +_DROPIN = _LANE_CENSUS_DIR / "onex-disk-gc.service.d" / "20-lane-census.conf" +_INSTALLER = _LANE_CENSUS_DIR / "install-lane-census.sh" + + +def test_dropin_targets_shared_disk_gc_service() -> None: + """The drop-in lives under onex-disk-gc.service.d (shared unit), not a new unit.""" + assert _DROPIN.exists(), f"missing drop-in: {_DROPIN}" + assert _DROPIN.parent.name == "onex-disk-gc.service.d" + body = _DROPIN.read_text(encoding="utf-8") + assert "[Service]" in body + assert "ExecStart=" in body + + +def test_dropin_invokes_lane_census_check() -> None: + body = _DROPIN.read_text(encoding="utf-8") + assert "scripts/lane-census-check.sh" in body + + +def test_no_second_timer_unit_shipped() -> None: + """Coordination rule: do NOT add a second timer. No .timer file in this dir.""" + timers = list(_LANE_CENSUS_DIR.glob("*.timer")) + assert not timers, ( + f"lane-census must share the onex-disk-gc.timer, not add its own: {timers}" + ) + # And no standalone lane-census .service either. + services = [ + p + for p in _LANE_CENSUS_DIR.glob("*.service") + if p.name != "onex-disk-gc.service" + ] + assert not services, f"unexpected standalone service unit(s): {services}" + + +def test_installer_depends_on_base_unit() -> None: + """Installer fails fast if the base onex-disk-gc.service is not installed.""" + body = _INSTALLER.read_text(encoding="utf-8") + assert "onex-disk-gc.service" in body + assert "install-disk-gc.sh" in body # points operator at OMN-13008's installer + assert "daemon-reload" in body