feat(OMN-13011): lane census reconciliation ratchet

contracts/OMN-13011.yaml

@@ -0,0 +1,89 @@
+# OMN-13011 contract file for omnibase_infra deploy-gate.
+# The canonical ticket contract and PASS receipts live in onex_change_control.
+schema_version: "1.0.0"
+ticket_id: "OMN-13011"
+title: "LANE CENSUS RECONCILIATION ratchet — declared desired-state per lane, scheduled + on-demand, drift = auto-ticket"
+summary: >-
+  The class fix for the recurring lane-drift regression. Nothing reconciled the
+  declared desired state of a runtime lane against what is actually running, so
+  the same failure kept recurring with zero signal: volume config drift
+  (OMN-12945), WORKER_REPLICAS silent zero (OMN-12988/12990), and on 2026-06-11
+  prod runtime containers plus the broker network were silently absent for hours
+  during demo prep. This ships a per-lane DESIRED-STATE census: (a) DECLARED in a
+  versioned lane manifest (deploy/lane-census/lane-manifest.yaml) — container set,
+  network, replicas, image-tag pattern per lane (stability-test/prod/judge/dev),
+  derived from the canonical compose lane files; (b) RECONCILED on a schedule on
+  .201 by sharing the OMN-13008 systemd timer (a drop-in 4th ExecStart on
+  onex-disk-gc.service, never a second timer) and on-demand via runtime_sweep;
+  (c) drift = a typed bus event + Linear auto-ticket naming exactly what is
+  missing/extra (container_absent, network_detached, replicas_zero,
+  unexpected_container, oneshot_failed/stuck, image_tag_mismatch). Fail-fast,
+  no warn-only mode (gates-block policy). Builds on the OMN-12988 deploy-agent
+  RUNTIME census (deploy-time) as the complementary steady-state reconciler.
+is_seam_ticket: false
+interface_change: false
+interfaces_touched: []
+evidence_requirements:
+  - kind: "tests"
+    description: "Pure-planner drift tests incl. the 2026-06-11 prod red fixture pass"
+    command: >-
+      uv run pytest tests/unit/scripts/test_lane_census_plan.py -q
+  - kind: "tests"
+    description: "Event-builder, manifest-parity, dry-run, and timer-dropin tests pass"
+    command: >-
+      uv run pytest
+      tests/unit/scripts/test_lane_census_event.py
+      tests/unit/scripts/test_lane_census_manifest_parity.py
+      tests/unit/scripts/test_lane_census_dry_run.py
+      tests/unit/scripts/test_lane_census_timer_dropin.py -q
+  - kind: "ci"
+    description: "New source passes ruff and mypy --strict"
+    command: >-
+      uv run ruff check scripts/lane_census_plan.py scripts/lane_census_event.py
+      && uv run mypy scripts/lane_census_plan.py scripts/lane_census_event.py --strict
+  - kind: "manual"
+    description: >-
+      On .201, after OMN-13008's install-disk-gc.sh, run
+      deploy/lane-census/install-lane-census.sh and confirm via
+      `systemctl --user cat onex-disk-gc.service` that the lane-census ExecStart
+      is the 4th pass on the shared unit (no second timer)
+emergency_bypass:
+  enabled: false
+  justification: ""
+  follow_up_ticket_id: ""
+dod_evidence:
+  - id: "dod-planner-red-fixture"
+    description: "Pure planner reports network_detached + container_absent for the 2026-06-11 prod outage"
+    source: "manual"
+    checks:
+      - check_type: "command"
+        check_value: >-
+          uv run pytest tests/unit/scripts/test_lane_census_plan.py -q
+  - id: "dod-event-and-ticket"
+    description: "Typed drift event builds stable alert_key + ticket title/body naming exactly what is wrong"
+    source: "manual"
+    checks:
+      - check_type: "command"
+        check_value: >-
+          uv run pytest tests/unit/scripts/test_lane_census_event.py -q
+  - id: "dod-manifest-parity"
+    description: "Lane manifest is locked in step with the compose lane files; no service declares replicas 0"
+    source: "manual"
+    checks:
+      - check_type: "command"
+        check_value: >-
+          uv run pytest tests/unit/scripts/test_lane_census_manifest_parity.py -q
+  - id: "dod-dry-run-failfast"
+    description: "Dry-run never publishes; drift exits 30; no broker => no localhost-default publish"
+    source: "manual"
+    checks:
+      - check_type: "command"
+        check_value: >-
+          uv run pytest tests/unit/scripts/test_lane_census_dry_run.py -q
+  - id: "dod-shared-timer"
+    description: "Lane-census shares the OMN-13008 onex-disk-gc.service via drop-in; no second timer"
+    source: "manual"
+    checks:
+      - check_type: "command"
+        check_value: >-
+          uv run pytest tests/unit/scripts/test_lane_census_timer_dropin.py -q

deploy/lane-census/install-lane-census.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc.
+# SPDX-License-Identifier: MIT
+
+# install-lane-census.sh — Register the lane-census reconcile on the SHARED
+# onex-disk-gc.service systemd USER timer (OMN-13011).
+#
+# COORDINATION (OMN-13008 / PR #1952): the disk-GC PR owns onex-disk-gc.service +
+# onex-disk-gc.timer. This installer does NOT create a second timer — it drops a
+# systemd override (onex-disk-gc.service.d/20-lane-census.conf) that appends the
+# lane-census ExecStart to that one unit, then reloads. The reconcile runs on the
+# same hourly tick as the GC/watermark passes.
+#
+# Prerequisite: OMN-13008's `bash deploy/disk-gc/install-disk-gc.sh` must have run
+# first (it installs the base onex-disk-gc.service/.timer). If the base unit is
+# absent this installer fails fast and tells you to run the disk-gc installer.
+#
+# Usage (on 192.168.86.201, user scope, no sudo):
+#   bash deploy/lane-census/install-lane-census.sh             # install drop-in + reload
+#   bash deploy/lane-census/install-lane-census.sh --uninstall
+#   bash deploy/lane-census/install-lane-census.sh --status
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+DROPIN_SRC="${SCRIPT_DIR}/onex-disk-gc.service.d/20-lane-census.conf"
+USER_UNIT_DIR="${HOME}/.config/systemd/user"
+BASE_SERVICE="${USER_UNIT_DIR}/onex-disk-gc.service"
+DROPIN_DST_DIR="${USER_UNIT_DIR}/onex-disk-gc.service.d"
+DROPIN_DST="${DROPIN_DST_DIR}/20-lane-census.conf"
+
+if [[ "${1:-}" == "--uninstall" ]]; then
+  echo "Removing lane-census drop-in..."
+  rm -f "$DROPIN_DST"
+  systemctl --user daemon-reload 2>/dev/null || true
+  echo "Done. lane-census drop-in removed (base onex-disk-gc unit untouched)."
+  exit 0
+fi
+
+if [[ "${1:-}" == "--status" ]]; then
+  echo "Effective onex-disk-gc.service (base + drop-ins):"
+  systemctl --user cat onex-disk-gc.service --no-pager 2>/dev/null || \
+    echo "  (base unit not installed — run deploy/disk-gc/install-disk-gc.sh first)"
+  exit 0
+fi
+
+if [[ ! -f "$BASE_SERVICE" ]]; then
+  echo "ERROR: base onex-disk-gc.service not installed at $BASE_SERVICE" >&2
+  echo "Run OMN-13008's installer first:  bash deploy/disk-gc/install-disk-gc.sh" >&2
+  exit 1
+fi
+
+echo "Installing lane-census drop-in onto the shared onex-disk-gc.service..."
+chmod +x "${SCRIPT_DIR}/../../scripts/lane-census-check.sh" 2>/dev/null || true
+mkdir -p "$DROPIN_DST_DIR"
+cp "$DROPIN_SRC" "$DROPIN_DST"
+systemctl --user daemon-reload
+
+echo ""
+echo "Done. lane-census reconcile registered on the shared onex-disk-gc timer."
+echo "  Drop-in: $DROPIN_DST"
+echo ""
+echo "Verify:   systemctl --user cat onex-disk-gc.service   # base + lane-census ExecStart"
+echo "Run now:  systemctl --user start onex-disk-gc.service"
+echo "Logs:     journalctl --user -u onex-disk-gc.service -f"
+echo "On-demand: bash scripts/lane-census-check.sh --json"

deploy/lane-census/lane-manifest.yaml

@@ -0,0 +1,182 @@
+# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc.
+# SPDX-License-Identifier: MIT
+#
+# lane-manifest.yaml — Versioned DESIRED-STATE census for the .201 runtime lanes (OMN-13011).
+#
+# THE CLASS FIX. We kept regressing the same way — nothing reconciled desired vs
+# actual lane state:
+#   - volume config drift (OMN-12945 family)
+#   - WORKER_REPLICAS silent zero (OMN-12988 / OMN-12990)
+#   - 2026-06-11: prod runtime containers + broker network silently absent for
+#     hours during demo prep, with ZERO alert.
+#
+# This file is the single source of truth for what each lane MUST look like in
+# steady state. scripts/lane_census_plan.py reads it and diffs it against the
+# live `docker ps` / `docker network ls` inventory; scripts/lane-census-check.sh
+# drives the reconcile, emits a typed bus event, and fails fast (non-zero) on any
+# drift. Nothing about lane shape is hardcoded in the scripts — it all lives here.
+#
+# Maintenance: the container_name and network values below are derived from the
+# canonical compose lane files (docker/docker-compose.<lane>.yml `container_name`
+# fields and the `networks:` block). When a lane gains/loses a container or
+# renames its network, update BOTH the compose file and this manifest in the same
+# PR. tests/unit/scripts/test_lane_census_manifest_parity.py asserts the manifest
+# stays in lock-step with the compose files so the manifest can never silently
+# drift away from the lane it is supposed to police.
+#
+# Container kinds:
+#   service  — long-running; MUST be in `docker ps` Running state. Absence = drift.
+#   oneshot  — run-to-completion migration/init container; expected to have Exited
+#              (0). It is NOT drift for a oneshot to be absent (compose removes it)
+#              or Exited; it IS drift for a oneshot to be Running-stuck or Exited
+#              non-zero. Tracked, not required-running.
+#
+# replicas: the REQUIRED running count for a service container. 1 unless a lane
+# explicitly scales out. A service whose replicas resolve to 0 is itself drift
+# (the WORKER_REPLICAS silent-zero regression): declaring replicas: 0 here is
+# forbidden by the parity test.
+#
+# image_tag_pattern: a regex every running container's image tag must match for
+# the lane. Guards against a lane silently running a stale/wrong generation. The
+# special token "${RUNTIME_TAG}" is resolved at check time from the deploy-agent
+# resolved runtime version; absence of a resolved tag relaxes the check to the
+# default pattern (never a hard failure on an unresolvable tag — that is a
+# separate signal, not a census drift).
+
+schema_version: "1.0.0"
+
+# Default image-tag pattern applied to a lane that does not override it.
+default_image_tag_pattern: '.+'
+
+lanes:
+  stability-test:
+    compose_file: docker/docker-compose.stability-test.yml
+    compose_project: omnibase-infra-stability-test
+    network: omnibase-infra-stability-test-network
+    image_tag_pattern: '.+'
+    services:
+      - name: omnibase-infra-stability-test-postgres
+        kind: service
+        replicas: 1
+      - name: omnibase-infra-stability-test-redpanda
+        kind: service
+        replicas: 1
+      - name: omnibase-infra-stability-test-redpanda-partition-cap
+        kind: oneshot
+      - name: omnibase-infra-stability-test-valkey
+        kind: service
+        replicas: 1
+      - name: omnibase-infra-stability-test-keycloak
+        kind: service
+        replicas: 1
+      - name: omnibase-infra-stability-test-forward-migration
+        kind: oneshot
+      - name: omnibase-infra-stability-test-migration-gate
+        kind: oneshot
+      - name: omnibase-infra-stability-test-intelligence-migration
+        kind: oneshot
+      - name: omninode-stability-test-runtime
+        kind: service
+        replicas: 1
+      - name: omninode-stability-test-runtime-effects
+        kind: service
+        replicas: 1
+      - name: omnimarket-stability-test-projection-api
+        kind: service
+        replicas: 1
+      # The worker that OMN-12988 / OMN-12990 silently scaled to 0. Declared
+      # replicas: 1 here so its absence (replicas 0 => no container) is loud drift.
+      - name: omninode-stability-test-runtime-worker
+        kind: service
+        replicas: 1
+
+  prod:
+    compose_file: docker/docker-compose.prod.yml
+    compose_project: omnibase-infra-prod
+    network: omnibase-infra-prod-network
+    image_tag_pattern: '.+'
+    services:
+      - name: omnibase-infra-prod-postgres
+        kind: service
+        replicas: 1
+      - name: omnibase-infra-prod-redpanda
+        kind: service
+        replicas: 1
+      - name: omnibase-infra-prod-redpanda-partition-cap
+        kind: oneshot
+      - name: omnibase-infra-prod-valkey
+        kind: service
+        replicas: 1
+      - name: omnibase-infra-prod-forward-migration
+        kind: oneshot
+      - name: omnibase-infra-prod-migration-gate
+        kind: oneshot
+      - name: omnibase-infra-prod-intelligence-migration
+        kind: oneshot
+      # The four runtime services that were silently absent on 2026-06-11.
+      - name: omninode-prod-runtime
+        kind: service
+        replicas: 1
+      - name: omninode-prod-runtime-effects
+        kind: service
+        replicas: 1
+      - name: omnimarket-prod-projection-api
+        kind: service
+        replicas: 1
+      - name: omninode-prod-runtime-worker
+        kind: service
+        replicas: 1
+
+  judge:
+    compose_file: docker/docker-compose.judge.yml
+    compose_project: omnibase-infra-judge
+    network: omnibase-infra-judge-network
+    image_tag_pattern: '.+'
+    services:
+      - name: omnibase-infra-judge-postgres
+        kind: service
+        replicas: 1
+      - name: omnibase-infra-judge-redpanda
+        kind: service
+        replicas: 1
+      - name: omnibase-infra-judge-redpanda-partition-cap
+        kind: oneshot
+      - name: omnibase-infra-judge-valkey
+        kind: service
+        replicas: 1
+      - name: omnibase-infra-judge-forward-migration
+        kind: oneshot
+      - name: omnibase-infra-judge-migration-gate
+        kind: oneshot
+      - name: omnibase-infra-judge-intelligence-migration
+        kind: oneshot
+      - name: omninode-judge-runtime
+        kind: service
+        replicas: 1
+      - name: omninode-judge-runtime-effects
+        kind: service
+        replicas: 1
+      - name: omnimarket-judge-projection-api
+        kind: service
+        replicas: 1
+
+  # The dev lane (compose project `omnibase-infra`, the default infra stack) uses
+  # generated compose with non-prefixed container names; its census is the
+  # deploy-agent RUNTIME-scope verifier (OMN-12988). The lane manifest treats dev
+  # as OPTIONAL (reconcile-when-present): dev is a developer lane that is legitimately
+  # down most of the time, so its absence is not a steady-state regression. When dev
+  # IS up, the listed services must be running. This avoids a noisy false-positive
+  # ticket every time a developer stops their local stack.
+  dev:
+    compose_file: docker/docker-compose.generated.yml
+    compose_project: omnibase-infra
+    network: omnibase-infra_default
+    image_tag_pattern: '.+'
+    optional: true
+    services:
+      - name: omninode-runtime
+        kind: service
+        replicas: 1
+      - name: omninode-runtime-effects
+        kind: service
+        replicas: 1

deploy/lane-census/onex-disk-gc.service.d/20-lane-census.conf

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc.
+# SPDX-License-Identifier: MIT
+#
+# 20-lane-census.conf — systemd drop-in extending the SHARED onex-disk-gc.service
+# with the lane-census reconcile pass (OMN-13011).
+#
+# COORDINATION (OMN-13008 / PR #1952): the disk-GC PR owns onex-disk-gc.service +
+# onex-disk-gc.timer. Rather than install a SECOND timer (per the ticket: "share
+# the timer unit rather than adding a second"), this drop-in appends a 4th
+# ExecStart to that ONE unit. systemd runs drop-ins after the base unit's own
+# ExecStart lines, so the lane-census reconcile runs after the GC passes on the
+# same hourly tick.
+#
+# Why a drop-in instead of editing the base .service: the base unit lives in a
+# separate in-flight PR. A drop-in lets both PRs land independently with zero
+# merge collision on the unit file. When OMN-13008 merges, install-lane-census.sh
+# copies this drop-in into ~/.config/systemd/user/onex-disk-gc.service.d/ and
+# reloads — no second timer, no duplicate schedule.
+#
+# Install (on .201, user scope, after OMN-13008's install-disk-gc.sh has run):
+#   bash deploy/lane-census/install-lane-census.sh
+#
+# Check:
+#   systemctl --user cat onex-disk-gc.service    # shows base + this drop-in
+#   journalctl --user -u onex-disk-gc.service -f
+
+[Service]
+# Reconcile every declared lane against actual. Non-zero exit on drift is the
+# expected maintenance signal (the watermark pass uses the same '-' fail-soft
+# convention so one drift does not fail the whole maintenance unit). The drift
+# event is published to the bus inside the script when KAFKA_BOOTSTRAP_SERVERS
+# is present.
+ExecStart=-/bin/bash %h/Code/omni_home/omnibase_infra/scripts/lane-census-check.sh