feat(OMN-18949): gate census drift in CI, and report live lane drift to a person

.github/workflows/lane-census-staleness.yml

@@ -14,7 +14,21 @@
 #      A census older than 7 days means the .201 timer stopped or the snapshot
 #      was never committed — either is a documentation failure.
 #
-#   2. ENV-VS-LIVE DRIFT (E-5 / OMN-13018): lane-manifest.yaml compose_project
+#   2. CENSUS-VS-MANIFEST DRIFT (OMN-18949): the committed snapshot and the
+#      committed manifest must describe the same world. A finding reporting a
+#      container as undeclared that the manifest now declares, a snapshot that
+#      miscounts its own findings, a lane one file knows and the other does
+#      not, and a manifest lane the table generator has no entry for are each
+#      a hard failure. It deliberately does NOT require drift_count to be
+#      zero: live drift on a mutable lane is a normal reportable state, and a
+#      gate refusing every PR over it would be switched off within a day.
+#
+#      Until this step existed, NOTHING refused on census drift at all. The
+#      committed snapshot carried five findings, severity warning, and the
+#      only surface that read them rendered the prose string
+#      "5 drift item(s) — see census" into a doctrine file.
+#
+#   3. ENV-VS-LIVE DRIFT (E-5 / OMN-13018): lane-manifest.yaml compose_project
 #      names, network declarations, and bootstrap server config must agree with
 #      what the compose files actually declare. Divergence means the manifest
 #      is policing the wrong lane.
@@ -31,6 +45,7 @@ on:
       - "deploy/lane-census/**"
       - "docker/docker-compose*.yml"
       - "scripts/check_lane_census_age.py"
+      - "scripts/check_lane_census_drift.py"
       - "scripts/check_lane_env_drift.py"
       - "scripts/generate_claude_lane_block.py"
   pull_request:
@@ -39,6 +54,7 @@ on:
       - "deploy/lane-census/**"
       - "docker/docker-compose*.yml"
       - "scripts/check_lane_census_age.py"
+      - "scripts/check_lane_census_drift.py"
       - "scripts/check_lane_env_drift.py"
       - "scripts/generate_claude_lane_block.py"
   workflow_dispatch:
@@ -94,6 +110,18 @@ jobs:
             --max-age-days "${MAX_AGE_DAYS}" \
             --snapshot deploy/lane-census/census-snapshot.json
 
+      - name: Check census-vs-manifest drift (OMN-18949)
+        run: |
+          echo "================================================================"
+          echo "Census-vs-Manifest Drift Gate — OMN-18949"
+          echo "The committed snapshot and the committed manifest must describe"
+          echo "the same world. A non-zero drift_count is NOT a failure; a"
+          echo "finding that contradicts the manifest is."
+          echo "================================================================"
+          uv run python scripts/check_lane_census_drift.py \
+            --manifest deploy/lane-census/lane-manifest.yaml \
+            --snapshot deploy/lane-census/census-snapshot.json
+
       - name: Check E-5 env-vs-manifest drift (OMN-13018)
         run: |
           echo "================================================================"
@@ -104,12 +132,39 @@ jobs:
             --manifest deploy/lane-census/lane-manifest.yaml \
             --compose-dir docker/
 
-      - name: Verify lane table generator runs without error
+      - name: Verify the lane table generator renders every declared lane
         run: |
           echo "================================================================"
-          echo "Lane Table Generator Smoke Test — OMN-13034"
-          echo "generate_claude_lane_block.py must produce valid output."
+          echo "Lane Table Generator — OMN-13034, hardened by OMN-18949"
+          echo "This step was a bare smoke test: it ran the generator and"
+          echo "compared its output to nothing, while the module's docstring"
+          echo "claimed CI failed when CLAUDE.md drifted. It never did."
+          echo "The drift step above now asserts the generator has an explicit"
+          echo "entry for every manifest lane, which is the half of that claim"
+          echo "this repository can enforce without checking out another one."
           echo "================================================================"
           uv run python scripts/generate_claude_lane_block.py \
             --manifest deploy/lane-census/lane-manifest.yaml \
-            --snapshot deploy/lane-census/census-snapshot.json
+            --snapshot deploy/lane-census/census-snapshot.json > /tmp/lane-block.md
+          echo "--- rendered block ---"
+          cat /tmp/lane-block.md
+          # One row per declared lane, counted from the manifest rather than
+          # from the output, so a generator that silently dropped a lane fails
+          # here instead of rendering a shorter table that still looks whole.
+          declared=$(uv run python -c "import yaml,sys; print(len(yaml.safe_load(open('deploy/lane-census/lane-manifest.yaml'))['lanes']))")
+          rendered=$(grep -c '^| [a-z]' /tmp/lane-block.md || true)
+          echo "declared lanes: ${declared}  rendered rows: ${rendered}"
+          if [ "${declared}" -ne "${rendered}" ]; then
+            echo "FAIL: the generated table has ${rendered} row(s) for ${declared} declared lane(s)." >&2
+            exit 1
+          fi
+
+      # RESIDUAL, stated rather than implied (OMN-18949). The generated block
+      # itself lives in omni_home/CLAUDE.md, a DIFFERENT repository this job
+      # does not check out, so `--check` is NOT run against it here. Closing
+      # that half needs a token with contents:read on that repository, and
+      # making a path-filtered PR gate depend on a cross-repo fetch trades one
+      # silent failure for an availability risk on unrelated pull requests.
+      # The comparison exists and is unit-tested (`--check <path>`); the
+      # scheduled surface that should run it is the 6-hourly refresh workflow,
+      # which already mints an app token. Not done here, and not claimed.

deploy/maintenance/omninode-system-slack-report.sh

@@ -967,6 +967,128 @@ check_fleet_failures() {
   printf '%s\n' "$out"
 }
 
+# OMN-18949: surface LANE CENSUS drift -- the live hourly census on this host
+# against the committed lane manifest.
+#
+# WHY IT LIVES HERE. Nothing refused on census drift anywhere. The hourly timer
+# on this host writes a snapshot; the committed snapshot in the repository
+# carried five findings and severity warning; the only surface that read either
+# rendered the prose string "N drift item(s) -- see census" into a doctrine
+# file, where the next refresh normalised it away. The CI gate added by the
+# same ticket catches the two COMMITTED files disagreeing, which is a different
+# question from the LIVE lane topology having moved -- CI cannot see this
+# host's docker daemon at all.
+#
+# "More than one tick" is not implemented here and must not be: CONFIRM_TICKS
+# above already requires a key to hold the same status for two consecutive
+# ticks before it pages, and CLEAR_TICKS absorbs the flap on the way back. A
+# second, private counter in this function would be a second answer to a
+# question the state machine already answers, and the two would disagree.
+#
+# Rows are `census|STATUS|key|detail`, read by `row_status()` at column 2 and
+# de-duplicated by `row_key()` as `census|<key>`. Every "could not look"
+# outcome is a row naming the reason: an unreadable census is never an empty
+# green, because an absent finding set and an unevaluated one are the two
+# states this family exists to tell apart.
+check_census_drift() {
+  local live manifest checker python_bin repo
+  repo="${OMNINODE_REPO_ROOT:-$HOME/Code/omni_home/omnibase_infra}"
+  live="${OMNINODE_CENSUS_LIVE_SNAPSHOT:-$HOME/.local/state/onex/census-snapshot.json}"
+  manifest="${OMNINODE_CENSUS_MANIFEST:-$repo/deploy/lane-census/lane-manifest.yaml}"
+  checker="${OMNINODE_CENSUS_CHECKER:-$repo/scripts/check_lane_census_drift.py}"
+  python_bin="${OMNINODE_CENSUS_PYTHON:-python3}"
+
+  if [[ "${OMNINODE_CENSUS_PROBE_ENABLED:-1}" != "1" ]]; then
+    return 0
+  fi
+  if [[ ! -r "$live" ]]; then
+    printf 'census|WARNING|snapshot|no live census at %s; the hourly lane-census timer may have stopped\n' "$live"
+    return 0
+  fi
+  if [[ ! -r "$manifest" ]]; then
+    printf 'census|WARNING|manifest|lane manifest unreadable at %s; live census cannot be judged\n' "$manifest"
+    return 0
+  fi
+
+  # Staleness first. A snapshot that stopped being written keeps reporting the
+  # last topology it saw, which reads as a healthy census indefinitely -- the
+  # exact failure the age gate exists for on the committed copy, and one this
+  # host had no reader for on the live copy.
+  local emitted age_h stale_h
+  stale_h="${OMNINODE_CENSUS_STALE_HOURS:-6}"
+  emitted=$("$python_bin" - "$live" <<'PY' 2>/dev/null || true
+import datetime as dt, json, sys
+try:
+    d = json.load(open(sys.argv[1]))
+    t = dt.datetime.fromisoformat(d["emitted_at"])
+    if t.tzinfo is None:
+        t = t.replace(tzinfo=dt.timezone.utc)
+    print(int((dt.datetime.now(dt.timezone.utc) - t).total_seconds() // 3600))
+except Exception:
+    pass
+PY
+)
+  if [[ -z "$emitted" ]]; then
+    printf 'census|WARNING|snapshot|live census at %s has no readable emitted_at; its age is unknown\n' "$live"
+    return 0
+  fi
+  age_h="$emitted"
+  if (( age_h >= stale_h )); then
+    printf 'census|WARNING|snapshot|live census is %sh old (>= %sh); the hourly timer has stopped writing\n' "$age_h" "$stale_h"
+    return 0
+  fi
+
+  # Agreement with the manifest, evaluated by the SAME module the CI gate runs,
+  # pointed at the live snapshot instead of the committed one. A second
+  # implementation here would be a second verdict that could disagree with CI's.
+  local json rc errfile err
+  if [[ -r "$checker" ]] && command -v "$python_bin" >/dev/null 2>&1; then
+    # stderr is CAPTURED, not discarded (CLAUDE.md rule 16). A suppressed error
+    # here returns an empty result that reads exactly like a clean census, and
+    # the reason it failed is the only thing that makes the row actionable --
+    # "exit 1" and "no module named yaml" are the same row without it. The
+    # interpreter is overridable because this checker needs PyYAML and the
+    # host's bare python3 may not have it.
+    errfile=$(mktemp)
+    json=$(timeout "${OMNINODE_CENSUS_PROBE_TIMEOUT:-60}" "$python_bin" "$checker" \
+             --manifest "$manifest" --snapshot "$live" --json 2>"$errfile")
+    rc=$?
+    err=$(tail -n1 "$errfile" 2>/dev/null | tr -d '\r' | cut -c1-160)
+    rm -f "$errfile"
+    if (( rc == 2 )) || [[ -z "$json" ]]; then
+      printf 'census|WARNING|manifest|drift checker could not evaluate the live census (exit %s)%s\n' \
+        "$rc" "${err:+: $err}"
+      return 0
+    fi
+    if (( rc == 1 )); then
+      local summary
+      summary=$("$python_bin" -c 'import json,sys; d=json.load(sys.stdin); f=d["findings"]; print("%d disagreement(s): %s" % (len(f), "; ".join("%s on %s (%s)" % (x["kind"], x["lane"], x["subject"]) for x in f[:3])))' <<<"$json" 2>/dev/null)
+      printf 'census|CRITICAL|manifest|the LIVE census contradicts the committed manifest -- %s\n' "${summary:-unparseable checker output}"
+      return 0
+    fi
+  else
+    printf 'census|WARNING|manifest|drift checker missing or unreadable at %s\n' "$checker"
+    return 0
+  fi
+
+  # Live topology drift. Distinct from the disagreement above: the manifest and
+  # the census agree about what is DECLARED, and the host is running something
+  # else. It is a WARNING, not a critical, because a mutable lane is allowed to
+  # carry drift -- what is not allowed is nobody being told.
+  local drift lanes
+  drift=$("$python_bin" -c 'import json,sys; print(len(json.load(open(sys.argv[1])).get("findings",[])))' "$live" 2>/dev/null || echo "")
+  if [[ -z "$drift" ]]; then
+    printf 'census|WARNING|drift|live census findings unreadable; drift state unknown\n'
+    return 0
+  fi
+  if (( drift > 0 )); then
+    lanes=$("$python_bin" -c 'import json,sys; print(",".join(sorted({f.get("lane","?") for f in json.load(open(sys.argv[1])).get("findings",[])})))' "$live" 2>/dev/null || echo "?")
+    printf 'census|WARNING|drift|%s drift item(s) on lane(s) %s, census %sh old\n' "$drift" "$lanes" "$age_h"
+    return 0
+  fi
+  printf 'census|OK|drift|no lane drift; census %sh old and agrees with the manifest\n' "$age_h"
+}
+
 collect() {
   local now host root data root_status data_status running unhealthy restarting dead created
   local dangling named_dangling anonymous_dangling docker_status docker_detail
@@ -1067,6 +1189,7 @@ collect() {
     check_ci_required_contexts
     check_runner_tree_converge
     check_fleet_failures
+    check_census_drift
   }
 }
 
@@ -1238,7 +1361,7 @@ recovered_keys=$(awk -F'\t' '$1=="RECOVERED" { print $2 }' <<<"$decisions")
 
 format_digest() {
   local title="$1"
-  local lines endpoint_lines ci_lines tree_lines fleet_lines issue_lines
+  local lines endpoint_lines ci_lines tree_lines fleet_lines census_lines issue_lines
   lines=$(awk -F'|' '$1=="disk" {printf "- `%s`: %s, %s, %s (%s)\n", $3, $4, $5, $6, $2} $1=="docker" {printf "- Docker `%s`: %s (%s)\n", $3, $4, $2}' <<<"$snapshot")
   # STARTING is rendered here with the rest (OMN-18435). A booting lane that
   # vanished from this section would be the BLIND direction of monitor failure,
@@ -1261,9 +1384,14 @@ format_digest() {
   # rendering as an absent section a reader takes for health.
   fleet_lines=$(awk -F'|' '$1=="fleet" {printf "- `%s`: %s (%s)\n", $3, $4, $2}' <<<"$snapshot")
   [[ -n "$fleet_lines" ]] || fleet_lines="- No fleet failure rows this tick"
+  # OMN-18949. Same reasoning as the heartbeats above: a section that prints
+  # nothing when the probe did not run is indistinguishable from one printing
+  # nothing because there is nothing wrong.
+  census_lines=$(awk -F'|' '$1=="census" {printf "- `%s`: %s (%s)\n", $3, $4, $2}' <<<"$snapshot")
+  [[ -n "$census_lines" ]] || census_lines="- No lane census rows this tick"
   # `next` keeps the three row shapes mutually exclusive so a `ci` row cannot
   # also be rendered by the generic column-2 branch below it.
-  issue_lines=$(awk -F'|' '($1=="ci" || $1=="runner-tree" || $1=="fleet") && ($2=="WARNING" || $2=="CRITICAL") {printf "- %s `%s`: %s\n", $2, $3, $4; next} $2=="WARNING" || $2=="CRITICAL" {printf "- %s `%s`: %s %s %s\n", $2, $3, $4, $5, $6; next} $1=="WARNING" || $1=="CRITICAL" {printf "- %s `%s`: HTTP %s %s\n", $1, $2, $3, $4}' <<<"$snapshot")
+  issue_lines=$(awk -F'|' '($1=="ci" || $1=="runner-tree" || $1=="fleet" || $1=="census") && ($2=="WARNING" || $2=="CRITICAL") {printf "- %s `%s`: %s\n", $2, $3, $4; next} $2=="WARNING" || $2=="CRITICAL" {printf "- %s `%s`: %s %s %s\n", $2, $3, $4, $5, $6; next} $1=="WARNING" || $1=="CRITICAL" {printf "- %s `%s`: HTTP %s %s\n", $1, $2, $3, $4}' <<<"$snapshot")
   if [[ -z "$issue_lines" ]]; then
     issue_lines="- No active warning/critical checks"
   fi
@@ -1282,6 +1410,8 @@ $ci_lines
 $tree_lines
 *Fleet failures*
 $fleet_lines
+*Lane census*
+$census_lines
 *Active issues*
 $issue_lines
 MSG