wip(OMN-16964): parked link-5 replay-core branch (handoff to Lakshman)

.github/workflows/chain-canary.yml

@@ -280,6 +280,47 @@ jobs:
                   handle.write(f'{key}={value}\n')
           "
 
+      # OMN-16964. Link 5's source, resolved the same way and with the same
+      # deliberate asymmetry as the step above: an unresolvable declaration
+      # WARNS and continues rather than killing the job, because the node
+      # already renders a missing instrument honestly (SKIPPED_NOT_CONFIGURED
+      # against OMN-16025 link 5, which is RED), and stopping here would throw
+      # away the four links the probe CAN prove.
+      #
+      # Only the NAME is exported. The DSN is injected as a secret on the probe
+      # step under that same name -- and for this leg no NEW secret exists:
+      # `ledger_chain` and `delegation_workflow_state` are both in the
+      # omnibase_infra database and both readable by `chain_canary_reader`, so
+      # the lane declares the SAME variable name for both legs today. The two
+      # declarations stay separate so either can be repointed without moving
+      # the other; only the value they currently resolve to is shared.
+      - name: Resolve the declared ledger-readback DSN reference
+        id: ledger
+        env:
+          BUS_LANE: ${{ github.event.inputs.bus_lane || 'dev' }}
+        run: |
+          set -euo pipefail
+          uv run python -c "
+          import os, pathlib
+          from omnibase_infra.nodes.node_chain_canary_effect.lane_transport import (
+              load_lane_ledger_readback, ledger_readback_env,
+          )
+          lane = os.environ['BUS_LANE']
+          try:
+              declaration = load_lane_ledger_readback(
+                  pathlib.Path('.ci-bus-overlay/config/ci_bus_lanes.yaml'), lane
+              )
+          except ValueError as exc:
+              print(f'::warning::no ledger readback declared for lane {lane!r}: {exc}')
+              print('OMN-16025 link 5 will report SKIPPED_NOT_CONFIGURED, which is RED.')
+              raise SystemExit(0)
+          print(f'lane {lane!r} declares its ledger DSN under '
+                f'{declaration.dsn_env} (a NAME; the value is a job secret)')
+          with open(os.environ['GITHUB_ENV'], 'a', encoding='utf-8') as handle:
+              for key, value in ledger_readback_env(declaration).items():
+                  handle.write(f'{key}={value}\n')
+          "
+
       - name: Fire one live delegation through the dev-lane ingress
         env:
           # Inside the deploy-runner container `localhost` is the container
@@ -329,6 +370,17 @@ jobs:
           # argv is world-readable through /proc, and the dispatch step echoes
           # its own broker values into this log.
           CHAIN_CANARY_PROJECTION_DSN: ${{ secrets.CHAIN_CANARY_PROJECTION_DSN }}
+          # OMN-16964: what a COMPLETE chain must carry, end to end. Declared
+          # here rather than inferred from whatever the ledger returned,
+          # because the scope's own wording is "no gaps tolerated silently" --
+          # a chain short by a hop must not look complete simply because every
+          # row present was well-formed. These are the ordered topics one
+          # delegation traverses, per the node contracts that publish them.
+          EXPECTED_LEDGER_HOPS: >-
+            onex.cmd.omnimarket.delegate-skill.v1,
+            onex.cmd.omnibase-infra.delegation-routing-request.v1,
+            onex.evt.omnibase-infra.routing-decision.v1,
+            onex.evt.omnimarket.delegate-skill-completed.v1
           # The omnimarket drift guard (OMN-14060) refuses `onex skill`
           # whenever $OMNI_HOME/omnimarket exists and the venv's omnimarket
           # is not a git co-install. The deploy runner sets OMNI_HOME to its
@@ -359,13 +411,16 @@ jobs:
           # The NAME is safe to print and is the one thing worth printing: it
           # is what a reader needs to check the declaration against the secret.
           echo "projection dsn var: ${CHAIN_CANARY_PROJECTION_DSN_ENV:-<undeclared>}"
+          echo "ledger dsn var:     ${CHAIN_CANARY_LEDGER_DSN_ENV:-<undeclared>}"
           uv run onex skill chain_canary \
             --probe-url "${PROBE_URL}" \
             --task-type "${TASK_TYPE}" \
             --budget-ms "${BUDGET_MS}" \
             --terminal-bootstrap-servers "${TERMINAL_BOOTSTRAP}" \
             --quarantine-bootstrap-servers "${QUARANTINE_BOOTSTRAP}" \
             --projection-dsn-env "${CHAIN_CANARY_PROJECTION_DSN_ENV:-}" \
+            --ledger-dsn-env "${CHAIN_CANARY_LEDGER_DSN_ENV:-}" \
+            --expected-ledger-hops "${EXPECTED_LEDGER_HOPS}" \
             > chain-canary-receipt.json
 
       - name: Publish receipt and set the verdict

deploy/lane-census/census-snapshot.json

@@ -3,7 +3,7 @@
   "event_type": "lane-census-drift",
   "topic": "onex.evt.infra.lane-census-drift.v1",
   "host": "omninode-pc",
-  "emitted_at": "2026-09-02T12:28:09.943586+00:00",
+  "emitted_at": "2026-09-10T03:26:28.616542+00:00",
   "severity": "critical",
   "lanes_checked": [
     "stability-test",
@@ -12,7 +12,7 @@
     "dev",
     "lakshman"
   ],
-  "drift_count": 2,
+  "drift_count": 4,
   "findings": [
     {
       "lane": "prod",
@@ -21,6 +21,20 @@
       "detail": "required service container 'omninode-prod-runtime-worker' is not running (desired replicas=1); lane 'prod' is degraded",
       "severity": "critical"
     },
+    {
+      "lane": "dev",
+      "kind": "unexpected_container",
+      "container": "onex-api",
+      "detail": "container 'onex-api' carries com.omninode.lane='dev' but is not declared in the lane manifest",
+      "severity": "warning"
+    },
+    {
+      "lane": "dev",
+      "kind": "unexpected_container",
+      "container": "omninode-infra-routing-decisions-consumer",
+      "detail": "container 'omninode-infra-routing-decisions-consumer' carries com.omninode.lane='dev' but is not declared in the lane manifest",
+      "severity": "warning"
+    },
     {
       "lane": "dev",
       "kind": "unexpected_container",
@@ -29,7 +43,7 @@
       "severity": "warning"
     }
   ],
-  "alert_key": "lane-census-drift:omninode-pc:b6127966f3696f12",
-  "ticket_title": "fix(infra): lane drift [dev, prod] \u2014 1x container_absent, 1x unexpected_container",
-  "ticket_body": "## Lane census drift detected\n\nHost: `omninode-pc`\nLanes checked: stability-test, prod, judge, dev, lakshman\n\nThe desired-state lane census (deploy/lane-census/lane-manifest.yaml)\ndoes not match the live runtime. Drift items below name exactly what is\nmissing or extra. This is the regression class the lane-census ratchet\n(OMN-13011) exists to catch \u2014 it would have fired on 2026-06-11 when prod\nruntime containers and the broker network were silently absent.\n\n## Findings\n\n- **[critical] container_absent** `omninode-prod-runtime-worker` (prod): required service container 'omninode-prod-runtime-worker' is not running (desired replicas=1); lane 'prod' is degraded\n- **[warning] unexpected_container** `omnibase-infra-forward-migration` (dev): container 'omnibase-infra-forward-migration' carries com.omninode.lane='dev' but is not declared in the lane manifest\n\n## Action\nBring the lane back to declared state (compose up the absent containers / reattach the network), or update the lane manifest in the same PR if the desired state legitimately changed. Do not silence the check."
+  "alert_key": "lane-census-drift:omninode-pc:0464669b44a0ee80",
+  "ticket_title": "fix(infra): lane drift [dev, prod] \u2014 1x container_absent, 3x unexpected_container",
+  "ticket_body": "## Lane census drift detected\n\nHost: `omninode-pc`\nLanes checked: stability-test, prod, judge, dev, lakshman\n\nThe desired-state lane census (deploy/lane-census/lane-manifest.yaml)\ndoes not match the live runtime. Drift items below name exactly what is\nmissing or extra. This is the regression class the lane-census ratchet\n(OMN-13011) exists to catch \u2014 it would have fired on 2026-06-11 when prod\nruntime containers and the broker network were silently absent.\n\n## Findings\n\n- **[critical] container_absent** `omninode-prod-runtime-worker` (prod): required service container 'omninode-prod-runtime-worker' is not running (desired replicas=1); lane 'prod' is degraded\n- **[warning] unexpected_container** `onex-api` (dev): container 'onex-api' carries com.omninode.lane='dev' but is not declared in the lane manifest\n- **[warning] unexpected_container** `omninode-infra-routing-decisions-consumer` (dev): container 'omninode-infra-routing-decisions-consumer' carries com.omninode.lane='dev' but is not declared in the lane manifest\n- **[warning] unexpected_container** `omnibase-infra-forward-migration` (dev): container 'omnibase-infra-forward-migration' carries com.omninode.lane='dev' but is not declared in the lane manifest\n\n## Action\nBring the lane back to declared state (compose up the absent containers / reattach the network), or update the lane manifest in the same PR if the desired state legitimately changed. Do not silence the check."
 }

docker/docker-compose.infra.yml

@@ -521,9 +521,17 @@ services:
       # the read-only instrument identity node_chain_canary_effect uses for its
       # OMN-16025 link-2 projection readback. Empty-means-skip, exactly as on
       # the two keys above: a lane with no canary provisioned leaves the role
-      # absent, migration 104 skips its grants with a NOTICE, and the canary
+      # absent, migration 105 skips its grants with a NOTICE, and the canary
       # reports skipped_not_configured -- red and honest, never a green from a
       # check that never ran.
+      #
+      # OMN-16964 correction: this comment used to name "migration 104" as the
+      # file granting this role. Ordinal 104 is BURNED (OMN-17923 retired
+      # 104_create_validator_ro_role.sql and the record forbids reuse), and no
+      # file ever issued the link-2 grant -- it is live on the .201 dev lane
+      # only because it was applied out of band, which is filed as OMN-18115.
+      # 105_create_ledger_chain.sql issues the LINK-5 grant reproducibly; the
+      # link-2 grant is still unissued by any migration in this repo.
       CHAIN_CANARY_READER_PASSWORD: ${CHAIN_CANARY_READER_PASSWORD:-}
     ports:
       - "${POSTGRES_EXTERNAL_PORT:-5436}:5432"