fix(OMN-14968): dev-lane runtime-worker replicas resolve fail-closed per lane (silent-zero blocked every dev deploy at RT-6)

docker/README.md

@@ -193,8 +193,13 @@ docker compose -f docker-compose.infra.yml --profile runtime up -d --build
 # Scale workers manually
 docker compose -f docker-compose.infra.yml --profile runtime up -d --scale runtime-worker=4
 
-# Or set via environment variable
-WORKER_REPLICAS=8 docker compose -f docker-compose.infra.yml --profile runtime up -d
+# Or set via environment variable. NOTE (OMN-14968): the knob is the
+# lane-prefixed DEV_WORKER_REPLICAS, and it is fail-closed -- the render aborts
+# if it is unset, so source the ledgered policy env (which pins it to 1)
+# rather than relying on a default. The bare WORKER_REPLICAS name no longer
+# exists; it silently resolved to 0 and dropped the worker.
+source runtime-policy.env  # supplies DEV_WORKER_REPLICAS=1
+DEV_WORKER_REPLICAS=8 docker compose -f docker-compose.infra.yml --profile runtime up -d
 ```
 
 #### Consul Profile (Service Discovery)

docker/docker-compose.infra.yml

@@ -981,9 +981,25 @@ services:
   # retries (5) because Kafka consumer groups are joined sequentially at
   # startup. Autoheal uses a workspace-wide 2400s grace period so main,
   # effects, and workers can all finish contract wiring before restart checks.
-  # OMN-7569: Default replicas=0 — workers join the same per-topic consumer groups
-  # as the main runtime on 1-partition topics, causing a constant rebalance storm.
-  # Scale to 0 until the kernel supports profile-based subscription filtering.
+  # OMN-7569 (SUPERSEDED by OMN-14968): this block used to default replicas to 0
+  # via a bare ${WORKER_REPLICAS:-0}, because workers joined the same per-topic
+  # consumer groups as the main runtime on 1-partition topics and caused a constant
+  # rebalance storm — "scale to 0 UNTIL the kernel supports profile-based
+  # subscription filtering". That filtering shipped: each lane process now carries a
+  # disjoint capability set (this service resolves DEV_RUNTIME_WORKER_CAPABILITIES =
+  # workflow.dispatch,contract.update,runtime.worker, disjoint from the main
+  # runtime's market.skill-proof,workflow.orchestration,runtime.main), and OMN-12990
+  # pinned worker replicas to 1 for EVERY lane in
+  # contracts/services/runtime_policy.contract.yaml.
+  #
+  # OMN-12988 / OMN-12990 converted the stability-test and prod overlays to the
+  # lane-prefixed fail-closed form but left this base line bare, so the dev lane
+  # (which has no overlay — this file IS the dev lane) still resolved to 0: a plain
+  # `up -d --no-deps runtime-worker` exited 0 creating nothing, while
+  # deploy-runtime.sh's RUNTIME_SERVICES / RT-6 deploy readback requires a running
+  # container, aborting and auto-restoring every dev-lane deploy. deploy.replicas
+  # below is now lane-prefixed and fail-closed, exactly like the sibling
+  # DEV_RUNTIME_WORKER_* vars in this same service block.
   runtime-worker:
     !!merge <<: *runtime-base
     profiles: ["runtime", "full"]
@@ -1022,7 +1038,7 @@ services:
       start_period: 1200s
     deploy:
       mode: replicated
-      replicas: ${WORKER_REPLICAS:-0}
+      replicas: ${DEV_WORKER_REPLICAS:?runtime policy contract must set DEV_WORKER_REPLICAS (source docker/runtime-policy.env)}
       resources:
         limits:
           cpus: '0.5'

docker/docker-compose.prod.yml

@@ -236,8 +236,9 @@ services:
       - "com.omninode.runtime.id=prod-worker"
     deploy:
       # OMN-12990: fail-fast on the ledgered config value. Without this override
-      # the prod worker inherited the base ${WORKER_REPLICAS:-0} default and a
-      # plain recreate would silently run zero workers. Sourcing
+      # the prod worker inherited the base compose's bare ${WORKER_REPLICAS:-0}
+      # default (lane-prefixed and fail-closed since OMN-14968) and a plain
+      # recreate would silently run zero workers. Sourcing
       # docker/runtime-policy.env supplies PROD_WORKER_REPLICAS=1; a recreate
       # without it now fails loudly instead of dropping the worker with no signal.
       replicas: ${PROD_WORKER_REPLICAS:?runtime policy contract must set PROD_WORKER_REPLICAS (source docker/runtime-policy.env)}

docker/docker-compose.stability-test.yml

@@ -367,7 +367,8 @@ services:
       - "com.omninode.runtime.id=stability-test-worker"
     deploy:
       # OMN-12990: fail-fast on the ledgered config value, NOT a soft :-1
-      # default. The base compose sets ${WORKER_REPLICAS:-0}; a plain recreate
+      # default (the base compose kept a bare ${WORKER_REPLICAS:-0} until
+      # OMN-14968 made it lane-prefixed and fail-closed too); a plain recreate
       # that omits the policy env would silently drop the worker. Sourcing
       # docker/runtime-policy.env (rendered from runtime_policy.contract.yaml)
       # supplies STABILITY_TEST_WORKER_REPLICAS=1; a recreate without it now

docker/env-example-full.txt

@@ -183,8 +183,13 @@ POSTGRES_PASSWORD=__REPLACE_WITH_SECURE_PASSWORD__
 # =============================================================================
 # Worker Runtime Service Configuration
 # =============================================================================
-# Number of worker replicas (for horizontal scaling)
-# WORKER_REPLICAS=2
+# Number of worker replicas (for horizontal scaling). OMN-14968: this is the
+# lane-prefixed DEV_WORKER_REPLICAS and it is `:?`-required in
+# docker-compose.infra.yml -- the compose render FAILS when it is unset rather
+# than silently scaling the worker to zero. The ledgered value is rendered into
+# docker/runtime-policy.env from contracts/services/runtime_policy.contract.yaml;
+# set it here only to override that pin.
+# DEV_WORKER_REPLICAS=2
 
 # Kafka topics for worker runtime
 # ONEX_WORKER_INPUT_TOPIC=worker-requests

docs/env-example-full.txt

@@ -658,8 +658,11 @@ ONEX_LOG_LEVEL=INFO
 # RUNTIME_MAIN_PORT=8085
 # RUNTIME_EFFECTS_PORT=8086
 
-# Number of worker replicas for horizontal scaling
-# WORKER_REPLICAS=2
+# Number of worker replicas for horizontal scaling. OMN-14968: the knob is the
+# lane-prefixed DEV_WORKER_REPLICAS, `:?`-required in docker-compose.infra.yml
+# (the render fails closed when unset instead of silently rendering 0 replicas).
+# The ledgered pin lives in docker/runtime-policy.env.
+# DEV_WORKER_REPLICAS=2
 
 # =============================================================================
 # Agent Actions Consumer Configuration (Observability Profile)

docs/runbooks/stability-test-runtime-lane.md

@@ -128,8 +128,10 @@ uv run pytest tests/integration/infra/test_stability_test_runtime_compose_render
 
 ## Worker Replica Census
 
-The base compose sets the runtime-worker deploy replicas to
-`${WORKER_REPLICAS:-0}` — a soft default of **zero**. The stability lane's
+The base compose used to set the runtime-worker deploy replicas to a bare
+`${WORKER_REPLICAS:-0}` — a soft default of **zero** that no surface exported
+(OMN-14968 replaced it with the lane-prefixed, fail-closed
+`${DEV_WORKER_REPLICAS:?...}`). The stability lane's
 required state includes a running worker (`GATE_ZERO_PROOF.md`: 4 runtime
 containers — main, effects, worker, projection-api). Any plain
 `docker compose up`/`recreate` that does not supply the worker replica count

scripts/deploy-agent/tests/unit/test_runtime_worker_census.py

@@ -4,9 +4,11 @@
 
 The stability-test lane's required state includes a running ``runtime-worker``
 container (4-container census: main, effects, worker, projection-api). The base
-``docker-compose.infra.yml`` defaults the worker to ``replicas: 0``
-(``${WORKER_REPLICAS:-0}``), so a plain compose ``up``/recreate that does not
-set replicas silently drops the worker — zero errors, zero signal.
+``docker-compose.infra.yml`` used to default the worker to ``replicas: 0`` via a
+bare ``${WORKER_REPLICAS:-0}``, so a plain compose ``up``/recreate that did not
+set replicas silently dropped the worker — zero errors, zero signal. OMN-14968
+made that base line lane-prefixed and fail-closed (``${DEV_WORKER_REPLICAS:?}``)
+as well; the census ratchets below stay the runtime-side backstop.
 
 Two ratchets guard against that:
 
@@ -47,13 +49,26 @@ def test_runtime_worker_in_runtime_scope_census() -> None:
     assert "runtime-worker" in services_for_scope(Scope.FULL)
 
 
-def test_stability_override_pins_worker_replicas_to_literal_one() -> None:
-    """The stability override pins worker ``replicas: 1`` as a literal.
+def test_stability_override_pins_worker_replicas_fail_closed() -> None:
+    """The stability override pins worker replicas fail-closed on the policy value.
 
-    Env indirection (``${STABILITY_TEST_WORKER_REPLICAS:-1}``) is a silent-drop
-    surface: an exported ``STABILITY_TEST_WORKER_REPLICAS=0`` or a future edit
-    removing the ``:-1`` fallback would scale the worker to 0 with no signal.
-    The pin must be the literal integer ``1``.
+    A SOFT env indirection (``${STABILITY_TEST_WORKER_REPLICAS:-1}``) is the
+    silent-drop surface: an exported ``STABILITY_TEST_WORKER_REPLICAS=0``, or a
+    future edit removing the ``:-1`` fallback, scales the worker to 0 with no
+    signal.
+
+    OMN-12988 first closed that with a LITERAL ``1``. OMN-12990 then deliberately
+    replaced the literal with the ledgered, fail-fast
+    ``${STABILITY_TEST_WORKER_REPLICAS:?...}`` form so the value comes from
+    ``contracts/services/runtime_policy.contract.yaml`` and a recreate that omits
+    the policy env ABORTS instead of dropping the worker. This test kept asserting
+    the superseded literal and has been RED since 2026-06-23 — invisible because
+    ``scripts/deploy-agent/tests/`` is a separate pytest root that the repo suite
+    does not collect. Corrected under OMN-14968, which applied the same fail-closed
+    form to the base compose (dev lane) as ``${DEV_WORKER_REPLICAS:?...}``.
+
+    Accepted: the literal ``1``, or a ``:?`` fail-fast reference to the ledgered
+    lane value. Rejected: any ``:-`` soft default, and a literal ``0``.
     """
     # The stability override uses docker compose custom tags (!override,
     # !!merge) that yaml.safe_load cannot parse, so assert against the raw text:
@@ -80,8 +95,13 @@ def test_stability_override_pins_worker_replicas_to_literal_one() -> None:
     )
 
     value = replicas_line.group(1)
-    assert value == "1", (
-        "stability-test runtime-worker must pin deploy.replicas to the literal "
-        f"integer 1 (no env-interpolation default that could silently scale to "
-        f"0), got {value!r}"
+    assert value == "1" or value.startswith("${STABILITY_TEST_WORKER_REPLICAS:?"), (
+        "stability-test runtime-worker must pin deploy.replicas either to the "
+        "literal integer 1 or to the ledgered policy value with the ':?' "
+        "fail-fast form (OMN-12990); a soft ':-' default can silently scale the "
+        f"worker to 0, got {value!r}"
+    )
+    assert ":-" not in value, (
+        "a soft ':-' default re-opens the silent-drop hole: a recreate that omits "
+        f"the policy env would scale the worker without any signal, got {value!r}"
     )

scripts/render_runtime_policy_env.py

@@ -116,8 +116,9 @@ def render_env(contract: ModelRuntimePolicyContract) -> dict[str, str]:
             if process_name == "worker":
                 # OMN-12990: pin the worker replica count in the ledgered config
                 # surface so a plain compose recreate cannot silently drop the
-                # worker via the base ${WORKER_REPLICAS:-0} default. The lane
-                # overrides reference ${<PROFILE>_WORKER_REPLICAS:?...} fail-fast.
+                # worker. Every lane compose surface -- the base infra file for
+                # dev (OMN-14968) and the stability-test/prod overlays -- resolves
+                # ${<PROFILE>_WORKER_REPLICAS:?...} fail-fast against these values.
                 env[f"{profile_prefix}_WORKER_REPLICAS"] = str(process.replicas)
             env[f"{prefix}_CAPABILITIES"] = ",".join(process.capabilities)
             env[f"{prefix}_BIFROST_VERIFY_ENDPOINTS"] = _bifrost_text(

src/omnibase_infra/runtime/models/model_runtime_process_policy.py

@@ -24,12 +24,13 @@ class ModelRuntimeProcessPolicy(BaseModel):
     omnimemory_memgraph_host: str = ""
     publish_introspection: bool = False
     # OMN-12990: contract-declared replica count for this runtime process.
-    # The base compose worker deploy default is ${WORKER_REPLICAS:-0}, which
-    # silently scales the worker to zero on a plain compose recreate. Each lane
-    # pins this value via {PROFILE}_WORKER_REPLICAS (rendered into
-    # runtime-policy.env) and the lane overrides reference it fail-fast, so a
-    # recreate that omits the policy env fails loudly instead of dropping the
-    # worker with no signal.
+    # The base compose worker deploy default used to be a bare
+    # ${WORKER_REPLICAS:-0}, which silently scaled the worker to zero on a plain
+    # compose recreate. Each lane pins this value via {PROFILE}_WORKER_REPLICAS
+    # (rendered into runtime-policy.env); every lane compose surface now
+    # references it fail-fast -- the base infra file for dev (OMN-14968) and the
+    # stability-test/prod overlays -- so a recreate that omits the policy env
+    # fails loudly instead of dropping the worker with no signal.
     replicas: int = Field(default=1, ge=1)
 
     @field_validator("capabilities")

tests/ci/test_runtime_policy_contract.py

@@ -183,9 +183,11 @@ def test_compose_consumes_policy_env_instead_of_hardcoded_policy_values() -> Non
 def test_worker_replicas_pinned_in_contract_for_every_lane() -> None:
     """OMN-12990: each lane's worker process declares an explicit replica pin.
 
-    The base compose default is ``${WORKER_REPLICAS:-0}`` (silent drop). The
-    contract worker process must carry a contract-declared replica count >= 1 so
-    the rendered policy env preserves the worker on a lane recreate.
+    The contract worker process must carry a contract-declared replica count
+    >= 1 so the rendered policy env preserves the worker on a lane recreate.
+    Every lane's compose surface resolves that pin fail-closed (OMN-14968); a
+    lane whose contract dropped below 1 would push a zero-container lane through
+    a render that cannot detect it.
     """
     contract = _load_contract()
 
@@ -208,18 +210,34 @@ def test_worker_replicas_rendered_into_policy_env_for_every_lane() -> None:
 
 
 def test_runtime_worker_replicas_are_fail_fast_not_silent_default() -> None:
-    """OMN-12990: lane overrides reference the policy value with fail-fast ``:?``.
-
-    A soft ``:-1`` / ``:-0`` default would silently re-introduce the silent-drop
-    hole on any recreate that omitted the policy env. The base compose keeps its
-    ``${WORKER_REPLICAS:-0}`` default (that is the gap the overrides close), so a
-    plain recreate against the base alone still scales to zero — by design the
-    lane overrides MUST be applied, and they now abort loudly when the policy env
-    is missing rather than dropping the worker with no signal.
+    """OMN-12990 / OMN-14968: EVERY lane surface resolves replicas fail-fast.
+
+    A soft ``:-1`` / ``:-0`` default silently re-introduces the silent-drop hole
+    on any recreate that omits the policy env.
+
+    OMN-12990 converted the stability-test and prod overlays but left the base
+    infra compose on a BARE ``${WORKER_REPLICAS:-0}``. That bare name is exported
+    by no surface in this repo, so it always took the ``0`` branch — and the base
+    file with no overlay IS the dev lane, so the dev lane rendered a
+    zero-container worker, `up` created nothing, and the RT-6 deploy readback in
+    ``scripts/deploy-runtime.sh`` (whose ``RUNTIME_SERVICES`` includes
+    ``runtime-worker``) failed closed on every dev-lane deploy. OMN-14968 closed
+    it with the lane-prefixed ``${DEV_WORKER_REPLICAS:?...}`` form used by the
+    sibling ``DEV_RUNTIME_WORKER_*`` vars in the same service block.
     """
+    base_text = COMPOSE_PATH.read_text(encoding="utf-8")
     stability_text = STABILITY_COMPOSE_PATH.read_text(encoding="utf-8")
     prod_text = PROD_COMPOSE_PATH.read_text(encoding="utf-8")
 
+    assert "${DEV_WORKER_REPLICAS:?" in base_text, (
+        "the base infra compose (the dev lane's own compose file) must resolve "
+        "worker replicas fail-fast on the ledgered DEV_WORKER_REPLICAS"
+    )
+    assert "${DEV_WORKER_REPLICAS:-" not in base_text
+    assert "replicas: ${WORKER_REPLICAS" not in base_text, (
+        "the bare WORKER_REPLICAS name is exported by no surface in this repo; "
+        "it always resolved to the silent 0 default (OMN-14968)"
+    )
     assert "${STABILITY_TEST_WORKER_REPLICAS:?" in stability_text, (
         "stability worker replicas must be fail-fast on the ledgered policy value"
     )

tests/integration/docker/test_docker_integration.py

@@ -1093,6 +1093,10 @@ def test_compose_defines_resource_reservations(
     "DEV_RUNTIME_WORKER_CAPABILITIES": "workflow.dispatch,contract.update,runtime.worker",
     "DEV_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_JSON": _SECRET_RESOLVER_CONFIG_JSON,
     "DEV_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_PATH": _SECRET_RESOLVER_CONFIG_PATH,
+    # OMN-14968: runtime-worker's deploy.replicas is `:?`-required on the
+    # lane-prefixed policy value (it was a bare ${WORKER_REPLICAS:-0} that no
+    # surface exported, so the dev lane silently rendered zero replicas).
+    "DEV_WORKER_REPLICAS": "1",
     "OMNIMEMORY_ENABLED": "false",
     "OMNIMEMORY_MEMGRAPH_PORT": "7687",
     "ONEX_ACTIVE_RUNTIME_PACKAGES": "omnibase_infra,omnimarket",