fix(OMN-11068): fail healthcheck for degraded runtime

contracts/OMN-11068.yaml

@@ -0,0 +1,49 @@
+# OMN-11068 contract file for omnibase_infra deploy-gate.
+# The canonical ticket contract and PASS receipts live in onex_change_control.
+schema_version: "1.0.0"
+ticket_id: "OMN-11068"
+title: "fix(OMN-11068): fail runtime health probes when runtime is degraded unavailable"
+summary: >-
+  Runtime /health now returns a failing HTTP status while RuntimeHostProcess is not attached or explicitly reports is_running=false, preventing Docker healthchecks from marking the stability-test runtime healthy while the runtime is pending or stopped.
+is_seam_ticket: false
+interface_change: true
+interfaces_touched:
+  - public_api
+evidence_requirements:
+  - kind: "tests"
+    description: "Runtime health endpoint returns HTTP 503 for runtime_pending and is_running=false while preserving running degraded HTTP 200"
+    command: >-
+      uv run pytest tests/unit/runtime/test_service_health.py tests/integration/test_omn_9741_health_liveness.py -q
+  - kind: "tests"
+    description: "Stability-test runtime compose lane inherits curl --fail healthchecks that fail on HTTP 503"
+    command: >-
+      uv run pytest tests/unit/infra/test_stability_test_runtime_lane.py tests/integration/infra/test_stability_test_runtime_compose_render.py -q
+  - kind: "ci"
+    description: "CI gates pass on omnibase_infra PR"
+    command: "gh pr checks {pr} --repo OmniNode-ai/omnibase_infra"
+emergency_bypass:
+  enabled: false
+  justification: ""
+  follow_up_ticket_id: ""
+dod_evidence:
+  - id: "dod-runtime-health-http-status"
+    description: "Runtime health endpoint fails Docker probes for runtime_pending and is_running=false"
+    source: "manual"
+    checks:
+      - check_type: "command"
+        check_value: >-
+          uv run pytest tests/unit/runtime/test_service_health.py tests/integration/test_omn_9741_health_liveness.py -q
+  - id: "dod-stability-compose-healthcheck"
+    description: "Stability-test runtime compose render inherits curl --fail /health probes"
+    source: "manual"
+    checks:
+      - check_type: "command"
+        check_value: >-
+          uv run pytest tests/unit/infra/test_stability_test_runtime_lane.py tests/integration/infra/test_stability_test_runtime_compose_render.py -q
+  - id: "dod-post-merge-deploy-validation"
+    description: "Runtime healthcheck rollout requires normal post-merge deploy validation before claiming live runtime health"
+    source: "manual"
+    checks:
+      - check_type: "command"
+        check_value: >-
+          deploy omninode-stability-test-runtime after merge, then verify docker compose healthcheck fails while /health reports runtime_attached=false or is_running=false and succeeds only after runtime attachment reports is_running=true

docker/Dockerfile.runtime

@@ -446,7 +446,9 @@ USER omniinfra
 # This should match the port your runtime listens on for health checks
 EXPOSE 8085
 
-# Health check using the runtime's health endpoint
+# Health check using the runtime's health endpoint. The endpoint returns a
+# failing status while runtime_attached=false or is_running=false, so plain
+# curl --fail cannot mark a degraded runtime as Docker healthy.
 # - interval: Check every 30 seconds
 # - timeout: Fail if check takes more than 10 seconds
 # - start-period: Allow 120 seconds for startup (Kafka consumer init is slow)

docker/docker-compose.infra.yml

@@ -664,10 +664,11 @@ services:
   # ONEX Runtime - Effects Handler (Runtime Profile)
   # ==========================================================================
   # Handles effect nodes for external service interactions.
-  # OMN-9741: Effects startup performs long serial Kafka subscription joins.
-  # /health is liveness and returns degraded HTTP 200 while startup is in
-  # progress; strict readiness belongs to /ready. The extended health window
-  # prevents autoheal from recycling a live process before startup completes.
+  # OMN-9741/OMN-11068: Effects startup performs long serial Kafka subscription
+  # joins. /health is liveness, but runtime_pending and is_running=false return
+  # HTTP 503 so Docker cannot report a degraded runtime as healthy. The extended
+  # health window prevents autoheal from recycling a live process before startup
+  # completes.
   # Unhealthy detection latency = start_period + (interval * retries) = 1800s + 150s = 1950s.
   # AUTOHEAL_START_PERIOD (2400s) must exceed this window to avoid premature restarts.
   runtime-effects:

src/omnibase_infra/runtime/service_runtime_host_process.py

@@ -5068,7 +5068,8 @@ async def health_check(self) -> dict[str, object]:
 
         # Degraded state:
         # - running with failed handlers (reduced functionality)
-        # - actively starting with a live event bus (liveness OK, not ready yet)
+        # - actively starting with a live event bus (structured degraded state;
+        #   ServiceHealth still fails Docker probes until is_running=True)
         startup_in_progress = (
             self._is_starting and not self._is_running and event_bus_healthy
         )

src/omnibase_infra/services/service_health.py

@@ -581,8 +581,8 @@ async def start(self) -> None:
 
         HTTP Status Codes:
             /health:
-                - 200: Healthy or degraded (container operational)
-                - 503: Unhealthy (container should be restarted)
+                - 200: Healthy, or degraded while the runtime is running
+                - 503: Unhealthy, runtime not attached, or runtime not running
             /ready:
                 - 200: Ready (all consumers subscribed with partition assignments)
                 - 503: Not ready (starting up or lost assignments)
@@ -824,9 +824,9 @@ async def _handle_health(self, request: web.Request) -> web.Response:
             - unhealthy: Critical failure, return HTTP 503
 
         Degraded State HTTP 200 Design Decision:
-            Degraded containers intentionally return HTTP 200 to keep them in service
-            rotation. This is a deliberate design choice that prioritizes investigation
-            over automatic restarts.
+            Degraded containers only return HTTP 200 when the runtime is attached and
+            running. A startup-pending or stopped runtime returns HTTP 503 so Docker
+            healthchecks cannot report success while the runtime is unavailable.
 
             Rationale:
                 1. Automatic restarts may mask recurring issues that need investigation
@@ -854,8 +854,8 @@ async def _handle_health(self, request: web.Request) -> web.Response:
         Returns:
             JSON response with health status information. The HTTP status code
             indicates container health to orchestration platforms:
-                - HTTP 200: Container is healthy or degraded (operational)
-                - HTTP 503: Container is unhealthy (restart recommended)
+                - HTTP 200: Runtime is healthy or running with degraded components
+                - HTTP 503: Runtime is unhealthy, pending attachment, or stopped
 
         Response Format (Success):
             {
@@ -865,6 +865,7 @@ async def _handle_health(self, request: web.Request) -> web.Response:
                     "healthy": bool,
                     "degraded": bool,
                     "is_running": bool,
+                    "runtime_attached": bool,
                     "is_draining": bool,  // True during graceful shutdown drain
                     "pending_message_count": int,  // In-flight messages
                     "handlers": {...},
@@ -928,7 +929,7 @@ async def _handle_health(self, request: web.Request) -> web.Response:
                 )
                 return web.Response(
                     text=response.model_dump_json(exclude_none=True),
-                    status=200,
+                    status=503,
                     content_type="application/json",
                 )
 
@@ -955,43 +956,44 @@ async def _handle_health(self, request: web.Request) -> web.Response:
             is_degraded = bool(health_details.get("degraded", False))
             startup_in_progress = bool(health_details.get("startup_in_progress", False))
             is_running = bool(health_details.get("is_running", False))
+            runtime_attached = health_details.get("runtime_attached", True) is not False
+            runtime_operational = (
+                runtime_attached and health_details.get("is_running") is not False
+            )
             event_bus_healthy = bool(health_details.get("event_bus_healthy", False))
 
-            if is_healthy:
+            if is_healthy and runtime_operational:
                 status: Literal["healthy", "degraded", "unhealthy"] = "healthy"
                 http_status = 200
             elif is_degraded:
-                # DESIGN DECISION: Degraded status returns HTTP 200 (not 503)
+                # DESIGN DECISION: Running degraded status returns HTTP 200 (not 503)
                 #
                 # Rationale: Degraded containers remain in service rotation to allow
                 # operators to investigate issues without triggering automatic restarts.
                 # The "degraded" status in the response body indicates reduced functionality
-                # while keeping the container operational for Docker/Kubernetes probes.
+                # while keeping the running container operational for Docker/Kubernetes probes.
+                # If the runtime is not attached or reports is_running=false, return HTTP
+                # 503 so Docker cannot mark a non-operational runtime healthy.
                 #
                 # Why HTTP 200 instead of 503:
                 #   1. Prevents cascading failures if multiple containers degrade together
                 #   2. Reduced functionality is often better than no functionality
                 #   3. Automatic restarts may mask recurring issues needing investigation
                 #   4. Operators can monitor "degraded" status via metrics/alerts
                 #
-                # Alternative considered: HTTP 503 would remove degraded containers from
-                # load balancer rotation while keeping liveness probes passing. Rejected
-                # because it reduces capacity during partial outages when degraded
-                # containers may still serve valuable traffic.
-                #
                 # Customization: To remove degraded containers from rotation, either:
                 #   - Subclass ServiceHealth and override _handle_health()
                 #   - Configure load balancer to inspect response body "status" field
                 #   - Change http_status below to 503 if restart-on-degrade is preferred
                 status = "degraded"
-                http_status = 200
+                http_status = 200 if runtime_operational else 503
             else:
                 status = "unhealthy"
                 http_status = 503
 
             self._log_health_transition(
                 status=status,
-                runtime_attached=True,
+                runtime_attached=runtime_attached,
                 startup_in_progress=startup_in_progress,
                 is_running=is_running,
                 event_bus_healthy=event_bus_healthy,
@@ -1002,6 +1004,7 @@ async def _handle_health(self, request: web.Request) -> web.Response:
             enriched_details = dict(health_details)
             enriched_details["degraded"] = is_degraded
             enriched_details["startup_in_progress"] = startup_in_progress
+            enriched_details["runtime_attached"] = runtime_attached
             enriched_details["components"] = {
                 name: comp.model_dump(mode="json", exclude_none=True)
                 for name, comp in components.items()

tests/integration/infra/test_stability_test_runtime_compose_render.py

@@ -302,6 +302,20 @@ def test_stability_lane_render_inherits_release_build_source() -> None:
         }
 
 
+@pytest.mark.integration
+def test_stability_lane_render_inherits_failing_runtime_healthcheck() -> None:
+    rendered_config = _compose_config_json()
+    services = rendered_config["services"]
+
+    for service_name in REQUIRED_RUNTIME_SERVICES:
+        assert services[service_name]["healthcheck"]["test"] == [
+            "CMD",
+            "curl",
+            "-sf",
+            "http://localhost:8085/health",
+        ]
+
+
 @pytest.mark.integration
 def test_stability_lane_render_does_not_expose_production_ports_or_services() -> None:
     rendered_config = _compose_config_json()

tests/integration/test_omn_9741_health_liveness.py

@@ -1,12 +1,14 @@
 # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc.
 # SPDX-License-Identifier: MIT
 
-"""Integration proof for OMN-9741 effects health liveness during startup.
+"""Integration proof for runtime health liveness during startup.
 
 OMN-9741 fixes the health liveness contract: when the runtime is not yet
-attached (subscription startup in progress), the /health endpoint must return
-HTTP 200 with status=degraded rather than HTTP 503, so Docker/autoheal does
-not recycle a live effects process during long Kafka subscription setup.
+attached, the /health endpoint can expose structured runtime_pending details
+before RuntimeHostProcess exists.
+
+OMN-11068 tightens the Docker probe contract: runtime_pending must return HTTP
+503 so Docker health cannot report success while runtime_attached=false.
 
 Ticket: OMN-9741
 Integration Test Coverage gate: OMN-7005 (hard gate since 2026-04-13).
@@ -25,22 +27,21 @@
 
 @pytest.mark.integration
 @pytest.mark.asyncio
-async def test_health_endpoint_returns_200_degraded_when_runtime_not_attached() -> None:
-    """Health endpoint must return HTTP 200 degraded when runtime is not yet attached.
+async def test_health_endpoint_returns_503_degraded_when_runtime_not_attached() -> None:
+    """Health endpoint must fail Docker probes when runtime is not yet attached.
 
-    This is the liveness invariant for OMN-9741: Docker autoheal kills containers
-    on HTTP 503. During Kafka subscription startup, runtime is None — the endpoint
-    must return degraded (HTTP 200), not unhealthy (HTTP 503).
+    This is the OMN-11068 masking regression: plain curl -sf Docker healthchecks
+    must fail while runtime_attached=false and startup_phase=runtime_pending.
     """
     container = MagicMock()
     server = ServiceHealth(container=container)
     mock_request = MagicMock(spec=web.Request)
 
     response = await server._handle_health(mock_request)
 
-    assert response.status == 200, (
-        f"Expected HTTP 200 (degraded) when runtime not attached, got {response.status}. "
-        "Docker autoheal would kill a container returning 503 during startup."
+    assert response.status == 503, (
+        f"Expected HTTP 503 (degraded) when runtime not attached, got {response.status}. "
+        "Docker healthchecks must not pass while runtime_attached=false."
     )
     body = json.loads(response.text)
     assert body["status"] == "degraded", (
@@ -81,8 +82,8 @@ async def test_health_endpoint_returns_200_healthy_after_attach_runtime() -> Non
 
 @pytest.mark.integration
 @pytest.mark.asyncio
-async def test_health_endpoint_returns_200_degraded_during_runtime_startup() -> None:
-    """Health endpoint returns degraded 200 when runtime reports startup_in_progress."""
+async def test_health_endpoint_returns_503_degraded_until_runtime_is_running() -> None:
+    """Health endpoint fails Docker probes while attached runtime is not running."""
     mock_runtime = MagicMock()
     mock_runtime.health_check = AsyncMock(
         return_value={
@@ -100,7 +101,8 @@ async def test_health_endpoint_returns_200_degraded_during_runtime_startup() ->
 
     response = await server._handle_health(mock_request)
 
-    assert response.status == 200
+    assert response.status == 503
     body = json.loads(response.text)
     assert body["status"] == "degraded"
     assert body["details"]["startup_in_progress"] is True
+    assert body["details"]["is_running"] is False

tests/unit/infra/test_stability_test_runtime_lane.py

@@ -10,6 +10,7 @@
 import yaml
 
 REPO_ROOT = Path(__file__).resolve().parents[3]
+BASE_COMPOSE_FILE = REPO_ROOT / "docker" / "docker-compose.infra.yml"
 OVERLAY_FILE = REPO_ROOT / "docker" / "docker-compose.stability-test.yml"
 RUNBOOK_FILE = REPO_ROOT / "docs" / "runbooks" / "stability-test-runtime-lane.md"
 
@@ -88,6 +89,13 @@ def _load_overlay() -> dict:
     return overlay
 
 
+def _load_base_compose() -> dict:
+    base_text = BASE_COMPOSE_FILE.read_text(encoding="utf-8")
+    base = yaml.load(base_text, Loader=yaml.SafeLoader)
+    assert isinstance(base, dict)
+    return base
+
+
 def _labels_by_key(labels: list[str]) -> dict[str, str]:
     label_map: dict[str, str] = {}
     for label in labels:
@@ -246,6 +254,22 @@ def test_stability_lane_runtime_services_preserve_image_runtime_contracts() -> N
         assert any(volume.endswith(":/app/skills:ro") for volume in volumes)
 
 
+@pytest.mark.unit
+def test_stability_lane_runtime_healthchecks_fail_on_http_503() -> None:
+    """Stability lane inherits curl --fail probes from runtime compose services."""
+    base = _load_base_compose()
+    overlay = _load_overlay()
+
+    for service_name in RUNTIME_SERVICES:
+        assert "healthcheck" not in overlay["services"][service_name]
+        assert base["services"][service_name]["healthcheck"]["test"] == [
+            "CMD",
+            "curl",
+            "-sf",
+            "http://localhost:8085/health",
+        ]
+
+
 @pytest.mark.unit
 def test_stability_lane_core_infra_names_are_distinct() -> None:
     overlay = _load_overlay()

tests/unit/runtime/test_service_health.py

@@ -335,7 +335,7 @@ async def test_health_endpoint_logs_runtime_pending_transition(self) -> None:
         with patch("omnibase_infra.services.service_health.logger.info") as mock_info:
             response = await server._handle_health(mock_request)
 
-        assert response.status == 200
+        assert response.status == 503
         transition_logs = [
             call
             for call in mock_info.call_args_list
@@ -347,8 +347,10 @@ async def test_health_endpoint_logs_runtime_pending_transition(self) -> None:
         assert transition_logs[0].kwargs["extra"]["is_running"] is False
 
     @pytest.mark.asyncio
-    async def test_health_endpoint_treats_attached_startup_as_degraded(self) -> None:
-        """Test /health returns degraded 200 when runtime reports startup_in_progress."""
+    async def test_health_endpoint_fails_when_attached_runtime_is_not_running(
+        self,
+    ) -> None:
+        """Test /health returns failing degraded when runtime reports not running."""
         mock_runtime = MagicMock()
         mock_runtime.health_check = AsyncMock(
             return_value={
@@ -365,12 +367,13 @@ async def test_health_endpoint_treats_attached_startup_as_degraded(self) -> None
 
         response = await server._handle_health(mock_request)
 
-        assert response.status == 200
+        assert response.status == 503
         response_text = response.text
         assert response_text is not None
         assert '"status":"degraded"' in response_text
         assert '"startup_in_progress":true' in response_text
         assert '"degraded":true' in response_text
+        assert '"runtime_attached":true' in response_text
 
 
 @pytest.mark.unit
@@ -954,8 +957,8 @@ def test_container_preserved_when_runtime_is_none_after_init(self) -> None:
         assert server.container is server.container
 
     @pytest.mark.asyncio
-    async def test_health_endpoint_degrades_when_runtime_not_resolved(self) -> None:
-        """Container-only startup should still expose a live /health endpoint."""
+    async def test_health_endpoint_fails_when_runtime_not_resolved(self) -> None:
+        """Container-only startup must not satisfy Docker healthchecks."""
         mock_container = MagicMock(spec=ModelONEXContainer)
         server = ServiceHealth(container=mock_container)
         server._is_running = True
@@ -964,7 +967,7 @@ async def test_health_endpoint_degrades_when_runtime_not_resolved(self) -> None:
 
         response = await server._handle_health(mock_request)
 
-        assert response.status == 200
+        assert response.status == 503
         response_text = response.text
         assert response_text is not None
         assert '"status":"degraded"' in response_text