jonahgabriel · Jul 9, 2026 · Jul 9, 2026 · Jul 9, 2026 · Jul 9, 2026 · Jul 9, 2026
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -317,14 +317,9 @@ jobs:
   infra-node-handler-ownership:
     needs: occ-preflight
     name: Infra Node Handler Ownership
-    # OMNI_RUNNER_SELECTOR_V1 — trusted CI defaults to self-hosted omnibase-ci; public forks default to ubuntu-latest
-    runs-on: >-
-      ${{
-        (github.event_name == 'pull_request' &&
-         github.event.pull_request.head.repo.full_name != github.repository)
-        && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]')
-        || fromJSON((github.event_name == 'merge_group' && vars.OMNI_REQUIRED_CI_RUNS_ON_JSON) || vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]')
-      }}
+    # Pure repository ownership lint. Keep it hosted so self-hosted saturation
+    # cannot cancel this short job and falsely fail CI Summary.
+    runs-on: ubuntu-latest
     timeout-minutes: 5
 
     steps:

diff --git a/.github/workflows/url-authority-gate.yml b/.github/workflows/url-authority-gate.yml
@@ -65,6 +65,10 @@ jobs:
         && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]')
         || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]')
       }}
+    env:
+      # Avoid reusing a corrupted self-hosted uv wheel cache; this gate has hit
+      # repeat grpcio hash mismatches from runner-local cache state.
+      UV_NO_CACHE: "1"
     steps:
       - name: Checkout
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -86,13 +90,20 @@ jobs:
 
       - name: Run url-authority gate (full-repo, against frozen baseline)
         run: |
-          uv run --with "omnibase-core @ git+https://github.com/OmniNode-ai/omnibase_core.git@8a53a063bd28f643d08b4cbbc6dd5c7c9f6435df" \
-            python -m omnibase_core.validation.validator_url_authority \
-            --all --repo omnibase_infra --repo-root .
+          for attempt in 1 2 3; do
+            uv run --with "omnibase-core @ git+https://github.com/OmniNode-ai/omnibase_core.git@8a53a063bd28f643d08b4cbbc6dd5c7c9f6435df" \
+              python -m omnibase_core.validation.validator_url_authority \
+              --all --repo omnibase_infra --repo-root . && break
+            status=$?
+            if [ "$attempt" = "3" ]; then
+              exit "$status"
+            fi
+            sleep "$((attempt * 15))"
+          done
 
       - name: Assert baseline did not grow (anti-gaming)
         run: |
-          uv run --with "omnibase-core @ git+https://github.com/OmniNode-ai/omnibase_core.git@8a53a063bd28f643d08b4cbbc6dd5c7c9f6435df" python - <<'PY'
+          cat > /tmp/url_authority_baseline_check.py <<'PY'
           import json
           import subprocess
           import sys
@@ -141,3 +152,12 @@ jobs:
               f"omnibase_infra subset {len(head_repo)} (<= prior {len(prior_repo)})."
           )
           PY
+          for attempt in 1 2 3; do
+            uv run --with "omnibase-core @ git+https://github.com/OmniNode-ai/omnibase_core.git@8a53a063bd28f643d08b4cbbc6dd5c7c9f6435df" \
+              python /tmp/url_authority_baseline_check.py && break
+            status=$?
+            if [ "$attempt" = "3" ]; then
+              exit "$status"
+            fi
+            sleep "$((attempt * 15))"
+          done
diff --git a/docker/runners/runner-monitor.sh b/docker/runners/runner-monitor.sh
@@ -401,6 +401,12 @@ for i in $(seq 1 "$EXPECTED_RUNNERS"); do
         stuck_created_list+=("${name}: Docker Created but never started — orphaned mid-recreate")
     fi
 
+    if [[ "${docker_state}" == Up* ]] && [[ "${docker_state}" != *"(healthy)"* ]] && runner_has_local_listener_evidence "${name}"; then
+        github_degraded_list+=("${name}: Docker health ${docker_state} ignored; local listener evidence present")
+        healthy=$((healthy + 1))
+        continue
+    fi
+
     if [[ "${docker_state}" != *"(healthy)"* ]] || [[ "${docker_state}" != Up* ]]; then
         unhealthy_list+=("${name}: Docker ${docker_state}")
         continue
@@ -693,14 +699,18 @@ auto_bounce() {
 # Compare with previous state and alert on transitions
 # ---------------------------------------------------------------------------
 prev_unhealthy_count=0
+prev_alert_count=0
 if [[ -f "$STATE_FILE" ]]; then
     prev_unhealthy_count=$(jq -r '.unhealthy_count // 0' "$STATE_FILE" 2>/dev/null || echo 0)
+    prev_alert_count=$(jq -r '.alert_count // 0' "$STATE_FILE" 2>/dev/null || echo 0)
 fi
 
 current_unhealthy_count=${#unhealthy_list[@]}
 wedge_count=${#wedge_list[@]}
 crashloop_count=${#crashloop_list[@]}
 stuck_created_count=${#stuck_created_list[@]}
+offline_idle_recreate_count=${#offline_idle_recreate_list[@]}
+missing_container_count=${#missing_container_list[@]}
 offline_first_seen_json="{}"
 if [[ -n "${offline_first_seen_lines}" ]]; then
     offline_first_seen_json=$(printf '%s' "${offline_first_seen_lines}" | jq -Rn '
@@ -715,13 +725,30 @@ if [[ -n "${offline_first_seen_lines}" ]]; then
     ')
 fi
 
+remediation_targets="$(collect_remediation_targets)"
+remediation_target_count=0
+if [[ -n "${remediation_targets// /}" ]]; then
+    remediation_target_count=$(wc -w <<< "${remediation_targets}")
+fi
+current_alert_count="${remediation_target_count}"
+if [[ "${github_api_failed}" == true ]]; then
+    current_alert_count=$((current_alert_count + 1))
+fi
+if [[ "${docker_ok}" != true ]]; then
+    current_alert_count=$((current_alert_count + 1))
+fi
+
 # Write current state
 jq -n \
     --argjson healthy "$healthy" \
     --argjson unhealthy_count "$current_unhealthy_count" \
+    --argjson alert_count "$current_alert_count" \
+    --argjson remediation_target_count "$remediation_target_count" \
     --argjson wedge_count "$wedge_count" \
     --argjson crashloop_count "$crashloop_count" \
     --argjson stuck_created_count "$stuck_created_count" \
+    --argjson offline_idle_recreate_count "$offline_idle_recreate_count" \
+    --argjson missing_container_count "$missing_container_count" \
     --argjson online "$online_count" \
     --argjson busy "$busy_count" \
     --argjson queued_age "$queued_age" \
@@ -736,9 +763,13 @@ jq -n \
     '{
         healthy: $healthy,
         unhealthy_count: $unhealthy_count,
+        alert_count: $alert_count,
+        remediation_target_count: $remediation_target_count,
         wedge_count: $wedge_count,
         crashloop_count: $crashloop_count,
         stuck_created_count: $stuck_created_count,
+        offline_idle_recreate_count: $offline_idle_recreate_count,
+        missing_container_count: $missing_container_count,
         online: $online,
         busy: $busy,
         oldest_queued_job_age_seconds: $queued_age,
@@ -756,7 +787,6 @@ jq -n \
 # Alert logic — only on state transitions
 # ---------------------------------------------------------------------------
 
-remediation_targets="$(collect_remediation_targets)"
 safe_bounce_block=""
 if [[ -n "${remediation_targets// /}" ]]; then
     safe_bounce_block="$(render_safe_bounce_cmd "${remediation_targets}")"
@@ -780,10 +810,10 @@ if [[ "${stuck_created_count}" -gt 0 ]]; then
     special_findings+="$(printf '%s\n' "${stuck_created_list[@]}")"$'\n'
 fi
 
-if [[ $current_unhealthy_count -gt 0 ]] && [[ $prev_unhealthy_count -eq 0 ]]; then
-    # Transition: all-good → something broken
+if [[ $current_alert_count -gt 0 ]] && [[ $prev_alert_count -eq 0 ]]; then
+    # Transition: no actionable alert -> actionable repair/monitor failure.
     detail=$(printf '%s\n' "${unhealthy_list[@]}")
-    msg="*[RUNNER ALERT]* ${current_unhealthy_count}/${EXPECTED_RUNNERS} runners unhealthy
+    msg="*[RUNNER ALERT]* ${current_alert_count} actionable runner-fleet issue(s); ${current_unhealthy_count}/${EXPECTED_RUNNERS} raw unhealthy
 
 \`\`\`
 ${detail}
@@ -808,13 +838,14 @@ ${safe_bounce_block}
 \`\`\`"
     fi
     slack_post "${msg}" "danger"
-    log "ALERT: ${current_unhealthy_count} runners unhealthy (was 0). wedge=${wedge_count} crashloop=${crashloop_count} stuck_created=${stuck_created_count}. Slack notified."
+    log "ALERT: ${current_alert_count} actionable issue(s), ${current_unhealthy_count} raw unhealthy (previous actionable 0). wedge=${wedge_count} crashloop=${crashloop_count} stuck_created=${stuck_created_count} offline_idle_recreate=${offline_idle_recreate_count}. Slack notified."
     auto_bounce "${remediation_targets}"
 
-elif [[ $current_unhealthy_count -gt 0 ]] && [[ $prev_unhealthy_count -gt 0 ]] && [[ $current_unhealthy_count -ne $prev_unhealthy_count ]]; then
-    # Transition: bad → worse or bad → partially recovered
+elif [[ $current_alert_count -gt 0 ]] && [[ $prev_alert_count -gt 0 ]] && [[ $current_alert_count -ne $prev_alert_count ]]; then
+    # Transition: actionable issue count changed. Raw GitHub/Docker drift count
+    # can churn under saturation; do not page on that noise.
     detail=$(printf '%s\n' "${unhealthy_list[@]}")
-    msg="*[RUNNER UPDATE]* ${current_unhealthy_count}/${EXPECTED_RUNNERS} runners unhealthy (was ${prev_unhealthy_count})
+    msg="*[RUNNER UPDATE]* ${current_alert_count} actionable runner-fleet issue(s) (was ${prev_alert_count}); ${current_unhealthy_count}/${EXPECTED_RUNNERS} raw unhealthy
 
 \`\`\`
 ${detail}
@@ -837,20 +868,21 @@ ${safe_bounce_block}
 \`\`\`"
     fi
     slack_post "${msg}" "warning"
-    log "UPDATE: ${current_unhealthy_count} unhealthy (was ${prev_unhealthy_count}). wedge=${wedge_count} crashloop=${crashloop_count} stuck_created=${stuck_created_count}. Slack notified."
+    log "UPDATE: ${current_alert_count} actionable issue(s) (was ${prev_alert_count}), ${current_unhealthy_count} raw unhealthy. wedge=${wedge_count} crashloop=${crashloop_count} stuck_created=${stuck_created_count} offline_idle_recreate=${offline_idle_recreate_count}. Slack notified."
     auto_bounce "${remediation_targets}"
 
-elif [[ $current_unhealthy_count -eq 0 ]] && [[ $prev_unhealthy_count -gt 0 ]]; then
-    # Transition: broken → all recovered
-    slack_post "*[RUNNER RECOVERED]* All ${EXPECTED_RUNNERS} runners healthy
+elif [[ $current_alert_count -eq 0 ]] && [[ $prev_alert_count -gt 0 ]]; then
+    # Transition: actionable alert cleared. Raw drift may remain and is logged.
+    slack_post "*[RUNNER RECOVERED]* No actionable runner-fleet issues remain
 
+Healthy: ${healthy}/${EXPECTED_RUNNERS} | Raw unhealthy: ${current_unhealthy_count}
 Online: ${online_count} | Busy: ${busy_count}
 Docker socket: $([ "$docker_ok" = true ] && echo 'OK' || echo 'FAILED')
 Host: ${RUNNER_HOST}" "good"
-    log "RECOVERED: All ${EXPECTED_RUNNERS} runners healthy. Slack notified."
+    log "RECOVERED: actionable alert count cleared; ${current_unhealthy_count} raw unhealthy remain. Slack notified."
 
 else
     # No state change — silent
-    log "OK: ${healthy}/${EXPECTED_RUNNERS} healthy, ${current_unhealthy_count} unhealthy (wedge=${wedge_count} crashloop=${crashloop_count} stuck_created=${stuck_created_count}, no change)."
+    log "OK: ${healthy}/${EXPECTED_RUNNERS} healthy, ${current_unhealthy_count} raw unhealthy, ${current_alert_count} actionable (wedge=${wedge_count} crashloop=${crashloop_count} stuck_created=${stuck_created_count} offline_idle_recreate=${offline_idle_recreate_count}, no Slack change)."
     auto_bounce "${remediation_targets}"
 fi
diff --git a/tests/unit/observability/runner_health/test_runner_monitor_wedge_detection.py b/tests/unit/observability/runner_health/test_runner_monitor_wedge_detection.py
@@ -195,6 +195,7 @@ def _make_mock_bin(
 {_queued_jobs_json(created_at=queued_job_created_at)}
 JSON
         elif [[ "${{url}}" == *"slack.com"* ]]; then
+          echo "slack $*" >> "${{MOCK_SLACK_CALLLOG:-/dev/null}}"
           echo '{{"ok":true}}'
         else
           echo ""
@@ -274,6 +275,7 @@ def _run_monitor(
     fleet_config = tmp_path / "runner_fleet.yaml"
     _write_fleet_config(fleet_config)
     call_log = tmp_path / "docker-calls.log"
+    slack_log = tmp_path / "slack-calls.log"
     if previous_state is not None:
         state_file.write_text(json.dumps(previous_state), encoding="utf-8")
 
@@ -287,6 +289,7 @@ def _run_monitor(
         "SLACK_CHANNEL_ID": "C-test",
         "RUNNER_GITHUB_TOKEN": "ghp-test",
         "MOCK_DOCKER_CALLLOG": str(call_log),
+        "MOCK_SLACK_CALLLOG": str(slack_log),
         # Wedge threshold low enough that the controlled queued-age trips it.
         "WEDGE_QUEUE_AGE_SECONDS": "600",
         "WEDGE_WATCH_REPOS": "OmniNode-ai/omnibase_infra",
@@ -336,6 +339,9 @@ def _run_monitor(
     state["_docker_calls"] = (
         call_log.read_text(encoding="utf-8") if call_log.exists() else ""
     )
+    state["_slack_calls"] = (
+        slack_log.read_text(encoding="utf-8") if slack_log.exists() else ""
+    )
     return state
 
 
@@ -614,6 +620,68 @@ def test_github_offline_with_local_listener_evidence_is_not_unhealthy(
     assert "OFFLINE-IDLE" not in str(state["offline_idle_bounce_names"]), state
 
 
+def test_docker_unhealthy_with_local_listener_evidence_is_degraded_not_alerted(
+    tmp_path: Path,
+) -> None:
+    """Docker health can lag while the runner listener is still processing jobs.
+
+    A locally live listener is degraded evidence, not an actionable fleet page.
+    """
+    _require_tools()
+    bindir = tmp_path / "bin"
+    _scenario_bin(
+        bindir,
+        status="online",
+        busy=True,
+        docker_status="Up 6 hours (unhealthy)",
+        restart_count=0,
+        docker_logs="2026-07-09T16:20:28Z: Running job: still-active",
+        queued=False,
+    )
+    state = _run_monitor(tmp_path, bindir)
+
+    assert _int(state, "unhealthy_count") == 0, state
+    assert _int(state, "alert_count") == 0, state
+    assert "Docker health Up 6 hours (unhealthy) ignored" in str(
+        state["github_degraded_names"]
+    ), state
+    assert state["_slack_calls"] == "", state
+
+
+def test_raw_github_offline_churn_does_not_send_slack_without_actionable_target(
+    tmp_path: Path,
+) -> None:
+    """Raw unhealthy count can churn when GitHub reports offline-idle runners.
+
+    The monitor should keep recording the drift while suppressing Slack until a
+    runner becomes an actionable remediation target.
+    """
+    _require_tools()
+    bindir = tmp_path / "bin"
+    _scenario_bin(
+        bindir,
+        status="offline",
+        busy=False,
+        docker_status="Up 6 hours (healthy)",
+        restart_count=0,
+        queued=False,
+    )
+    state = _run_monitor(
+        tmp_path,
+        bindir,
+        previous_state={
+            "unhealthy_count": 1,
+            "alert_count": 0,
+            "offline_first_seen": {},
+        },
+    )
+
+    assert _int(state, "unhealthy_count") == TEST_FLEET_COUNT, state
+    assert _int(state, "alert_count") == 0, state
+    assert "OFFLINE-IDLE" in str(state["offline_idle_bounce_names"]), state
+    assert state["_slack_calls"] == "", state
+
+
 def test_persistent_offline_idle_runner_auto_bounces_when_locally_idle(
     tmp_path: Path,
 ) -> None: