fix(OMN-15617): resolve bash>=5 explicitly instead of trusting PATH order

scripts/ci/resolve_modern_bash.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc.
+# SPDX-License-Identifier: MIT
+#
+# resolve_modern_bash.sh -- OMN-15617: explicit, PATH-order-independent
+# resolution of a bash interpreter new enough for `declare -A`.
+#
+# Root cause: on stickybeatz-studio (.200, the rule-11a default gate host),
+# non-interactive ssh sessions resolve `bash` / `env bash` to the system
+# 3.2.57 shell (no associative arrays) even though a modern bash 5.x sits at
+# /opt/homebrew/bin/bash -- it simply is not first on PATH for that session
+# class. Callers that spawn `bash <script-using-declare--A>` via bare "bash"
+# silently run the wrong interpreter and fail with a bash syntax error deep
+# inside the script, not a resolvable "wrong bash" diagnostic.
+#
+# This script performs the resolution ONCE, explicitly, independent of PATH
+# order, and is the single source of truth callers (the pre-push canary and
+# the runner-monitor.sh pytest harness) both use -- so they can never drift.
+#
+# MUST remain runnable under bash 3.2 itself (no `declare -A`, no `[[ ]]`
+# reliance beyond what 3.2 supports) -- interpreter resolution cannot
+# presuppose the thing it is resolving.
+#
+# Output contract: on success, prints the resolved interpreter's absolute
+# path to stdout and exits 0. On failure, prints nothing to stdout, prints a
+# pointed ERROR + REMEDIATION message to stderr, and exits 1. Never a silent
+# fallback and never a quiet skip -- an unresolvable modern bash is a hard
+# failure the caller must surface.
+#
+# Env overrides:
+#   OMNIBASE_INFRA_BASH_BIN       explicit interpreter path to try FIRST.
+#   OMNIBASE_INFRA_MIN_BASH_MAJOR minimum BASH_VERSINFO[0] required (default 5).
+
+set -euo pipefail
+
+MIN_MAJOR="${OMNIBASE_INFRA_MIN_BASH_MAJOR:-5}"
+
+# Prints the candidate's BASH_VERSINFO[0] on stdout, or nothing + nonzero exit
+# if the candidate is not an executable bash at all.
+bash_major_version() {
+  local bin="$1"
+  [ -x "$bin" ] || return 1
+  "$bin" -c 'printf "%s" "${BASH_VERSINFO[0]:-}"' 2> /dev/null
+}
+
+seen=""
+try_candidate() {
+  local bin="$1" major
+  case " ${seen} " in
+    *" ${bin} "*) return 1 ;;
+  esac
+  seen="${seen} ${bin}"
+  major="$(bash_major_version "$bin")" || return 1
+  [ -n "$major" ] || return 1
+  [ "$major" -ge "$MIN_MAJOR" ] || return 1
+  printf '%s\n' "$bin"
+  return 0
+}
+
+# Ordered candidate list: explicit override, the two brew prefixes (Apple
+# Silicon + Intel), then every "bash" found walking $PATH -- so a modern bash
+# installed somewhere non-standard is still picked up without PATH surgery.
+CANDIDATES=""
+if [ -n "${OMNIBASE_INFRA_BASH_BIN:-}" ]; then
+  CANDIDATES="${CANDIDATES} ${OMNIBASE_INFRA_BASH_BIN}"
+fi
+CANDIDATES="${CANDIDATES} /opt/homebrew/bin/bash /usr/local/bin/bash"
+
+old_ifs="$IFS"
+IFS=':'
+for _dir in $PATH; do
+  [ -n "$_dir" ] || continue
+  CANDIDATES="${CANDIDATES} ${_dir}/bash"
+done
+IFS="$old_ifs"
+
+for _candidate in $CANDIDATES; do
+  if try_candidate "$_candidate"; then
+    exit 0
+  fi
+done
+
+{
+  printf 'ERROR: no bash interpreter >= %s found (declare -A requires bash>=4; OMN-15617 requires >=%s).\n' "$MIN_MAJOR" "$MIN_MAJOR"
+  printf 'Checked: OMNIBASE_INFRA_BASH_BIN, /opt/homebrew/bin/bash, /usr/local/bin/bash, and every "bash" on PATH.\n'
+  printf 'REMEDIATION: install a modern bash (e.g. `brew install bash`) and/or set OMNIBASE_INFRA_BASH_BIN to its absolute path.\n'
+} >&2
+exit 1

scripts/hooks/prepush_smart_tests.sh

@@ -166,6 +166,30 @@ REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" \
          "run 'git push' from within the omnibase_infra repository"
 cd "$REPO_ROOT"
 
+# =============================================================================
+# bash>=5 canary (OMN-15617)
+# =============================================================================
+# On stickybeatz-studio (.200, the rule-11a default gate host), non-interactive
+# ssh sessions resolve `bash` to the system 3.2.57 shell even though a modern
+# bash 5.x sits at /opt/homebrew/bin/bash -- it just is not first on PATH for
+# that session class. runner-monitor.sh (exercised end-to-end by
+# tests/unit/observability/runner_health/test_runner_monitor_*.py) uses
+# `declare -A`, which bash 3.2 does not support, so those tests fail SILENTLY
+# on every push from this host class -- a bash syntax error deep inside a
+# subprocess, not a resolvable "wrong interpreter" diagnostic.
+#
+# Resolve a bash>=5 interpreter EXPLICITLY here, independent of PATH order,
+# via the same resolver the pytest harness uses (single source of truth --
+# scripts/ci/resolve_modern_bash.sh -- so the two can never drift apart), and
+# export it so the harness does not have to re-discover it. Fail LOUD with a
+# pointed remediation message if none is resolvable anywhere -- never a quiet
+# skip, never a silent fallback to whatever "bash" happens to resolve first.
+MODERN_BASH="$(bash "${REPO_ROOT}/scripts/ci/resolve_modern_bash.sh")" \
+  || die "no bash>=5 interpreter resolvable on this host" \
+         "install a modern bash (e.g. 'brew install bash') and/or set OMNIBASE_INFRA_BASH_BIN to its absolute path; see scripts/ci/resolve_modern_bash.sh"
+export OMNIBASE_INFRA_BASH_BIN="$MODERN_BASH"
+log "bash>=5 canary: resolved ${MODERN_BASH}"
+
 BASE_REF="${PREPUSH_BASE_REF:-origin/dev}"
 
 # Deterministic diff base: fetch the base ref best-effort so an online push gets

tests/unit/observability/runner_health/_resolve_modern_bash.py

@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc.
+# SPDX-License-Identifier: MIT
+"""Shared bash>=5 interpreter resolution for the runner-monitor.sh harness.
+
+OMN-15617: on stickybeatz-studio (.200, the rule-11a default gate host),
+non-interactive ssh resolves the system bash 3.2.57 first on PATH, silently
+failing every `declare -A` predicate in runner-monitor.sh. These tests drive
+the REAL script end-to-end (deliberately -- see the module docstrings in the
+two callers), so they need a real bash>=5, resolved explicitly rather than
+trusted to ambient PATH order.
+
+Deliberately NOT a Python re-implementation of the bash version check: it
+shells out to scripts/ci/resolve_modern_bash.sh, the single source of truth
+also used by the pre-push canary (scripts/hooks/prepush_smart_tests.sh), so
+the two can never drift apart (DRY, root CLAUDE.md operating discipline).
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).parents[4]
+RESOLVE_SCRIPT = REPO_ROOT / "scripts" / "ci" / "resolve_modern_bash.sh"
+
+
+def resolve_modern_bash() -> str:
+    """Return an absolute path to a bash>=5 interpreter, or fail loud.
+
+    Fails the test via ``pytest.fail`` (never a silent skip or a quiet
+    fallback to whatever "bash" resolves first on PATH) when no bash>=5
+    interpreter can be found anywhere -- that quiet-fallback shape is the
+    exact bug OMN-15617 exists to close.
+    """
+    if not RESOLVE_SCRIPT.is_file():
+        pytest.fail(f"resolver script missing: {RESOLVE_SCRIPT}")
+    # The resolver script itself is bash-3.2-safe by construction (it cannot
+    # presuppose the modern bash it is trying to find), so it is safe to
+    # invoke via the bare "bash" on whatever PATH this process inherited.
+    result = subprocess.run(
+        ["bash", str(RESOLVE_SCRIPT)],
+        capture_output=True,
+        text=True,
+        timeout=10,
+        check=False,
+    )
+    resolved = result.stdout.strip()
+    if result.returncode != 0 or not resolved:
+        pytest.fail(
+            "no bash>=5 interpreter resolvable for the runner-monitor.sh "
+            "harness (declare -A requires bash>=4; OMN-15617).\n"
+            f"resolver stderr:\n{result.stderr}"
+        )
+    return resolved

tests/unit/observability/runner_health/test_runner_monitor_auto_bounce.py

@@ -39,6 +39,10 @@
 
 import pytest
 
+from tests.unit.observability.runner_health._resolve_modern_bash import (
+    resolve_modern_bash,
+)
+
 REPO_ROOT = Path(__file__).parents[4]
 MONITOR_SCRIPT = REPO_ROOT / "docker" / "runners" / "runner-monitor.sh"
 
@@ -55,6 +59,11 @@ def _require_tools() -> None:
     for tool in ("bash", "jq", "flock"):
         if shutil.which(tool) is None:
             pytest.skip(f"{tool} not available; shell detection test requires it")
+    # runner-monitor.sh uses `declare -A` (bash>=4). OMN-15617: a bash that
+    # merely EXISTS on PATH (checked above) is not sufficient -- resolve one
+    # that is actually >=5, explicitly. Fails loud (never a silent skip) when
+    # none is resolvable anywhere.
+    resolve_modern_bash()
 
 
 def _write_exec(path: Path, body: str) -> None:
@@ -268,22 +277,27 @@ def _run_monitor(
     if extra_env:
         env.update(extra_env)
 
+    # OMN-15617: resolve a bash>=5 interpreter explicitly rather than trusting
+    # ambient PATH order (see the sibling wedge-detection module for the full
+    # rationale).
+    modern_bash = resolve_modern_bash()
+
     wrapper = tmp_path / "run.sh"
     wrapper.write_text(
         textwrap.dedent(
             f"""\
             #!/usr/bin/env bash
             set -euo pipefail
             sed 's#^STATE_FILE=.*#STATE_FILE="{state_file}"#' "{MONITOR_SCRIPT}" > "{tmp_path}/monitor.sh"
-            bash "{tmp_path}/monitor.sh"
+            "{modern_bash}" "{tmp_path}/monitor.sh"
             """
         ),
         encoding="utf-8",
     )
     wrapper.chmod(wrapper.stat().st_mode | stat.S_IEXEC)
 
     result = subprocess.run(
-        ["bash", str(wrapper)],
+        [modern_bash, str(wrapper)],
         env=env,
         capture_output=True,
         text=True,

tests/unit/observability/runner_health/test_runner_monitor_wedge_detection.py

@@ -35,6 +35,10 @@
 
 import pytest
 
+from tests.unit.observability.runner_health._resolve_modern_bash import (
+    resolve_modern_bash,
+)
+
 REPO_ROOT = Path(__file__).parents[4]
 MONITOR_SCRIPT = REPO_ROOT / "docker" / "runners" / "runner-monitor.sh"
 FLEET_CONFIG = REPO_ROOT / "config" / "runner_fleet.yaml"
@@ -53,6 +57,13 @@ def _require_tools() -> None:
     for tool in ("bash", "jq"):
         if shutil.which(tool) is None:
             pytest.skip(f"{tool} not available; shell detection test requires it")
+    # runner-monitor.sh uses `declare -A` (bash>=4). Resolve a bash>=5
+    # interpreter explicitly here too -- a bash that merely EXISTS on PATH
+    # (checked above) is not sufficient; OMN-15617 is exactly the case where
+    # "bash" exists but resolves to the wrong (3.2) interpreter. Fails loud
+    # (pytest.fail via resolve_modern_bash), never a silent skip, when no
+    # bash>=5 is resolvable anywhere.
+    resolve_modern_bash()
 
 
 def _write_exec(path: Path, body: str) -> None:
@@ -303,6 +314,14 @@ def _run_monitor(
     if extra_env:
         env.update(extra_env)
 
+    # runner-monitor.sh uses `declare -A` (bash>=4). OMN-15617: resolve a
+    # bash>=5 interpreter EXPLICITLY rather than invoking bare "bash" and
+    # trusting PATH order -- on stickybeatz-studio (.200) non-interactive ssh
+    # resolves the system bash 3.2.57 first, which fails every predicate in
+    # this script silently (a bash syntax error, not a resolvable "wrong
+    # bash" diagnostic).
+    modern_bash = resolve_modern_bash()
+
     # The script reads STATE_FILE via a hardcoded default; override by editing
     # the env the script honors. The script defines STATE_FILE internally, so we
     # pass it through a wrapper that exports our value first.
@@ -315,15 +334,15 @@ def _run_monitor(
             # The script hardcodes STATE_FILE; rewrite that one line on the fly
             # into a temp copy so the test controls the output path.
             sed 's#^STATE_FILE=.*#STATE_FILE="{state_file}"#' "{MONITOR_SCRIPT}" > "{tmp_path}/monitor.sh"
-            bash "{tmp_path}/monitor.sh"
+            "{modern_bash}" "{tmp_path}/monitor.sh"
             """
         ),
         encoding="utf-8",
     )
     wrapper.chmod(wrapper.stat().st_mode | stat.S_IEXEC)
 
     result = subprocess.run(
-        ["bash", str(wrapper)],
+        [modern_bash, str(wrapper)],
         env=env,
         capture_output=True,
         text=True,