diff --git a/scripts/ci/resolve_modern_bash.sh b/scripts/ci/resolve_modern_bash.sh new file mode 100755 index 0000000000..6cfefe94c2 --- /dev/null +++ b/scripts/ci/resolve_modern_bash.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# resolve_modern_bash.sh -- OMN-15617: explicit, PATH-order-independent +# resolution of a bash interpreter new enough for `declare -A`. +# +# Root cause: on stickybeatz-studio (.200, the rule-11a default gate host), +# non-interactive ssh sessions resolve `bash` / `env bash` to the system +# 3.2.57 shell (no associative arrays) even though a modern bash 5.x sits at +# /opt/homebrew/bin/bash -- it simply is not first on PATH for that session +# class. Callers that spawn `bash ` via bare "bash" +# silently run the wrong interpreter and fail with a bash syntax error deep +# inside the script, not a resolvable "wrong bash" diagnostic. +# +# This script performs the resolution ONCE, explicitly, independent of PATH +# order, and is the single source of truth callers (the pre-push canary and +# the runner-monitor.sh pytest harness) both use -- so they can never drift. +# +# MUST remain runnable under bash 3.2 itself (no `declare -A`, no `[[ ]]` +# reliance beyond what 3.2 supports) -- interpreter resolution cannot +# presuppose the thing it is resolving. +# +# Output contract: on success, prints the resolved interpreter's absolute +# path to stdout and exits 0. On failure, prints nothing to stdout, prints a +# pointed ERROR + REMEDIATION message to stderr, and exits 1. Never a silent +# fallback and never a quiet skip -- an unresolvable modern bash is a hard +# failure the caller must surface. +# +# Env overrides: +# OMNIBASE_INFRA_BASH_BIN explicit interpreter path to try FIRST. +# OMNIBASE_INFRA_MIN_BASH_MAJOR minimum BASH_VERSINFO[0] required (default 5). + +set -euo pipefail + +MIN_MAJOR="${OMNIBASE_INFRA_MIN_BASH_MAJOR:-5}" + +# Prints the candidate's BASH_VERSINFO[0] on stdout, or nothing + nonzero exit +# if the candidate is not an executable bash at all. +bash_major_version() { + local bin="$1" + [ -x "$bin" ] || return 1 + "$bin" -c 'printf "%s" "${BASH_VERSINFO[0]:-}"' 2> /dev/null +} + +seen="" +try_candidate() { + local bin="$1" major + case " ${seen} " in + *" ${bin} "*) return 1 ;; + esac + seen="${seen} ${bin}" + major="$(bash_major_version "$bin")" || return 1 + [ -n "$major" ] || return 1 + [ "$major" -ge "$MIN_MAJOR" ] || return 1 + printf '%s\n' "$bin" + return 0 +} + +# Ordered candidate list: explicit override, the two brew prefixes (Apple +# Silicon + Intel), then every "bash" found walking $PATH -- so a modern bash +# installed somewhere non-standard is still picked up without PATH surgery. +CANDIDATES="" +if [ -n "${OMNIBASE_INFRA_BASH_BIN:-}" ]; then + CANDIDATES="${CANDIDATES} ${OMNIBASE_INFRA_BASH_BIN}" +fi +CANDIDATES="${CANDIDATES} /opt/homebrew/bin/bash /usr/local/bin/bash" + +old_ifs="$IFS" +IFS=':' +for _dir in $PATH; do + [ -n "$_dir" ] || continue + CANDIDATES="${CANDIDATES} ${_dir}/bash" +done +IFS="$old_ifs" + +for _candidate in $CANDIDATES; do + if try_candidate "$_candidate"; then + exit 0 + fi +done + +{ + printf 'ERROR: no bash interpreter >= %s found (declare -A requires bash>=4; OMN-15617 requires >=%s).\n' "$MIN_MAJOR" "$MIN_MAJOR" + printf 'Checked: OMNIBASE_INFRA_BASH_BIN, /opt/homebrew/bin/bash, /usr/local/bin/bash, and every "bash" on PATH.\n' + printf 'REMEDIATION: install a modern bash (e.g. `brew install bash`) and/or set OMNIBASE_INFRA_BASH_BIN to its absolute path.\n' +} >&2 +exit 1 diff --git a/scripts/hooks/prepush_smart_tests.sh b/scripts/hooks/prepush_smart_tests.sh index b994e231a1..b54468219b 100755 --- a/scripts/hooks/prepush_smart_tests.sh +++ b/scripts/hooks/prepush_smart_tests.sh @@ -166,6 +166,30 @@ REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" \ "run 'git push' from within the omnibase_infra repository" cd "$REPO_ROOT" +# ============================================================================= +# bash>=5 canary (OMN-15617) +# ============================================================================= +# On stickybeatz-studio (.200, the rule-11a default gate host), non-interactive +# ssh sessions resolve `bash` to the system 3.2.57 shell even though a modern +# bash 5.x sits at /opt/homebrew/bin/bash -- it just is not first on PATH for +# that session class. runner-monitor.sh (exercised end-to-end by +# tests/unit/observability/runner_health/test_runner_monitor_*.py) uses +# `declare -A`, which bash 3.2 does not support, so those tests fail SILENTLY +# on every push from this host class -- a bash syntax error deep inside a +# subprocess, not a resolvable "wrong interpreter" diagnostic. +# +# Resolve a bash>=5 interpreter EXPLICITLY here, independent of PATH order, +# via the same resolver the pytest harness uses (single source of truth -- +# scripts/ci/resolve_modern_bash.sh -- so the two can never drift apart), and +# export it so the harness does not have to re-discover it. Fail LOUD with a +# pointed remediation message if none is resolvable anywhere -- never a quiet +# skip, never a silent fallback to whatever "bash" happens to resolve first. +MODERN_BASH="$(bash "${REPO_ROOT}/scripts/ci/resolve_modern_bash.sh")" \ + || die "no bash>=5 interpreter resolvable on this host" \ + "install a modern bash (e.g. 'brew install bash') and/or set OMNIBASE_INFRA_BASH_BIN to its absolute path; see scripts/ci/resolve_modern_bash.sh" +export OMNIBASE_INFRA_BASH_BIN="$MODERN_BASH" +log "bash>=5 canary: resolved ${MODERN_BASH}" + BASE_REF="${PREPUSH_BASE_REF:-origin/dev}" # Deterministic diff base: fetch the base ref best-effort so an online push gets diff --git a/tests/unit/observability/runner_health/_resolve_modern_bash.py b/tests/unit/observability/runner_health/_resolve_modern_bash.py new file mode 100644 index 0000000000..72cddc3627 --- /dev/null +++ b/tests/unit/observability/runner_health/_resolve_modern_bash.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Shared bash>=5 interpreter resolution for the runner-monitor.sh harness. + +OMN-15617: on stickybeatz-studio (.200, the rule-11a default gate host), +non-interactive ssh resolves the system bash 3.2.57 first on PATH, silently +failing every `declare -A` predicate in runner-monitor.sh. These tests drive +the REAL script end-to-end (deliberately -- see the module docstrings in the +two callers), so they need a real bash>=5, resolved explicitly rather than +trusted to ambient PATH order. + +Deliberately NOT a Python re-implementation of the bash version check: it +shells out to scripts/ci/resolve_modern_bash.sh, the single source of truth +also used by the pre-push canary (scripts/hooks/prepush_smart_tests.sh), so +the two can never drift apart (DRY, root CLAUDE.md operating discipline). +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parents[4] +RESOLVE_SCRIPT = REPO_ROOT / "scripts" / "ci" / "resolve_modern_bash.sh" + + +def resolve_modern_bash() -> str: + """Return an absolute path to a bash>=5 interpreter, or fail loud. + + Fails the test via ``pytest.fail`` (never a silent skip or a quiet + fallback to whatever "bash" resolves first on PATH) when no bash>=5 + interpreter can be found anywhere -- that quiet-fallback shape is the + exact bug OMN-15617 exists to close. + """ + if not RESOLVE_SCRIPT.is_file(): + pytest.fail(f"resolver script missing: {RESOLVE_SCRIPT}") + # The resolver script itself is bash-3.2-safe by construction (it cannot + # presuppose the modern bash it is trying to find), so it is safe to + # invoke via the bare "bash" on whatever PATH this process inherited. + result = subprocess.run( + ["bash", str(RESOLVE_SCRIPT)], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + resolved = result.stdout.strip() + if result.returncode != 0 or not resolved: + pytest.fail( + "no bash>=5 interpreter resolvable for the runner-monitor.sh " + "harness (declare -A requires bash>=4; OMN-15617).\n" + f"resolver stderr:\n{result.stderr}" + ) + return resolved diff --git a/tests/unit/observability/runner_health/test_runner_monitor_auto_bounce.py b/tests/unit/observability/runner_health/test_runner_monitor_auto_bounce.py index cdf89e487f..c5dd0ee4e1 100644 --- a/tests/unit/observability/runner_health/test_runner_monitor_auto_bounce.py +++ b/tests/unit/observability/runner_health/test_runner_monitor_auto_bounce.py @@ -39,6 +39,10 @@ import pytest +from tests.unit.observability.runner_health._resolve_modern_bash import ( + resolve_modern_bash, +) + REPO_ROOT = Path(__file__).parents[4] MONITOR_SCRIPT = REPO_ROOT / "docker" / "runners" / "runner-monitor.sh" @@ -55,6 +59,11 @@ def _require_tools() -> None: for tool in ("bash", "jq", "flock"): if shutil.which(tool) is None: pytest.skip(f"{tool} not available; shell detection test requires it") + # runner-monitor.sh uses `declare -A` (bash>=4). OMN-15617: a bash that + # merely EXISTS on PATH (checked above) is not sufficient -- resolve one + # that is actually >=5, explicitly. Fails loud (never a silent skip) when + # none is resolvable anywhere. + resolve_modern_bash() def _write_exec(path: Path, body: str) -> None: @@ -268,6 +277,11 @@ def _run_monitor( if extra_env: env.update(extra_env) + # OMN-15617: resolve a bash>=5 interpreter explicitly rather than trusting + # ambient PATH order (see the sibling wedge-detection module for the full + # rationale). + modern_bash = resolve_modern_bash() + wrapper = tmp_path / "run.sh" wrapper.write_text( textwrap.dedent( @@ -275,7 +289,7 @@ def _run_monitor( #!/usr/bin/env bash set -euo pipefail sed 's#^STATE_FILE=.*#STATE_FILE="{state_file}"#' "{MONITOR_SCRIPT}" > "{tmp_path}/monitor.sh" - bash "{tmp_path}/monitor.sh" + "{modern_bash}" "{tmp_path}/monitor.sh" """ ), encoding="utf-8", @@ -283,7 +297,7 @@ def _run_monitor( wrapper.chmod(wrapper.stat().st_mode | stat.S_IEXEC) result = subprocess.run( - ["bash", str(wrapper)], + [modern_bash, str(wrapper)], env=env, capture_output=True, text=True, diff --git a/tests/unit/observability/runner_health/test_runner_monitor_wedge_detection.py b/tests/unit/observability/runner_health/test_runner_monitor_wedge_detection.py index db5ccf6073..9aea6d8bcf 100644 --- a/tests/unit/observability/runner_health/test_runner_monitor_wedge_detection.py +++ b/tests/unit/observability/runner_health/test_runner_monitor_wedge_detection.py @@ -35,6 +35,10 @@ import pytest +from tests.unit.observability.runner_health._resolve_modern_bash import ( + resolve_modern_bash, +) + REPO_ROOT = Path(__file__).parents[4] MONITOR_SCRIPT = REPO_ROOT / "docker" / "runners" / "runner-monitor.sh" FLEET_CONFIG = REPO_ROOT / "config" / "runner_fleet.yaml" @@ -53,6 +57,13 @@ def _require_tools() -> None: for tool in ("bash", "jq"): if shutil.which(tool) is None: pytest.skip(f"{tool} not available; shell detection test requires it") + # runner-monitor.sh uses `declare -A` (bash>=4). Resolve a bash>=5 + # interpreter explicitly here too -- a bash that merely EXISTS on PATH + # (checked above) is not sufficient; OMN-15617 is exactly the case where + # "bash" exists but resolves to the wrong (3.2) interpreter. Fails loud + # (pytest.fail via resolve_modern_bash), never a silent skip, when no + # bash>=5 is resolvable anywhere. + resolve_modern_bash() def _write_exec(path: Path, body: str) -> None: @@ -303,6 +314,14 @@ def _run_monitor( if extra_env: env.update(extra_env) + # runner-monitor.sh uses `declare -A` (bash>=4). OMN-15617: resolve a + # bash>=5 interpreter EXPLICITLY rather than invoking bare "bash" and + # trusting PATH order -- on stickybeatz-studio (.200) non-interactive ssh + # resolves the system bash 3.2.57 first, which fails every predicate in + # this script silently (a bash syntax error, not a resolvable "wrong + # bash" diagnostic). + modern_bash = resolve_modern_bash() + # The script reads STATE_FILE via a hardcoded default; override by editing # the env the script honors. The script defines STATE_FILE internally, so we # pass it through a wrapper that exports our value first. @@ -315,7 +334,7 @@ def _run_monitor( # The script hardcodes STATE_FILE; rewrite that one line on the fly # into a temp copy so the test controls the output path. sed 's#^STATE_FILE=.*#STATE_FILE="{state_file}"#' "{MONITOR_SCRIPT}" > "{tmp_path}/monitor.sh" - bash "{tmp_path}/monitor.sh" + "{modern_bash}" "{tmp_path}/monitor.sh" """ ), encoding="utf-8", @@ -323,7 +342,7 @@ def _run_monitor( wrapper.chmod(wrapper.stat().st_mode | stat.S_IEXEC) result = subprocess.run( - ["bash", str(wrapper)], + [modern_bash, str(wrapper)], env=env, capture_output=True, text=True,