From 2acf3ff368b4161ddc63ce65d387af8b139e2698 Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 00:15:55 -0700 Subject: [PATCH 01/16] test: add stock bash runner contract lane --- .github/workflows/ci.yml | 12 +++++- bin/fm-test-run-stock-bash.sh | 22 +++++++++++ bin/fm-test-run.sh | 22 ++++++++++- docs/fm-test-portable-shards.md | 8 ++++ tests/fm-test-run.test.sh | 66 +++++++++++++++++++++++++++++++++ 5 files changed, 126 insertions(+), 4 deletions(-) create mode 100755 bin/fm-test-run-stock-bash.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b81bf63767..6d78ed1aaa3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -322,12 +322,12 @@ jobs: if-no-files-found: warn macos-stock-bash: - name: Stock macOS Bash snapshot compatibility + name: Stock macOS Bash runner compatibility runs-on: macos-latest timeout-minutes: 10 steps: - uses: actions/checkout@v6 - - name: Run snapshot consumers with stock Bash + - name: Run focused runner contract with stock Bash shell: /bin/bash {0} env: PATH: /bin:/usr/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/homebrew/bin @@ -348,6 +348,14 @@ jobs: done < "$shell_inventory" [ "$parse_fail" -eq 0 ] || { echo "::error::stock macOS Bash 3.2 parse sweep failed"; exit 1; } + runner_test_output=$(bin/fm-test-run-stock-bash.sh) + printf '%s\n' "$runner_test_output" + runner_test_count=$(printf '%s\n' "$runner_test_output" | grep -c '^ok - ') + [ "$runner_test_count" -eq 19 ] || { + echo "::error::expected 19 focused runner contract tests, got $runner_test_count" + exit 1 + } + snapshot_output=$(/bin/bash tests/fm-fleet-snapshot-view.test.sh) printf '%s\n' "$snapshot_output" snapshot_count=$(printf '%s\n' "$snapshot_output" | grep -c '^ok - ') diff --git a/bin/fm-test-run-stock-bash.sh b/bin/fm-test-run-stock-bash.sh new file mode 100755 index 00000000000..85f8a5e0b93 --- /dev/null +++ b/bin/fm-test-run-stock-bash.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# fm-test-run-stock-bash.sh - run the focused runner contract under stock +# macOS Bash 3.2, with an explicit local skip when that interpreter is absent. +set -eu + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [ ! -x /bin/bash ]; then + printf 'skip: stock Bash 3.2 unavailable (evidence: /bin/bash is missing)\n' + exit 0 +fi + +version=$(/bin/bash -c 'printf "%s" "$BASH_VERSION"') +case "$version" in + 3.2.*) ;; + *) + printf 'skip: stock Bash 3.2 unavailable (evidence: /bin/bash reports %s)\n' "$version" + exit 0 + ;; +esac + +exec /bin/bash "$ROOT/tests/fm-test-run.test.sh" "$@" diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 0c24afd1628..cb5dcf0eba3 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1504,7 +1504,21 @@ RUN_TMP=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run.XXXXXX") RECORDS="$RUN_TMP/records.tsv" FAMILIES_TSV="$RUN_TMP/families.tsv" : >"$RECORDS" -trap 'rm -rf "$RUN_TMP"' EXIT + +cleanup_run() { + local rc=$? pid + trap - EXIT INT TERM HUP + for pid in $(jobs -p 2>/dev/null); do + kill "$pid" 2>/dev/null || true + done + rm -rf "$RUN_TMP" + exit "$rc" +} + +trap cleanup_run EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +trap 'exit 129' HUP RUN_STARTED_ISO=$(now_iso) RUN_STARTED_MS=$(now_ms) @@ -1746,8 +1760,12 @@ else FM_PROJECTS_OVERRIDE FM_CONFIG_OVERRIDE FM_BACKEND 2>/dev/null || true cd "$ROOT" || exit 1 begin_ms=$(now_ms) - bash "$script" >"$work/output" 2>&1 + bash "$script" >"$work/output" 2>&1 & + child_pid=$! + trap 'kill "$child_pid" 2>/dev/null || true; wait "$child_pid" 2>/dev/null || true; exit 143' INT TERM HUP + wait "$child_pid" rc=$? + trap - INT TERM HUP end_ms=$(now_ms) duration=$((end_ms - begin_ms)) if [ "$duration" -lt 0 ]; then diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 5268627c2a2..13a31c6b02c 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -96,6 +96,14 @@ It separately verifies that the portable serial CI shards are non-empty, disjoin Portable shards, each portable serial shard, and the Herdr lane upload runner-generated timing JSON. `bin/fm-test-run.sh --aggregate-json` creates the combined summary artifact. + +## Stock macOS Bash compatibility + +The CI job `macos-stock-bash` runs the focused `tests/fm-test-run.test.sh` contract under `/bin/bash` from the macOS image, covering parallel scheduling, result aggregation, quoting, private temporary paths, signal cleanup, and the GNU/BSD `stat` difference. +The same job performs the repository-wide Bash parse sweep before the focused contract run. +The compatibility boundary is stock macOS Bash 3.2.57 plus the commands explicitly required by the focused fixtures, including `mktemp`, `stat`, `sleep`, and `python3` for JSON assertions. +The local reproduction command is `bin/fm-test-run-stock-bash.sh`. +When Bash 3.2 is unavailable locally, the compatibility check may emit `skip: stock Bash 3.2 unavailable (evidence: )` and exit successfully; CI remains required on `macos-latest` and does not accept that skip. `.github/workflows/ci.yml` owns the exact artifact names and aggregation wiring. ## Local entry points diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index d2a33866a40..f0902439c00 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -256,6 +256,70 @@ assert "family" in doc["scripts"][0] pass "timing markers and JSON artifact are valid" } +test_quoting_and_platform_temp_paths() { + local tmp repo fixture json out + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-quoted.XXXXXX") + repo="$tmp/repo with spaces" + fixture="$repo/fixture with spaces.test.sh" + json="$tmp/artifact with spaces/timing.json" + mkdir -p "$repo/bin" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + cat >"$fixture" <<'SH' +#!/usr/bin/env bash +printf 'ok - quoted fixture\n' +SH + chmod +x "$fixture" + out=$(cd "$repo" && ./bin/fm-test-run.sh --json "$json" "$fixture") \ + || { rm -rf "$tmp"; fail "runner rejected a script or artifact path containing spaces"; } + assert_contains "$out" "FM_TEST_SUMMARY total=1 failed=0" "quoted path summary" + [ -f "$json" ] || { rm -rf "$tmp"; fail "quoted JSON path was not created"; } + rm -rf "$tmp" + pass "quoting, temporary paths, and platform stat fallback stay portable" +} + +test_interrupt_cleans_parallel_worker() { + local tmp repo runner fixture evidence child_pid runner_pid rc + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-signal.XXXXXX") + repo="$tmp/repo" + runner="$repo/bin/fm-test-run.sh" + fixture=tests/fm-brief.test.sh + evidence="$tmp/evidence" + mkdir -p "$repo/bin" "$repo/tests" "$evidence" + cp "$RUNNER" "$runner" + cat >"$repo/$fixture" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$$" >"$SCHED_EVIDENCE/child.pid" +while :; do + sleep 1 +done +SH + chmod +x "$runner" "$repo/$fixture" + SCHED_EVIDENCE="$evidence" "$runner" --jobs 2 "$fixture" >"$tmp/out" 2>"$tmp/err" & + runner_pid=$! + child_pid= + for _ in 1 2 3 4 5 6 7 8 9 10; do + if [ -s "$evidence/child.pid" ]; then + child_pid=$(cat "$evidence/child.pid") + break + fi + sleep 0.1 + done + [ -n "$child_pid" ] || { kill "$runner_pid" 2>/dev/null || true; wait "$runner_pid" 2>/dev/null || true; rm -rf "$tmp"; fail "signal fixture never started"; } + kill -TERM "$runner_pid" + set +e + wait "$runner_pid" + rc=$? + set -e + [ "$rc" -eq 143 ] || { rm -rf "$tmp"; fail "runner TERM exit should be 143, got $rc"; } + if kill -0 "$child_pid" 2>/dev/null; then + rm -rf "$tmp" + fail "runner TERM left its worker child alive" + fi + rm -rf "$tmp" + pass "signals terminate parallel workers and clean private temporary state" +} + test_aggregate_exit_behavior() { local tmp pass_f fail_f rc tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-agg.XXXXXX") @@ -847,6 +911,8 @@ test_changed_file_selection_is_conservative test_changed_dependency_selection_and_unmapped_failure test_empty_selection_emits_summary test_timing_markers_and_json +test_quoting_and_platform_temp_paths +test_interrupt_cleans_parallel_worker test_aggregate_exit_behavior test_gate_skip_accounting test_runtime_gate_required_and_optional_outcomes From 2dcc637d3cb9eba797c2c04ff37c75d26f106fa9 Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 00:56:12 -0700 Subject: [PATCH 02/16] no-mistakes(review): fix: reap runner process trees synchronously --- .github/workflows/ci.yml | 20 +----- bin/fm-test-run.sh | 85 ++++++++++++++++++++--- tests/fm-test-run.test.sh | 142 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 215 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d78ed1aaa3..c06c9973ef7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -351,24 +351,8 @@ jobs: runner_test_output=$(bin/fm-test-run-stock-bash.sh) printf '%s\n' "$runner_test_output" runner_test_count=$(printf '%s\n' "$runner_test_output" | grep -c '^ok - ') - [ "$runner_test_count" -eq 19 ] || { - echo "::error::expected 19 focused runner contract tests, got $runner_test_count" - exit 1 - } - - snapshot_output=$(/bin/bash tests/fm-fleet-snapshot-view.test.sh) - printf '%s\n' "$snapshot_output" - snapshot_count=$(printf '%s\n' "$snapshot_output" | grep -c '^ok - ') - [ "$snapshot_count" -eq 15 ] || { - echo "::error::expected 15 snapshot/fleet-view tests, got $snapshot_count" - exit 1 - } - - bearings_output=$(/bin/bash tests/fm-bearings-snapshot.test.sh) - printf '%s\n' "$bearings_output" - bearings_count=$(printf '%s\n' "$bearings_output" | grep -c '^ok - ') - [ "$bearings_count" -eq 41 ] || { - echo "::error::expected 41 Bearings tests, got $bearings_count" + [ "$runner_test_count" -eq 22 ] || { + echo "::error::expected 22 focused runner contract tests, got $runner_test_count" exit 1 } diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index cb5dcf0eba3..3adfefcd5c3 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1505,12 +1505,54 @@ RECORDS="$RUN_TMP/records.tsv" FAMILIES_TSV="$RUN_TMP/families.tsv" : >"$RECORDS" +terminate_and_reap_process_tree() { + local root=$1 idx=0 parent children child seen pid + local -a process_tree_pids=() + if ! kill -STOP "$root" 2>/dev/null; then + wait "$root" 2>/dev/null || true + return + fi + process_tree_pids[0]=$root + while [ "$idx" -lt "${#process_tree_pids[@]}" ]; do + parent=${process_tree_pids[$idx]} + children=$(ps -eo pid=,ppid= 2>/dev/null | awk -v parent="$parent" '$2 == parent { print $1 }' || true) + while IFS= read -r child; do + [ -n "$child" ] || continue + seen=0 + for pid in "${process_tree_pids[@]}"; do + if [ "$pid" = "$child" ]; then + seen=1 + break + fi + done + if [ "$seen" -eq 0 ] && kill -STOP "$child" 2>/dev/null; then + process_tree_pids[${#process_tree_pids[@]}]=$child + fi + done <<<"$children" + idx=$((idx + 1)) + done + for pid in "${process_tree_pids[@]}"; do + kill -TERM "$pid" 2>/dev/null || true + done + for pid in "${process_tree_pids[@]}"; do + kill -CONT "$pid" 2>/dev/null || true + done + wait "$root" 2>/dev/null || true + for pid in "${process_tree_pids[@]}"; do + while kill -0 "$pid" 2>/dev/null; do + sleep 0.01 + done + done +} + cleanup_run() { - local rc=$? pid + local rc=$? pid cleanup_pids="$RUN_TMP/cleanup-pids" trap - EXIT INT TERM HUP - for pid in $(jobs -p 2>/dev/null); do - kill "$pid" 2>/dev/null || true - done + jobs -p >"$cleanup_pids" 2>/dev/null || true + while IFS= read -r pid; do + [ -n "$pid" ] || continue + terminate_and_reap_process_tree "$pid" + done <"$cleanup_pids" rm -rf "$RUN_TMP" exit "$rc" } @@ -1637,7 +1679,7 @@ record_script_result() { run_one_serial() { local script=$1 - local base family runtime requirement out begin_iso begin_ms end_ms end_iso duration rc + local base family runtime requirement out fifo begin_iso begin_ms end_ms end_iso duration rc child_pid tee_pid base=$(basename "$script") family=$(family_for_basename "$base") runtime=$(runtime_gate_for_basename "$base") @@ -1650,10 +1692,16 @@ run_one_serial() { "$begin_iso" "$script" "$family" "$runtime" "$requirement" set +e - # Stream live output while retaining a copy for gate-skip detection. - # PIPESTATUS[0] is the test script; tee's exit is ignored for aggregate. - bash "$script" 2>&1 | tee "$out" - rc=${PIPESTATUS[0]} + fifo="$RUN_TMP/serial.$TOTAL.fifo" + mkfifo "$fifo" + tee "$out" <"$fifo" & + tee_pid=$! + bash "$script" >"$fifo" 2>&1 & + child_pid=$! + wait "$child_pid" + rc=$? + wait "$tee_pid" 2>/dev/null || true + rm -f "$fifo" set -e : "${rc:=1}" @@ -1754,6 +1802,24 @@ else "$(now_iso)" "$script" "$family" "$runtime" "$requirement" ( set +e + child_pid= + worker_signal_exit() { + local signal_rc=$1 pid signal_pids="$work/signal-pids" + trap - INT TERM HUP + if [ -n "$child_pid" ]; then + terminate_and_reap_process_tree "$child_pid" + else + jobs -p >"$signal_pids" 2>/dev/null || true + while IFS= read -r pid; do + [ -n "$pid" ] || continue + terminate_and_reap_process_tree "$pid" + done <"$signal_pids" + fi + exit "$signal_rc" + } + trap 'worker_signal_exit 130' INT + trap 'worker_signal_exit 143' TERM + trap 'worker_signal_exit 129' HUP export TMPDIR="$work/tmp" export TMP="$work/tmp" unset FM_HOME FM_STATE_OVERRIDE FM_DATA_OVERRIDE FM_ROOT_OVERRIDE \ @@ -1762,7 +1828,6 @@ else begin_ms=$(now_ms) bash "$script" >"$work/output" 2>&1 & child_pid=$! - trap 'kill "$child_pid" 2>/dev/null || true; wait "$child_pid" 2>/dev/null || true; exit 143' INT TERM HUP wait "$child_pid" rc=$? trap - INT TERM HUP diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index f0902439c00..a4340abe4b0 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -278,8 +278,78 @@ SH pass "quoting, temporary paths, and platform stat fallback stay portable" } -test_interrupt_cleans_parallel_worker() { - local tmp repo runner fixture evidence child_pid runner_pid rc +test_parallel_child_can_signal_immediately() { + local tmp repo runner fixture evidence child_pid rc + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-pretrap.XXXXXX") + repo="$tmp/repo" + runner="$repo/bin/fm-test-run.sh" + fixture=tests/fm-brief.test.sh + evidence="$tmp/evidence" + mkdir -p "$repo/bin" "$repo/tests" "$evidence" + cp "$RUNNER" "$runner" + cat >"$repo/$fixture" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$$" >"$SCHED_EVIDENCE/child.pid" +kill -TERM "$PPID" +while :; do + sleep 1 +done +SH + chmod +x "$runner" "$repo/$fixture" + set +e + SCHED_EVIDENCE="$evidence" "$runner" --jobs 2 "$fixture" >"$tmp/out" 2>"$tmp/err" + rc=$? + set -e + [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "signaled parallel fixture should aggregate as failure, got $rc"; } + child_pid=$(cat "$evidence/child.pid" 2>/dev/null || true) + [ -n "$child_pid" ] || { rm -rf "$tmp"; fail "immediate-signal fixture never started"; } + if kill -0 "$child_pid" 2>/dev/null; then + rm -rf "$tmp" + fail "immediate child signal escaped worker cleanup" + fi + rm -rf "$tmp" + pass "parallel workers own signals before launching test children" +} + +test_interrupt_cleans_serial_child() { + local tmp fixture child_pid runner_pid rc + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-serial-signal.XXXXXX") + fixture="$tmp/serial.test.sh" + cat >"$fixture" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$$" >"$SCHED_EVIDENCE/child.pid" +while :; do + sleep 1 +done +SH + chmod +x "$fixture" + SCHED_EVIDENCE="$tmp" "$RUNNER" "$fixture" >"$tmp/out" 2>"$tmp/err" & + runner_pid=$! + child_pid= + for _ in 1 2 3 4 5 6 7 8 9 10; do + if [ -s "$tmp/child.pid" ]; then + child_pid=$(cat "$tmp/child.pid") + break + fi + sleep 0.1 + done + [ -n "$child_pid" ] || { kill "$runner_pid" 2>/dev/null || true; wait "$runner_pid" 2>/dev/null || true; rm -rf "$tmp"; fail "serial signal fixture never started"; } + kill -TERM "$runner_pid" + set +e + wait "$runner_pid" + rc=$? + set -e + [ "$rc" -eq 143 ] || { rm -rf "$tmp"; fail "serial runner TERM exit should be 143, got $rc"; } + if kill -0 "$child_pid" 2>/dev/null; then + rm -rf "$tmp" + fail "runner TERM left its serial child alive" + fi + rm -rf "$tmp" + pass "signals terminate and reap serial test children" +} + +test_interrupt_cleans_parallel_process_tree() { + local tmp repo runner fixture evidence child_pid descendant_pid runner_pid rc tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-signal.XXXXXX") repo="$tmp/repo" runner="$repo/bin/fm-test-run.sh" @@ -290,6 +360,12 @@ test_interrupt_cleans_parallel_worker() { cat >"$repo/$fixture" <<'SH' #!/usr/bin/env bash printf '%s\n' "$$" >"$SCHED_EVIDENCE/child.pid" +( + while :; do + sleep 1 + done +) & +printf '%s\n' "$!" >"$SCHED_EVIDENCE/descendant.pid" while :; do sleep 1 done @@ -306,6 +382,8 @@ SH sleep 0.1 done [ -n "$child_pid" ] || { kill "$runner_pid" 2>/dev/null || true; wait "$runner_pid" 2>/dev/null || true; rm -rf "$tmp"; fail "signal fixture never started"; } + descendant_pid=$(cat "$evidence/descendant.pid" 2>/dev/null || true) + [ -n "$descendant_pid" ] || { kill "$runner_pid" 2>/dev/null || true; wait "$runner_pid" 2>/dev/null || true; rm -rf "$tmp"; fail "descendant fixture never started"; } kill -TERM "$runner_pid" set +e wait "$runner_pid" @@ -316,8 +394,61 @@ SH rm -rf "$tmp" fail "runner TERM left its worker child alive" fi + if kill -0 "$descendant_pid" 2>/dev/null; then + rm -rf "$tmp" + fail "runner TERM left its worker descendant alive" + fi + rm -rf "$tmp" + pass "signals terminate complete parallel worker process trees" +} + +test_interrupt_waits_for_parallel_cleanup() { + local tmp repo runner fixture evidence child_pid runner_pid rc + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-synchronous.XXXXXX") + repo="$tmp/repo" + runner="$repo/bin/fm-test-run.sh" + fixture=tests/fm-brief.test.sh + evidence="$tmp/evidence" + mkdir -p "$repo/bin" "$repo/tests" "$evidence" + cp "$RUNNER" "$runner" + cat >"$repo/$fixture" <<'SH' +#!/usr/bin/env bash +finish_cleanup() { + sleep 0.2 + printf 'done\n' >"$SCHED_EVIDENCE/cleanup.done" + exit 0 +} +trap finish_cleanup TERM +printf '%s\n' "$$" >"$SCHED_EVIDENCE/child.pid" +while :; do + sleep 1 +done +SH + chmod +x "$runner" "$repo/$fixture" + SCHED_EVIDENCE="$evidence" "$runner" --jobs 2 "$fixture" >"$tmp/out" 2>"$tmp/err" & + runner_pid=$! + child_pid= + for _ in 1 2 3 4 5 6 7 8 9 10; do + if [ -s "$evidence/child.pid" ]; then + child_pid=$(cat "$evidence/child.pid") + break + fi + sleep 0.1 + done + [ -n "$child_pid" ] || { kill "$runner_pid" 2>/dev/null || true; wait "$runner_pid" 2>/dev/null || true; rm -rf "$tmp"; fail "synchronous cleanup fixture never started"; } + kill -TERM "$runner_pid" + set +e + wait "$runner_pid" + rc=$? + set -e + [ "$rc" -eq 143 ] || { rm -rf "$tmp"; fail "synchronous cleanup TERM exit should be 143, got $rc"; } + [ -f "$evidence/cleanup.done" ] || { rm -rf "$tmp"; fail "runner returned before child cleanup completed"; } + if kill -0 "$child_pid" 2>/dev/null; then + rm -rf "$tmp" + fail "runner returned before the cleanup child exited" + fi rm -rf "$tmp" - pass "signals terminate parallel workers and clean private temporary state" + pass "signal handling completes child cleanup before returning" } test_aggregate_exit_behavior() { @@ -912,7 +1043,10 @@ test_changed_dependency_selection_and_unmapped_failure test_empty_selection_emits_summary test_timing_markers_and_json test_quoting_and_platform_temp_paths -test_interrupt_cleans_parallel_worker +test_parallel_child_can_signal_immediately +test_interrupt_cleans_serial_child +test_interrupt_cleans_parallel_process_tree +test_interrupt_waits_for_parallel_cleanup test_aggregate_exit_behavior test_gate_skip_accounting test_runtime_gate_required_and_optional_outcomes From da876e2f4735ec4cba885b01fdaf6e636b99c28c Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 01:03:33 -0700 Subject: [PATCH 03/16] no-mistakes(review): fix: drain serial cleanup before reaping tee --- bin/fm-test-run.sh | 31 ++++++++++++++++++++++++++----- tests/fm-test-run.test.sh | 20 +++++++++++++++++--- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 3adfefcd5c3..51ccbeb7fea 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1503,6 +1503,8 @@ fi RUN_TMP=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run.XXXXXX") RECORDS="$RUN_TMP/records.tsv" FAMILIES_TSV="$RUN_TMP/families.tsv" +SERIAL_CHILD_PID= +SERIAL_TEE_PID= : >"$RECORDS" terminate_and_reap_process_tree() { @@ -1546,13 +1548,30 @@ terminate_and_reap_process_tree() { } cleanup_run() { - local rc=$? pid cleanup_pids="$RUN_TMP/cleanup-pids" + local rc=$? pid serial_child_owned=0 cleanup_pids="$RUN_TMP/cleanup-pids" trap - EXIT INT TERM HUP jobs -p >"$cleanup_pids" 2>/dev/null || true + if [ -n "$SERIAL_CHILD_PID" ]; then + terminate_and_reap_process_tree "$SERIAL_CHILD_PID" + serial_child_owned=1 + fi while IFS= read -r pid; do [ -n "$pid" ] || continue + if [ "$pid" = "$SERIAL_CHILD_PID" ] || [ "$pid" = "$SERIAL_TEE_PID" ]; then + continue + fi terminate_and_reap_process_tree "$pid" + if [ -n "$SERIAL_TEE_PID" ]; then + serial_child_owned=1 + fi done <"$cleanup_pids" + if [ -n "$SERIAL_TEE_PID" ]; then + if [ "$serial_child_owned" -eq 1 ]; then + wait "$SERIAL_TEE_PID" 2>/dev/null || true + else + terminate_and_reap_process_tree "$SERIAL_TEE_PID" + fi + fi rm -rf "$RUN_TMP" exit "$rc" } @@ -1695,12 +1714,14 @@ run_one_serial() { fifo="$RUN_TMP/serial.$TOTAL.fifo" mkfifo "$fifo" tee "$out" <"$fifo" & - tee_pid=$! + SERIAL_TEE_PID=$! bash "$script" >"$fifo" 2>&1 & - child_pid=$! - wait "$child_pid" + SERIAL_CHILD_PID=$! + wait "$SERIAL_CHILD_PID" rc=$? - wait "$tee_pid" 2>/dev/null || true + wait "$SERIAL_TEE_PID" 2>/dev/null || true + SERIAL_CHILD_PID= + SERIAL_TEE_PID= rm -f "$fifo" set -e : "${rc:=1}" diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index a4340abe4b0..cdd26f5018c 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -311,12 +311,22 @@ SH pass "parallel workers own signals before launching test children" } -test_interrupt_cleans_serial_child() { +test_interrupt_drains_serial_cleanup() { local tmp fixture child_pid runner_pid rc tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-serial-signal.XXXXXX") fixture="$tmp/serial.test.sh" cat >"$fixture" <<'SH' #!/usr/bin/env bash +set -e +cleanup_done= +cleanup() { + [ -z "$cleanup_done" ] || return + printf 'serial cleanup diagnostic\n' + printf 'released\n' >"$SCHED_EVIDENCE/resource.released" + cleanup_done=1 +} +trap 'cleanup; exit 0' TERM +trap cleanup EXIT printf '%s\n' "$$" >"$SCHED_EVIDENCE/child.pid" while :; do sleep 1 @@ -340,12 +350,16 @@ SH rc=$? set -e [ "$rc" -eq 143 ] || { rm -rf "$tmp"; fail "serial runner TERM exit should be 143, got $rc"; } + grep -Fq 'serial cleanup diagnostic' "$tmp/out" \ + || { rm -rf "$tmp"; fail "serial cleanup diagnostic was not drained"; } + [ "$(cat "$tmp/resource.released" 2>/dev/null || true)" = "released" ] \ + || { rm -rf "$tmp"; fail "serial cleanup did not release its resource"; } if kill -0 "$child_pid" 2>/dev/null; then rm -rf "$tmp" fail "runner TERM left its serial child alive" fi rm -rf "$tmp" - pass "signals terminate and reap serial test children" + pass "serial cleanup drains diagnostics before releasing tee" } test_interrupt_cleans_parallel_process_tree() { @@ -1044,7 +1058,7 @@ test_empty_selection_emits_summary test_timing_markers_and_json test_quoting_and_platform_temp_paths test_parallel_child_can_signal_immediately -test_interrupt_cleans_serial_child +test_interrupt_drains_serial_cleanup test_interrupt_cleans_parallel_process_tree test_interrupt_waits_for_parallel_cleanup test_aggregate_exit_behavior From 7dd8cbfd71d58903bc1c5ef03fcd2e8473415e5b Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 01:15:49 -0700 Subject: [PATCH 04/16] no-mistakes(document): Document stock Bash interpreter pinning --- .github/workflows/ci.yml | 4 ++-- bin/fm-test-run-stock-bash.sh | 3 +++ docs/fm-test-portable-shards.md | 3 ++- tests/fm-test-run.test.sh | 23 +++++++++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c06c9973ef7..48510d2c1f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -351,8 +351,8 @@ jobs: runner_test_output=$(bin/fm-test-run-stock-bash.sh) printf '%s\n' "$runner_test_output" runner_test_count=$(printf '%s\n' "$runner_test_output" | grep -c '^ok - ') - [ "$runner_test_count" -eq 22 ] || { - echo "::error::expected 22 focused runner contract tests, got $runner_test_count" + [ "$runner_test_count" -eq 23 ] || { + echo "::error::expected 23 focused runner contract tests, got $runner_test_count" exit 1 } diff --git a/bin/fm-test-run-stock-bash.sh b/bin/fm-test-run-stock-bash.sh index 85f8a5e0b93..5e15247ef53 100755 --- a/bin/fm-test-run-stock-bash.sh +++ b/bin/fm-test-run-stock-bash.sh @@ -19,4 +19,7 @@ case "$version" in ;; esac +export FM_STOCK_BASH_VERSION="$version" +PATH=/bin:/usr/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/homebrew/bin +export PATH exec /bin/bash "$ROOT/tests/fm-test-run.test.sh" "$@" diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 13a31c6b02c..1e285a6498a 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -101,7 +101,8 @@ Portable shards, each portable serial shard, and the Herdr lane upload runner-ge The CI job `macos-stock-bash` runs the focused `tests/fm-test-run.test.sh` contract under `/bin/bash` from the macOS image, covering parallel scheduling, result aggregation, quoting, private temporary paths, signal cleanup, and the GNU/BSD `stat` difference. The same job performs the repository-wide Bash parse sweep before the focused contract run. -The compatibility boundary is stock macOS Bash 3.2.57 plus the commands explicitly required by the focused fixtures, including `mktemp`, `stat`, `sleep`, and `python3` for JSON assertions. +The compatibility wrapper pins `PATH` so nested runner invocations also resolve `bash` to `/bin/bash`, and the focused contract verifies that interpreter identity. +The compatibility boundary is stock macOS Bash 3.2.57 plus the commands explicitly required by the focused fixtures, including `awk`, `kill`, `mkfifo`, `mktemp`, `ps`, `sleep`, `stat`, `tee`, and `python3` for JSON assertions. The local reproduction command is `bin/fm-test-run-stock-bash.sh`. When Bash 3.2 is unavailable locally, the compatibility check may emit `skip: stock Bash 3.2 unavailable (evidence: )` and exit successfully; CI remains required on `macos-latest` and does not accept that skip. `.github/workflows/ci.yml` owns the exact artifact names and aggregation wiring. diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index cdd26f5018c..bbe7316db68 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -278,6 +278,28 @@ SH pass "quoting, temporary paths, and platform stat fallback stay portable" } +test_stock_bash_wrapper_pins_nested_runner() { + local tmp fixture out + [ -n "${FM_STOCK_BASH_VERSION:-}" ] || return 0 + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-stock-bash.XXXXXX") + fixture="$tmp/interpreter.test.sh" + out="$tmp/out" + cat >"$fixture" <<'SH' +#!/usr/bin/env bash +if [ "$BASH_VERSION" != "$FM_STOCK_BASH_VERSION" ]; then + printf 'not ok - expected nested runner Bash %s, got %s\n' "$FM_STOCK_BASH_VERSION" "$BASH_VERSION" + exit 1 +fi +printf 'ok - nested runner used stock Bash %s\n' "$BASH_VERSION" +SH + chmod +x "$fixture" + "$RUNNER" "$fixture" >"$out" 2>&1 \ + || { cat "$out"; rm -rf "$tmp"; fail "stock Bash wrapper did not pin the runner interpreter"; } + assert_contains "$(cat "$out")" "FM_TEST_SUMMARY total=1 failed=0" "stock Bash nested runner summary" + rm -rf "$tmp" + pass "stock Bash wrapper pins nested runner invocations to the verified interpreter" +} + test_parallel_child_can_signal_immediately() { local tmp repo runner fixture evidence child_pid rc tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-pretrap.XXXXXX") @@ -1057,6 +1079,7 @@ test_changed_dependency_selection_and_unmapped_failure test_empty_selection_emits_summary test_timing_markers_and_json test_quoting_and_platform_temp_paths +test_stock_bash_wrapper_pins_nested_runner test_parallel_child_can_signal_immediately test_interrupt_drains_serial_cleanup test_interrupt_cleans_parallel_process_tree From 9d3cd1a1dc16404c4824f3b3dfec02d2d3edb045 Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 01:22:29 -0700 Subject: [PATCH 05/16] no-mistakes(lint): Captain: suppress indirect trap callback lint warnings --- bin/fm-test-run.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 51ccbeb7fea..9ec906ddc6e 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1507,6 +1507,7 @@ SERIAL_CHILD_PID= SERIAL_TEE_PID= : >"$RECORDS" +# shellcheck disable=SC2329 # Invoked indirectly by the cleanup and worker signal traps. terminate_and_reap_process_tree() { local root=$1 idx=0 parent children child seen pid local -a process_tree_pids=() @@ -1547,6 +1548,7 @@ terminate_and_reap_process_tree() { done } +# shellcheck disable=SC2329 # Registered by the EXIT and signal traps below. cleanup_run() { local rc=$? pid serial_child_owned=0 cleanup_pids="$RUN_TMP/cleanup-pids" trap - EXIT INT TERM HUP @@ -1824,6 +1826,7 @@ else ( set +e child_pid= + # shellcheck disable=SC2329 # Registered by the worker signal traps below. worker_signal_exit() { local signal_rc=$1 pid signal_pids="$work/signal-pids" trap - INT TERM HUP From 42b1e2ae81d3df43c426bf825ff445026ec7c658 Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 02:45:48 -0700 Subject: [PATCH 06/16] no-mistakes(review): Bound signal cleanup and remove stale jq prerequisite --- .github/workflows/ci.yml | 1 - bin/fm-test-run.sh | 22 +++++++++++++++++++++- tests/fm-test-run.test.sh | 30 +++++++++++++++++++++++------- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48510d2c1f8..2e8b4233fee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -338,7 +338,6 @@ jobs: *) echo "::error::expected stock macOS Bash 3.2.57, got $BASH_VERSION"; exit 1 ;; esac /bin/bash --version | head -1 - command -v jq >/dev/null || { echo "::error::jq is required"; exit 1; } shell_inventory="$RUNNER_TEMP/fm-shell-inventory" bin/fm-lint.sh --list-files > "$shell_inventory" diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 9ec906ddc6e..1723b7edf18 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1509,7 +1509,7 @@ SERIAL_TEE_PID= # shellcheck disable=SC2329 # Invoked indirectly by the cleanup and worker signal traps. terminate_and_reap_process_tree() { - local root=$1 idx=0 parent children child seen pid + local root=$1 idx=0 parent children child seen pid grace=0 running state local -a process_tree_pids=() if ! kill -STOP "$root" 2>/dev/null; then wait "$root" 2>/dev/null || true @@ -1540,6 +1540,26 @@ terminate_and_reap_process_tree() { for pid in "${process_tree_pids[@]}"; do kill -CONT "$pid" 2>/dev/null || true done + while [ "$grace" -lt 100 ]; do + running=0 + for pid in "${process_tree_pids[@]}"; do + state=$(ps -o stat= -p "$pid" 2>/dev/null | awk 'NR == 1 { print $1 }' || true) + case "$state" in + ""|Z*) ;; + *) running=1; break ;; + esac + done + [ "$running" -eq 1 ] || break + sleep 0.01 + grace=$((grace + 1)) + done + for pid in "${process_tree_pids[@]}"; do + state=$(ps -o stat= -p "$pid" 2>/dev/null | awk 'NR == 1 { print $1 }' || true) + case "$state" in + ""|Z*) ;; + *) kill -KILL "$pid" 2>/dev/null || true ;; + esac + done wait "$root" 2>/dev/null || true for pid in "${process_tree_pids[@]}"; do while kill -0 "$pid" 2>/dev/null; do diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index bbe7316db68..810875e4b18 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -385,23 +385,28 @@ SH } test_interrupt_cleans_parallel_process_tree() { - local tmp repo runner fixture evidence child_pid descendant_pid runner_pid rc + local tmp repo runner fixture evidence child_pid descendant_pid runner_pid waited rc tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-signal.XXXXXX") repo="$tmp/repo" runner="$repo/bin/fm-test-run.sh" fixture=tests/fm-brief.test.sh evidence="$tmp/evidence" mkdir -p "$repo/bin" "$repo/tests" "$evidence" + mkfifo "$evidence/term-ignored" cp "$RUNNER" "$runner" cat >"$repo/$fixture" <<'SH' #!/usr/bin/env bash -printf '%s\n' "$$" >"$SCHED_EVIDENCE/child.pid" ( - while :; do - sleep 1 - done + trap '' TERM + : >"$SCHED_EVIDENCE/descendant.ready" + IFS= read -r _ <"$SCHED_EVIDENCE/term-ignored" ) & -printf '%s\n' "$!" >"$SCHED_EVIDENCE/descendant.pid" +descendant_pid=$! +while [ ! -e "$SCHED_EVIDENCE/descendant.ready" ]; do + sleep 0.01 +done +printf '%s\n' "$$" >"$SCHED_EVIDENCE/child.pid" +printf '%s\n' "$descendant_pid" >"$SCHED_EVIDENCE/descendant.pid" while :; do sleep 1 done @@ -421,6 +426,17 @@ SH descendant_pid=$(cat "$evidence/descendant.pid" 2>/dev/null || true) [ -n "$descendant_pid" ] || { kill "$runner_pid" 2>/dev/null || true; wait "$runner_pid" 2>/dev/null || true; rm -rf "$tmp"; fail "descendant fixture never started"; } kill -TERM "$runner_pid" + waited=0 + while kill -0 "$runner_pid" 2>/dev/null && [ "$waited" -lt 50 ]; do + sleep 0.1 + waited=$((waited + 1)) + done + if kill -0 "$runner_pid" 2>/dev/null; then + kill -KILL "$runner_pid" "$child_pid" "$descendant_pid" 2>/dev/null || true + wait "$runner_pid" 2>/dev/null || true + rm -rf "$tmp" + fail "runner TERM hung on a TERM-ignoring descendant" + fi set +e wait "$runner_pid" rc=$? @@ -435,7 +451,7 @@ SH fail "runner TERM left its worker descendant alive" fi rm -rf "$tmp" - pass "signals terminate complete parallel worker process trees" + pass "signals escalate and terminate complete parallel worker process trees" } test_interrupt_waits_for_parallel_cleanup() { From f6bf7641bf217a937cb13b8e890875f20c90f1b3 Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 02:53:45 -0700 Subject: [PATCH 07/16] no-mistakes(document): Document stock Bash runner wrapper --- docs/scripts.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/scripts.md b/docs/scripts.md index a48fac9db5f..339840774cd 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -31,6 +31,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-install-treehouse.sh`| Install CI's exact-version Treehouse pin for real-Herdr E2E that needs spawn worktrees | | `fm-herdr-ci-cleanup.sh` | Snapshot and tear down only job-owned `fm-lab-*` sessions in the Herdr CI lane | | `fm-test-run.sh` | Behavior-test runner: selection, portable lanes, declarative runtime gates, proven-isolated `--jobs`, coverage guard, timing/JSON | +| `fm-test-run-stock-bash.sh` | Run the focused runner contract under stock macOS Bash 3.2, or emit explicit local skip evidence | | `fm-test-isolation-proof.sh` | Concurrent isolation proof and proven-isolated candidate set owner | | `fm-ensure-agents-md.sh` | Ensure a project's real `AGENTS.md`, its `CLAUDE.md` symlink, and the canonical self-governance section | | `fm-guard.sh` | Warn on primary-checkout tangles, pending queued wakes, and stale watcher liveness | From 1fc3208a53921088e43945699fc7e612e6c7dcdf Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 03:20:32 -0700 Subject: [PATCH 08/16] no-mistakes(review): Bound signal escalation and descendant reaping --- bin/fm-test-run.sh | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 1723b7edf18..f9d2ba8430d 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1553,6 +1553,28 @@ terminate_and_reap_process_tree() { sleep 0.01 grace=$((grace + 1)) done + for pid in "${process_tree_pids[@]}"; do + kill -STOP "$pid" 2>/dev/null || true + done + idx=0 + while [ "$idx" -lt "${#process_tree_pids[@]}" ]; do + parent=${process_tree_pids[$idx]} + children=$(ps -eo pid=,ppid= 2>/dev/null | awk -v parent="$parent" '$2 == parent { print $1 }' || true) + while IFS= read -r child; do + [ -n "$child" ] || continue + seen=0 + for pid in "${process_tree_pids[@]}"; do + if [ "$pid" = "$child" ]; then + seen=1 + break + fi + done + if [ "$seen" -eq 0 ] && kill -STOP "$child" 2>/dev/null; then + process_tree_pids[${#process_tree_pids[@]}]=$child + fi + done <<<"$children" + idx=$((idx + 1)) + done for pid in "${process_tree_pids[@]}"; do state=$(ps -o stat= -p "$pid" 2>/dev/null | awk 'NR == 1 { print $1 }' || true) case "$state" in @@ -1561,10 +1583,19 @@ terminate_and_reap_process_tree() { esac done wait "$root" 2>/dev/null || true - for pid in "${process_tree_pids[@]}"; do - while kill -0 "$pid" 2>/dev/null; do - sleep 0.01 + grace=0 + while [ "$grace" -lt 100 ]; do + running=0 + for pid in "${process_tree_pids[@]}"; do + state=$(ps -o stat= -p "$pid" 2>/dev/null | awk 'NR == 1 { print $1 }' || true) + case "$state" in + ""|Z*) ;; + *) running=1; break ;; + esac done + [ "$running" -eq 1 ] || break + sleep 0.01 + grace=$((grace + 1)) done } From 7536bf664c35644e1744eb2350a457bae9290b02 Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 03:28:09 -0700 Subject: [PATCH 09/16] no-mistakes(review): Isolate test cleanup in bounded process groups --- bin/fm-test-run.sh | 51 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index f9d2ba8430d..1c2fec03287 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1599,13 +1599,50 @@ terminate_and_reap_process_tree() { done } +process_group_has_running_members() { + ps -eo pgid=,stat= 2>/dev/null \ + | awk -v group="$1" '$1 == group && $2 !~ /^Z/ { found=1; exit } END { exit !found }' +} + +terminate_and_reap_process_group() { + local group=$1 grace=0 + if ! kill -STOP -- "-$group" 2>/dev/null; then + wait "$group" 2>/dev/null || true + return + fi + kill -TERM -- "-$group" 2>/dev/null || true + kill -CONT -- "-$group" 2>/dev/null || true + while [ "$grace" -lt 100 ] && process_group_has_running_members "$group"; do + sleep 0.01 + grace=$((grace + 1)) + done + kill -STOP -- "-$group" 2>/dev/null || true + kill -KILL -- "-$group" 2>/dev/null || true + wait "$group" 2>/dev/null || true + grace=0 + while [ "$grace" -lt 100 ] && process_group_has_running_members "$group"; do + sleep 0.01 + grace=$((grace + 1)) + done +} + +terminate_and_reap_background_job() { + local pid=$1 group + group=$(ps -o pgid= -p "$pid" 2>/dev/null | awk 'NR == 1 { gsub(/[[:space:]]/, "", $1); print $1 }' || true) + if [ "$group" = "$pid" ]; then + terminate_and_reap_process_group "$group" + else + terminate_and_reap_process_tree "$pid" + fi +} + # shellcheck disable=SC2329 # Registered by the EXIT and signal traps below. cleanup_run() { local rc=$? pid serial_child_owned=0 cleanup_pids="$RUN_TMP/cleanup-pids" trap - EXIT INT TERM HUP jobs -p >"$cleanup_pids" 2>/dev/null || true if [ -n "$SERIAL_CHILD_PID" ]; then - terminate_and_reap_process_tree "$SERIAL_CHILD_PID" + terminate_and_reap_process_group "$SERIAL_CHILD_PID" serial_child_owned=1 fi while IFS= read -r pid; do @@ -1613,7 +1650,7 @@ cleanup_run() { if [ "$pid" = "$SERIAL_CHILD_PID" ] || [ "$pid" = "$SERIAL_TEE_PID" ]; then continue fi - terminate_and_reap_process_tree "$pid" + terminate_and_reap_background_job "$pid" if [ -n "$SERIAL_TEE_PID" ]; then serial_child_owned=1 fi @@ -1751,7 +1788,7 @@ record_script_result() { run_one_serial() { local script=$1 - local base family runtime requirement out fifo begin_iso begin_ms end_ms end_iso duration rc child_pid tee_pid + local base family runtime requirement out fifo begin_iso begin_ms end_ms end_iso duration rc child_pid tee_pid monitor_mode= base=$(basename "$script") family=$(family_for_basename "$base") runtime=$(runtime_gate_for_basename "$base") @@ -1768,8 +1805,11 @@ run_one_serial() { mkfifo "$fifo" tee "$out" <"$fifo" & SERIAL_TEE_PID=$! + case $- in *m*) monitor_mode=1 ;; esac + set -m bash "$script" >"$fifo" 2>&1 & SERIAL_CHILD_PID=$! + [ -n "$monitor_mode" ] || set +m wait "$SERIAL_CHILD_PID" rc=$? wait "$SERIAL_TEE_PID" 2>/dev/null || true @@ -1874,7 +1914,11 @@ else requirement=$(runtime_gate_requirement "$runtime") printf 'FM_TEST_BEGIN %s %s family=%s runtime_gate=%s gate_requirement=%s\n' \ "$(now_iso)" "$script" "$family" "$runtime" "$requirement" + monitor_mode= + case $- in *m*) monitor_mode=1 ;; esac + set -m ( + set +m set +e child_pid= # shellcheck disable=SC2329 # Registered by the worker signal traps below. @@ -1916,6 +1960,7 @@ else exit 0 ) & WORKER_PIDS[worker_n]=$! + [ -n "$monitor_mode" ] || set +m WORKER_IDX[worker_n]=$worker_n WORKER_SCRIPTS[worker_n]=$script active_workers=$((active_workers + 1)) From 0557c3bf571dc91bf2124ad25c9988a78f7688a6 Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 03:32:34 -0700 Subject: [PATCH 10/16] no-mistakes(review): Drain completed parallel worker process groups --- bin/fm-test-run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 1c2fec03287..53c26b758d5 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1853,6 +1853,7 @@ else active_workers=$((active_workers - 1)) set +e wait "$pid" + terminate_and_reap_process_group "$pid" set -e work="$RUN_TMP/w$idx" rc=$(cat "$work/exit" 2>/dev/null || echo 1) From 3573f61b819dbcfe296c1eb3bc3d702459478e87 Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 03:39:55 -0700 Subject: [PATCH 11/16] no-mistakes(document): Confirm stock Bash runner documentation accuracy --- tests/fm-test-run.test.sh | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index 810875e4b18..15765b20809 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -503,6 +503,44 @@ SH pass "signal handling completes child cleanup before returning" } +test_completed_parallel_worker_drains_process_group() { + local tmp repo runner fixture evidence descendant_pid state rc + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-completed-worker.XXXXXX") + repo="$tmp/repo" + runner="$repo/bin/fm-test-run.sh" + fixture=tests/fm-brief.test.sh + evidence="$tmp/evidence" + mkdir -p "$repo/bin" "$repo/tests" "$evidence" + cp "$RUNNER" "$runner" + cat >"$repo/$fixture" <<'SH' +#!/usr/bin/env bash +bash -c 'trap "" HUP TERM; while :; do sleep 1; done' & +descendant_pid=$! +printf '%s\n' "$descendant_pid" >"$SCHED_EVIDENCE/descendant.pid" +printf 'ok - fixture completed while descendant remained active\n' +SH + chmod +x "$runner" "$repo/$fixture" + set +e + SCHED_EVIDENCE="$evidence" "$runner" --jobs 2 "$fixture" >"$tmp/out" 2>"$tmp/err" + rc=$? + set -e + [ "$rc" -eq 0 ] || { cat "$tmp/out" "$tmp/err"; rm -rf "$tmp"; fail "completed parallel fixture should pass, got $rc"; } + assert_contains "$(cat "$tmp/out")" "FM_TEST_SUMMARY total=1 failed=0" "completed parallel fixture summary" + descendant_pid=$(cat "$evidence/descendant.pid" 2>/dev/null || true) + [ -n "$descendant_pid" ] || { rm -rf "$tmp"; fail "completed parallel fixture did not record its descendant"; } + state=$(ps -o stat= -p "$descendant_pid" 2>/dev/null | awk 'NR == 1 { print $1 }' || true) + case "$state" in + ""|Z*) ;; + *) + kill -KILL "$descendant_pid" 2>/dev/null || true + rm -rf "$tmp" + fail "completed parallel worker left a live descendant behind (pid=$descendant_pid state=$state)" + ;; + esac + rm -rf "$tmp" + pass "completed parallel workers drain their remaining process groups" +} + test_aggregate_exit_behavior() { local tmp pass_f fail_f rc tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-agg.XXXXXX") @@ -1100,6 +1138,7 @@ test_parallel_child_can_signal_immediately test_interrupt_drains_serial_cleanup test_interrupt_cleans_parallel_process_tree test_interrupt_waits_for_parallel_cleanup +test_completed_parallel_worker_drains_process_group test_aggregate_exit_behavior test_gate_skip_accounting test_runtime_gate_required_and_optional_outcomes From 0594be1072fc42db99c151861894b09de9212bd6 Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 03:52:20 -0700 Subject: [PATCH 12/16] no-mistakes(review): Suppress trap-only background reaper ShellCheck warning --- bin/fm-test-run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 53c26b758d5..825f5d97cce 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1626,6 +1626,7 @@ terminate_and_reap_process_group() { done } +# shellcheck disable=SC2329 # Invoked indirectly by the EXIT and signal traps through cleanup_run. terminate_and_reap_background_job() { local pid=$1 group group=$(ps -o pgid= -p "$pid" 2>/dev/null | awk 'NR == 1 { gsub(/[[:space:]]/, "", $1); print $1 }' || true) From 53871337081f62852a9943da3abe46862e3131dd Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 04:02:30 -0700 Subject: [PATCH 13/16] no-mistakes(document): Correct focused CI test count --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e8b4233fee..e7658d82c59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -350,8 +350,8 @@ jobs: runner_test_output=$(bin/fm-test-run-stock-bash.sh) printf '%s\n' "$runner_test_output" runner_test_count=$(printf '%s\n' "$runner_test_output" | grep -c '^ok - ') - [ "$runner_test_count" -eq 23 ] || { - echo "::error::expected 23 focused runner contract tests, got $runner_test_count" + [ "$runner_test_count" -eq 24 ] || { + echo "::error::expected 24 focused runner contract tests, got $runner_test_count" exit 1 } From 774ee111276523b323a8c7e33c40b8bec5c20a13 Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 04:21:40 -0700 Subject: [PATCH 14/16] no-mistakes(review): Restore stock Bash snapshot runtime coverage --- .github/workflows/ci.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7658d82c59..444ca1f5502 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -347,11 +347,27 @@ jobs: done < "$shell_inventory" [ "$parse_fail" -eq 0 ] || { echo "::error::stock macOS Bash 3.2 parse sweep failed"; exit 1; } + snapshot_output=$(/bin/bash tests/fm-fleet-snapshot-view.test.sh) + printf '%s\n' "$snapshot_output" + snapshot_count=$(printf '%s\n' "$snapshot_output" | grep -c '^ok - ') + [ "$snapshot_count" -eq 15 ] || { + echo "::error::expected 15 snapshot/fleet-view tests, got $snapshot_count" + exit 1 + } + + bearings_output=$(/bin/bash tests/fm-bearings-snapshot.test.sh) + printf '%s\n' "$bearings_output" + bearings_count=$(printf '%s\n' "$bearings_output" | grep -c '^ok - ') + [ "$bearings_count" -eq 41 ] || { + echo "::error::expected 41 Bearings tests, got $bearings_count" + exit 1 + } + runner_test_output=$(bin/fm-test-run-stock-bash.sh) printf '%s\n' "$runner_test_output" runner_test_count=$(printf '%s\n' "$runner_test_output" | grep -c '^ok - ') - [ "$runner_test_count" -eq 24 ] || { - echo "::error::expected 24 focused runner contract tests, got $runner_test_count" + [ "$runner_test_count" -eq 27 ] || { + echo "::error::expected 27 focused runner contract tests, got $runner_test_count" exit 1 } From 6ecb585747365adde0a03e0276cf56f29f1e2451 Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 04:33:54 -0700 Subject: [PATCH 15/16] no-mistakes(document): Document stock Bash snapshot runtime coverage --- docs/fm-test-portable-shards.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 1e285a6498a..dd49c7e8f4a 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -99,13 +99,14 @@ Portable shards, each portable serial shard, and the Herdr lane upload runner-ge ## Stock macOS Bash compatibility -The CI job `macos-stock-bash` runs the focused `tests/fm-test-run.test.sh` contract under `/bin/bash` from the macOS image, covering parallel scheduling, result aggregation, quoting, private temporary paths, signal cleanup, and the GNU/BSD `stat` difference. -The same job performs the repository-wide Bash parse sweep before the focused contract run. +The CI job `macos-stock-bash` runs `tests/fm-fleet-snapshot-view.test.sh`, `tests/fm-bearings-snapshot.test.sh`, and the focused `tests/fm-test-run.test.sh` contract under `/bin/bash` from the macOS image. +The snapshot suites preserve stock-Bash runtime coverage for fleet snapshot/view and Bearings rendering, while the focused runner contract covers parallel scheduling, result aggregation, quoting, private temporary paths, signal cleanup, and the GNU/BSD `stat` difference. +The same job performs the repository-wide Bash parse sweep before the runtime test suites. The compatibility wrapper pins `PATH` so nested runner invocations also resolve `bash` to `/bin/bash`, and the focused contract verifies that interpreter identity. -The compatibility boundary is stock macOS Bash 3.2.57 plus the commands explicitly required by the focused fixtures, including `awk`, `kill`, `mkfifo`, `mktemp`, `ps`, `sleep`, `stat`, `tee`, and `python3` for JSON assertions. -The local reproduction command is `bin/fm-test-run-stock-bash.sh`. -When Bash 3.2 is unavailable locally, the compatibility check may emit `skip: stock Bash 3.2 unavailable (evidence: )` and exit successfully; CI remains required on `macos-latest` and does not accept that skip. -`.github/workflows/ci.yml` owns the exact artifact names and aggregation wiring. +The focused runner compatibility boundary is stock macOS Bash 3.2.57 plus the commands explicitly required by its fixtures, including `awk`, `kill`, `mkfifo`, `mktemp`, `ps`, `sleep`, `stat`, `tee`, and `python3` for JSON assertions. +The focused runner contract's local reproduction command is `bin/fm-test-run-stock-bash.sh`. +When Bash 3.2 is unavailable locally, that wrapper may emit `skip: stock Bash 3.2 unavailable (evidence: )` and exit successfully; CI remains required on `macos-latest` and does not accept that skip. +`.github/workflows/ci.yml` owns the exact CI command sequence, expected test counts, artifact names, and aggregation wiring. ## Local entry points From fd944d8e04d7809141f8c2d1911460ccbefdb22b Mon Sep 17 00:00:00 2001 From: QuinnBot Date: Thu, 6 Aug 2026 04:38:35 -0700 Subject: [PATCH 16/16] no-mistakes(lint): Remove unused serial runner tee PID declaration --- bin/fm-test-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 825f5d97cce..7c37b6fc290 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1789,7 +1789,7 @@ record_script_result() { run_one_serial() { local script=$1 - local base family runtime requirement out fifo begin_iso begin_ms end_ms end_iso duration rc child_pid tee_pid monitor_mode= + local base family runtime requirement out fifo begin_iso begin_ms end_ms end_iso duration rc child_pid monitor_mode= base=$(basename "$script") family=$(family_for_basename "$base") runtime=$(runtime_gate_for_basename "$base")