Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ Those sleeps look like recoverable overhead - `fm-watch-triage.test.sh` alone is
Sampling less often does not remove that wait, it only delays detection: raising the interval to 0.5s and charging each sample proportionally measured `fm-watch-triage.test.sh` at 435s and 440s against 390s and 393s for the unchanged script, back to back on 2026-09-03, because each of its ~40 poll-cycle waits and ~73 process-exit waits paid up to half a second more.
Some of those loops are also catching a transient rather than waiting for a settled condition, so a coarser sample can step over the state they assert on.
Discover tests by listing `tests/*.test.sh`: each is a self-contained bash script named `<subject>.test.sh`, and its header comment describes what it covers, so pass one to `bin/fm-test-run.sh` to focus on a subject with canonical timing output.
Shared test helpers live in `tests/lib.sh` (reporters, temp roots, git fixtures), `tests/fixtures.sh` (fake toolchain and spawn-world builders), `tests/wake-helpers.sh`, and `tests/secondmate-helpers.sh`.
Shared test helpers live in `tests/lib.sh` (reporters, temp roots, git fixtures), `tests/fixtures.sh` (fake toolchain and spawn-world builders), `tests/wake-helpers.sh`, `tests/secondmate-helpers.sh`, and `tests/git-config-helpers.sh` (fixture Git isolation from the host's global and system configuration, already sourced by `tests/lib.sh` and `tests/herdr-test-safety.sh`; a suite that sources neither must source it itself before its first Git operation so a direct invocation stays isolated).
Source those instead of copying a fake toolchain into a new suite.
A fixture may shorten a production timeout to keep a failure path prompt, but never below what the real work inside that window costs on a loaded machine: a fork, an exec, a lock acquisition, a beacon publication, or a first-poll check.
Where a case's assertion is not about the timeout itself, give that window headroom over the measured loaded cost, and bound the test's own waiting with iteration-counted poll loops, which stretch under load where a wall-clock budget does not.
Expand Down
35 changes: 31 additions & 4 deletions bin/fm-test-run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,10 @@
# live-capability (a live-harness guard governed by fm_live_gate, which records
# unavailable tools and explicit policy skips; see tests/lib.sh), or none.
#
# Every selected script runs isolated from the host's global and system Git
# configuration, including one that sources no test helper of its own;
# tests/git-config-helpers.sh owns that contract and its limits.
#
# Family labels, the changed-file map, and production portable-shard composition
# live in this script only (one owner). The proven-isolated candidate set remains
# owned by bin/fm-test-isolation-proof.sh; portable parallel shards are a
Expand Down Expand Up @@ -1139,12 +1143,14 @@ select_family() {
[ "$found" -eq 1 ] || die "no tests mapped to family '$want'"
}

families_for_test_reference() {
local needle=$1 s
families_for_test_reference() { # <needle>...
local s needle
local found=0
local -a needles=()
for needle in "$@"; do needles+=(-e "$needle"); done
while IFS= read -r s; do
[ -n "$s" ] || continue
if grep -Fq "$needle" "$s"; then
if grep -Fq "${needles[@]}" "$s"; then
family_for_basename "$(basename "$s")"
found=1
fi
Expand Down Expand Up @@ -1221,12 +1227,21 @@ families_for_changed_path() {
# resolution in the caller; emit a marker family of __script__
printf '%s\n' "__script__:$(basename "$path")"
;;
bin/fm-test-run.sh|bin/fm-test-isolation-proof.sh)
bin/fm-test-run.sh)
# Deliberately the WHOLE family, not just the two contract tests. This
# runner executes every pure-contract-unit script, so a change to it is
# only proven by running them: its own contract test passing says the
# runner's logic is right, not that the suite it drives still runs.
printf '%s\n' pure-contract-unit
# Only this script wraps each suite in run_script_bounded's fixture Git
# isolation, and only a standalone-family script proves it.
printf '%s\n' "__script__:fm-test-fixtures.test.sh"
;;
bin/fm-test-isolation-proof.sh)
# Same reason as the runner above: the proof drives every
# pure-contract-unit script. It runs each candidate directly, never
# through run_script_bounded, so it cannot regress fixture Git isolation.
printf '%s\n' pure-contract-unit
;;
bin/backends/herdr*|bin/fm-herdr-lab.sh|tests/herdr-test-safety.sh)
printf '%s\n' real-herdr-gated
Expand Down Expand Up @@ -1432,6 +1447,12 @@ families_for_changed_path() {
docs/configuration.md|docs/supervision-protocols/*)
printf '%s\n' pure-contract-unit
;;
tests/git-config-helpers.sh)
# The reference scan is not transitive, so match the two helpers that
# source this one as well: most suites inherit it only through them.
families_for_test_reference git-config-helpers.sh lib.sh herdr-test-safety.sh \
|| printf '%s\n' "__unmapped__:$path"
;;
tests/lib.sh|tests/*-helpers.sh|tests/fixtures.sh)
families_for_test_reference "$(basename "$path")" \
|| printf '%s\n' "__unmapped__:$path"
Expand Down Expand Up @@ -2179,6 +2200,12 @@ record_script_result() {
# because an unbounded suite is what silently outruns its caller's budget.
run_script_bounded() { # <script> <out> <stream> <id>
local script=$1 out=$2 stream=$3 id=$4
# Declaring the variables local first keeps the helper's export scoped to this
# call and its child script, so the runner's own environment is left as the
# caller had it.
local GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM
# shellcheck source=tests/git-config-helpers.sh
. "$ROOT/tests/git-config-helpers.sh" || return
local rc
: "$id"
set +e
Expand Down
6 changes: 3 additions & 3 deletions docs/fm-test-isolation-proof.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,10 +119,10 @@ Both `bin/fm-test-run.sh` and the current proof harness therefore order concurre
| 1 | `FM_ISOLATION_SUMMARY total=32 failed=0 concurrency=4 duration_ms=161837` |
| 2 | `FM_ISOLATION_SUMMARY total=32 failed=0 concurrency=4 duration_ms=156462` |

This family is what a change to `bin/fm-test-run.sh` itself selects, so it decides that selection's wall clock.
Before admission, 14 of its scripts fell to the serial tail and the 33-script selection measured 327.3s against a 300s budget: the concurrent group was 19 scripts totalling 273.4s while the tail alone was 215.7s, dominated by `fm-calm-pi-extension` (77.5s), `fm-vendor-auth-probe` (51.0s), and `fm-muse-harness` (39.7s).
The current runner-change selection is owned by [`bin/fm-test-run.sh`](../bin/fm-test-run.sh)'s changed-file map.
Before admission, 14 of the family's scripts fell to the serial tail and the 33-script selection measured 327.3s against a 300s budget: the concurrent group was 19 scripts totalling 273.4s while the tail alone was 215.7s, dominated by `fm-calm-pi-extension` (77.5s), `fm-vendor-auth-probe` (51.0s), and `fm-muse-harness` (39.7s).
Admitting the family moves that tail into the bounded concurrent group.
Current runner-file selection was verified on 2026-08-28 with the runner and its tests bound to each measured Bash version.
The then-current runner-file selection was verified on 2026-08-28 with the runner and its tests bound to each measured Bash version.
Because the runner uses `#!/usr/bin/env bash` and invokes each test with `bash` from `PATH`, the stock macOS measurement used `PATH=/bin:$PATH bin/fm-test-run.sh --changed --max-wall-ms 300000` so both resolved to `/bin/bash` 3.2.57.
Two runs selected all 33 scripts, passed the five-minute result check in 153.5s and 166.8s, and reported the same two failures as `main`: `tests/fm-muse-harness.test.sh` and `tests/fm-composer-lib.test.sh`.
With Bash 5.3.9 on `PATH`, three runs of `bin/fm-test-run.sh --changed --max-wall-ms 300000` selected the same 33 scripts, completed with 0 failures, and reported 163.8s, 172.0s, and 166.9s.
Expand Down
2 changes: 2 additions & 0 deletions tests/fm-gitignore-config.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
set -u

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck source=tests/git-config-helpers.sh
. "$ROOT/tests/git-config-helpers.sh"

fail() {
printf 'not ok - %s\n' "$1" >&2
Expand Down
141 changes: 141 additions & 0 deletions tests/fm-test-fixtures.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,153 @@
# filesystem effects - never on helper source text. Migrated spawn suites cover
# fm_test_run_spawn through the real fm-spawn.sh; this file pins the shared
# primitives and stubs those suites use.
#
# It is also the fixture Git-config isolation regression, with host signing
# armed on a scratch config file: it drives every entry point that must reach
# tests/git-config-helpers.sh - the shared helpers, bin/fm-test-run.sh's
# per-suite wrapper, and the standalone scripts runnable without a live vendor.
# That helper's header owns the contract and the layers it leaves in force.
set -u

# shellcheck source=tests/fixtures.sh
. "$(dirname "${BASH_SOURCE[0]}")/fixtures.sh"

TMP_ROOT=$(fm_test_tmproot fm-test-fixtures)

test_git_config_isolation() (
local dir="$TMP_ROOT/git-config" helper jobs timeout fakebin rc
mkdir -p "$dir/runner/bin" "$dir/runner/tests"
git init -q "$dir/caller"
git -C "$dir/caller" config commit.gpgsign false
cd "$dir/caller" || exit 1
cp "$ROOT/bin/fm-test-run.sh" "$ROOT/bin/fm-timeout-lib.sh" "$dir/runner/bin/"
cp "$ROOT/tests/git-config-helpers.sh" "$dir/runner/tests/"
fakebin=$(fm_fakebin "$dir/standalone")
fm_fake_exit0 "$fakebin" pi
cat > "$fakebin/tmux" <<'SH'
#!/usr/bin/env bash
set -eu
while [ "$#" -gt 0 ]; do
if [ "$1" = -c ]; then
git -C "$2" log -1 --format=%s > "${FM_TEST_STANDALONE_COMMIT:?}"
exit 1
fi
shift
done
SH
chmod +x "$fakebin/tmux"
cat > "$dir/runner/tests/fm-test-run.test.sh" <<'SH'
#!/usr/bin/env bash
set -eu
repo=$(mktemp -d "${TMPDIR:-/tmp}/fm-git-runner.XXXXXX")
trap 'rm -rf "$repo"' EXIT
git init -q "$repo"
git -C "$repo" config user.name 'Runner Fixture'
git -C "$repo" config user.email runner@example.invalid
git -C "$repo" commit -q --allow-empty -m initial
[ "$(git -C "$repo" log -1 --format='%s:%an:%ae')" = 'initial:Runner Fixture:runner@example.invalid' ]
[ "$(git -C "$repo" config --get fixture.input)" = preserved ]
[ "$(GIT_CONFIG_GLOBAL="$FM_TEST_GIT_CONFIG" git config --global --get commit.gpgsign)" = true ]
SH
chmod +x "$dir/runner/tests/fm-test-run.test.sh"
export GIT_CONFIG_GLOBAL="$dir/global" GIT_CONFIG_SYSTEM="$dir/system"
export GIT_CONFIG_NOSYSTEM=0
unset GIT_CONFIG_COUNT GIT_CONFIG_PARAMETERS

# A failing signer exposes inherited config without requiring GPG or keys.
arm_host_signing() { # <scope>: only this layer carries the failing signer
: > "$dir/global"
: > "$dir/system"
git config --file "$dir/$1" commit.gpgsign true
git config --file "$dir/$1" gpg.format openpgp
git config --file "$dir/$1" gpg.program /usr/bin/false
cp "$dir/$1" "$dir/expected"
}

assert_helper_isolates() { # <helper> <scope>
bash -eus -- "$ROOT/tests/$1.sh" "$dir/$2-$1" "$dir/$2" <<'SH' || exit 1
. "$1"
fm_git_init_commit "$2"
[ "$(git -C "$2" log -1 --format=%s)" = initial ] || fail "fixture has no initial commit"
fm_git_identity
# Child Git processes and direct commits inherit the same isolation.
bash -eu -c 'git -C "$1" commit -q --allow-empty -m child' _ "$2"
# Repository-local config and explicit command inputs remain authoritative.
git -C "$2" config commit.gpgsign true
git -C "$2" config gpg.program /usr/bin/false
if git -C "$2" commit -q --allow-empty -m signed > "$2/signing.log" 2>&1; then
fail "repository-local signing config was ignored"
fi
assert_grep 'gpg failed to sign' "$2/signing.log" "local signing was not attempted"
git -C "$2" -c commit.gpgsign=false commit -q --allow-empty -m explicit
GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=commit.gpgsign GIT_CONFIG_VALUE_0=false \
git -C "$2" commit -q --allow-empty -m environment
# A config test can deliberately supply its own global file after sourcing.
[ "$(GIT_CONFIG_GLOBAL="$3" git config --global --get commit.gpgsign)" = true ] || fail "explicit global config was ignored"
SH
}

assert_host_config_still_governs() { # <scope>
# Sourcing in test subprocesses cannot change the caller or its config files.
[ "$(git config --"$1" --get commit.gpgsign)" = true ] || fail "caller lost signing preference"
cmp -s "$dir/$1" "$dir/expected" || fail "host config file was changed"
git init -q "$dir/$1-outside"
if git -C "$dir/$1-outside" -c user.name=test -c user.email=test@example.invalid \
commit -q --allow-empty -m outside > "$dir/outside.log" 2>&1; then
fail "commit outside fixtures bypassed signing"
fi
assert_grep 'gpg failed to sign' "$dir/outside.log" "outside commit did not attempt signing"
}

# Every fixture entry point, once. Each only has to reach the shared helper;
# which layers that helper neutralizes is the helper's own property, settled
# by the system-layer case below.
arm_host_signing global
for helper in lib fixtures secondmate-helpers wake-helpers; do
assert_helper_isolates "$helper" global
done
bash -eus -- "$ROOT/tests/herdr-test-safety.sh" "$dir/global-herdr" <<'SH' || exit 1
. "$1"
git init -q "$2"
git -C "$2" -c user.name=test -c user.email=test@example.invalid \
commit -q --allow-empty -m initial
[ "$(git -C "$2" log -1 --format=%s)" = initial ]
SH
for jobs in 1 2; do
for timeout in 0 30; do
GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=fixture.input GIT_CONFIG_VALUE_0=preserved \
FM_TEST_GIT_CONFIG="$dir/global" \
"$dir/runner/bin/fm-test-run.sh" --jobs "$jobs" --per-script-timeout-secs "$timeout" \
tests/fm-test-run.test.sh > "$dir/runner.log" 2>&1 \
|| fail "runner inherited global config (jobs=$jobs, timeout=$timeout): $(cat "$dir/runner.log")"
assert_grep 'FM_TEST_SUMMARY total=1 failed=0 skipped_gate=0' "$dir/runner.log" \
"runner did not execute the Git fixture"
done
done
rc=0
FM_SESSIONSTART_INSTRUCTION_REFRESH_LIVE_E2E=1 FM_SESSIONSTART_INSTRUCTION_REFRESH_REF=HEAD \
FM_SESSIONSTART_INSTRUCTION_REFRESH_EXPECT=updated \
FM_TEST_STANDALONE_COMMIT="$dir/global-standalone-commit" PATH="$fakebin:$PATH" \
bash "$ROOT/tests/fm-sessionstart-instruction-refresh-live-e2e.test.sh" \
> "$dir/standalone.log" 2>&1 || rc=$?
[ "$rc" = 1 ] || fail "standalone fixture did not stop at the tmux launch"
assert_grep 'could not start isolated Pi session' "$dir/standalone.log" \
"standalone fixture failed before the tmux launch: $(cat "$dir/standalone.log")"
[ "$(cat "$dir/global-standalone-commit")" = 'test: initial instruction contract' ] \
|| fail "standalone fixture did not create its initial commit"
bash "$ROOT/tests/fm-gitignore-config.test.sh" > "$dir/gitignore.log" 2>&1 \
|| fail "standalone gitignore fixture inherited global config: $(cat "$dir/gitignore.log")"
assert_host_config_still_governs global

# The system layer is the shared helper's other half: one entry point settles
# it, and the caller still signing proves the layer was genuinely armed.
arm_host_signing system
assert_helper_isolates lib system
assert_host_config_still_governs system

pass "runner and shared helpers isolate host Git config and preserve explicit config and outside commits"
)

test_touch_epoch_preserves_repeated_dst_hour() {
local TZ=Europe/Paris epoch path actual
export TZ
Expand Down Expand Up @@ -139,6 +279,7 @@ test_spawn_home_layout() {
pass "spawn-home layout writes harness pin, beat, and brief"
}

test_git_config_isolation || fail "Git fixture config isolation"
test_touch_epoch_preserves_repeated_dst_hour
test_no_mistakes_version_constant
test_no_mistakes_init_doctor_markers
Expand Down
Loading
Loading