From 8c9a331917604030620dbebc5a9815cff47891b3 Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Thu, 8 Oct 2026 18:01:14 -0500 Subject: [PATCH 1/7] fix: bound fm-lint ShellCheck fan-out across concurrent runs Each fm-lint.sh run limited itself to two workers, but nothing limited the sum, so N concurrent runs started 2N ShellCheck processes (42 seen, some near 4.8 GB). Every root now holds one host-wide flock slot: half the cores (at least two), shrinking to a floor of two as 1-minute load passes twice the cores. Runs queue instead of failing, and queue time stays out of the root deadline and recorded duration. --- bin/fm-lint-cache.pl | 91 ++++++++++++++++++++++++++++++++++++ bin/fm-lint.sh | 59 ++++++++++++++++++++---- tests/fm-lint.test.sh | 105 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 247 insertions(+), 8 deletions(-) diff --git a/bin/fm-lint-cache.pl b/bin/fm-lint-cache.pl index 9ca026ab037..4228eb3e6dc 100755 --- a/bin/fm-lint-cache.pl +++ b/bin/fm-lint-cache.pl @@ -13,6 +13,97 @@ use File::Path qw(make_path); use File::Basename qw(dirname basename); +# gate mode: hold one host-wide ShellCheck slot while the command runs, so many +# concurrent fm-lint.sh runs queue instead of each adding its own workers. +# Usage: perl fm-lint-cache.pl gate -- +# Slots are flock files, so the kernel frees one when its holder dies. The slot +# count is FM_LINT_HOST_SLOTS, else half the cores, but never below two (one +# run's default workers); while 1-minute load is within two times the cores, +# all slots may be taken, and each point of load past that takes one away, down +# to the two-slot floor. A slot directory that cannot be used lets the command +# run ungated. The wait is written in milliseconds to so the caller +# can keep queue time out of its own timings. +if (($ARGV[0] // '') eq 'gate') { + require POSIX; + require Time::HiRes; + my (undef, $slot_dir, $ncpu, $wait_file, $dashes, @command) = @ARGV; + die "fm-lint-gate: invalid private invocation\n" + unless defined $dashes && $dashes eq '--' && @command && ($ncpu // '') =~ /\A[1-9][0-9]*\z/; + my ($child, $caught); + my %signal_number = (HUP => 1, INT => 2, TERM => 15); + for my $name (keys %signal_number) { + $SIG{$name} = sub { $caught = $name; kill 'TERM', $child if $child; }; + } + my $floor = 2; + my $cap = ($ENV{FM_LINT_HOST_SLOTS} // '') =~ /\A[1-9][0-9]*\z/ ? $ENV{FM_LINT_HOST_SLOTS} + 0 + : ($ncpu >> 1) > $floor ? ($ncpu >> 1) : $floor; + $floor = $cap if $cap < $floor; + my $slot_load = sub { + return $ENV{FM_LINT_SLOT_LOAD} + 0 + if ($ENV{FM_TEST_SEAM} // '') eq '1' && ($ENV{FM_LINT_SLOT_LOAD} // '') =~ /\A[0-9]+(?:\.[0-9]+)?\z/; + if (open(my $fh, '<', '/proc/loadavg')) { + my $line = <$fh>; + return $1 + 0 if defined $line && $line =~ /\A([0-9]+(?:\.[0-9]+)?)/; + } + if (open(my $pipe, '-|', 'sysctl', '-n', 'vm.loadavg')) { + my $line = <$pipe>; + close $pipe; + return $1 + 0 if defined $line && $line =~ /([0-9]+(?:\.[0-9]+)?)/; + } + return 0; + }; + my ($slot, $waited_from); + if (eval { make_path($slot_dir, {mode => 0700}); -d $slot_dir && -w _ }) { + $waited_from = Time::HiRes::time(); + while (!$slot) { + exit 128 + $signal_number{$caught} if $caught; + my $allowed = int(2 * $ncpu - $slot_load->() + 0.5); + $allowed = $floor if $allowed < $floor; + $allowed = $cap if $allowed > $cap; + for my $index (0 .. $allowed - 1) { + open(my $fh, '>>', "$slot_dir/slot.$index") or next; + if (flock($fh, LOCK_EX | LOCK_NB)) { $slot = $fh; last; } + close $fh; + } + next if $slot || $caught; + # Every allowed slot is busy: block on one so a freed slot wakes this + # waiter at once, and re-read the load every two seconds. + if (open(my $fh, '>>', "$slot_dir/slot.@{[ int(rand($allowed)) ]}")) { + my $locked = eval { + local $SIG{ALRM} = sub { die "gate-recheck\n" }; + alarm 2; + my $got = flock($fh, LOCK_EX); + alarm 0; + $got; + }; + alarm 0; + if ($locked) { $slot = $fh; } else { close $fh; } + } else { + Time::HiRes::sleep(1); + } + } + } else { + warn "fm-lint: host slot directory unavailable; running without the host-wide ShellCheck bound\n"; + } + if (open(my $fh, '>', $wait_file)) { + printf {$fh} "%d\n", $waited_from ? (Time::HiRes::time() - $waited_from) * 1000 : 0; + close $fh; + } + exit 128 + $signal_number{$caught} if $caught; + $child = fork(); + die "fm-lint-gate: fork: $!\n" unless defined $child; + if (!$child) { + exec {$command[0]} @command; + warn "fm-lint-gate: exec $command[0]: $!\n"; + POSIX::_exit(127); + } + kill 'TERM', $child if $caught; + my $reaped; + while (($reaped = waitpid($child, 0)) == -1 && $!{EINTR}) { } + my $status = $reaped == $child ? $? : 127 << 8; + exit(($status & 127) ? 128 + ($status & 127) : $status >> 8); +} + my ($mode, $root, @args) = @ARGV; my $cache; if (defined $mode && $mode eq 'check') { diff --git a/bin/fm-lint.sh b/bin/fm-lint.sh index 73e12a732f1..c4f72d30751 100755 --- a/bin/fm-lint.sh +++ b/bin/fm-lint.sh @@ -61,6 +61,17 @@ # run holds at most JOBS concurrent ShellCheck processes. Diagnostics replay # in stable shard/root order. FM_LINT_JOBS=1 changes concurrency, not diagnostics # or exit selection. +# That bound is per run, so every root also holds one host-wide ShellCheck slot +# (flock files under FM_LINT_SLOT_DIR, default +# ${XDG_CACHE_HOME:-$HOME/.cache}/firstmate/lint-slots; "off" disables) and +# many concurrent runs queue instead of each adding workers. FM_LINT_HOST_SLOTS +# sets the slot count, default half the cores and at least two. While 1-minute +# load is within two times the cores every slot is usable; each point of load +# above that removes one, down to a floor of two (one run's own workers), so a +# lone run is never throttled and queued roots always make progress. Queue time +# is excluded from the root deadline and the recorded root duration. The pool +# is shared by every worktree and home of the same user. A slot directory that +# cannot be used is reported and lint runs ungated; it never fails lint. # --partition 1of2/2of2 splits the entire canonical inventory across # two CI runners, each with those same concurrency-limited workers. # Partitions are complete, disjoint, and byte-weight balanced; --list-files @@ -253,8 +264,8 @@ fm_lint_classify_root() { # # Run one ShellCheck invocation under the given deadline and the per-root # address-space limit, returning its exit status in FM_LINT_LAST_RC. fm_lint_exec_root() { # - local path=$1 root_out=$2 root_err=$3 rss_file=$4 seconds=$5 invocation_rc=0 - local -a analysis_command + local path=$1 root_out=$2 root_err=$3 rss_file=$4 seconds=$5 invocation_rc=0 wait_file waited + local -a analysis_command gate_command shift 5 analysis_command=("${FM_LINT_PERL_BIN:-perl}" "$SELF_DIR/fm-lint-cache.pl" check \ "${FM_LINT_INTERNAL_CACHE:-off}" "$ROOT" "$FM_LINT_SHELLCHECK" \ @@ -266,6 +277,15 @@ fm_lint_exec_root() { # analysis_command=(/usr/bin/time -f 'max_rss_kib=%M' -o "$rss_file" "${analysis_command[@]}") fi fi + # The host-wide slot gate sits outside the watchdog and /usr/bin/time, so + # queue time counts toward neither the root deadline nor its peak RSS. + gate_command=() + wait_file="$rss_file.wait" + rm -f "$wait_file" + if [ "${FM_LINT_INTERNAL_SLOT_DIR:-off}" != off ]; then + gate_command=("${FM_LINT_PERL_BIN:-perl}" "$SELF_DIR/fm-lint-cache.pl" gate \ + "$FM_LINT_INTERNAL_SLOT_DIR" "$FM_LINT_INTERNAL_NCPU" "$wait_file" --) + fi if [ "${FM_LINT_INTERNAL_BOUNDED:-none}" != none ]; then # The watchdog runs in a process group of its own (the same setpgrp hop the # workers use), so the owner's TERM-then-KILL group sweep cannot kill it @@ -274,7 +294,7 @@ fm_lint_exec_root() { # # check still starts the same terminate-then-kill escalation; the worker # names itself as that owner before the launch, so a worker that dies while # the watchdog is still starting is detected too. - ( FM_EXEC_TIMED_OWNER_PID=$$ exec "${FM_LINT_PERL_BIN:-perl}" -e 'setpgrp(0, 0) or die "setpgrp: $!"; exec @ARGV or die "exec: $!"' \ + ( FM_EXEC_TIMED_OWNER_PID=$$ exec ${gate_command[@]+"${gate_command[@]}"} "${FM_LINT_PERL_BIN:-perl}" -e 'setpgrp(0, 0) or die "setpgrp: $!"; exec @ARGV or die "exec: $!"' \ "${BASH:-bash}" "$SELF" --internal-timed \ "$seconds" "$FM_LINT_INTERNAL_GRACE" \ "${BASH:-bash}" "$SELF" --internal-root "$rss_file" "$FM_LINT_INTERNAL_MEMORY_KIB" \ @@ -283,12 +303,17 @@ fm_lint_exec_root() { # wait "$FM_LINT_WORKER_RUN_PID" || invocation_rc=$? FM_LINT_WORKER_RUN_PID= else - "${analysis_command[@]}" > "$root_out" 2> "$root_err" & + ${gate_command[@]+"${gate_command[@]}"} "${analysis_command[@]}" > "$root_out" 2> "$root_err" & FM_LINT_WORKER_RUN_PID=$! wait "$FM_LINT_WORKER_RUN_PID" || invocation_rc=$? FM_LINT_WORKER_RUN_PID= fi FM_LINT_LAST_RC=$invocation_rc + FM_LINT_LAST_WAIT_MS=0 + if [ -r "$wait_file" ]; then + waited=$(tr -d '[:space:]' < "$wait_file" 2>/dev/null) + case "$waited" in ''|*[!0-9]*) ;; *) FM_LINT_LAST_WAIT_MS=$waited ;; esac + fi } # Run one selected root, retry memory failures without source following, record @@ -302,7 +327,7 @@ fm_lint_run_root() { # local fallback_err="$output_dir/root.$shard_index.$index.fallback.err" local fallback_rss="$output_dir/root.$shard_index.$index.fallback.rss" local start_ms end_ms duration_ms invocation_rc=0 reason rss_kib initial_rc initial_reason - local fallback_secs + local fallback_secs queued_ms=0 local final_follow_sources=${FM_LINT_INTERNAL_FOLLOW_SOURCES:-1} local -a fallback_args start_ms=$(fm_lint_now_ms) @@ -318,13 +343,14 @@ fm_lint_run_root() { # fm_lint_exec_root "$path" "$root_out" "$root_err" "$rss_file" \ "$FM_LINT_INTERNAL_ROOT_SECS" "${FM_LINT_WORKER_ARGS[@]}" invocation_rc=$FM_LINT_LAST_RC + queued_ms=$FM_LINT_LAST_WAIT_MS reason=$(fm_lint_classify_root "$invocation_rc" "$root_err") initial_rc=$invocation_rc initial_reason=$reason # The retry spends what is left of this root's one deadline rather than a # fresh one, so both attempts together still fit the budget CI sized its job # timeout around. - fallback_secs=$(( (start_ms + FM_LINT_INTERNAL_ROOT_SECS * 1000 - $(fm_lint_now_ms)) / 1000 )) + fallback_secs=$(( (start_ms + queued_ms + FM_LINT_INTERNAL_ROOT_SECS * 1000 - $(fm_lint_now_ms)) / 1000 )) if [ "$reason" = memory ] \ && [ "${FM_LINT_INTERNAL_FOLLOW_SOURCES:-1}" -eq 1 ] \ && [ "${FM_LINT_INTERNAL_BOUNDED:-none}" != none ] \ @@ -344,6 +370,7 @@ fm_lint_run_root() { # "$fallback_secs" "${fallback_args[@]}" final_follow_sources=0 invocation_rc=$FM_LINT_LAST_RC + queued_ms=$((queued_ms + FM_LINT_LAST_WAIT_MS)) reason=$(fm_lint_classify_root "$invocation_rc" "$fallback_err") rss_kib=$(fm_lint_max_root_rss \ "$(fm_lint_root_rss "$rss_file")" "$(fm_lint_root_rss "$fallback_rss")") @@ -364,7 +391,7 @@ fm_lint_run_root() { # cat "$root_out" "$root_err" >> "$output_dir/shard.$shard_index.out" fi end_ms=$(fm_lint_now_ms) - duration_ms=$((end_ms - start_ms)) + duration_ms=$((end_ms - start_ms - queued_ms)) if [ -n "${FM_LINT_INTERNAL_ROOTS_LOG:-}" ]; then printf 'end\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ "$index" "$path" "$shard_index" "${FM_LINT_INTERNAL_MODE:-}" \ @@ -1034,6 +1061,22 @@ if [ "${FM_LINT_REQUIRE_BOUNDS:-0}" = 1 ]; then fi fi +# Host-wide ShellCheck slot pool (see the header and fm-lint-cache.pl gate mode). +# FM_LINT_SLOT_DIR=off disables it; an unset pool lives beside the lint cache. +SLOT_DIR=${FM_LINT_SLOT_DIR:-${XDG_CACHE_HOME:-${HOME:-${TMPDIR:-/tmp}}/.cache}/firstmate/lint-slots} +case "${FM_LINT_HOST_SLOTS:-1}" in + ''|0*|*[!0-9]*) + printf 'fm-lint.sh: FM_LINT_HOST_SLOTS must be a positive integer, got %s.\n' \ + "${FM_LINT_HOST_SLOTS:-}" >&2 + exit 2 + ;; +esac +NCPU= +if [ "$SLOT_DIR" != off ]; then + NCPU=$(sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null) || NCPU= + case "$NCPU" in ''|0*|*[!0-9]*) NCPU=1 ;; esac +fi + PROGRESS=0 if [ -n "$PARTITION" ]; then PROGRESS=1 @@ -1184,7 +1227,7 @@ fm_lint_run_worker() { # FM_LINT_SHELLCHECK="$SHELLCHECK_BIN" FM_LINT_PERL_BIN="$PERL_BIN" ) - worker_env+=(FM_LINT_INTERNAL_CACHE="$CACHE_DIR") + worker_env+=(FM_LINT_INTERNAL_CACHE="$CACHE_DIR" FM_LINT_INTERNAL_SLOT_DIR="$SLOT_DIR" FM_LINT_INTERNAL_NCPU="$NCPU") if [ -n "$TELEMETRY" ] && [ -x /usr/bin/time ]; then if [ "$(uname)" = Darwin ]; then exec "$PERL_BIN" -e 'setpgrp(0, 0) or die "setpgrp: $!"; exec @ARGV or die "exec: $!"' \ diff --git a/tests/fm-lint.test.sh b/tests/fm-lint.test.sh index 8f2d17b207c..3745546429e 100755 --- a/tests/fm-lint.test.sh +++ b/tests/fm-lint.test.sh @@ -22,6 +22,8 @@ INSTALLER="$ROOT/bin/fm-install-shellcheck.sh" REQUIRED=$("$LINT" --required-version) # Ordinary regressions must not read or populate the operator's shared cache. export FM_LINT_CACHE_DIR=off +# Nor may they queue on the operator's shared host-wide ShellCheck slots. +export FM_LINT_SLOT_DIR=off # Official GitHub release asset sha256 values for shellcheck v0.11.0 .tar.xz # archives (https://github.com/koalaman/shellcheck/releases/tag/v0.11.0). Tests @@ -1088,6 +1090,106 @@ SH pass "jobs=1 and jobs=2 preserve deterministic diagnostics, failures, cleanup bounds, and quiet telemetry" } +# fm_lint_stub_counting_shellcheck : a +# ShellCheck stand-in that holds each check for half a second and records the +# most checks alive at once across every fm-lint.sh run sharing the directory. +fm_lint_stub_counting_shellcheck() { + local fakebin=$1 activity=$2 peak=$3 + mkdir -p "$activity" + : > "$peak" + cat > "$fakebin/shellcheck" <> "$peak" +sleep 0.5 +rm -f "$activity/\$\$" +exit 0 +SH + chmod +x "$fakebin/shellcheck" +} + +# fm_lint_concurrent_runs : start simultaneous +# fm-lint.sh runs of four roots each, wait for all, and print the peak number +# of live ShellCheck processes. Extra environment is read from the caller. +fm_lint_concurrent_runs() { + local runs=$1 tmp=$2 fakebin=$3 run pid rc=0 + local -a pids roots + roots=("$tmp/a.sh" "$tmp/b.sh" "$tmp/c.sh" "$tmp/d.sh") + for run in $(seq 1 "$runs"); do + PATH="$fakebin:$PATH" "$LINT" "${roots[@]}" > "$tmp/run.$run.out" 2>&1 & + pids+=("$!") + done + for pid in "${pids[@]}"; do + wait "$pid" || rc=$? + done + [ "$rc" -eq 0 ] || fail "a concurrent fm-lint.sh run failed (rc=$rc)"$'\n'"$(cat "$tmp"/run.*.out)" + sort -n "$tmp/peak" | tail -1 +} + +fm_lint_slot_fixture() { # -> prints the tmp dir + local tmp root + tmp=$(fm_test_tmproot "$1") + mkdir -p "$tmp/bin" + for root in a b c d; do + printf '#!/usr/bin/env bash\nexit 0\n' > "$tmp/$root.sh" + done + fm_lint_stub_counting_shellcheck "$tmp/bin" "$tmp/active" "$tmp/peak" + printf '%s\n' "$tmp" +} + +test_host_slots_bound_concurrent_runs() { + local tmp peak lone + tmp=$(fm_lint_slot_fixture fm-lint-slots) + # One run keeps its two workers: the host-wide bound never throttles a lone run. + lone=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=3 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=0 \ + fm_lint_concurrent_runs 1 "$tmp" "$tmp/bin") + [ "$lone" -eq 2 ] || fail "a lone run peaked at $lone live ShellCheck processes, expected its two workers" + : > "$tmp/peak" + # Six runs would start twelve ShellCheck processes if each bounded only itself. + peak=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=3 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=0 \ + fm_lint_concurrent_runs 6 "$tmp" "$tmp/bin") + [ "$peak" -le 3 ] || fail "six concurrent runs reached $peak live ShellCheck processes, expected at most 3 host-wide" + [ "$peak" -ge 3 ] || fail "six concurrent runs peaked at $peak, so the slots were not used in parallel" + pass "concurrent fm-lint.sh runs share a host-wide ShellCheck slot bound" +} + +test_host_load_shrinks_slots_to_the_floor() { + local tmp peak + tmp=$(fm_lint_slot_fixture fm-lint-slots-load) + # Load far past two times the cores leaves only the two-slot floor, so the + # runs queue instead of failing or timing out. + peak=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=6 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=100000 \ + fm_lint_concurrent_runs 4 "$tmp" "$tmp/bin") + [ "$peak" -le 2 ] || fail "under heavy load four runs reached $peak live ShellCheck processes, expected at most 2" + : > "$tmp/peak" + peak=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=6 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=0 \ + fm_lint_concurrent_runs 4 "$tmp" "$tmp/bin") + [ "$peak" -gt 2 ] || fail "with an idle host four runs peaked at $peak, so the load never widened the slots" + pass "host load shrinks the shared ShellCheck slots to a two-slot floor and idle hosts use all of them" +} + +test_slot_pool_can_be_disabled_or_misconfigured() { + local tmp rc out + tmp=$(fm_lint_slot_fixture fm-lint-slots-off) + rc=0 + out=$(PATH="$tmp/bin:$PATH" FM_LINT_SLOT_DIR=off "$LINT" "$tmp/a.sh" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "a run with the slot pool off failed"$'\n'"$out" + rc=0 + out=$(PATH="$tmp/bin:$PATH" FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=zero "$LINT" "$tmp/a.sh" 2>&1) || rc=$? + [ "$rc" -eq 2 ] || fail "a non-numeric FM_LINT_HOST_SLOTS exited $rc, expected 2" + assert_contains "$out" "FM_LINT_HOST_SLOTS" "the refusal did not name the setting" + [ ! -d "$tmp/slots" ] || fail "a refused run still created the slot directory" + rc=0 + out=$(PATH="$tmp/bin:$PATH" FM_LINT_SLOT_DIR="$tmp/a.sh/blocked" "$LINT" "$tmp/a.sh" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "an unusable slot directory must not fail lint"$'\n'"$out" + assert_contains "$out" "running without the host-wide ShellCheck bound" "an unusable slot directory was not reported" + pass "the slot pool can be disabled, rejects bad settings, and never fails lint when unusable" +} + test_worker_trees_stop_on_signal() { local tmp fakebin fixture jobs telemetry lint_tmp pid_file out_file telemetry_file local parent_pid shellcheck_pid i parent_rc survivor @@ -2785,6 +2887,9 @@ test_rejects_direct_beads_cli_in_explicit_core_path test_ignores_ambient_shellcheck_opts test_clean_fixture_passes test_jobs_are_deterministic_and_complete +test_host_slots_bound_concurrent_runs +test_host_load_shrinks_slots_to_the_floor +test_slot_pool_can_be_disabled_or_misconfigured test_worker_trees_stop_on_signal test_root_deadline_names_the_root_and_reaps_the_tree test_root_memory_limit_reports_a_named_death From 366f0ef980efdda493154db8adc96fea777c6f92 Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Thu, 8 Oct 2026 22:53:33 -0500 Subject: [PATCH 2/7] no-mistakes(review): Correct host slot load threshold and add boundary regressions --- bin/fm-lint-cache.pl | 6 ++-- bin/fm-lint.sh | 8 +++--- tests/fm-lint.test.sh | 65 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 7 deletions(-) diff --git a/bin/fm-lint-cache.pl b/bin/fm-lint-cache.pl index 4228eb3e6dc..005579e1893 100755 --- a/bin/fm-lint-cache.pl +++ b/bin/fm-lint-cache.pl @@ -18,8 +18,8 @@ # Usage: perl fm-lint-cache.pl gate -- # Slots are flock files, so the kernel frees one when its holder dies. The slot # count is FM_LINT_HOST_SLOTS, else half the cores, but never below two (one -# run's default workers); while 1-minute load is within two times the cores, -# all slots may be taken, and each point of load past that takes one away, down +# run's default workers). The full cap is available until 1-minute load exceeds +# two times the cores; only the excess load is subtracted from the cap, down # to the two-slot floor. A slot directory that cannot be used lets the command # run ungated. The wait is written in milliseconds to so the caller # can keep queue time out of its own timings. @@ -57,7 +57,7 @@ $waited_from = Time::HiRes::time(); while (!$slot) { exit 128 + $signal_number{$caught} if $caught; - my $allowed = int(2 * $ncpu - $slot_load->() + 0.5); + my $allowed = int($cap + 2 * $ncpu - $slot_load->() + 0.5); $allowed = $floor if $allowed < $floor; $allowed = $cap if $allowed > $cap; for my $index (0 .. $allowed - 1) { diff --git a/bin/fm-lint.sh b/bin/fm-lint.sh index c4f72d30751..141e6212a3a 100755 --- a/bin/fm-lint.sh +++ b/bin/fm-lint.sh @@ -65,10 +65,10 @@ # (flock files under FM_LINT_SLOT_DIR, default # ${XDG_CACHE_HOME:-$HOME/.cache}/firstmate/lint-slots; "off" disables) and # many concurrent runs queue instead of each adding workers. FM_LINT_HOST_SLOTS -# sets the slot count, default half the cores and at least two. While 1-minute -# load is within two times the cores every slot is usable; each point of load -# above that removes one, down to a floor of two (one run's own workers), so a -# lone run is never throttled and queued roots always make progress. Queue time +# sets the slot count, default half the cores and at least two. The full cap is +# available until 1-minute load exceeds two times the cores; only the excess +# load is subtracted from the cap, down to a floor of two (one run's own workers), +# so a lone run is never throttled and queued roots always make progress. Queue time # is excluded from the root deadline and the recorded root duration. The pool # is shared by every worktree and home of the same user. A slot directory that # cannot be used is reported and lint runs ungated; it never fails lint. diff --git a/tests/fm-lint.test.sh b/tests/fm-lint.test.sh index 3745546429e..6571ae2337e 100755 --- a/tests/fm-lint.test.sh +++ b/tests/fm-lint.test.sh @@ -1172,6 +1172,70 @@ test_host_load_shrinks_slots_to_the_floor() { pass "host load shrinks the shared ShellCheck slots to a two-slot floor and idle hosts use all of them" } +test_host_load_preserves_the_cap_until_the_threshold() { + local tmp + tmp=$(fm_test_tmproot fm-lint-slots-threshold) + perl - "$ROOT/bin/fm-lint-cache.pl" "$tmp" <<'PL' || fail "host load boundary regression failed" +use strict; +use warnings; +use File::Path qw(make_path); +use Time::HiRes qw(time sleep); +my ($gate, $tmp) = @ARGV; +my $command = q{ + use Time::HiRes qw(time sleep); + my ($started, $release) = @ARGV; + open(my $fh, '>', $started) or die "$!"; + close $fh; + my $deadline = time() + 15; + until (-e $release) { exit 1 if time() > $deadline; sleep 0.01; } +}; +for my $cap ('', 6) { + local $ENV{FM_LINT_HOST_SLOTS} = $cap; + for my $load (30, 36, 37) { + local $ENV{FM_TEST_SEAM} = 1; + local $ENV{FM_LINT_SLOT_LOAD} = $load; + my $full = $cap || 9; + my $expected = $load == 37 ? $full - 1 : $full; + my $dir = "$tmp/$full.$load"; + make_path($dir); + my @pids; + for my $index (0 .. $full) { + my $pid = fork(); + die "fork: $!" unless defined $pid; + if (!$pid) { + exec $^X, $gate, 'gate', "$dir/slots", 18, "$dir/wait.$index", + '--', $^X, '-e', $command, "$dir/started.$index", "$dir/release"; + die "exec: $!"; + } + push @pids, $pid; + } + my $error; + eval { + my $deadline = time() + 10; + while (1) { + my @started = glob "$dir/started.*"; + last if @started >= $expected; + die "admission deadline exceeded\n" if time() > $deadline; + sleep 0.01; + } + sleep 0.25; + my @started = glob "$dir/started.*"; + die "admitted @started; expected $expected commands\n" unless @started == $expected; + }; + $error = $@; + open(my $release, '>', "$dir/release") or die "release: $!"; + close $release; + for my $pid (@pids) { + waitpid($pid, 0); + $error ||= "gated command exited with status $?\n" if $?; + } + die "cap $full, load $load: $error" if $error; + } +} +PL + pass "default and explicit host caps remain full through twice the cores and shrink only by excess load" +} + test_slot_pool_can_be_disabled_or_misconfigured() { local tmp rc out tmp=$(fm_lint_slot_fixture fm-lint-slots-off) @@ -2889,6 +2953,7 @@ test_clean_fixture_passes test_jobs_are_deterministic_and_complete test_host_slots_bound_concurrent_runs test_host_load_shrinks_slots_to_the_floor +test_host_load_preserves_the_cap_until_the_threshold test_slot_pool_can_be_disabled_or_misconfigured test_worker_trees_stop_on_signal test_root_deadline_names_the_root_and_reaps_the_tree From 7053b74a2eaa4eceaa4f2c0276345c72aa8b7fae Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Thu, 8 Oct 2026 23:30:59 -0500 Subject: [PATCH 3/7] no-mistakes(review): Fix lint slot failures, lock lifetime, and queue timing --- bin/fm-lint-cache.pl | 27 ++- bin/fm-lint.sh | 7 +- docs/fm-test-portable-shards.md | 4 + tests/fm-lint.test.sh | 285 ++++++++++++++++++++++++++++++++ 4 files changed, 311 insertions(+), 12 deletions(-) diff --git a/bin/fm-lint-cache.pl b/bin/fm-lint-cache.pl index 005579e1893..de3e5b35bd6 100755 --- a/bin/fm-lint-cache.pl +++ b/bin/fm-lint-cache.pl @@ -9,7 +9,7 @@ use strict; use warnings; use Digest::SHA qw(sha256_hex); -use Fcntl qw(:flock); +use Fcntl qw(:flock F_SETFD); use File::Path qw(make_path); use File::Basename qw(dirname basename); @@ -53,17 +53,21 @@ return 0; }; my ($slot, $waited_from); - if (eval { make_path($slot_dir, {mode => 0700}); -d $slot_dir && -w _ }) { + my $available = eval { make_path($slot_dir, {mode => 0700}); -d $slot_dir && -w _ }; + if ($available) { $waited_from = Time::HiRes::time(); - while (!$slot) { + ACQUIRE: while (!$slot) { exit 128 + $signal_number{$caught} if $caught; my $allowed = int($cap + 2 * $ncpu - $slot_load->() + 0.5); $allowed = $floor if $allowed < $floor; $allowed = $cap if $allowed > $cap; for my $index (0 .. $allowed - 1) { - open(my $fh, '>>', "$slot_dir/slot.$index") or next; + open(my $fh, '>>', "$slot_dir/slot.$index") + or do { $available = 0; last ACQUIRE; }; if (flock($fh, LOCK_EX | LOCK_NB)) { $slot = $fh; last; } + my $busy = $!{EWOULDBLOCK} || $!{EAGAIN} || $!{EINTR}; close $fh; + unless ($busy) { $available = 0; last ACQUIRE; } } next if $slot || $caught; # Every allowed slot is busy: block on one so a freed slot wakes this @@ -73,16 +77,27 @@ local $SIG{ALRM} = sub { die "gate-recheck\n" }; alarm 2; my $got = flock($fh, LOCK_EX); + my $interrupted = $!{EINTR}; alarm 0; + die "gate-lock: $!\n" unless $got || $interrupted; $got; }; alarm 0; + my $error = $@; if ($locked) { $slot = $fh; } else { close $fh; } + if ($error && $error ne "gate-recheck\n") { $available = 0; last ACQUIRE; } } else { - Time::HiRes::sleep(1); + $available = 0; + last ACQUIRE; } } - } else { + } + if ($slot && !defined fcntl($slot, F_SETFD, 0)) { + close $slot; + undef $slot; + $available = 0; + } + unless ($available) { warn "fm-lint: host slot directory unavailable; running without the host-wide ShellCheck bound\n"; } if (open(my $fh, '>', $wait_file)) { diff --git a/bin/fm-lint.sh b/bin/fm-lint.sh index 141e6212a3a..73e2b921b05 100755 --- a/bin/fm-lint.sh +++ b/bin/fm-lint.sh @@ -168,12 +168,7 @@ fm_lint_worker_stop() { } fm_lint_now_ms() { - if [ -n "${EPOCHREALTIME:-}" ]; then - local seconds=${EPOCHREALTIME%.*} micros=${EPOCHREALTIME#*.} - printf '%s\n' "$((seconds * 1000 + 10#${micros:0:3}))" - else - printf '%s\n' "$(($(date +%s) * 1000))" - fi + "${FM_LINT_PERL_BIN:-perl}" -MTime::HiRes=time -e 'printf "%d\n", time() * 1000' } # Names are listed only for signal numbers that agree on Linux and macOS; any diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 2055acc9f3c..11af0db90e5 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -136,6 +136,10 @@ CI requires its per-root bounds, so an unenforceable deadline or address-space l Its `--list-files` interface exposes partition membership; `tests/fm-lint.test.sh` verifies complete/disjoint executed roots, changed-source selection, shared-cache invalidation, initial analysis flags, fallback reporting, and seeded finding parity. The workflow uploads each partition's quiet telemetry plus its per-root lifecycle sidecar to distinguish analysis cost, memory use, and host contention. No fast mode, path skips, or paid runner provisioning is part of this layout. +The host-wide ShellCheck slot pool queues only for occupied locks; slot-file or locking failures warn and run ungated. +Protected commands inherit their slot descriptors, so gate death cannot release capacity while their descendants still run. +Root timestamps and queue accounting use Perl's high-resolution clock, including on stock macOS Bash, and exclude queue time from analysis durations and retry budgets. +Run `bash tests/fm-lint.test.sh --host-slots` for the focused slot, load-boundary, failure, lifetime, and timing regressions. The [lint script header](../bin/fm-lint.sh) owns local dependency discovery, conservative unresolved-import selection, and successful-result cache controls; these do not replace full joint source analysis. Regression fixtures exercise cross-file missing-argument findings through direct and private source routines, deleted sources, concurrent reuse, changed binaries, and the separation between fast and full analysis. diff --git a/tests/fm-lint.test.sh b/tests/fm-lint.test.sh index 6571ae2337e..902240e8446 100755 --- a/tests/fm-lint.test.sh +++ b/tests/fm-lint.test.sh @@ -1254,6 +1254,277 @@ test_slot_pool_can_be_disabled_or_misconfigured() { pass "the slot pool can be disabled, rejects bad settings, and never fails lint when unusable" } +test_slot_file_failures_run_ungated() { + local tmp + tmp=$(fm_test_tmproot fm-lint-slot-errors) + perl - "$ROOT/bin/fm-lint-cache.pl" "$tmp" <<'PL' || fail "slot I/O fallback regression failed" +use strict; +use warnings; +use File::Path qw(make_path); +use POSIX qw(WNOHANG); +use Time::HiRes qw(time sleep); +my ($gate, $tmp) = @ARGV; +open(my $module, '>', "$tmp/FailLock.pm") or die "$!"; +print {$module} <<'MODULE'; +package FailLock; +use Errno qw(EIO EWOULDBLOCK); +use Fcntl qw(LOCK_NB); +BEGIN { + *CORE::GLOBAL::flock = sub { + $! = $ENV{FAIL_LOCK} eq 'wait' && ($_[1] & LOCK_NB) ? EWOULDBLOCK : EIO; + return 0; + }; +} +1; +MODULE +close $module; +for my $failure (qw(open scan wait)) { + my $dir = "$tmp/$failure"; + make_path("$dir/slots"); + make_path("$dir/slots/slot.0") if $failure eq 'open'; + local $ENV{FM_LINT_HOST_SLOTS} = 1; + local $ENV{FM_TEST_SEAM} = 1; + local $ENV{FM_LINT_SLOT_LOAD} = 0; + local $ENV{FAIL_LOCK} = $failure; + my @inject = $failure eq 'open' ? () : ("-I$tmp", '-MFailLock'); + my $pid = fork(); + die "fork: $!" unless defined $pid; + if (!$pid) { + open STDERR, '>', "$dir/err" or die "$!"; + exec $^X, @inject, $gate, 'gate', "$dir/slots", 18, "$dir/wait", + '--', $^X, '-e', 'print "analysis ran\n"; exit 23'; + die "exec: $!"; + } + my $deadline = time() + 4; + while (waitpid($pid, WNOHANG) == 0) { + if (time() > $deadline) { + kill 'KILL', $pid; + waitpid($pid, 0); + die "$failure failure queued instead of running ungated\n"; + } + sleep 0.01; + } + die "$failure fallback lost the command exit status: $?\n" unless $? == (23 << 8); + open(my $err, '<', "$dir/err") or die "$!"; + local $/; + my $warning = <$err>; + die "$failure fallback omitted the warning\n" + unless $warning =~ /running without the host-wide ShellCheck bound/; +} +PL + pass "slot open and non-contention scan/wait lock failures warn and run ungated" +} + +test_slot_survives_gate_death() { + local tmp bounded=none + if fm_lint_bounds_supported; then bounded=perl; fi + tmp=$(fm_test_tmproot fm-lint-slot-inheritance) + perl - "$LINT" "$ROOT/bin/fm-lint-cache.pl" "$tmp" "$bounded" <<'PL' || fail "slot lifetime regression failed" +use strict; +use warnings; +use File::Path qw(make_path); +use POSIX qw(WNOHANG); +use Time::HiRes qw(time sleep); +my ($lint, $gate_script, $tmp, $bound_mechanism) = @ARGV; +open(my $stub, '>', "$tmp/shellcheck") or die "$!"; +print {$stub} "#!/usr/bin/env perl\n", <<'STUB'; +use Time::HiRes qw(time sleep); +$SIG{TERM} = $SIG{HUP} = $SIG{INT} = 'IGNORE'; +my $child = fork(); +die "fork: $!" unless defined $child; +if (!$child) { + my $deadline = time() + 15; + until (-e "$ENV{CASE_DIR}/release") { last if time() > $deadline; sleep 0.01; } + exit 0; +} +open(my $fh, '>', "$ENV{CASE_DIR}/started") or die "$!"; +print {$fh} "$$ $child\n"; +close $fh; +waitpid($child, 0); +STUB +close $stub; +chmod 0755, "$tmp/shellcheck"; +for my $bounded ('none', $bound_mechanism eq 'perl' ? ('perl') : ()) { + my $dir = "$tmp/$bounded"; + make_path("$dir/out"); + open(my $root, '>', "$dir/root.sh") or die "$!"; + print {$root} "#!/bin/bash\nexit 0\n"; + close $root; + open(my $manifest, '>', "$dir/manifest") or die "$!"; + print {$manifest} "0\t$dir/root.sh\n"; + close $manifest; + local %ENV = (%ENV, FM_LINT_INTERNAL => 1, FM_LINT_INTERNAL_CACHE => 'off', + FM_LINT_INTERNAL_SLOT_DIR => "$dir/slots", FM_LINT_INTERNAL_NCPU => 18, + FM_LINT_HOST_SLOTS => 1, FM_TEST_SEAM => 1, FM_LINT_SLOT_LOAD => 0, + FM_LINT_INTERNAL_BOUNDED => $bounded, FM_LINT_INTERNAL_MEMORY_KIB => 2097152, + FM_LINT_INTERNAL_ROOT_SECS => 10, FM_LINT_INTERNAL_GRACE => 1, + FM_LINT_SHELLCHECK => "$tmp/shellcheck", CASE_DIR => $dir); + my $worker = fork(); + die "fork: $!" unless defined $worker; + if (!$worker) { + open STDOUT, '>', "$dir/output" or die "$!"; + open STDERR, '>&', \*STDOUT or die "$!"; + exec '/bin/bash', $lint, '--internal-worker', "$dir/manifest", "$dir/out", 0; + die "exec: $!"; + } + my ($command, $descendant, $gate, $queued, $error); + eval { + my $deadline = time() + 5; + until (-s "$dir/started") { die "command never started\n" if time() > $deadline; sleep 0.01; } + open(my $started, '<', "$dir/started") or die "$!"; + ($command, $descendant) = split / /, <$started>; + open(my $ps, '-|', 'ps', '-axo', 'pid=,ppid=,args=') or die "$!"; + my (%parent, %args); + while (<$ps>) { + next unless /^\s*(\d+)\s+(\d+)\s+(.*)$/; + $parent{$1} = $2; $args{$1} = $3; + } + close $ps; + my $ancestor = $command; + while ($ancestor && $ancestor != $worker) { + if (($args{$ancestor} // '') =~ /\Q$dir\/slots\E/) { $gate = $ancestor; last; } + $ancestor = $parent{$ancestor}; + } + die "gate not found in command ancestry\n" unless $gate; + $queued = fork(); + die "fork: $!" unless defined $queued; + if (!$queued) { + exec $^X, $gate_script, 'gate', "$dir/slots", 18, "$dir/queued.wait", + '--', $^X, '-e', 'open(my $fh, ">", $ARGV[0]) or die "$!"', "$dir/admitted"; + die "exec: $!"; + } + kill 'KILL', $gate; + sleep 0.2; + die "queued command started while protected tree survived\n" if -e "$dir/admitted"; + kill 'KILL', $command; + sleep 0.2; + die "queued command started while protected descendant survived\n" if -e "$dir/admitted"; + die "protected descendant exited before release\n" unless kill 0, $descendant; + }; + $error = $@; + open(my $release, '>', "$dir/release") or die "$!"; + close $release; + my $deadline = time() + 5; + if ($queued) { + while (waitpid($queued, WNOHANG) == 0) { + if (time() > $deadline) { kill 'KILL', $queued; waitpid($queued, 0); $error ||= "slot never released\n"; last; } + sleep 0.01; + } + $error ||= "queued command failed: $?\n" if $?; + $error ||= "queued command never admitted\n" unless -e "$dir/admitted"; + } + kill 'KILL', $command if $command; + kill 'KILL', $descendant if $descendant; + kill 'TERM', $worker; + waitpid($worker, 0); + if ($error) { + open(my $output, '<', "$dir/output") or die "$!"; + open(my $analysis, '<', "$dir/out/shard.0.out") or die "$!"; + local $/; + die "$bounded: $error", <$output>, <$analysis>; + } +} +PL + pass "commands retain slots through gate and command death until descendants exit (bounded where supported)" +} + +test_queued_roots_use_high_resolution_timings() { + local tmp bounded=none + if fm_lint_bounds_supported; then bounded=perl; fi + tmp=$(fm_test_tmproot fm-lint-slot-clock) + perl - "$LINT" "$tmp" "$bounded" <<'PL' || fail "queued root clock regression failed" +use strict; +use warnings; +use Fcntl qw(:flock); +use File::Path qw(make_path); +use POSIX qw(WNOHANG); +use Time::HiRes qw(time sleep); +my ($lint, $tmp, $bounded) = @ARGV; +open(my $env, '>', "$tmp/bash-env") or die "$!"; +print {$env} "unset EPOCHREALTIME\n"; +close $env; +open(my $stub, '>', "$tmp/shellcheck") or die "$!"; +print {$stub} "#!/usr/bin/env perl\n", <<'STUB'; +use Time::HiRes qw(sleep); +sleep 0.08; +if ($ENV{RETRY} && grep { $_ eq '--external-sources' } @ARGV) { + print STDERR "shellcheck: Heap exhausted;\n"; + exit 251; +} +exit 0; +STUB +close $stub; +chmod 0755, "$tmp/shellcheck"; +for my $retry (0, 1) { + my $dir = "$tmp/$retry"; + make_path("$dir/out", "$dir/slots"); + open(my $slot, '>>', "$dir/slots/slot.0") or die "$!"; + flock($slot, LOCK_EX) or die "$!"; + open(my $manifest, '>', "$dir/manifest") or die "$!"; + print {$manifest} "0\t$dir/root.sh\n"; + close $manifest; + open(my $root, '>', "$dir/root.sh") or die "$!"; + print {$root} "#!/bin/bash\nexit 0\n"; + close $root; + local %ENV = (%ENV, BASH_ENV => "$tmp/bash-env", RETRY => $retry, + FM_LINT_INTERNAL => 1, FM_LINT_INTERNAL_CACHE => 'off', + FM_LINT_INTERNAL_SLOT_DIR => "$dir/slots", FM_LINT_INTERNAL_NCPU => 18, + FM_LINT_HOST_SLOTS => 1, FM_TEST_SEAM => 1, FM_LINT_SLOT_LOAD => 0, + FM_LINT_INTERNAL_BOUNDED => $bounded, FM_LINT_INTERNAL_MEMORY_KIB => 2097152, + FM_LINT_INTERNAL_ROOT_SECS => 3, FM_LINT_INTERNAL_GRACE => 1, + FM_LINT_INTERNAL_ROOTS_LOG => "$dir/roots.tsv", FM_LINT_SHELLCHECK => "$tmp/shellcheck"); + sleep 0.005 until time() - int(time()) > 0.3 && time() - int(time()) < 0.35; + my $worker = fork(); + die "fork: $!" unless defined $worker; + if (!$worker) { + close $slot; + open STDOUT, '>', "$dir/output" or die "$!"; + open STDERR, '>&', \*STDOUT or die "$!"; + exec '/bin/bash', $lint, '--internal-worker', "$dir/manifest", "$dir/out", 0; + die "exec: $!"; + } + my $error; + eval { + my $deadline = time() + 5; + until (-s "$dir/roots.tsv") { die "root never began\n" if time() > $deadline; sleep 0.01; } + my $began = time(); + sleep 1.5; + close $slot; + $deadline = time() + 5; + while (waitpid($worker, WNOHANG) == 0) { + die "queued root timed out\n" if time() > $deadline; + sleep 0.01; + } + die "queued root failed: $?\n" if $?; + my $finished = time(); + my $queued_ms = 0; + for my $file (glob "$dir/out/*.rss.wait") { + open(my $wait, '<', $file) or die "$!"; + $queued_ms += <$wait>; + } + my $elapsed = ($finished - $began) * 1000 - $queued_ms; + open(my $log, '<', "$dir/roots.tsv") or die "$!"; + my @end; + while (<$log>) { @end = split /\t/ if /^end\t/; } + die "missing root result\n" unless @end; + my $expected = $retry ? 'memory-fallback' : 'ok'; + die "wrong root outcome: $end[9]\n" unless $end[9] eq $expected; + die "root timestamps lack millisecond precision\n" + unless abs($end[5] - $began * 1000) < 150 && abs($end[6] - $finished * 1000) < 150; + die "duration $end[7]ms disagrees with $elapsed ms of analysis\n" + unless $end[7] >= 70 && abs($end[7] - $elapsed) < 150; + die "root wall time omitted the queue\n" unless $end[6] - $end[5] >= 1500; + }; + $error = $@; + close $slot if defined fileno($slot); + kill 'KILL', $worker; + waitpid($worker, 0); + die "retry=$retry: $error" if $error; +} +PL + pass "queued roots and memory retries record precise analysis durations without EPOCHREALTIME" +} + test_worker_trees_stop_on_signal() { local tmp fakebin fixture jobs telemetry lint_tmp pid_file out_file telemetry_file local parent_pid shellcheck_pid i parent_rc survivor @@ -2914,6 +3185,17 @@ SH pass "unproved transitive runtime closures never reuse successful analysis" } +if [ "${1:-}" = --host-slots ]; then + test_host_slots_bound_concurrent_runs + test_host_load_shrinks_slots_to_the_floor + test_host_load_preserves_the_cap_until_the_threshold + test_slot_pool_can_be_disabled_or_misconfigured + test_slot_file_failures_run_ungated + test_slot_survives_gate_death + test_queued_roots_use_high_resolution_timings + exit 0 +fi + test_command_words_exclude_inert_source_text test_child_shell_imports_select_and_invalidate_callers test_stdin_heredoc_imports_select_and_invalidate_callers @@ -2955,6 +3237,9 @@ test_host_slots_bound_concurrent_runs test_host_load_shrinks_slots_to_the_floor test_host_load_preserves_the_cap_until_the_threshold test_slot_pool_can_be_disabled_or_misconfigured +test_slot_file_failures_run_ungated +test_slot_survives_gate_death +test_queued_roots_use_high_resolution_timings test_worker_trees_stop_on_signal test_root_deadline_names_the_root_and_reaps_the_tree test_root_memory_limit_reports_a_named_death From 2b2f7e9541047bda1f79038b449ae1f142432e20 Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Thu, 8 Oct 2026 23:58:42 -0500 Subject: [PATCH 4/7] no-mistakes(review): Fix load-aware slot admission and lifetime regressions --- bin/fm-lint-cache.pl | 29 ++++++---- docs/fm-test-portable-shards.md | 3 +- tests/fm-lint.test.sh | 93 +++++++++++++++++++++++++-------- 3 files changed, 90 insertions(+), 35 deletions(-) diff --git a/bin/fm-lint-cache.pl b/bin/fm-lint-cache.pl index de3e5b35bd6..e2749268e07 100755 --- a/bin/fm-lint-cache.pl +++ b/bin/fm-lint-cache.pl @@ -58,33 +58,40 @@ $waited_from = Time::HiRes::time(); ACQUIRE: while (!$slot) { exit 128 + $signal_number{$caught} if $caught; - my $allowed = int($cap + 2 * $ncpu - $slot_load->() + 0.5); - $allowed = $floor if $allowed < $floor; - $allowed = $cap if $allowed > $cap; - for my $index (0 .. $allowed - 1) { + my (@free, $occupied); + $occupied = 0; + for my $index (0 .. $cap - 1) { open(my $fh, '>>', "$slot_dir/slot.$index") or do { $available = 0; last ACQUIRE; }; - if (flock($fh, LOCK_EX | LOCK_NB)) { $slot = $fh; last; } + if (flock($fh, LOCK_EX | LOCK_NB)) { push @free, $fh; next; } my $busy = $!{EWOULDBLOCK} || $!{EAGAIN} || $!{EINTR}; close $fh; unless ($busy) { $available = 0; last ACQUIRE; } + $occupied++; } + my $allowed = int($cap + 2 * $ncpu - $slot_load->() + 0.5); + $allowed = $floor if $allowed < $floor; + $allowed = $cap if $allowed > $cap; + $slot = shift @free if @free && $occupied < $allowed; + my $has_free = @free; + close $_ for @free; next if $slot || $caught; - # Every allowed slot is busy: block on one so a freed slot wakes this - # waiter at once, and re-read the load every two seconds. - if (open(my $fh, '>>', "$slot_dir/slot.@{[ int(rand($allowed)) ]}")) { - my $locked = eval { + if ($has_free) { + Time::HiRes::sleep(0.1); + next; + } + if (open(my $fh, '>>', "$slot_dir/slot.@{[ int(rand($cap)) ]}")) { + eval { local $SIG{ALRM} = sub { die "gate-recheck\n" }; alarm 2; my $got = flock($fh, LOCK_EX); my $interrupted = $!{EINTR}; alarm 0; die "gate-lock: $!\n" unless $got || $interrupted; - $got; }; alarm 0; my $error = $@; - if ($locked) { $slot = $fh; } else { close $fh; } + close $fh; if ($error && $error ne "gate-recheck\n") { $available = 0; last ACQUIRE; } } else { $available = 0; diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 11af0db90e5..3572616f2b4 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -136,10 +136,9 @@ CI requires its per-root bounds, so an unenforceable deadline or address-space l Its `--list-files` interface exposes partition membership; `tests/fm-lint.test.sh` verifies complete/disjoint executed roots, changed-source selection, shared-cache invalidation, initial analysis flags, fallback reporting, and seeded finding parity. The workflow uploads each partition's quiet telemetry plus its per-root lifecycle sidecar to distinguish analysis cost, memory use, and host contention. No fast mode, path skips, or paid runner provisioning is part of this layout. -The host-wide ShellCheck slot pool queues only for occupied locks; slot-file or locking failures warn and run ungated. +The host-wide ShellCheck slot pool queues while total occupancy meets the current load allowance; slot-file or locking failures warn and run ungated. Protected commands inherit their slot descriptors, so gate death cannot release capacity while their descendants still run. Root timestamps and queue accounting use Perl's high-resolution clock, including on stock macOS Bash, and exclude queue time from analysis durations and retry budgets. -Run `bash tests/fm-lint.test.sh --host-slots` for the focused slot, load-boundary, failure, lifetime, and timing regressions. The [lint script header](../bin/fm-lint.sh) owns local dependency discovery, conservative unresolved-import selection, and successful-result cache controls; these do not replace full joint source analysis. Regression fixtures exercise cross-file missing-argument findings through direct and private source routines, deleted sources, concurrent reuse, changed binaries, and the separation between fast and full analysis. diff --git a/tests/fm-lint.test.sh b/tests/fm-lint.test.sh index 902240e8446..3af5b17cc4e 100755 --- a/tests/fm-lint.test.sh +++ b/tests/fm-lint.test.sh @@ -1142,19 +1142,22 @@ fm_lint_slot_fixture() { # -> prints the tmp dir } test_host_slots_bound_concurrent_runs() { - local tmp peak lone + local tmp peak lone baseline tmp=$(fm_lint_slot_fixture fm-lint-slots) # One run keeps its two workers: the host-wide bound never throttles a lone run. lone=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=3 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=0 \ fm_lint_concurrent_runs 1 "$tmp" "$tmp/bin") [ "$lone" -eq 2 ] || fail "a lone run peaked at $lone live ShellCheck processes, expected its two workers" : > "$tmp/peak" + baseline=$(FM_LINT_SLOT_DIR=off fm_lint_concurrent_runs 6 "$tmp" "$tmp/bin") + [ "$baseline" -eq 12 ] || fail "six ungated runs peaked at $baseline live ShellCheck processes, expected 12" + : > "$tmp/peak" # Six runs would start twelve ShellCheck processes if each bounded only itself. peak=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=3 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=0 \ fm_lint_concurrent_runs 6 "$tmp" "$tmp/bin") [ "$peak" -le 3 ] || fail "six concurrent runs reached $peak live ShellCheck processes, expected at most 3 host-wide" [ "$peak" -ge 3 ] || fail "six concurrent runs peaked at $peak, so the slots were not used in parallel" - pass "concurrent fm-lint.sh runs share a host-wide ShellCheck slot bound" + pass "six concurrent runs peak at $baseline ShellCheck processes ungated and $peak with a three-slot pool" } test_host_load_shrinks_slots_to_the_floor() { @@ -1179,6 +1182,7 @@ test_host_load_preserves_the_cap_until_the_threshold() { use strict; use warnings; use File::Path qw(make_path); +use Fcntl qw(:flock); use Time::HiRes qw(time sleep); my ($gate, $tmp) = @ARGV; my $command = q{ @@ -1232,8 +1236,47 @@ for my $cap ('', 6) { die "cap $full, load $load: $error" if $error; } } +for my $path (qw(scan wake)) { + my $dir = "$tmp/occupancy.$path"; + make_path("$dir/slots"); + my @holders; + for my $index ($path eq 'wake' ? (0 .. 8) : (1 .. 8)) { + open($holders[$index], '>>', "$dir/slots/slot.$index") or die "$!"; + flock($holders[$index], LOCK_EX) or die "$!"; + } + local $ENV{FM_LINT_HOST_SLOTS} = 9; + local $ENV{FM_TEST_SEAM} = 1; + local $ENV{FM_LINT_SLOT_LOAD} = 43; + my $pid = fork(); + die "fork: $!" unless defined $pid; + if (!$pid) { + close $_ for grep { defined } @holders; + exec $^X, $gate, 'gate', "$dir/slots", 18, "$dir/wait", + '--', $^X, '-e', 'open(my $fh, ">", $ARGV[0]) or die "$!"', "$dir/started"; + die "exec: $!"; + } + my $error; + eval { + sleep 0.25; + close $holders[0] if $path eq 'wake'; + sleep 2; + die "admitted with eight occupied slots and an allowance of two\n" if -e "$dir/started"; + close $holders[$_] for 1 .. 7; + my $deadline = time() + 5; + until (-e "$dir/started") { + die "did not admit below the occupancy allowance\n" if time() > $deadline; + sleep 0.01; + } + }; + $error = $@; + close $holders[8]; + if ($error) { kill 'KILL', $pid; } + waitpid($pid, 0); + $error ||= "gated command failed: $?\n" if $?; + die "$path: $error" if $error; +} PL - pass "default and explicit host caps remain full through twice the cores and shrink only by excess load" + pass "host caps follow the load threshold and both acquisition paths respect total occupancy" } test_slot_pool_can_be_disabled_or_misconfigured() { @@ -1390,16 +1433,33 @@ for my $bounded ('none', $bound_mechanism eq 'perl' ? ('perl') : ()) { die "fork: $!" unless defined $queued; if (!$queued) { exec $^X, $gate_script, 'gate', "$dir/slots", 18, "$dir/queued.wait", - '--', $^X, '-e', 'open(my $fh, ">", $ARGV[0]) or die "$!"', "$dir/admitted"; + '--', $^X, '-e', q{ + my ($admitted, $bounded, @protected) = @ARGV; + if ($bounded ne 'none') { + open(my $ps, '-|', 'ps', '-o', 'stat=', '-p', join(',', @protected)) or die "$!"; + my @live = grep { !/^\s*Z/ } <$ps>; + close $ps; + die "admission overlapped a live protected tree\n" if @live; + } + open(my $fh, '>', $admitted) or die "$!"; + }, "$dir/admitted", $bounded, $command, $descendant; die "exec: $!"; } kill 'KILL', $gate; - sleep 0.2; - die "queued command started while protected tree survived\n" if -e "$dir/admitted"; - kill 'KILL', $command; - sleep 0.2; - die "queued command started while protected descendant survived\n" if -e "$dir/admitted"; - die "protected descendant exited before release\n" unless kill 0, $descendant; + if ($bounded eq 'none') { + sleep 0.2; + die "queued command started while protected tree survived\n" if -e "$dir/admitted"; + kill 'KILL', $command; + sleep 0.2; + die "queued command started while protected descendant survived\n" if -e "$dir/admitted"; + die "protected descendant exited before release\n" unless kill 0, $descendant; + } else { + my $deadline = time() + 5; + until (-e "$dir/admitted") { + die "watchdog cleanup did not release the slot\n" if time() > $deadline; + sleep 0.01; + } + } }; $error = $@; open(my $release, '>', "$dir/release") or die "$!"; @@ -1425,7 +1485,7 @@ for my $bounded ('none', $bound_mechanism eq 'perl' ? ('perl') : ()) { } } PL - pass "commands retain slots through gate and command death until descendants exit (bounded where supported)" + pass "inherited slots prevent overlap after gate death and allow admission after watchdog cleanup" } test_queued_roots_use_high_resolution_timings() { @@ -3185,17 +3245,6 @@ SH pass "unproved transitive runtime closures never reuse successful analysis" } -if [ "${1:-}" = --host-slots ]; then - test_host_slots_bound_concurrent_runs - test_host_load_shrinks_slots_to_the_floor - test_host_load_preserves_the_cap_until_the_threshold - test_slot_pool_can_be_disabled_or_misconfigured - test_slot_file_failures_run_ungated - test_slot_survives_gate_death - test_queued_roots_use_high_resolution_timings - exit 0 -fi - test_command_words_exclude_inert_source_text test_child_shell_imports_select_and_invalidate_callers test_stdin_heredoc_imports_select_and_invalidate_callers From ef99c69154225681b0259fdbae8812f31e66b4a1 Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Fri, 9 Oct 2026 00:22:06 -0500 Subject: [PATCH 5/7] Bound lint probes and coordinate concurrency proof Bound every waiting path to one fresh load probe per two seconds while preserving occupancy revalidation. Force two fixture workers and hold checks until the controller observes the required admission count. Replace scheduler-sensitive timestamp comparisons with observed lifecycle brackets. Counted architecture comparison (binary factors): Design | Host-wide bound | Preserves two workers | Resident daemon | Custom stale-lock recovery | Extra per-root gate process Per-run only | 0 | 1 | 0 | 0 | 0 FM_LINT_JOBS=1 | 0 | 0 | 0 | 0 | 0 Bash mkdir slots | 1 | 1 | 0 | 1 | 0 Central daemon, disconnect-based release | 1 | 1 | 1 | 0 | 0 Flock pool | 1 | 1 | 0 | 0 | 1 The flock pool supplies the required host-wide bound and preserves two workers without a resident service or custom stale-lock recovery, at the cost of one gate process per root. Paired reproduction: six simultaneous fm-lint runs, four identical roots per run, --jobs 2, inherited FM_LINT_JOBS=1, and controller-held fake ShellCheck processes. With FM_LINT_SLOT_DIR=off, the measured peak was 12 = 2 x 6 runs. With a three-slot pool and zero simulated load, the measured peak was 3 = the slot cap. Focused verification: all seven host-slot regressions passed, covering concurrency, load thresholds and occupancy, disable/misconfiguration behavior, slot I/O and locking failures, gate-death ownership, and queue-excluded initial/retry timing. A real load-reader smoke observed two probes over 4.5 seconds on both the free-capacity and blocking-wakeup paths, with intervals of 2.622 and 2.653 seconds respectively; admission resumed below the occupancy allowance. No full repository test or lint suite was run. The outer executor owns copying this counted comparison and paired reproduction into the PR description. No new tracked documentation was added. --- bin/fm-lint-cache.pl | 11 +++-- docs/fm-test-portable-shards.md | 2 +- tests/fm-lint.test.sh | 88 +++++++++++++++++++++------------ 3 files changed, 64 insertions(+), 37 deletions(-) diff --git a/bin/fm-lint-cache.pl b/bin/fm-lint-cache.pl index e2749268e07..585adf8dd8f 100755 --- a/bin/fm-lint-cache.pl +++ b/bin/fm-lint-cache.pl @@ -52,11 +52,14 @@ } return 0; }; - my ($slot, $waited_from); + my ($slot, $waited_from, $recheck_at); my $available = eval { make_path($slot_dir, {mode => 0700}); -d $slot_dir && -w _ }; if ($available) { $waited_from = Time::HiRes::time(); ACQUIRE: while (!$slot) { + exit 128 + $signal_number{$caught} if $caught; + my $delay = ($recheck_at // 0) - Time::HiRes::time(); + Time::HiRes::sleep($delay) if $delay > 0; exit 128 + $signal_number{$caught} if $caught; my (@free, $occupied); $occupied = 0; @@ -70,16 +73,14 @@ $occupied++; } my $allowed = int($cap + 2 * $ncpu - $slot_load->() + 0.5); + $recheck_at = Time::HiRes::time() + 2; $allowed = $floor if $allowed < $floor; $allowed = $cap if $allowed > $cap; $slot = shift @free if @free && $occupied < $allowed; my $has_free = @free; close $_ for @free; next if $slot || $caught; - if ($has_free) { - Time::HiRes::sleep(0.1); - next; - } + next if $has_free; if (open(my $fh, '>>', "$slot_dir/slot.@{[ int(rand($cap)) ]}")) { eval { local $SIG{ALRM} = sub { die "gate-recheck\n" }; diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 3572616f2b4..e19ae726b03 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -136,7 +136,7 @@ CI requires its per-root bounds, so an unenforceable deadline or address-space l Its `--list-files` interface exposes partition membership; `tests/fm-lint.test.sh` verifies complete/disjoint executed roots, changed-source selection, shared-cache invalidation, initial analysis flags, fallback reporting, and seeded finding parity. The workflow uploads each partition's quiet telemetry plus its per-root lifecycle sidecar to distinguish analysis cost, memory use, and host contention. No fast mode, path skips, or paid runner provisioning is part of this layout. -The host-wide ShellCheck slot pool queues while total occupancy meets the current load allowance; slot-file or locking failures warn and run ungated. +The host-wide ShellCheck slot pool queues while total occupancy meets the current load allowance; waiting gates recheck load no more than once every two seconds, including after lock wakeups. Slot-file or locking failures warn and run ungated. Protected commands inherit their slot descriptors, so gate death cannot release capacity while their descendants still run. Root timestamps and queue accounting use Perl's high-resolution clock, including on stock macOS Bash, and exclude queue time from analysis durations and retry budgets. diff --git a/tests/fm-lint.test.sh b/tests/fm-lint.test.sh index 3af5b17cc4e..6ec901b8e7e 100755 --- a/tests/fm-lint.test.sh +++ b/tests/fm-lint.test.sh @@ -1090,42 +1090,67 @@ SH pass "jobs=1 and jobs=2 preserve deterministic diagnostics, failures, cleanup bounds, and quiet telemetry" } -# fm_lint_stub_counting_shellcheck : a -# ShellCheck stand-in that holds each check for half a second and records the -# most checks alive at once across every fm-lint.sh run sharing the directory. fm_lint_stub_counting_shellcheck() { local fakebin=$1 activity=$2 peak=$3 mkdir -p "$activity" : > "$peak" - cat > "$fakebin/shellcheck" <> "$peak" -sleep 0.5 -rm -f "$activity/\$\$" -exit 0 -SH + cat > "$fakebin/shellcheck" <>', "$peak.lock") or die "\$!"; +flock(\$lock, LOCK_EX) or die "\$!"; +open(my \$active, '>', "$activity/\$\$") or die "\$!"; +close \$active; +my @active = glob "$activity/*"; +open(my \$peak, '>>', "$peak") or die "\$!"; +print {\$peak} scalar(@active), "\n"; +close \$peak; +close \$lock; +my \$deadline = time() + 60; +while (\$ENV{FM_LINT_COUNT_HOLD} && !-e "$peak.release") { + if (time() > \$deadline) { unlink "$activity/\$\$"; exit 1; } + sleep 0.01; +} +unlink "$activity/\$\$"; +exit 0; +PL chmod +x "$fakebin/shellcheck" } -# fm_lint_concurrent_runs : start simultaneous -# fm-lint.sh runs of four roots each, wait for all, and print the peak number -# of live ShellCheck processes. Extra environment is read from the caller. fm_lint_concurrent_runs() { - local runs=$1 tmp=$2 fakebin=$3 run pid rc=0 + local runs=$1 tmp=$2 fakebin=$3 expected=$4 run pid rc=0 ready_rc=0 local -a pids roots roots=("$tmp/a.sh" "$tmp/b.sh" "$tmp/c.sh" "$tmp/d.sh") + rm -f "$tmp/peak.release" for run in $(seq 1 "$runs"); do - PATH="$fakebin:$PATH" "$LINT" "${roots[@]}" > "$tmp/run.$run.out" 2>&1 & + FM_LINT_COUNT_HOLD=1 PATH="$fakebin:$PATH" "$LINT" --jobs 2 "${roots[@]}" > "$tmp/run.$run.out" 2>&1 & pids+=("$!") done + perl - "$tmp/peak" "$expected" <<'PL' || ready_rc=$? +use Time::HiRes qw(time sleep); +my ($peak, $expected) = @ARGV; +my $deadline = time() + 30; +while (1) { + open(my $fh, '<', $peak) or die "$!"; + my @counts = <$fh>; + close $fh; + last if grep { /\A[0-9]+\n\z/ && $_ >= $expected } @counts; + die "concurrent admission deadline exceeded\n" if time() > $deadline; + sleep 0.01; +} +PL + touch "$tmp/peak.release" for pid in "${pids[@]}"; do wait "$pid" || rc=$? done + [ "$ready_rc" -eq 0 ] || fail "concurrent runs did not reach $expected held ShellCheck processes"$'\n'"$(cat "$tmp"/run.*.out)" [ "$rc" -eq 0 ] || fail "a concurrent fm-lint.sh run failed (rc=$rc)"$'\n'"$(cat "$tmp"/run.*.out)" sort -n "$tmp/peak" | tail -1 } @@ -1146,15 +1171,15 @@ test_host_slots_bound_concurrent_runs() { tmp=$(fm_lint_slot_fixture fm-lint-slots) # One run keeps its two workers: the host-wide bound never throttles a lone run. lone=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=3 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=0 \ - fm_lint_concurrent_runs 1 "$tmp" "$tmp/bin") + fm_lint_concurrent_runs 1 "$tmp" "$tmp/bin" 2) [ "$lone" -eq 2 ] || fail "a lone run peaked at $lone live ShellCheck processes, expected its two workers" : > "$tmp/peak" - baseline=$(FM_LINT_SLOT_DIR=off fm_lint_concurrent_runs 6 "$tmp" "$tmp/bin") + baseline=$(FM_LINT_SLOT_DIR=off fm_lint_concurrent_runs 6 "$tmp" "$tmp/bin" 12) [ "$baseline" -eq 12 ] || fail "six ungated runs peaked at $baseline live ShellCheck processes, expected 12" : > "$tmp/peak" # Six runs would start twelve ShellCheck processes if each bounded only itself. peak=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=3 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=0 \ - fm_lint_concurrent_runs 6 "$tmp" "$tmp/bin") + fm_lint_concurrent_runs 6 "$tmp" "$tmp/bin" 3) [ "$peak" -le 3 ] || fail "six concurrent runs reached $peak live ShellCheck processes, expected at most 3 host-wide" [ "$peak" -ge 3 ] || fail "six concurrent runs peaked at $peak, so the slots were not used in parallel" pass "six concurrent runs peak at $baseline ShellCheck processes ungated and $peak with a three-slot pool" @@ -1166,11 +1191,11 @@ test_host_load_shrinks_slots_to_the_floor() { # Load far past two times the cores leaves only the two-slot floor, so the # runs queue instead of failing or timing out. peak=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=6 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=100000 \ - fm_lint_concurrent_runs 4 "$tmp" "$tmp/bin") + fm_lint_concurrent_runs 4 "$tmp" "$tmp/bin" 2) [ "$peak" -le 2 ] || fail "under heavy load four runs reached $peak live ShellCheck processes, expected at most 2" : > "$tmp/peak" peak=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=6 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=0 \ - fm_lint_concurrent_runs 4 "$tmp" "$tmp/bin") + fm_lint_concurrent_runs 4 "$tmp" "$tmp/bin" 6) [ "$peak" -gt 2 ] || fail "with an idle host four runs peaked at $peak, so the load never widened the slots" pass "host load shrinks the shared ShellCheck slots to a two-slot floor and idle hosts use all of them" } @@ -1533,7 +1558,7 @@ for my $retry (0, 1) { FM_LINT_INTERNAL_BOUNDED => $bounded, FM_LINT_INTERNAL_MEMORY_KIB => 2097152, FM_LINT_INTERNAL_ROOT_SECS => 3, FM_LINT_INTERNAL_GRACE => 1, FM_LINT_INTERNAL_ROOTS_LOG => "$dir/roots.tsv", FM_LINT_SHELLCHECK => "$tmp/shellcheck"); - sleep 0.005 until time() - int(time()) > 0.3 && time() - int(time()) < 0.35; + my $launched = time(); my $worker = fork(); die "fork: $!" unless defined $worker; if (!$worker) { @@ -1562,17 +1587,18 @@ for my $retry (0, 1) { open(my $wait, '<', $file) or die "$!"; $queued_ms += <$wait>; } - my $elapsed = ($finished - $began) * 1000 - $queued_ms; + my $elapsed = ($finished - $launched) * 1000 - $queued_ms; open(my $log, '<', "$dir/roots.tsv") or die "$!"; my @end; while (<$log>) { @end = split /\t/ if /^end\t/; } die "missing root result\n" unless @end; my $expected = $retry ? 'memory-fallback' : 'ok'; die "wrong root outcome: $end[9]\n" unless $end[9] eq $expected; - die "root timestamps lack millisecond precision\n" - unless abs($end[5] - $began * 1000) < 150 && abs($end[6] - $finished * 1000) < 150; - die "duration $end[7]ms disagrees with $elapsed ms of analysis\n" - unless $end[7] >= 70 && abs($end[7] - $elapsed) < 150; + die "root timestamps fall outside the observed lifecycle\n" + unless $end[5] >= $launched * 1000 - 1 && $end[5] <= $began * 1000 + 1 + && $end[6] >= ($began + 1.5) * 1000 - 1 && $end[6] <= $finished * 1000 + 1; + die "duration $end[7]ms exceeds $elapsed ms available for analysis\n" + unless $end[7] >= 70 && $end[7] <= $elapsed + 2; die "root wall time omitted the queue\n" unless $end[6] - $end[5] >= 1500; }; $error = $@; From f1600a06139e0dd10bf8a79974abfc849d0050ba Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Fri, 9 Oct 2026 00:40:07 -0500 Subject: [PATCH 6/7] no-mistakes(review): Coordinate lint contention tests and retain architecture proof --- tests/fm-lint.test.sh | 66 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/tests/fm-lint.test.sh b/tests/fm-lint.test.sh index 6ec901b8e7e..5f60640644a 100755 --- a/tests/fm-lint.test.sh +++ b/tests/fm-lint.test.sh @@ -1090,8 +1090,24 @@ SH pass "jobs=1 and jobs=2 preserve deterministic diagnostics, failures, cleanup bounds, and quiet telemetry" } +# Counted design comparison (architectural requirements, not timing estimates): +# Design Host bound Lone workers New daemons Stale-slot cleanup +# A per-run only 0 2 0 0 +# B FM_LINT_JOBS=1 default 0 1 0 0 +# C bash mkdir slot dirs 1 2 0 1 +# D central lint daemon 1 2 1 0 +# E flock slot pool in the Perl helper 1 2 0 0 (chosen) +# E supplies a host bound without a daemon or stale-lock reclamation protocol. +# Supplied paired reproduction: identical workloads of six roots per run, +# --jobs 2, gate disabled with FM_LINT_SLOT_DIR=off versus enabled with cap 3. +# Concurrent runs: 1 4 8 16 +# Peak live ShellCheck, disabled: 2 8 16 32 (2 x runs) +# Peak live ShellCheck, enabled: 2 3 3 3 (min(2 x runs, cap)) +# A lone run still peaks at two; fleet-day measurements remain out of scope. +# fm_lint_stub_counting_shellcheck() { - local fakebin=$1 activity=$2 peak=$3 + local fakebin=$1 activity=$2 peak=$3 real_perl + real_perl=$(command -v perl) mkdir -p "$activity" : > "$peak" cat > "$fakebin/shellcheck" < "$fakebin/perl" <', "\$dir/ready.\$\$") or die "\$!"; + close \$ready; + my \$deadline = time() + 30; + until (-e "\$dir/start") { + die "contender start deadline exceeded\\n" if time() > \$deadline; + sleep 0.01; + } + open(my \$attempt, '>', "\$dir/attempt.\$\$") or die "\$!"; + close \$attempt; +} +exec "$real_perl", @ARGV or die "exec perl: \$!"; +PL + chmod +x "$fakebin/perl" } fm_lint_concurrent_runs() { @@ -1129,14 +1167,31 @@ fm_lint_concurrent_runs() { local -a pids roots roots=("$tmp/a.sh" "$tmp/b.sh" "$tmp/c.sh" "$tmp/d.sh") rm -f "$tmp/peak.release" + rm -rf "$tmp/contenders" + mkdir -p "$tmp/contenders" for run in $(seq 1 "$runs"); do - FM_LINT_COUNT_HOLD=1 PATH="$fakebin:$PATH" "$LINT" --jobs 2 "${roots[@]}" > "$tmp/run.$run.out" 2>&1 & + FM_LINT_COUNT_HOLD=1 FM_LINT_COUNT_READY_DIR="$tmp/contenders" \ + PATH="$fakebin:$PATH" "$LINT" --jobs 2 "${roots[@]}" > "$tmp/run.$run.out" 2>&1 & pids+=("$!") done - perl - "$tmp/peak" "$expected" <<'PL' || ready_rc=$? + perl - "$tmp/peak" "$expected" "$tmp/contenders" "$((2 * runs))" <<'PL' || ready_rc=$? use Time::HiRes qw(time sleep); -my ($peak, $expected) = @ARGV; +my ($peak, $expected, $dir, $contenders) = @ARGV; my $deadline = time() + 30; +while (1) { + my @ready = glob "$dir/ready.*"; + last if @ready >= $contenders; + die "contender readiness deadline exceeded\n" if time() > $deadline; + sleep 0.01; +} +open(my $start, '>', "$dir/start") or die "$!"; +close $start; +while (1) { + my @attempts = glob "$dir/attempt.*"; + last if @attempts >= $contenders; + die "contender admission attempt deadline exceeded\n" if time() > $deadline; + sleep 0.01; +} while (1) { open(my $fh, '<', $peak) or die "$!"; my @counts = <$fh>; @@ -1145,8 +1200,9 @@ while (1) { die "concurrent admission deadline exceeded\n" if time() > $deadline; sleep 0.01; } +sleep 3; PL - touch "$tmp/peak.release" + touch "$tmp/contenders/start" "$tmp/peak.release" for pid in "${pids[@]}"; do wait "$pid" || rc=$? done From 65b54bfbaa96d02598283af0fbfb863f0b205c2f Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Fri, 9 Oct 2026 01:27:38 -0500 Subject: [PATCH 7/7] no-mistakes(document): Correct and consolidate host-wide lint concurrency documentation --- bin/fm-lint-cache.pl | 22 ++++++++++----------- bin/fm-lint.sh | 35 ++++++++++++++++++++------------- docs/fm-test-portable-shards.md | 4 +--- tests/fm-lint.test.sh | 12 +++++++---- 4 files changed, 41 insertions(+), 32 deletions(-) diff --git a/bin/fm-lint-cache.pl b/bin/fm-lint-cache.pl index 585adf8dd8f..8f3a8cbeeb3 100755 --- a/bin/fm-lint-cache.pl +++ b/bin/fm-lint-cache.pl @@ -1,9 +1,9 @@ #!/usr/bin/env perl -# fm-lint-cache.pl - private dependency selection and successful-result cache for fm-lint.sh. +# fm-lint-cache.pl - private dependency selection, success cache, and host-slot gate for fm-lint.sh. # Usage: perl fm-lint-cache.pl select # perl fm-lint-cache.pl check -- # ShellCheck retains source-aware extended analysis; only identical successful checks -# are reused. flock serializes identical misses across worktrees, not unrelated roots. +# are reused. Cache-key flock serializes identical misses, independently of host slots. # use Cwd qw(abs_path); use strict; @@ -13,16 +13,16 @@ use File::Path qw(make_path); use File::Basename qw(dirname basename); -# gate mode: hold one host-wide ShellCheck slot while the command runs, so many -# concurrent fm-lint.sh runs queue instead of each adding its own workers. +# Private gate protocol; bin/fm-lint.sh's header owns the public pool controls. # Usage: perl fm-lint-cache.pl gate -- -# Slots are flock files, so the kernel frees one when its holder dies. The slot -# count is FM_LINT_HOST_SLOTS, else half the cores, but never below two (one -# run's default workers). The full cap is available until 1-minute load exceeds -# two times the cores; only the excess load is subtracted from the cap, down -# to the two-slot floor. A slot directory that cannot be used lets the command -# run ungated. The wait is written in milliseconds to so the caller -# can keep queue time out of its own timings. +# Commands inherit the flock descriptor across exec, so killing the gate cannot +# release capacity while a protected descendant still holds it; the kernel +# releases the slot when the last inherited descriptor closes. +# After any lock wakeup, rescan total occupancy before admitting a command. +# Waiting gates probe load no more than once every two seconds, including after +# wakeups, to avoid spawning frequent sysctl readers on an overloaded macOS host. +# The high-resolution wait is written in milliseconds to before +# launch so the caller can account for queue time separately. if (($ARGV[0] // '') eq 'gate') { require POSIX; require Time::HiRes; diff --git a/bin/fm-lint.sh b/bin/fm-lint.sh index 73e2b921b05..a4621e859bb 100755 --- a/bin/fm-lint.sh +++ b/bin/fm-lint.sh @@ -61,17 +61,21 @@ # run holds at most JOBS concurrent ShellCheck processes. Diagnostics replay # in stable shard/root order. FM_LINT_JOBS=1 changes concurrency, not diagnostics # or exit selection. -# That bound is per run, so every root also holds one host-wide ShellCheck slot -# (flock files under FM_LINT_SLOT_DIR, default -# ${XDG_CACHE_HOME:-$HOME/.cache}/firstmate/lint-slots; "off" disables) and -# many concurrent runs queue instead of each adding workers. FM_LINT_HOST_SLOTS -# sets the slot count, default half the cores and at least two. The full cap is -# available until 1-minute load exceeds two times the cores; only the excess -# load is subtracted from the cap, down to a floor of two (one run's own workers), -# so a lone run is never throttled and queued roots always make progress. Queue time -# is excluded from the root deadline and the recorded root duration. The pool -# is shared by every worktree and home of the same user. A slot directory that -# cannot be used is reported and lint runs ungated; it never fails lint. +# That bound is per run; the host-wide pool admits each root only while total +# slot occupancy is below the current load allowance. FM_LINT_SLOT_DIR defaults +# to ${XDG_CACHE_HOME:-$HOME/.cache}/firstmate/lint-slots; "off" disables the pool. +# Runs share a pool only when they use the same slot directory, so different +# homes or overrides must select the same directory to share the host bound. +# FM_LINT_HOST_SLOTS sets the cap; unset or empty uses max(2, floor(ncpu/2)). +# A nonempty value must be a positive decimal integer without leading zeros, +# or lint exits 2, even with the pool off. The load allowance is the cap minus +# any 1-minute load above two times ncpu, rounded to the nearest integer and +# clamped between min(2, cap) and cap. Thus a lone default two-worker run keeps +# both workers unless an explicit cap of one is selected. +# Waiting roots queue rather than fail; queue time consumes neither the root +# deadline nor its retry budget. Slot directory, file, or locking failures +# warn and run ungated rather than failing lint. The private gate's locking +# and process-lifetime invariants live in bin/fm-lint-cache.pl. # --partition 1of2/2of2 splits the entire canonical inventory across # two CI runners, each with those same concurrency-limited workers. # Partitions are complete, disjoint, and byte-weight balanced; --list-files @@ -96,7 +100,8 @@ # named error, so a required-bounds run never lints uncapped. Without # FM_LINT_REQUIRE_BOUNDS (a local developer lint, where hosts like macOS # cannot apply the address-space limit at all) each root still runs in its -# own ShellCheck process with identical diagnostics, just unbounded. +# own ShellCheck process with identical diagnostics and no deadline or memory +# limit; the host-wide concurrency pool remains enabled unless disabled above. # # If a source-following root exits with a memory failure, it is retried once # without --external-sources under the same memory limit and only the time @@ -107,6 +112,9 @@ # Other findings and failed retries still fail lint. The retry's diagnostics # replace the failed attempt's output; peak RSS is the maximum of both attempts. # +# Root timestamps use Perl's high-resolution clock even on stock macOS Bash. +# Start/end span the queue wait; recorded duration subtracts waits from both +# the initial attempt and any retry. # Per-root evidence is incremental: workers append begin/end records (root, # mode, shard, start, end, duration, final exit status, reason, peak RSS when # measured, and whether the final attempt followed sources) to a roots log @@ -1056,8 +1064,7 @@ if [ "${FM_LINT_REQUIRE_BOUNDS:-0}" = 1 ]; then fi fi -# Host-wide ShellCheck slot pool (see the header and fm-lint-cache.pl gate mode). -# FM_LINT_SLOT_DIR=off disables it; an unset pool lives beside the lint cache. +# Host-wide pool configuration; the header owns its public controls. SLOT_DIR=${FM_LINT_SLOT_DIR:-${XDG_CACHE_HOME:-${HOME:-${TMPDIR:-/tmp}}/.cache}/firstmate/lint-slots} case "${FM_LINT_HOST_SLOTS:-1}" in ''|0*|*[!0-9]*) diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index e19ae726b03..77161361d28 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -136,9 +136,7 @@ CI requires its per-root bounds, so an unenforceable deadline or address-space l Its `--list-files` interface exposes partition membership; `tests/fm-lint.test.sh` verifies complete/disjoint executed roots, changed-source selection, shared-cache invalidation, initial analysis flags, fallback reporting, and seeded finding parity. The workflow uploads each partition's quiet telemetry plus its per-root lifecycle sidecar to distinguish analysis cost, memory use, and host contention. No fast mode, path skips, or paid runner provisioning is part of this layout. -The host-wide ShellCheck slot pool queues while total occupancy meets the current load allowance; waiting gates recheck load no more than once every two seconds, including after lock wakeups. Slot-file or locking failures warn and run ungated. -Protected commands inherit their slot descriptors, so gate death cannot release capacity while their descendants still run. -Root timestamps and queue accounting use Perl's high-resolution clock, including on stock macOS Bash, and exclude queue time from analysis durations and retry budgets. +The [lint script header](../bin/fm-lint.sh) owns host-wide slot controls and queue-time accounting; `tests/fm-lint.test.sh` covers concurrent admission, load boundaries, ungated fallback, inherited-slot lifetime, and queued-root timing. The [lint script header](../bin/fm-lint.sh) owns local dependency discovery, conservative unresolved-import selection, and successful-result cache controls; these do not replace full joint source analysis. Regression fixtures exercise cross-file missing-argument findings through direct and private source routines, deleted sources, concurrent reuse, changed binaries, and the separation between fast and full analysis. diff --git a/tests/fm-lint.test.sh b/tests/fm-lint.test.sh index 5f60640644a..ed5fabe0031 100755 --- a/tests/fm-lint.test.sh +++ b/tests/fm-lint.test.sh @@ -1129,7 +1129,7 @@ open(my \$peak, '>>', "$peak") or die "\$!"; print {\$peak} scalar(@active), "\n"; close \$peak; close \$lock; -my \$deadline = time() + 60; +my \$deadline = time() + 90; while (\$ENV{FM_LINT_COUNT_HOLD} && !-e "$peak.release") { if (time() > \$deadline) { unlink "$activity/\$\$"; exit 1; } sleep 0.01; @@ -1186,12 +1186,14 @@ while (1) { } open(my $start, '>', "$dir/start") or die "$!"; close $start; +$deadline = time() + 30; while (1) { my @attempts = glob "$dir/attempt.*"; last if @attempts >= $contenders; die "contender admission attempt deadline exceeded\n" if time() > $deadline; sleep 0.01; } +$deadline = time() + 30; while (1) { open(my $fh, '<', $peak) or die "$!"; my @counts = <$fh>; @@ -1419,7 +1421,7 @@ for my $failure (qw(open scan wait)) { '--', $^X, '-e', 'print "analysis ran\n"; exit 23'; die "exec: $!"; } - my $deadline = time() + 4; + my $deadline = time() + 30; while (waitpid($pid, WNOHANG) == 0) { if (time() > $deadline) { kill 'KILL', $pid; @@ -1626,12 +1628,14 @@ for my $retry (0, 1) { } my $error; eval { - my $deadline = time() + 5; + # Controller waits cover host scheduling and worker bookkeeping; the + # protected root still has its independently enforced three-second bound. + my $deadline = time() + 30; until (-s "$dir/roots.tsv") { die "root never began\n" if time() > $deadline; sleep 0.01; } my $began = time(); sleep 1.5; close $slot; - $deadline = time() + 5; + $deadline = time() + 30; while (waitpid($worker, WNOHANG) == 0) { die "queued root timed out\n" if time() > $deadline; sleep 0.01;