diff --git a/bin/fm-lint-cache.pl b/bin/fm-lint-cache.pl index 9ca026ab037..8f3a8cbeeb3 100755 --- a/bin/fm-lint-cache.pl +++ b/bin/fm-lint-cache.pl @@ -1,18 +1,132 @@ #!/usr/bin/env perl -# fm-lint-cache.pl - private dependency selection and successful-result cache for fm-lint.sh. +# fm-lint-cache.pl - private dependency selection, success cache, and host-slot gate for fm-lint.sh. # Usage: perl fm-lint-cache.pl select # perl fm-lint-cache.pl check -- # ShellCheck retains source-aware extended analysis; only identical successful checks -# are reused. flock serializes identical misses across worktrees, not unrelated roots. +# are reused. Cache-key flock serializes identical misses, independently of host slots. # use Cwd qw(abs_path); use strict; use warnings; use Digest::SHA qw(sha256_hex); -use Fcntl qw(:flock); +use Fcntl qw(:flock F_SETFD); use File::Path qw(make_path); use File::Basename qw(dirname basename); +# Private gate protocol; bin/fm-lint.sh's header owns the public pool controls. +# Usage: perl fm-lint-cache.pl gate -- +# Commands inherit the flock descriptor across exec, so killing the gate cannot +# release capacity while a protected descendant still holds it; the kernel +# releases the slot when the last inherited descriptor closes. +# After any lock wakeup, rescan total occupancy before admitting a command. +# Waiting gates probe load no more than once every two seconds, including after +# wakeups, to avoid spawning frequent sysctl readers on an overloaded macOS host. +# The high-resolution wait is written in milliseconds to before +# launch so the caller can account for queue time separately. +if (($ARGV[0] // '') eq 'gate') { + require POSIX; + require Time::HiRes; + my (undef, $slot_dir, $ncpu, $wait_file, $dashes, @command) = @ARGV; + die "fm-lint-gate: invalid private invocation\n" + unless defined $dashes && $dashes eq '--' && @command && ($ncpu // '') =~ /\A[1-9][0-9]*\z/; + my ($child, $caught); + my %signal_number = (HUP => 1, INT => 2, TERM => 15); + for my $name (keys %signal_number) { + $SIG{$name} = sub { $caught = $name; kill 'TERM', $child if $child; }; + } + my $floor = 2; + my $cap = ($ENV{FM_LINT_HOST_SLOTS} // '') =~ /\A[1-9][0-9]*\z/ ? $ENV{FM_LINT_HOST_SLOTS} + 0 + : ($ncpu >> 1) > $floor ? ($ncpu >> 1) : $floor; + $floor = $cap if $cap < $floor; + my $slot_load = sub { + return $ENV{FM_LINT_SLOT_LOAD} + 0 + if ($ENV{FM_TEST_SEAM} // '') eq '1' && ($ENV{FM_LINT_SLOT_LOAD} // '') =~ /\A[0-9]+(?:\.[0-9]+)?\z/; + if (open(my $fh, '<', '/proc/loadavg')) { + my $line = <$fh>; + return $1 + 0 if defined $line && $line =~ /\A([0-9]+(?:\.[0-9]+)?)/; + } + if (open(my $pipe, '-|', 'sysctl', '-n', 'vm.loadavg')) { + my $line = <$pipe>; + close $pipe; + return $1 + 0 if defined $line && $line =~ /([0-9]+(?:\.[0-9]+)?)/; + } + return 0; + }; + my ($slot, $waited_from, $recheck_at); + my $available = eval { make_path($slot_dir, {mode => 0700}); -d $slot_dir && -w _ }; + if ($available) { + $waited_from = Time::HiRes::time(); + ACQUIRE: while (!$slot) { + exit 128 + $signal_number{$caught} if $caught; + my $delay = ($recheck_at // 0) - Time::HiRes::time(); + Time::HiRes::sleep($delay) if $delay > 0; + exit 128 + $signal_number{$caught} if $caught; + my (@free, $occupied); + $occupied = 0; + for my $index (0 .. $cap - 1) { + open(my $fh, '>>', "$slot_dir/slot.$index") + or do { $available = 0; last ACQUIRE; }; + if (flock($fh, LOCK_EX | LOCK_NB)) { push @free, $fh; next; } + my $busy = $!{EWOULDBLOCK} || $!{EAGAIN} || $!{EINTR}; + close $fh; + unless ($busy) { $available = 0; last ACQUIRE; } + $occupied++; + } + my $allowed = int($cap + 2 * $ncpu - $slot_load->() + 0.5); + $recheck_at = Time::HiRes::time() + 2; + $allowed = $floor if $allowed < $floor; + $allowed = $cap if $allowed > $cap; + $slot = shift @free if @free && $occupied < $allowed; + my $has_free = @free; + close $_ for @free; + next if $slot || $caught; + next if $has_free; + if (open(my $fh, '>>', "$slot_dir/slot.@{[ int(rand($cap)) ]}")) { + eval { + local $SIG{ALRM} = sub { die "gate-recheck\n" }; + alarm 2; + my $got = flock($fh, LOCK_EX); + my $interrupted = $!{EINTR}; + alarm 0; + die "gate-lock: $!\n" unless $got || $interrupted; + }; + alarm 0; + my $error = $@; + close $fh; + if ($error && $error ne "gate-recheck\n") { $available = 0; last ACQUIRE; } + } else { + $available = 0; + last ACQUIRE; + } + } + } + if ($slot && !defined fcntl($slot, F_SETFD, 0)) { + close $slot; + undef $slot; + $available = 0; + } + unless ($available) { + warn "fm-lint: host slot directory unavailable; running without the host-wide ShellCheck bound\n"; + } + if (open(my $fh, '>', $wait_file)) { + printf {$fh} "%d\n", $waited_from ? (Time::HiRes::time() - $waited_from) * 1000 : 0; + close $fh; + } + exit 128 + $signal_number{$caught} if $caught; + $child = fork(); + die "fm-lint-gate: fork: $!\n" unless defined $child; + if (!$child) { + exec {$command[0]} @command; + warn "fm-lint-gate: exec $command[0]: $!\n"; + POSIX::_exit(127); + } + kill 'TERM', $child if $caught; + my $reaped; + while (($reaped = waitpid($child, 0)) == -1 && $!{EINTR}) { } + my $status = $reaped == $child ? $? : 127 << 8; + exit(($status & 127) ? 128 + ($status & 127) : $status >> 8); +} + my ($mode, $root, @args) = @ARGV; my $cache; if (defined $mode && $mode eq 'check') { diff --git a/bin/fm-lint.sh b/bin/fm-lint.sh index 73e12a732f1..a4621e859bb 100755 --- a/bin/fm-lint.sh +++ b/bin/fm-lint.sh @@ -61,6 +61,21 @@ # run holds at most JOBS concurrent ShellCheck processes. Diagnostics replay # in stable shard/root order. FM_LINT_JOBS=1 changes concurrency, not diagnostics # or exit selection. +# That bound is per run; the host-wide pool admits each root only while total +# slot occupancy is below the current load allowance. FM_LINT_SLOT_DIR defaults +# to ${XDG_CACHE_HOME:-$HOME/.cache}/firstmate/lint-slots; "off" disables the pool. +# Runs share a pool only when they use the same slot directory, so different +# homes or overrides must select the same directory to share the host bound. +# FM_LINT_HOST_SLOTS sets the cap; unset or empty uses max(2, floor(ncpu/2)). +# A nonempty value must be a positive decimal integer without leading zeros, +# or lint exits 2, even with the pool off. The load allowance is the cap minus +# any 1-minute load above two times ncpu, rounded to the nearest integer and +# clamped between min(2, cap) and cap. Thus a lone default two-worker run keeps +# both workers unless an explicit cap of one is selected. +# Waiting roots queue rather than fail; queue time consumes neither the root +# deadline nor its retry budget. Slot directory, file, or locking failures +# warn and run ungated rather than failing lint. The private gate's locking +# and process-lifetime invariants live in bin/fm-lint-cache.pl. # --partition 1of2/2of2 splits the entire canonical inventory across # two CI runners, each with those same concurrency-limited workers. # Partitions are complete, disjoint, and byte-weight balanced; --list-files @@ -85,7 +100,8 @@ # named error, so a required-bounds run never lints uncapped. Without # FM_LINT_REQUIRE_BOUNDS (a local developer lint, where hosts like macOS # cannot apply the address-space limit at all) each root still runs in its -# own ShellCheck process with identical diagnostics, just unbounded. +# own ShellCheck process with identical diagnostics and no deadline or memory +# limit; the host-wide concurrency pool remains enabled unless disabled above. # # If a source-following root exits with a memory failure, it is retried once # without --external-sources under the same memory limit and only the time @@ -96,6 +112,9 @@ # Other findings and failed retries still fail lint. The retry's diagnostics # replace the failed attempt's output; peak RSS is the maximum of both attempts. # +# Root timestamps use Perl's high-resolution clock even on stock macOS Bash. +# Start/end span the queue wait; recorded duration subtracts waits from both +# the initial attempt and any retry. # Per-root evidence is incremental: workers append begin/end records (root, # mode, shard, start, end, duration, final exit status, reason, peak RSS when # measured, and whether the final attempt followed sources) to a roots log @@ -157,12 +176,7 @@ fm_lint_worker_stop() { } fm_lint_now_ms() { - if [ -n "${EPOCHREALTIME:-}" ]; then - local seconds=${EPOCHREALTIME%.*} micros=${EPOCHREALTIME#*.} - printf '%s\n' "$((seconds * 1000 + 10#${micros:0:3}))" - else - printf '%s\n' "$(($(date +%s) * 1000))" - fi + "${FM_LINT_PERL_BIN:-perl}" -MTime::HiRes=time -e 'printf "%d\n", time() * 1000' } # Names are listed only for signal numbers that agree on Linux and macOS; any @@ -253,8 +267,8 @@ fm_lint_classify_root() { # # Run one ShellCheck invocation under the given deadline and the per-root # address-space limit, returning its exit status in FM_LINT_LAST_RC. fm_lint_exec_root() { # - local path=$1 root_out=$2 root_err=$3 rss_file=$4 seconds=$5 invocation_rc=0 - local -a analysis_command + local path=$1 root_out=$2 root_err=$3 rss_file=$4 seconds=$5 invocation_rc=0 wait_file waited + local -a analysis_command gate_command shift 5 analysis_command=("${FM_LINT_PERL_BIN:-perl}" "$SELF_DIR/fm-lint-cache.pl" check \ "${FM_LINT_INTERNAL_CACHE:-off}" "$ROOT" "$FM_LINT_SHELLCHECK" \ @@ -266,6 +280,15 @@ fm_lint_exec_root() { # analysis_command=(/usr/bin/time -f 'max_rss_kib=%M' -o "$rss_file" "${analysis_command[@]}") fi fi + # The host-wide slot gate sits outside the watchdog and /usr/bin/time, so + # queue time counts toward neither the root deadline nor its peak RSS. + gate_command=() + wait_file="$rss_file.wait" + rm -f "$wait_file" + if [ "${FM_LINT_INTERNAL_SLOT_DIR:-off}" != off ]; then + gate_command=("${FM_LINT_PERL_BIN:-perl}" "$SELF_DIR/fm-lint-cache.pl" gate \ + "$FM_LINT_INTERNAL_SLOT_DIR" "$FM_LINT_INTERNAL_NCPU" "$wait_file" --) + fi if [ "${FM_LINT_INTERNAL_BOUNDED:-none}" != none ]; then # The watchdog runs in a process group of its own (the same setpgrp hop the # workers use), so the owner's TERM-then-KILL group sweep cannot kill it @@ -274,7 +297,7 @@ fm_lint_exec_root() { # # check still starts the same terminate-then-kill escalation; the worker # names itself as that owner before the launch, so a worker that dies while # the watchdog is still starting is detected too. - ( FM_EXEC_TIMED_OWNER_PID=$$ exec "${FM_LINT_PERL_BIN:-perl}" -e 'setpgrp(0, 0) or die "setpgrp: $!"; exec @ARGV or die "exec: $!"' \ + ( FM_EXEC_TIMED_OWNER_PID=$$ exec ${gate_command[@]+"${gate_command[@]}"} "${FM_LINT_PERL_BIN:-perl}" -e 'setpgrp(0, 0) or die "setpgrp: $!"; exec @ARGV or die "exec: $!"' \ "${BASH:-bash}" "$SELF" --internal-timed \ "$seconds" "$FM_LINT_INTERNAL_GRACE" \ "${BASH:-bash}" "$SELF" --internal-root "$rss_file" "$FM_LINT_INTERNAL_MEMORY_KIB" \ @@ -283,12 +306,17 @@ fm_lint_exec_root() { # wait "$FM_LINT_WORKER_RUN_PID" || invocation_rc=$? FM_LINT_WORKER_RUN_PID= else - "${analysis_command[@]}" > "$root_out" 2> "$root_err" & + ${gate_command[@]+"${gate_command[@]}"} "${analysis_command[@]}" > "$root_out" 2> "$root_err" & FM_LINT_WORKER_RUN_PID=$! wait "$FM_LINT_WORKER_RUN_PID" || invocation_rc=$? FM_LINT_WORKER_RUN_PID= fi FM_LINT_LAST_RC=$invocation_rc + FM_LINT_LAST_WAIT_MS=0 + if [ -r "$wait_file" ]; then + waited=$(tr -d '[:space:]' < "$wait_file" 2>/dev/null) + case "$waited" in ''|*[!0-9]*) ;; *) FM_LINT_LAST_WAIT_MS=$waited ;; esac + fi } # Run one selected root, retry memory failures without source following, record @@ -302,7 +330,7 @@ fm_lint_run_root() { # local fallback_err="$output_dir/root.$shard_index.$index.fallback.err" local fallback_rss="$output_dir/root.$shard_index.$index.fallback.rss" local start_ms end_ms duration_ms invocation_rc=0 reason rss_kib initial_rc initial_reason - local fallback_secs + local fallback_secs queued_ms=0 local final_follow_sources=${FM_LINT_INTERNAL_FOLLOW_SOURCES:-1} local -a fallback_args start_ms=$(fm_lint_now_ms) @@ -318,13 +346,14 @@ fm_lint_run_root() { # fm_lint_exec_root "$path" "$root_out" "$root_err" "$rss_file" \ "$FM_LINT_INTERNAL_ROOT_SECS" "${FM_LINT_WORKER_ARGS[@]}" invocation_rc=$FM_LINT_LAST_RC + queued_ms=$FM_LINT_LAST_WAIT_MS reason=$(fm_lint_classify_root "$invocation_rc" "$root_err") initial_rc=$invocation_rc initial_reason=$reason # The retry spends what is left of this root's one deadline rather than a # fresh one, so both attempts together still fit the budget CI sized its job # timeout around. - fallback_secs=$(( (start_ms + FM_LINT_INTERNAL_ROOT_SECS * 1000 - $(fm_lint_now_ms)) / 1000 )) + fallback_secs=$(( (start_ms + queued_ms + FM_LINT_INTERNAL_ROOT_SECS * 1000 - $(fm_lint_now_ms)) / 1000 )) if [ "$reason" = memory ] \ && [ "${FM_LINT_INTERNAL_FOLLOW_SOURCES:-1}" -eq 1 ] \ && [ "${FM_LINT_INTERNAL_BOUNDED:-none}" != none ] \ @@ -344,6 +373,7 @@ fm_lint_run_root() { # "$fallback_secs" "${fallback_args[@]}" final_follow_sources=0 invocation_rc=$FM_LINT_LAST_RC + queued_ms=$((queued_ms + FM_LINT_LAST_WAIT_MS)) reason=$(fm_lint_classify_root "$invocation_rc" "$fallback_err") rss_kib=$(fm_lint_max_root_rss \ "$(fm_lint_root_rss "$rss_file")" "$(fm_lint_root_rss "$fallback_rss")") @@ -364,7 +394,7 @@ fm_lint_run_root() { # cat "$root_out" "$root_err" >> "$output_dir/shard.$shard_index.out" fi end_ms=$(fm_lint_now_ms) - duration_ms=$((end_ms - start_ms)) + duration_ms=$((end_ms - start_ms - queued_ms)) if [ -n "${FM_LINT_INTERNAL_ROOTS_LOG:-}" ]; then printf 'end\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ "$index" "$path" "$shard_index" "${FM_LINT_INTERNAL_MODE:-}" \ @@ -1034,6 +1064,21 @@ if [ "${FM_LINT_REQUIRE_BOUNDS:-0}" = 1 ]; then fi fi +# Host-wide pool configuration; the header owns its public controls. +SLOT_DIR=${FM_LINT_SLOT_DIR:-${XDG_CACHE_HOME:-${HOME:-${TMPDIR:-/tmp}}/.cache}/firstmate/lint-slots} +case "${FM_LINT_HOST_SLOTS:-1}" in + ''|0*|*[!0-9]*) + printf 'fm-lint.sh: FM_LINT_HOST_SLOTS must be a positive integer, got %s.\n' \ + "${FM_LINT_HOST_SLOTS:-}" >&2 + exit 2 + ;; +esac +NCPU= +if [ "$SLOT_DIR" != off ]; then + NCPU=$(sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null) || NCPU= + case "$NCPU" in ''|0*|*[!0-9]*) NCPU=1 ;; esac +fi + PROGRESS=0 if [ -n "$PARTITION" ]; then PROGRESS=1 @@ -1184,7 +1229,7 @@ fm_lint_run_worker() { # FM_LINT_SHELLCHECK="$SHELLCHECK_BIN" FM_LINT_PERL_BIN="$PERL_BIN" ) - worker_env+=(FM_LINT_INTERNAL_CACHE="$CACHE_DIR") + worker_env+=(FM_LINT_INTERNAL_CACHE="$CACHE_DIR" FM_LINT_INTERNAL_SLOT_DIR="$SLOT_DIR" FM_LINT_INTERNAL_NCPU="$NCPU") if [ -n "$TELEMETRY" ] && [ -x /usr/bin/time ]; then if [ "$(uname)" = Darwin ]; then exec "$PERL_BIN" -e 'setpgrp(0, 0) or die "setpgrp: $!"; exec @ARGV or die "exec: $!"' \ diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 2055acc9f3c..77161361d28 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -136,6 +136,7 @@ CI requires its per-root bounds, so an unenforceable deadline or address-space l Its `--list-files` interface exposes partition membership; `tests/fm-lint.test.sh` verifies complete/disjoint executed roots, changed-source selection, shared-cache invalidation, initial analysis flags, fallback reporting, and seeded finding parity. The workflow uploads each partition's quiet telemetry plus its per-root lifecycle sidecar to distinguish analysis cost, memory use, and host contention. No fast mode, path skips, or paid runner provisioning is part of this layout. +The [lint script header](../bin/fm-lint.sh) owns host-wide slot controls and queue-time accounting; `tests/fm-lint.test.sh` covers concurrent admission, load boundaries, ungated fallback, inherited-slot lifetime, and queued-root timing. The [lint script header](../bin/fm-lint.sh) owns local dependency discovery, conservative unresolved-import selection, and successful-result cache controls; these do not replace full joint source analysis. Regression fixtures exercise cross-file missing-argument findings through direct and private source routines, deleted sources, concurrent reuse, changed binaries, and the separation between fast and full analysis. diff --git a/tests/fm-lint.test.sh b/tests/fm-lint.test.sh index 8f2d17b207c..ed5fabe0031 100755 --- a/tests/fm-lint.test.sh +++ b/tests/fm-lint.test.sh @@ -22,6 +22,8 @@ INSTALLER="$ROOT/bin/fm-install-shellcheck.sh" REQUIRED=$("$LINT" --required-version) # Ordinary regressions must not read or populate the operator's shared cache. export FM_LINT_CACHE_DIR=off +# Nor may they queue on the operator's shared host-wide ShellCheck slots. +export FM_LINT_SLOT_DIR=off # Official GitHub release asset sha256 values for shellcheck v0.11.0 .tar.xz # archives (https://github.com/koalaman/shellcheck/releases/tag/v0.11.0). Tests @@ -1088,6 +1090,587 @@ SH pass "jobs=1 and jobs=2 preserve deterministic diagnostics, failures, cleanup bounds, and quiet telemetry" } +# Counted design comparison (architectural requirements, not timing estimates): +# Design Host bound Lone workers New daemons Stale-slot cleanup +# A per-run only 0 2 0 0 +# B FM_LINT_JOBS=1 default 0 1 0 0 +# C bash mkdir slot dirs 1 2 0 1 +# D central lint daemon 1 2 1 0 +# E flock slot pool in the Perl helper 1 2 0 0 (chosen) +# E supplies a host bound without a daemon or stale-lock reclamation protocol. +# Supplied paired reproduction: identical workloads of six roots per run, +# --jobs 2, gate disabled with FM_LINT_SLOT_DIR=off versus enabled with cap 3. +# Concurrent runs: 1 4 8 16 +# Peak live ShellCheck, disabled: 2 8 16 32 (2 x runs) +# Peak live ShellCheck, enabled: 2 3 3 3 (min(2 x runs, cap)) +# A lone run still peaks at two; fleet-day measurements remain out of scope. +# +fm_lint_stub_counting_shellcheck() { + local fakebin=$1 activity=$2 peak=$3 real_perl + real_perl=$(command -v perl) + mkdir -p "$activity" + : > "$peak" + cat > "$fakebin/shellcheck" <>', "$peak.lock") or die "\$!"; +flock(\$lock, LOCK_EX) or die "\$!"; +open(my \$active, '>', "$activity/\$\$") or die "\$!"; +close \$active; +my @active = glob "$activity/*"; +open(my \$peak, '>>', "$peak") or die "\$!"; +print {\$peak} scalar(@active), "\n"; +close \$peak; +close \$lock; +my \$deadline = time() + 90; +while (\$ENV{FM_LINT_COUNT_HOLD} && !-e "$peak.release") { + if (time() > \$deadline) { unlink "$activity/\$\$"; exit 1; } + sleep 0.01; +} +unlink "$activity/\$\$"; +exit 0; +PL + chmod +x "$fakebin/shellcheck" + cat > "$fakebin/perl" <', "\$dir/ready.\$\$") or die "\$!"; + close \$ready; + my \$deadline = time() + 30; + until (-e "\$dir/start") { + die "contender start deadline exceeded\\n" if time() > \$deadline; + sleep 0.01; + } + open(my \$attempt, '>', "\$dir/attempt.\$\$") or die "\$!"; + close \$attempt; +} +exec "$real_perl", @ARGV or die "exec perl: \$!"; +PL + chmod +x "$fakebin/perl" +} + +fm_lint_concurrent_runs() { + local runs=$1 tmp=$2 fakebin=$3 expected=$4 run pid rc=0 ready_rc=0 + local -a pids roots + roots=("$tmp/a.sh" "$tmp/b.sh" "$tmp/c.sh" "$tmp/d.sh") + rm -f "$tmp/peak.release" + rm -rf "$tmp/contenders" + mkdir -p "$tmp/contenders" + for run in $(seq 1 "$runs"); do + FM_LINT_COUNT_HOLD=1 FM_LINT_COUNT_READY_DIR="$tmp/contenders" \ + PATH="$fakebin:$PATH" "$LINT" --jobs 2 "${roots[@]}" > "$tmp/run.$run.out" 2>&1 & + pids+=("$!") + done + perl - "$tmp/peak" "$expected" "$tmp/contenders" "$((2 * runs))" <<'PL' || ready_rc=$? +use Time::HiRes qw(time sleep); +my ($peak, $expected, $dir, $contenders) = @ARGV; +my $deadline = time() + 30; +while (1) { + my @ready = glob "$dir/ready.*"; + last if @ready >= $contenders; + die "contender readiness deadline exceeded\n" if time() > $deadline; + sleep 0.01; +} +open(my $start, '>', "$dir/start") or die "$!"; +close $start; +$deadline = time() + 30; +while (1) { + my @attempts = glob "$dir/attempt.*"; + last if @attempts >= $contenders; + die "contender admission attempt deadline exceeded\n" if time() > $deadline; + sleep 0.01; +} +$deadline = time() + 30; +while (1) { + open(my $fh, '<', $peak) or die "$!"; + my @counts = <$fh>; + close $fh; + last if grep { /\A[0-9]+\n\z/ && $_ >= $expected } @counts; + die "concurrent admission deadline exceeded\n" if time() > $deadline; + sleep 0.01; +} +sleep 3; +PL + touch "$tmp/contenders/start" "$tmp/peak.release" + for pid in "${pids[@]}"; do + wait "$pid" || rc=$? + done + [ "$ready_rc" -eq 0 ] || fail "concurrent runs did not reach $expected held ShellCheck processes"$'\n'"$(cat "$tmp"/run.*.out)" + [ "$rc" -eq 0 ] || fail "a concurrent fm-lint.sh run failed (rc=$rc)"$'\n'"$(cat "$tmp"/run.*.out)" + sort -n "$tmp/peak" | tail -1 +} + +fm_lint_slot_fixture() { # -> prints the tmp dir + local tmp root + tmp=$(fm_test_tmproot "$1") + mkdir -p "$tmp/bin" + for root in a b c d; do + printf '#!/usr/bin/env bash\nexit 0\n' > "$tmp/$root.sh" + done + fm_lint_stub_counting_shellcheck "$tmp/bin" "$tmp/active" "$tmp/peak" + printf '%s\n' "$tmp" +} + +test_host_slots_bound_concurrent_runs() { + local tmp peak lone baseline + tmp=$(fm_lint_slot_fixture fm-lint-slots) + # One run keeps its two workers: the host-wide bound never throttles a lone run. + lone=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=3 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=0 \ + fm_lint_concurrent_runs 1 "$tmp" "$tmp/bin" 2) + [ "$lone" -eq 2 ] || fail "a lone run peaked at $lone live ShellCheck processes, expected its two workers" + : > "$tmp/peak" + baseline=$(FM_LINT_SLOT_DIR=off fm_lint_concurrent_runs 6 "$tmp" "$tmp/bin" 12) + [ "$baseline" -eq 12 ] || fail "six ungated runs peaked at $baseline live ShellCheck processes, expected 12" + : > "$tmp/peak" + # Six runs would start twelve ShellCheck processes if each bounded only itself. + peak=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=3 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=0 \ + fm_lint_concurrent_runs 6 "$tmp" "$tmp/bin" 3) + [ "$peak" -le 3 ] || fail "six concurrent runs reached $peak live ShellCheck processes, expected at most 3 host-wide" + [ "$peak" -ge 3 ] || fail "six concurrent runs peaked at $peak, so the slots were not used in parallel" + pass "six concurrent runs peak at $baseline ShellCheck processes ungated and $peak with a three-slot pool" +} + +test_host_load_shrinks_slots_to_the_floor() { + local tmp peak + tmp=$(fm_lint_slot_fixture fm-lint-slots-load) + # Load far past two times the cores leaves only the two-slot floor, so the + # runs queue instead of failing or timing out. + peak=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=6 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=100000 \ + fm_lint_concurrent_runs 4 "$tmp" "$tmp/bin" 2) + [ "$peak" -le 2 ] || fail "under heavy load four runs reached $peak live ShellCheck processes, expected at most 2" + : > "$tmp/peak" + peak=$(FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=6 FM_TEST_SEAM=1 FM_LINT_SLOT_LOAD=0 \ + fm_lint_concurrent_runs 4 "$tmp" "$tmp/bin" 6) + [ "$peak" -gt 2 ] || fail "with an idle host four runs peaked at $peak, so the load never widened the slots" + pass "host load shrinks the shared ShellCheck slots to a two-slot floor and idle hosts use all of them" +} + +test_host_load_preserves_the_cap_until_the_threshold() { + local tmp + tmp=$(fm_test_tmproot fm-lint-slots-threshold) + perl - "$ROOT/bin/fm-lint-cache.pl" "$tmp" <<'PL' || fail "host load boundary regression failed" +use strict; +use warnings; +use File::Path qw(make_path); +use Fcntl qw(:flock); +use Time::HiRes qw(time sleep); +my ($gate, $tmp) = @ARGV; +my $command = q{ + use Time::HiRes qw(time sleep); + my ($started, $release) = @ARGV; + open(my $fh, '>', $started) or die "$!"; + close $fh; + my $deadline = time() + 15; + until (-e $release) { exit 1 if time() > $deadline; sleep 0.01; } +}; +for my $cap ('', 6) { + local $ENV{FM_LINT_HOST_SLOTS} = $cap; + for my $load (30, 36, 37) { + local $ENV{FM_TEST_SEAM} = 1; + local $ENV{FM_LINT_SLOT_LOAD} = $load; + my $full = $cap || 9; + my $expected = $load == 37 ? $full - 1 : $full; + my $dir = "$tmp/$full.$load"; + make_path($dir); + my @pids; + for my $index (0 .. $full) { + my $pid = fork(); + die "fork: $!" unless defined $pid; + if (!$pid) { + exec $^X, $gate, 'gate', "$dir/slots", 18, "$dir/wait.$index", + '--', $^X, '-e', $command, "$dir/started.$index", "$dir/release"; + die "exec: $!"; + } + push @pids, $pid; + } + my $error; + eval { + my $deadline = time() + 10; + while (1) { + my @started = glob "$dir/started.*"; + last if @started >= $expected; + die "admission deadline exceeded\n" if time() > $deadline; + sleep 0.01; + } + sleep 0.25; + my @started = glob "$dir/started.*"; + die "admitted @started; expected $expected commands\n" unless @started == $expected; + }; + $error = $@; + open(my $release, '>', "$dir/release") or die "release: $!"; + close $release; + for my $pid (@pids) { + waitpid($pid, 0); + $error ||= "gated command exited with status $?\n" if $?; + } + die "cap $full, load $load: $error" if $error; + } +} +for my $path (qw(scan wake)) { + my $dir = "$tmp/occupancy.$path"; + make_path("$dir/slots"); + my @holders; + for my $index ($path eq 'wake' ? (0 .. 8) : (1 .. 8)) { + open($holders[$index], '>>', "$dir/slots/slot.$index") or die "$!"; + flock($holders[$index], LOCK_EX) or die "$!"; + } + local $ENV{FM_LINT_HOST_SLOTS} = 9; + local $ENV{FM_TEST_SEAM} = 1; + local $ENV{FM_LINT_SLOT_LOAD} = 43; + my $pid = fork(); + die "fork: $!" unless defined $pid; + if (!$pid) { + close $_ for grep { defined } @holders; + exec $^X, $gate, 'gate', "$dir/slots", 18, "$dir/wait", + '--', $^X, '-e', 'open(my $fh, ">", $ARGV[0]) or die "$!"', "$dir/started"; + die "exec: $!"; + } + my $error; + eval { + sleep 0.25; + close $holders[0] if $path eq 'wake'; + sleep 2; + die "admitted with eight occupied slots and an allowance of two\n" if -e "$dir/started"; + close $holders[$_] for 1 .. 7; + my $deadline = time() + 5; + until (-e "$dir/started") { + die "did not admit below the occupancy allowance\n" if time() > $deadline; + sleep 0.01; + } + }; + $error = $@; + close $holders[8]; + if ($error) { kill 'KILL', $pid; } + waitpid($pid, 0); + $error ||= "gated command failed: $?\n" if $?; + die "$path: $error" if $error; +} +PL + pass "host caps follow the load threshold and both acquisition paths respect total occupancy" +} + +test_slot_pool_can_be_disabled_or_misconfigured() { + local tmp rc out + tmp=$(fm_lint_slot_fixture fm-lint-slots-off) + rc=0 + out=$(PATH="$tmp/bin:$PATH" FM_LINT_SLOT_DIR=off "$LINT" "$tmp/a.sh" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "a run with the slot pool off failed"$'\n'"$out" + rc=0 + out=$(PATH="$tmp/bin:$PATH" FM_LINT_SLOT_DIR="$tmp/slots" FM_LINT_HOST_SLOTS=zero "$LINT" "$tmp/a.sh" 2>&1) || rc=$? + [ "$rc" -eq 2 ] || fail "a non-numeric FM_LINT_HOST_SLOTS exited $rc, expected 2" + assert_contains "$out" "FM_LINT_HOST_SLOTS" "the refusal did not name the setting" + [ ! -d "$tmp/slots" ] || fail "a refused run still created the slot directory" + rc=0 + out=$(PATH="$tmp/bin:$PATH" FM_LINT_SLOT_DIR="$tmp/a.sh/blocked" "$LINT" "$tmp/a.sh" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "an unusable slot directory must not fail lint"$'\n'"$out" + assert_contains "$out" "running without the host-wide ShellCheck bound" "an unusable slot directory was not reported" + pass "the slot pool can be disabled, rejects bad settings, and never fails lint when unusable" +} + +test_slot_file_failures_run_ungated() { + local tmp + tmp=$(fm_test_tmproot fm-lint-slot-errors) + perl - "$ROOT/bin/fm-lint-cache.pl" "$tmp" <<'PL' || fail "slot I/O fallback regression failed" +use strict; +use warnings; +use File::Path qw(make_path); +use POSIX qw(WNOHANG); +use Time::HiRes qw(time sleep); +my ($gate, $tmp) = @ARGV; +open(my $module, '>', "$tmp/FailLock.pm") or die "$!"; +print {$module} <<'MODULE'; +package FailLock; +use Errno qw(EIO EWOULDBLOCK); +use Fcntl qw(LOCK_NB); +BEGIN { + *CORE::GLOBAL::flock = sub { + $! = $ENV{FAIL_LOCK} eq 'wait' && ($_[1] & LOCK_NB) ? EWOULDBLOCK : EIO; + return 0; + }; +} +1; +MODULE +close $module; +for my $failure (qw(open scan wait)) { + my $dir = "$tmp/$failure"; + make_path("$dir/slots"); + make_path("$dir/slots/slot.0") if $failure eq 'open'; + local $ENV{FM_LINT_HOST_SLOTS} = 1; + local $ENV{FM_TEST_SEAM} = 1; + local $ENV{FM_LINT_SLOT_LOAD} = 0; + local $ENV{FAIL_LOCK} = $failure; + my @inject = $failure eq 'open' ? () : ("-I$tmp", '-MFailLock'); + my $pid = fork(); + die "fork: $!" unless defined $pid; + if (!$pid) { + open STDERR, '>', "$dir/err" or die "$!"; + exec $^X, @inject, $gate, 'gate', "$dir/slots", 18, "$dir/wait", + '--', $^X, '-e', 'print "analysis ran\n"; exit 23'; + die "exec: $!"; + } + my $deadline = time() + 30; + while (waitpid($pid, WNOHANG) == 0) { + if (time() > $deadline) { + kill 'KILL', $pid; + waitpid($pid, 0); + die "$failure failure queued instead of running ungated\n"; + } + sleep 0.01; + } + die "$failure fallback lost the command exit status: $?\n" unless $? == (23 << 8); + open(my $err, '<', "$dir/err") or die "$!"; + local $/; + my $warning = <$err>; + die "$failure fallback omitted the warning\n" + unless $warning =~ /running without the host-wide ShellCheck bound/; +} +PL + pass "slot open and non-contention scan/wait lock failures warn and run ungated" +} + +test_slot_survives_gate_death() { + local tmp bounded=none + if fm_lint_bounds_supported; then bounded=perl; fi + tmp=$(fm_test_tmproot fm-lint-slot-inheritance) + perl - "$LINT" "$ROOT/bin/fm-lint-cache.pl" "$tmp" "$bounded" <<'PL' || fail "slot lifetime regression failed" +use strict; +use warnings; +use File::Path qw(make_path); +use POSIX qw(WNOHANG); +use Time::HiRes qw(time sleep); +my ($lint, $gate_script, $tmp, $bound_mechanism) = @ARGV; +open(my $stub, '>', "$tmp/shellcheck") or die "$!"; +print {$stub} "#!/usr/bin/env perl\n", <<'STUB'; +use Time::HiRes qw(time sleep); +$SIG{TERM} = $SIG{HUP} = $SIG{INT} = 'IGNORE'; +my $child = fork(); +die "fork: $!" unless defined $child; +if (!$child) { + my $deadline = time() + 15; + until (-e "$ENV{CASE_DIR}/release") { last if time() > $deadline; sleep 0.01; } + exit 0; +} +open(my $fh, '>', "$ENV{CASE_DIR}/started") or die "$!"; +print {$fh} "$$ $child\n"; +close $fh; +waitpid($child, 0); +STUB +close $stub; +chmod 0755, "$tmp/shellcheck"; +for my $bounded ('none', $bound_mechanism eq 'perl' ? ('perl') : ()) { + my $dir = "$tmp/$bounded"; + make_path("$dir/out"); + open(my $root, '>', "$dir/root.sh") or die "$!"; + print {$root} "#!/bin/bash\nexit 0\n"; + close $root; + open(my $manifest, '>', "$dir/manifest") or die "$!"; + print {$manifest} "0\t$dir/root.sh\n"; + close $manifest; + local %ENV = (%ENV, FM_LINT_INTERNAL => 1, FM_LINT_INTERNAL_CACHE => 'off', + FM_LINT_INTERNAL_SLOT_DIR => "$dir/slots", FM_LINT_INTERNAL_NCPU => 18, + FM_LINT_HOST_SLOTS => 1, FM_TEST_SEAM => 1, FM_LINT_SLOT_LOAD => 0, + FM_LINT_INTERNAL_BOUNDED => $bounded, FM_LINT_INTERNAL_MEMORY_KIB => 2097152, + FM_LINT_INTERNAL_ROOT_SECS => 10, FM_LINT_INTERNAL_GRACE => 1, + FM_LINT_SHELLCHECK => "$tmp/shellcheck", CASE_DIR => $dir); + my $worker = fork(); + die "fork: $!" unless defined $worker; + if (!$worker) { + open STDOUT, '>', "$dir/output" or die "$!"; + open STDERR, '>&', \*STDOUT or die "$!"; + exec '/bin/bash', $lint, '--internal-worker', "$dir/manifest", "$dir/out", 0; + die "exec: $!"; + } + my ($command, $descendant, $gate, $queued, $error); + eval { + my $deadline = time() + 5; + until (-s "$dir/started") { die "command never started\n" if time() > $deadline; sleep 0.01; } + open(my $started, '<', "$dir/started") or die "$!"; + ($command, $descendant) = split / /, <$started>; + open(my $ps, '-|', 'ps', '-axo', 'pid=,ppid=,args=') or die "$!"; + my (%parent, %args); + while (<$ps>) { + next unless /^\s*(\d+)\s+(\d+)\s+(.*)$/; + $parent{$1} = $2; $args{$1} = $3; + } + close $ps; + my $ancestor = $command; + while ($ancestor && $ancestor != $worker) { + if (($args{$ancestor} // '') =~ /\Q$dir\/slots\E/) { $gate = $ancestor; last; } + $ancestor = $parent{$ancestor}; + } + die "gate not found in command ancestry\n" unless $gate; + $queued = fork(); + die "fork: $!" unless defined $queued; + if (!$queued) { + exec $^X, $gate_script, 'gate', "$dir/slots", 18, "$dir/queued.wait", + '--', $^X, '-e', q{ + my ($admitted, $bounded, @protected) = @ARGV; + if ($bounded ne 'none') { + open(my $ps, '-|', 'ps', '-o', 'stat=', '-p', join(',', @protected)) or die "$!"; + my @live = grep { !/^\s*Z/ } <$ps>; + close $ps; + die "admission overlapped a live protected tree\n" if @live; + } + open(my $fh, '>', $admitted) or die "$!"; + }, "$dir/admitted", $bounded, $command, $descendant; + die "exec: $!"; + } + kill 'KILL', $gate; + if ($bounded eq 'none') { + sleep 0.2; + die "queued command started while protected tree survived\n" if -e "$dir/admitted"; + kill 'KILL', $command; + sleep 0.2; + die "queued command started while protected descendant survived\n" if -e "$dir/admitted"; + die "protected descendant exited before release\n" unless kill 0, $descendant; + } else { + my $deadline = time() + 5; + until (-e "$dir/admitted") { + die "watchdog cleanup did not release the slot\n" if time() > $deadline; + sleep 0.01; + } + } + }; + $error = $@; + open(my $release, '>', "$dir/release") or die "$!"; + close $release; + my $deadline = time() + 5; + if ($queued) { + while (waitpid($queued, WNOHANG) == 0) { + if (time() > $deadline) { kill 'KILL', $queued; waitpid($queued, 0); $error ||= "slot never released\n"; last; } + sleep 0.01; + } + $error ||= "queued command failed: $?\n" if $?; + $error ||= "queued command never admitted\n" unless -e "$dir/admitted"; + } + kill 'KILL', $command if $command; + kill 'KILL', $descendant if $descendant; + kill 'TERM', $worker; + waitpid($worker, 0); + if ($error) { + open(my $output, '<', "$dir/output") or die "$!"; + open(my $analysis, '<', "$dir/out/shard.0.out") or die "$!"; + local $/; + die "$bounded: $error", <$output>, <$analysis>; + } +} +PL + pass "inherited slots prevent overlap after gate death and allow admission after watchdog cleanup" +} + +test_queued_roots_use_high_resolution_timings() { + local tmp bounded=none + if fm_lint_bounds_supported; then bounded=perl; fi + tmp=$(fm_test_tmproot fm-lint-slot-clock) + perl - "$LINT" "$tmp" "$bounded" <<'PL' || fail "queued root clock regression failed" +use strict; +use warnings; +use Fcntl qw(:flock); +use File::Path qw(make_path); +use POSIX qw(WNOHANG); +use Time::HiRes qw(time sleep); +my ($lint, $tmp, $bounded) = @ARGV; +open(my $env, '>', "$tmp/bash-env") or die "$!"; +print {$env} "unset EPOCHREALTIME\n"; +close $env; +open(my $stub, '>', "$tmp/shellcheck") or die "$!"; +print {$stub} "#!/usr/bin/env perl\n", <<'STUB'; +use Time::HiRes qw(sleep); +sleep 0.08; +if ($ENV{RETRY} && grep { $_ eq '--external-sources' } @ARGV) { + print STDERR "shellcheck: Heap exhausted;\n"; + exit 251; +} +exit 0; +STUB +close $stub; +chmod 0755, "$tmp/shellcheck"; +for my $retry (0, 1) { + my $dir = "$tmp/$retry"; + make_path("$dir/out", "$dir/slots"); + open(my $slot, '>>', "$dir/slots/slot.0") or die "$!"; + flock($slot, LOCK_EX) or die "$!"; + open(my $manifest, '>', "$dir/manifest") or die "$!"; + print {$manifest} "0\t$dir/root.sh\n"; + close $manifest; + open(my $root, '>', "$dir/root.sh") or die "$!"; + print {$root} "#!/bin/bash\nexit 0\n"; + close $root; + local %ENV = (%ENV, BASH_ENV => "$tmp/bash-env", RETRY => $retry, + FM_LINT_INTERNAL => 1, FM_LINT_INTERNAL_CACHE => 'off', + FM_LINT_INTERNAL_SLOT_DIR => "$dir/slots", FM_LINT_INTERNAL_NCPU => 18, + FM_LINT_HOST_SLOTS => 1, FM_TEST_SEAM => 1, FM_LINT_SLOT_LOAD => 0, + FM_LINT_INTERNAL_BOUNDED => $bounded, FM_LINT_INTERNAL_MEMORY_KIB => 2097152, + FM_LINT_INTERNAL_ROOT_SECS => 3, FM_LINT_INTERNAL_GRACE => 1, + FM_LINT_INTERNAL_ROOTS_LOG => "$dir/roots.tsv", FM_LINT_SHELLCHECK => "$tmp/shellcheck"); + my $launched = time(); + my $worker = fork(); + die "fork: $!" unless defined $worker; + if (!$worker) { + close $slot; + open STDOUT, '>', "$dir/output" or die "$!"; + open STDERR, '>&', \*STDOUT or die "$!"; + exec '/bin/bash', $lint, '--internal-worker', "$dir/manifest", "$dir/out", 0; + die "exec: $!"; + } + my $error; + eval { + # Controller waits cover host scheduling and worker bookkeeping; the + # protected root still has its independently enforced three-second bound. + my $deadline = time() + 30; + until (-s "$dir/roots.tsv") { die "root never began\n" if time() > $deadline; sleep 0.01; } + my $began = time(); + sleep 1.5; + close $slot; + $deadline = time() + 30; + while (waitpid($worker, WNOHANG) == 0) { + die "queued root timed out\n" if time() > $deadline; + sleep 0.01; + } + die "queued root failed: $?\n" if $?; + my $finished = time(); + my $queued_ms = 0; + for my $file (glob "$dir/out/*.rss.wait") { + open(my $wait, '<', $file) or die "$!"; + $queued_ms += <$wait>; + } + my $elapsed = ($finished - $launched) * 1000 - $queued_ms; + open(my $log, '<', "$dir/roots.tsv") or die "$!"; + my @end; + while (<$log>) { @end = split /\t/ if /^end\t/; } + die "missing root result\n" unless @end; + my $expected = $retry ? 'memory-fallback' : 'ok'; + die "wrong root outcome: $end[9]\n" unless $end[9] eq $expected; + die "root timestamps fall outside the observed lifecycle\n" + unless $end[5] >= $launched * 1000 - 1 && $end[5] <= $began * 1000 + 1 + && $end[6] >= ($began + 1.5) * 1000 - 1 && $end[6] <= $finished * 1000 + 1; + die "duration $end[7]ms exceeds $elapsed ms available for analysis\n" + unless $end[7] >= 70 && $end[7] <= $elapsed + 2; + die "root wall time omitted the queue\n" unless $end[6] - $end[5] >= 1500; + }; + $error = $@; + close $slot if defined fileno($slot); + kill 'KILL', $worker; + waitpid($worker, 0); + die "retry=$retry: $error" if $error; +} +PL + pass "queued roots and memory retries record precise analysis durations without EPOCHREALTIME" +} + test_worker_trees_stop_on_signal() { local tmp fakebin fixture jobs telemetry lint_tmp pid_file out_file telemetry_file local parent_pid shellcheck_pid i parent_rc survivor @@ -2785,6 +3368,13 @@ test_rejects_direct_beads_cli_in_explicit_core_path test_ignores_ambient_shellcheck_opts test_clean_fixture_passes test_jobs_are_deterministic_and_complete +test_host_slots_bound_concurrent_runs +test_host_load_shrinks_slots_to_the_floor +test_host_load_preserves_the_cap_until_the_threshold +test_slot_pool_can_be_disabled_or_misconfigured +test_slot_file_failures_run_ungated +test_slot_survives_gate_death +test_queued_roots_use_high_resolution_timings test_worker_trees_stop_on_signal test_root_deadline_names_the_root_and_reaps_the_tree test_root_memory_limit_reports_a_named_death