Skip to content

fix(bin): cut lint partition wall time with per-root source views and up to four workers #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
RajeshRajendiran merged 3 commits into from
Sep 21, 2026
Merged

fix(bin): cut lint partition wall time with per-root source views and up to four workers #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2abbba9
fix(bin): cut lint partition wall time with per-root source views and…
RajeshRajendiran Sep 21, 2026
eea62e3
no-mistakes(review): Dedupe only top-level source sites in lint view
RajeshRajendiran Sep 21, 2026
6b688cc
no-mistakes(document): Align fm-lint header with top-level-only dedupe
RajeshRajendiran Sep 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 143 additions & 46 deletions bin/fm-lint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,14 @@
# invocations in the core bin/ and bin/backends/ scripts so every configured
# backlog backend follows the same tasks-axi lifecycle path.
#
# Lint defaults to two bounded workers over two stable logical shards.
# Diagnostics replay in stable shard/root order. FM_LINT_JOBS=1 changes
# concurrency, not diagnostics or exit selection.
# Lint runs four stable logical shards on one to four bounded workers, by
# default as many as the machine has CPUs (at least two). Diagnostics replay in
# stable shard/root order. FM_LINT_JOBS=1 changes concurrency, not diagnostics
# or exit selection.
# Source-following roots are each analyzed against a private view of the
# repository that includes every library once (see fm_lint_build_view), because
# ShellCheck otherwise re-analyzes a library at every include site; only later
# top-level sites are collapsed, so scope-dependent sites still follow the library.
# --partition 1of2/2of2 splits the entire canonical inventory across
# two CI runners, each with those same bounded workers. Partitions are complete,
# disjoint, and byte-weight balanced; --list-files exposes their actual roots.
Expand All @@ -58,7 +63,7 @@
# fm-lint.sh lint the context-selected file set (see above)
# fm-lint.sh --fast [path]... local lint with extended analysis disabled
# fm-lint.sh <path>... lint explicit roots with the same config
# fm-lint.sh --jobs <1|2> [path]... override bounded worker count
# fm-lint.sh --jobs <1-4> [path]... override bounded worker count
# fm-lint.sh --partition <1of2|2of2> lint one full-rigor canonical CI partition
# fm-lint.sh --telemetry <path> ... write a quiet metrics snapshot
# fm-lint.sh --required-version print the ShellCheck pin
Expand All @@ -84,8 +89,82 @@ fm_lint_worker_stop() {
FM_LINT_WORKER_SHELLCHECK_PID=
}

# ShellCheck re-parses and re-analyzes a sourced file at every include site, so
# a library reached through several paths costs its full size once per path and
# a root's cost tracks its expanded include tree rather than the unique graph.
# The view builder gives one root a private mirror of the repository in which
# each library is included by directive once, at its first top-level site in
# ShellCheck's traversal order, and later top-level sites in non-test files
# become source=/dev/null. Indented sites (functions, subshells, conditionals)
# are scope-dependent, so they are neither deduped nor counted as a first site.
# Every function and variable stays visible where it first appeared, so the
# findings are identical while the work drops toward the unique graph. Test
# files keep every site because they source inside subshells, where a second
# include changes the findings. FM_LINT_DEDUPE_SOURCES=0 disables the view.
FM_LINT_VIEW_PERL=$(cat <<'PERL'
use strict;
use warnings;
use File::Basename qw(dirname);
use File::Path qw(make_path);
use Cwd qw(getcwd);
my ($root, $dest) = @ARGV;
my $cwd = getcwd();
my %seen = ($root => 1);
my %written;
sub visit {
my ($path) = @_;
return if $written{$path}++;
my @lines;
if (open(my $in, '<', $path)) { @lines = <$in>; close $in; }
for (my $i = 0; $i <= $#lines; $i++) {
next unless $lines[$i] =~ m{^#\s*shellcheck\s+source=(\S+)\s*$};
my $target = $1;
next if $target eq '/dev/null' or !-f $target;
my $j = $i + 1;
$j++ while $j <= $#lines and $lines[$j] =~ /^\s*(#.*)?$/;
next unless $j <= $#lines and $lines[$j] =~ /^(?:\.|source)\s/;
if ($seen{$target}) {
# Test files stay per-site: they source inside subshells whose scoping
# decides findings such as SC2031.
$lines[$i] =~ s{source=\S+}{source=/dev/null} unless $path =~ m{^tests/};
next;
}
$seen{$target} = 1;
visit($target);
}
my $out = "$dest/$path";
make_path(dirname($out));
open(my $o, '>', $out) or die "$out: $!";
print $o @lines;
close $o;
}
visit($root);
# Mirror the rest of the tree by symlink so sources reached without a
# directive still resolve exactly as in the real tree.
sub mirror {
my ($rel) = @_;
my $src = $rel eq '' ? $cwd : "$cwd/$rel";
my $dst = $rel eq '' ? $dest : "$dest/$rel";
opendir(my $d, $src) or return;
my @entries = grep { $_ ne '.' && $_ ne '..' && $_ ne '.git' } readdir $d;
closedir $d;
for my $e (@entries) {
my $r = $rel eq '' ? $e : "$rel/$e";
if (-d "$dst/$e" && !-l "$dst/$e") { mirror($r); next; }
next if -e "$dst/$e" || -l "$dst/$e";
symlink("$src/$e", "$dst/$e");
}
}
mirror('');
PERL
)

fm_lint_build_view() { # <root-path> <view-dir>
perl -e "$FM_LINT_VIEW_PERL" -- "$1" "$2"
}

fm_lint_worker() { # <manifest> <output-dir> <shard-index>
local manifest=$1 output_dir=$2 shard_index=$3 tab index path output invocation_rc rc=0
local manifest=$1 output_dir=$2 shard_index=$3 tab index path output invocation_rc view rc=0
local -a roots shellcheck_args
roots=()
tab=$(printf '\t')
Expand All @@ -109,23 +188,33 @@ fm_lint_worker() { # <manifest> <output-dir> <shard-index>
shellcheck_args+=(--extended-analysis=false)
fi
: > "$output.out"
if [ "${FM_LINT_INTERNAL_FOLLOW_SOURCES:-1}" -eq 1 ]; then
"$FM_LINT_SHELLCHECK" "${shellcheck_args[@]}" -- "${roots[@]}" >> "$output.out" 2>&1 &
for path in "${roots[@]}"; do
invocation_rc=0
view=
if [ "${FM_LINT_INTERNAL_FOLLOW_SOURCES:-1}" -eq 1 ] && [ "${FM_LINT_DEDUPE_SOURCES:-1}" -eq 1 ]; then
case "$path" in
/*|*..*) ;;
*)
if [ -f "$path" ]; then
view="$output_dir/view.$shard_index"
rm -rf "$view"
fm_lint_build_view "$path" "$view" || { rm -rf "$view"; view=; }
fi
;;
esac
fi
(
[ -z "$view" ] || cd "$view" || exit 2
exec "$FM_LINT_SHELLCHECK" "${shellcheck_args[@]}" -- "$path"
) >> "$output.out" 2>&1 &
FM_LINT_WORKER_SHELLCHECK_PID=$!
wait "$FM_LINT_WORKER_SHELLCHECK_PID" || rc=$?
wait "$FM_LINT_WORKER_SHELLCHECK_PID" || invocation_rc=$?
FM_LINT_WORKER_SHELLCHECK_PID=
else
for path in "${roots[@]}"; do
invocation_rc=0
"$FM_LINT_SHELLCHECK" "${shellcheck_args[@]}" -- "$path" >> "$output.out" 2>&1 &
FM_LINT_WORKER_SHELLCHECK_PID=$!
wait "$FM_LINT_WORKER_SHELLCHECK_PID" || invocation_rc=$?
FM_LINT_WORKER_SHELLCHECK_PID=
if [ "$rc" -eq 0 ] && [ "$invocation_rc" -ne 0 ]; then
rc=$invocation_rc
fi
done
fi
[ -z "$view" ] || rm -rf "$view"
if [ "$rc" -eq 0 ] && [ "$invocation_rc" -ne 0 ]; then
rc=$invocation_rc
fi
done
trap - HUP INT TERM
else
: > "$output.out"
Expand Down Expand Up @@ -398,7 +487,16 @@ fm_lint_run_backend_purity() {
}
}

JOBS=${FM_LINT_JOBS:-2}
# Default concurrency follows the machine but stays inside the 1-4 bound: one
# ShellCheck root holds over a gigabyte, so more workers than cores or four buys
# nothing. An unknown CPU count keeps the historical two.
JOBS=${FM_LINT_JOBS:-}
if [ -z "$JOBS" ]; then
JOBS=$(getconf _NPROCESSORS_ONLN 2>/dev/null || printf '2')
case "$JOBS" in ''|*[!0-9]*) JOBS=2 ;; esac
[ "$JOBS" -ge 2 ] || JOBS=2
[ "$JOBS" -le 4 ] || JOBS=4
fi
TELEMETRY=${FM_LINT_TELEMETRY:-}
FAST=0
ANALYSIS_MODE=full
Expand All @@ -408,7 +506,7 @@ LIST_FILES=0
while [ "$#" -gt 0 ]; do
case "$1" in
--jobs)
[ "$#" -ge 2 ] || { printf 'fm-lint.sh: --jobs requires 1 or 2.\n' >&2; exit 2; }
[ "$#" -ge 2 ] || { printf 'fm-lint.sh: --jobs requires 1 to 4.\n' >&2; exit 2; }
JOBS=$2
shift 2
;;
Expand Down Expand Up @@ -458,8 +556,8 @@ while [ "$#" -gt 0 ]; do
done

case "$JOBS" in
1|2) ;;
*) printf 'fm-lint.sh: jobs must be 1 or 2, got %s.\n' "$JOBS" >&2; exit 2 ;;
1|2|3|4) ;;
*) printf 'fm-lint.sh: jobs must be between 1 and 4, got %s.\n' "$JOBS" >&2; exit 2 ;;
esac

case "$PARTITION" in
Expand Down Expand Up @@ -667,7 +765,7 @@ trap 'exit 143' TERM
WEIGHTS="$TMP_ROOT/weights"
OUTPUT_DIR="$TMP_ROOT/output"
mkdir -p "$OUTPUT_DIR"
SHARD_COUNT=2
SHARD_COUNT=4
worker=0
while [ "$worker" -lt "$SHARD_COUNT" ]; do
: > "$TMP_ROOT/manifest.$worker"
Expand All @@ -676,16 +774,18 @@ done

fm_lint_root_weights > "$WEIGHTS" || exit $?

# Largest-first deterministic greedy assignment keeps the two bounded workers
# Largest-first deterministic greedy assignment keeps the bounded workers
# balanced without affecting replay order. Direct bytes are a stable portable
# proxy after the expensive dynamic adapter source fan-out is cut.
WORKER_LOADS=(0 0)
WORKER_LOADS=(0 0 0 0)
LC_ALL=C sort -t "$TAB" -k1,1nr -k2,2n "$WEIGHTS" > "$WEIGHTS.sorted"
while IFS="$TAB" read -r weight index path; do
worker=0
if [ "${WORKER_LOADS[1]}" -lt "${WORKER_LOADS[0]}" ]; then
worker=1
fi
for candidate in 1 2 3; do
if [ "${WORKER_LOADS[candidate]}" -lt "${WORKER_LOADS[worker]}" ]; then
worker=$candidate
fi
done
printf '%s\t%s\n' "$index" "$path" >> "$TMP_ROOT/manifest.$worker"
WORKER_LOADS[worker]=$((WORKER_LOADS[worker] + weight))
done < "$WEIGHTS.sorted"
Expand Down Expand Up @@ -773,23 +873,18 @@ fm_lint_wait_workers() {
done
}

if [ "$JOBS" -eq 1 ]; then
worker=0
while [ "$worker" -lt "$SHARD_COUNT" ]; do
fm_lint_start_worker "$worker"
fm_lint_wait_workers
worker=$((worker + 1))
done
else
worker=0
while [ "$worker" -lt "$SHARD_COUNT" ]; do
fm_lint_start_worker "$worker"
worker=$((worker + 1))
done
fm_lint_wait_workers
fi
worker=0
while [ "$worker" -lt "$SHARD_COUNT" ]; do
if [ "${#ACTIVE_PIDS[@]}" -ge "$JOBS" ]; then
wait "${ACTIVE_PIDS[0]}" 2>/dev/null || true
ACTIVE_PIDS=("${ACTIVE_PIDS[@]:1}")
fi
fm_lint_start_worker "$worker"
worker=$((worker + 1))
done
fm_lint_wait_workers

# Replay both stable shards in deterministic order and select the first nonzero
# Replay every stable shard in deterministic order and select the first nonzero
# shard status. ShellCheck processes every root in a shard after earlier findings.
overall_rc=0
worker=0
Expand Down Expand Up @@ -901,6 +996,8 @@ EOF
printf 'source_target_count\t%s\n' "$source_targets"
printf 'shard_1_weight_bytes\t%s\n' "${WORKER_LOADS[0]}"
printf 'shard_2_weight_bytes\t%s\n' "${WORKER_LOADS[1]:-0}"
printf 'shard_3_weight_bytes\t%s\n' "${WORKER_LOADS[2]:-0}"
printf 'shard_4_weight_bytes\t%s\n' "${WORKER_LOADS[3]:-0}"
printf 'wall_seconds\t%s\n' "$((TELEMETRY_END_EPOCH - TELEMETRY_START_EPOCH))"
printf 'worker_wall_sum_seconds\t%s\n' "$timing_worker_wall"
printf 'max_worker_wall_seconds\t%s\n' "$max_worker_wall"
Expand Down
2 changes: 1 addition & 1 deletion docs/fm-test-portable-shards.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ Portable shards, each portable serial shard, and the Herdr lane upload runner-ge

## Lint partitions and end-to-end latency

`bin/fm-lint.sh` owns two canonical CI partitions, each running the same full source-aware ShellCheck analysis with two bounded workers, pinned versions, workflow validation, and backend-purity checks.
`bin/fm-lint.sh` owns two canonical CI partitions, each running the same full source-aware ShellCheck analysis with up to four bounded workers, analyzing each root against a private view that includes every library once, pinned versions, workflow validation, and backend-purity checks.
Its `--list-files` interface exposes partition membership; `tests/fm-lint.test.sh` verifies complete/disjoint executed roots and unchanged analysis flags.
The workflow uploads each partition's quiet telemetry to distinguish analysis cost, memory use, and host contention.
No fast mode, path skips, reduced checks, or paid runner provisioning is part of this layout.
Expand Down
60 changes: 59 additions & 1 deletion tests/fm-lint.test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1230,8 +1230,11 @@ SH
assert_contains "$out_fail_1" "SC1007" "the first failing root diagnostic was lost"
assert_contains "$out_fail_1" "SC2086" "the later failing root diagnostic was lost"
rc_bad_jobs=0
FM_LINT_JOBS=3 "$LINT" "$good" >/dev/null 2>&1 || rc_bad_jobs=$?
FM_LINT_JOBS=5 "$LINT" "$good" >/dev/null 2>&1 || rc_bad_jobs=$?
[ "$rc_bad_jobs" -eq 2 ] || fail "the lint owner must reject unbounded worker counts"
local out_fail_4
out_fail_4=$(FM_LINT_JOBS=4 "$LINT" "$bad_a" "$bad_b" 2>&1) || true
[ "$out_fail_4" = "$out_fail_1" ] || fail "jobs=4 diagnostics differ from jobs=1"

telemetry_out=$(FM_LINT_JOBS=2 FM_LINT_TELEMETRY="$telemetry" "$LINT" "$good" 2>&1) \
|| fail "telemetry-enabled clean lint failed"
Expand All @@ -1257,6 +1260,60 @@ SH
pass "jobs=1 and jobs=2 preserve deterministic diagnostics, failures, cleanup bounds, and quiet telemetry"
}

test_source_view_keeps_cross_file_findings_and_follows_each_library_once() {
if ! pinned_ready; then
pass "SKIP (ShellCheck $REQUIRED not resolved): source view check"
return
fi
local tmp repo fakebin log out_view out_full
tmp=$(fm_test_tmproot fm-lint-view)
repo="$tmp/repo"
fakebin="$tmp/fakebin"
log="$tmp/shellcheck.log"
mkdir -p "$repo/bin" "$fakebin"
cp "$LINT" "$repo/bin/fm-lint.sh"
cat > "$repo/bin/lib-a.sh" <<'SH'
#!/usr/bin/env bash
helper_a() {
printf '%s\n' "$1"
}
SH
cat > "$repo/bin/lib-b.sh" <<'SH'
#!/usr/bin/env bash
# shellcheck source=bin/lib-a.sh
. bin/lib-a.sh
helper_b() {
printf 'b\n'
}
SH
# helper_a is called without arguments, so SC2119 exists only when the
# sourced definition is followed; lib-a is reached through both paths.
cat > "$repo/bin/root.sh" <<'SH'
#!/usr/bin/env bash
# shellcheck source=bin/lib-a.sh
. bin/lib-a.sh
# shellcheck source=bin/lib-b.sh
. bin/lib-b.sh
helper_a
helper_b
SH
cat > "$fakebin/shellcheck" <<SH
#!/usr/bin/env bash
[ "\$1" != --norc ] || pwd -P >> '$log'
exec '$(command -v shellcheck)' "\$@"
SH
chmod +x "$fakebin/shellcheck"
out_view=$(cd "$repo" && PATH="$fakebin:$PATH" FM_LINT_JOBS=1 bin/fm-lint.sh bin/root.sh 2>&1) || true
assert_contains "$out_view" "SC2119" "the source view lost a finding only cross-file analysis can produce"
[ -s "$log" ] || fail "the followed root was never analyzed"
! grep -qx "$(cd "$repo" && pwd -P)" "$log" || fail "the followed root was not analyzed against its own source view"
: > "$log"
out_full=$(cd "$repo" && PATH="$fakebin:$PATH" FM_LINT_JOBS=1 FM_LINT_DEDUPE_SOURCES=0 bin/fm-lint.sh bin/root.sh 2>&1) || true
[ "$out_view" = "$out_full" ] || fail "the source view changed the diagnostics against following every site"
grep -qx "$(cd "$repo" && pwd -P)" "$log" || fail "disabling the source view must analyze in place"
pass "the source view preserves cross-file findings and matches following every site"
}

test_worker_trees_stop_on_signal() {
local tmp fakebin fixture jobs telemetry lint_tmp pid_file out_file telemetry_file
local parent_pid shellcheck_pid i parent_rc survivor
Expand Down Expand Up @@ -1426,6 +1483,7 @@ test_rejects_direct_beads_cli_in_explicit_core_path
test_ignores_ambient_shellcheck_opts
test_clean_fixture_passes
test_jobs_are_deterministic_and_complete
test_source_view_keeps_cross_file_findings_and_follows_each_library_once
test_worker_trees_stop_on_signal
test_seeded_module_boundary_parity
test_changed_mode_lints_only_the_changed_file
Expand Down
Loading