From 5bcde16dfc3e8247647e9c4f1d3d72f4238d8795 Mon Sep 17 00:00:00 2001 From: Jeff Dickey <216188+jdx@users.noreply.github.com> Date: Wed, 15 Jul 2026 20:33:47 -0700 Subject: [PATCH 1/2] perf(release): add PGO builds for natively-trainable release targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports jdx/aube's three-phase rustc PGO pipeline to mise: build with -Cprofile-generate, train, rebuild with -Cprofile-use. The training workload is hermetic and offline — throwaway HOME with isolated MISE_*_DIRs exercising the startup-dominated hot paths: settings and config load, toolset resolution, hook-env (full run and the per-prompt early-exit fast path), env rendering, task listing, and exec. Enabled for the targets whose instrumented binaries can execute on their build runner: x86_64 linux gnu/musl (cross-built, trained on the amd64 host) and native macos-arm64. arm/armv7 Linux, macos-x64, and Windows keep plain serious builds. Measured on macOS arm64 against a same-source serious build (hyperfine -N, warm, isolated env): - mise version: 4.3ms -> 4.1ms - mise current: 6.9ms -> 6.3ms (1.10x) - hook-env early-exit fast path (per shell prompt): 3.6ms -> 3.3ms - binary size: 84.5MB -> 81.3MB; startup pages touched: 1182 -> 1079 Costs one extra serious-profile build per PGO target, so the linux tarball job budget goes from 45 to 80 minutes. Backing a target out is deleting its `pgo: 1` matrix line. Cross.toml now passes RUSTFLAGS through so the profile-generate/-use flags reach cargo inside cross containers, and pgo.bash bind-mounts the profile dir at the same path so -Cprofile-use resolves there. BOLT post-link optimization (Linux-only, layered on PGO) is a potential follow-up; aube's phase-4 flow is the template. Co-Authored-By: Claude Fable 5 --- .github/workflows/release.yml | 25 +++- Cross.toml | 3 + scripts/build-tarball.sh | 12 +- scripts/pgo.bash | 219 ++++++++++++++++++++++++++++++++++ 4 files changed, 256 insertions(+), 3 deletions(-) create mode 100755 scripts/pgo.bash diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9a78c6b4e0b..4b6575ff4f0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,9 @@ jobs: if: github.event_name != 'pull_request' || github.head_ref == 'release' name: build-tarball-${{matrix.name}} runs-on: namespace-profile-endev-linux-amd64-large - timeout-minutes: 45 + # PGO rows build the serious profile twice (instrument + optimize), + # so they need roughly double the old 45-minute budget. + timeout-minutes: 80 env: MINIO_AWS_ACCESS_KEY_ID: ${{ secrets.MINIO_AWS_ACCESS_KEY_ID }} MINIO_AWS_SECRET_ACCESS_KEY: ${{ secrets.MINIO_AWS_SECRET_ACCESS_KEY }} @@ -36,10 +38,16 @@ jobs: fail-fast: false matrix: include: + # pgo: the amd64 runner can execute x86_64 binaries, so those + # targets build via scripts/pgo.bash (instrument -> train -> + # rebuild with profile). arm targets can't run their + # instrumented binaries here and stay plain serious builds. - name: linux-x64 target: x86_64-unknown-linux-gnu + pgo: 1 - name: linux-x64-musl target: x86_64-unknown-linux-musl + pgo: 1 - name: linux-arm64 target: aarch64-unknown-linux-gnu - name: linux-arm64-musl @@ -54,10 +62,15 @@ jobs: uses: taiki-e/install-action@51cd0b8c0499559d9a4d75c0f5c67bec3a894ec8 # v2 with: tool: cross + - name: Install llvm-tools (PGO) + if: matrix.pgo + run: rustup component add llvm-tools - name: build-tarball ${{matrix.target}} uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4 + env: + MISE_PGO: ${{ matrix.pgo || '' }} with: - timeout_minutes: 45 + timeout_minutes: 75 max_attempts: 3 command: scripts/build-tarball.sh ${{matrix.target}} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -80,10 +93,13 @@ jobs: fail-fast: false matrix: include: + # pgo: only the native arm64 target — an x64 instrumented + # binary would train under Rosetta and skew the profile. - name: macos-x64 target: x86_64-apple-darwin - name: macos-arm64 target: aarch64-apple-darwin + pgo: 1 steps: - uses: apple-actions/import-codesign-certs@5142e029c445c10ffc7149d172e540235a065466 # v7 with: @@ -92,8 +108,13 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - name: Setup Rust target run: rustup target add ${{matrix.target}} + - name: Install llvm-tools (PGO) + if: matrix.pgo + run: rustup component add llvm-tools - name: build-tarball ${{matrix.target}} uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4 + env: + MISE_PGO: ${{ matrix.pgo || '' }} with: timeout_minutes: 90 max_attempts: 3 diff --git a/Cross.toml b/Cross.toml index c9b12ddfc3d..5b835101064 100644 --- a/Cross.toml +++ b/Cross.toml @@ -8,6 +8,9 @@ passthrough = [ "CARGO_TERM_COLOR", "AWS_LC_SYS_EXTERNAL_BINDGEN", "BINDGEN_EXTRA_CLANG_ARGS", + # scripts/pgo.bash passes -Cprofile-generate / -Cprofile-use through + # RUSTFLAGS; without this the flags never reach cargo in the container. + "RUSTFLAGS", ] # Explicitly use 0.2.5 images for GNU targets (Ubuntu 16.04, glibc 2.23) diff --git a/scripts/build-tarball.sh b/scripts/build-tarball.sh index 46c22cc1718..ab9b1be8fe1 100755 --- a/scripts/build-tarball.sh +++ b/scripts/build-tarball.sh @@ -83,7 +83,17 @@ if [[ $os == "macos" ]]; then export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET:-12.0} fi -if [[ $os == "linux" ]]; then +if [[ -n "${MISE_PGO:-}" ]]; then + # Profile-guided optimization: instrument, train against the hermetic + # offline workload in scripts/pgo.bash, rebuild with the profile. + # Only valid for targets whose binaries can execute on this machine. + build_tool=cargo + if [[ $os == "linux" ]]; then + build_tool=cross + fi + MISE_PGO_BUILD_TOOL="$build_tool" MISE_PGO_TARGET="$RUST_TRIPLE" \ + bash scripts/pgo.bash --no-default-features --features "$features" +elif [[ $os == "linux" ]]; then cross build --profile=serious --target "$RUST_TRIPLE" --no-default-features --features "$features" else cargo build --profile=serious --target "$RUST_TRIPLE" --no-default-features --features "$features" diff --git a/scripts/pgo.bash b/scripts/pgo.bash new file mode 100755 index 00000000000..d1c2cfcf5e3 --- /dev/null +++ b/scripts/pgo.bash @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# Profile-Guided Optimization build for mise. +# +# Three-phase rustc PGO flow, adapted from jdx/aube's benchmarks/pgo.bash: +# +# 1. Build mise with -Cprofile-generate (instrumented binary). +# 2. Train against a hermetic, offline workload covering the startup +# hot paths: settings/config load, toolset resolution, hook-env +# (both the full run and the per-prompt early-exit fast path), +# env rendering, task listing, and exec. No network, no registry — +# everything runs in a throwaway HOME with isolated MISE_*_DIR s. +# 3. Merge .profraw via the rustup toolchain's llvm-profdata, +# recompile with -Cprofile-use. +# +# The startup path dominates every mise invocation (shell prompts via +# hook-env, shims, version checks), so the training workload is biased +# toward short-lived invocations rather than long installs. +# +# Env hooks (mirroring aube's CI contract): +# MISE_PGO_PROFILE= cargo profile for both phases +# (default: serious, matching release +# tarballs). +# MISE_PGO_TARGET= cross-compilation target (default: +# host). The instrumented binary must be +# runnable on THIS machine for training — +# same-arch targets only. +# MISE_PGO_BUILD_TOOL= `cargo` (default) or `cross`. cross is +# used in CI for Linux targets so the +# binary keeps cross's older glibc +# baseline; the cross-built instrumented +# binary still runs on the host for +# training (older glibc is +# forward-compatible). +# MISE_PGO_SKIP_FINAL_BUILD=1 stop after merging .profraw, for CI +# setups that delegate the final build to +# a separate step picking up RUSTFLAGS. +# +# Additional arguments are passed through to both cargo invocations — +# scripts/build-tarball.sh uses this for --no-default-features +# --features "...". +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$REPO_ROOT" + +PGO_DATA_DIR="$REPO_ROOT/target/pgo-data" +PGO_PROFRAW_DIR="$PGO_DATA_DIR/profraw" +PGO_MERGED="$PGO_DATA_DIR/merged.profdata" + +PGO_PROFILE="${MISE_PGO_PROFILE:-serious}" +PGO_TARGET="${MISE_PGO_TARGET:-}" +PGO_BUILD_TOOL="${MISE_PGO_BUILD_TOOL:-cargo}" + +# target_arg stays unquoted at expansion sites: empty string disappears, +# "--target=foo" expands to one arg. Avoids bash 3.2 (macOS) array+set -u +# unbound-variable issues with "${arr[@]}". +target_arg="" +target_dir_part="" +if [ -n "$PGO_TARGET" ]; then + target_arg="--target=$PGO_TARGET" + target_dir_part="$PGO_TARGET/" +fi + +RUSTC_HOST="$(rustc -vV | sed -n 's|^host: ||p')" +RUSTC_SYSROOT="$(rustc --print sysroot)" +LLVM_PROFDATA="$RUSTC_SYSROOT/lib/rustlib/$RUSTC_HOST/bin/llvm-profdata" +if [ ! -x "$LLVM_PROFDATA" ]; then + echo "ERROR: llvm-profdata not found at $LLVM_PROFDATA" >&2 + echo " Install with: rustup component add llvm-tools" >&2 + exit 1 +fi + +mkdir -p "$PGO_PROFRAW_DIR" +rm -f "$PGO_PROFRAW_DIR"/*.profraw "$PGO_MERGED" + +# With MISE_PGO_BUILD_TOOL=cross, rustc runs inside a container that +# mounts the project at a container path, so when phase 3 reads +# -Cprofile-use= from RUSTFLAGS the file is invisible. +# Bind-mount PGO_DATA_DIR at the same host path inside the container so +# the RUSTFLAGS value resolves. Harmless for phase 1. +if [ "$PGO_BUILD_TOOL" = "cross" ]; then + export CROSS_CONTAINER_OPTS="${CROSS_CONTAINER_OPTS:-} -v $PGO_DATA_DIR:$PGO_DATA_DIR:rw" +fi + +# ---------- Phase 1: instrumented build ---------- +echo ">>> [1/3] Building instrumented binary ($PGO_BUILD_TOOL, profile=$PGO_PROFILE${PGO_TARGET:+, target=$PGO_TARGET})" +# shellcheck disable=SC2086 # intentional word-splitting on $target_arg +RUSTFLAGS="${RUSTFLAGS:-} -Cprofile-generate=$PGO_PROFRAW_DIR" \ + "$PGO_BUILD_TOOL" build --profile="$PGO_PROFILE" $target_arg --bin mise "$@" + +INSTRUMENTED_BIN="$REPO_ROOT/target/${target_dir_part}${PGO_PROFILE}/mise" +if [ ! -x "$INSTRUMENTED_BIN" ]; then + echo "ERROR: instrumented binary missing at $INSTRUMENTED_BIN" >&2 + exit 1 +fi + +# ---------- Phase 2: training ---------- +echo ">>> [2/3] Training against hermetic offline workload" + +train_dir="$(mktemp -d "${TMPDIR:-/tmp}/mise-pgo-train.XXXXXX")" +cleanup() { + rm -rf "$train_dir" +} +trap cleanup EXIT + +# Force the instrumented binary to write profraw to a path we control, +# regardless of what -Cprofile-generate= baked in at compile time +# (with cross, the compile-time path is a container path that doesn't +# resolve on the host). %m disambiguates per module; %p per process. +export LLVM_PROFILE_FILE="$PGO_PROFRAW_DIR/mise-%m-%p.profraw" + +# Hermetic project fixture: env vars (incl. a template), pinned tools +# (never resolved against the network), and a trivial task. MISE_OFFLINE +# is belt-and-braces — nothing in the workload should hit the network. +mkdir -p "$train_dir/home" "$train_dir/proj/subdir" +cat >"$train_dir/proj/mise.toml" <<'EOF' +[env] +TRAIN_FOO = "bar" +TRAIN_TEMPLATED = "{{ env.HOME }}/x" + +[tools] +node = "22.17.0" + +[tasks.hello] +run = "true" +EOF +echo "node 22.17.0" >"$train_dir/proj/subdir/.tool-versions" + +# shellcheck disable=SC2016 # the single-quoted script takes $bin/$proj as args +train() { + # Fresh isolated state per training pass so cold paths (state dirs, + # trust, caches) and warm paths both land in the profile. + local pass=$1 + local root="$train_dir/state.$pass" + mkdir -p "$root" + env -i PATH="$PATH" HOME="$train_dir/home" TMPDIR="${TMPDIR:-/tmp}" \ + LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \ + MISE_DATA_DIR="$root/data" MISE_CACHE_DIR="$root/cache" \ + MISE_CONFIG_DIR="$root/config" MISE_STATE_DIR="$root/state" \ + MISE_GLOBAL_CONFIG_FILE="$root/config/config.toml" \ + MISE_OFFLINE=1 MISE_HIDE_UPDATE_WARNING=1 \ + bash -c ' + set -e + bin=$1; proj=$2 + cd "$proj" + "$bin" trust --all >/dev/null 2>&1 || true + "$bin" version >/dev/null + "$bin" current >/dev/null 2>&1 || true + "$bin" ls >/dev/null 2>&1 || true + "$bin" env -s zsh >/dev/null 2>&1 || true + "$bin" env -s bash >/dev/null 2>&1 || true + "$bin" settings >/dev/null 2>&1 || true + "$bin" tasks ls >/dev/null 2>&1 || true + "$bin" install >/dev/null 2>&1 || true + "$bin" exec -- true >/dev/null 2>&1 || true + # hook-env: full run, then eval the session so the + # remaining runs take the per-prompt early-exit fast + # path — the single hottest path in real usage. + eval "$("$bin" hook-env -s bash 2>/dev/null)" || true + for _ in 1 2 3 4 5; do + "$bin" hook-env -s bash >/dev/null 2>&1 || true + done + # subdir with .tool-versions: idiomatic-file parsing + + # config hierarchy walk + cd subdir + "$bin" current >/dev/null 2>&1 || true + "$bin" hook-env -s bash >/dev/null 2>&1 || true + ' -- "$INSTRUMENTED_BIN" "$train_dir/proj" +} + +for pass in 1 2 3; do + echo " train: pass $pass" + train "$pass" +done +unset LLVM_PROFILE_FILE + +# Sanity check: confirm training actually wrote profraw. On cross-built +# targets a bad LLVM_PROFILE_FILE path silently produces zero files and +# llvm-profdata merges nothing. Fail loudly here instead. +profraw_count=$(find "$PGO_PROFRAW_DIR" -maxdepth 1 -name '*.profraw' -type f | wc -l | tr -d ' ') +if [ "$profraw_count" -eq 0 ]; then + echo "ERROR: no .profraw files written to $PGO_PROFRAW_DIR after training" >&2 + echo " Training ran but the instrumented binary did not record profile data." >&2 + exit 1 +fi +echo ">>> $profraw_count .profraw files collected" + +# ---------- Phase 3a: merge ---------- +echo ">>> [3/3] Merging profile data" +"$LLVM_PROFDATA" merge -o "$PGO_MERGED" "$PGO_PROFRAW_DIR" + +# Defense in depth: a version mismatch between the rustc that +# instrumented and the llvm-profdata that merges can be a 0-exit silent +# no-op; phase 3b would then fail with an opaque missing-file error. +if [ ! -f "$PGO_MERGED" ]; then + echo "ERROR: $PGO_MERGED was not produced by llvm-profdata merge" >&2 + exit 1 +fi +echo ">>> merged profile written: $(stat -c %s "$PGO_MERGED" 2>/dev/null || stat -f %z "$PGO_MERGED") bytes" + +if [ -n "${MISE_PGO_SKIP_FINAL_BUILD:-}" ]; then + echo ">>> Skipping final optimized build (MISE_PGO_SKIP_FINAL_BUILD=1)" + echo ">>> Profile ready at: $PGO_MERGED" + exit 0 +fi + +# ---------- Phase 3b: optimize ---------- +echo ">>> Rebuilding with -Cprofile-use" +# -Cllvm-args=-pgo-warn-missing-function=false: coverage gaps are +# expected (training can't exercise every path) and LLVM otherwise logs +# a note per uncovered symbol. Uncovered functions compile normally, +# just without PGO data. +# shellcheck disable=SC2086 # intentional word-splitting on $target_arg +RUSTFLAGS="${RUSTFLAGS:-} -Cprofile-use=$PGO_MERGED -Cllvm-args=-pgo-warn-missing-function=false" \ + "$PGO_BUILD_TOOL" build --profile="$PGO_PROFILE" $target_arg --bin mise "$@" + +echo ">>> PGO build complete: $INSTRUMENTED_BIN" +ls -lh "$INSTRUMENTED_BIN" From 005258e59e4053df628c009b8ba3c1d2ed520a70 Mon Sep 17 00:00:00 2001 From: Jeff Dickey <216188+jdx@users.noreply.github.com> Date: Wed, 15 Jul 2026 20:51:15 -0700 Subject: [PATCH 2/2] perf(release): scope RUSTFLAGS passthrough to PGO targets Review feedback: global [build.env] passthrough would silently inject host RUSTFLAGS into every cross build, including non-PGO arm targets. Scope it to the two x86_64 targets that PGO actually builds through cross. Also alias the phase-3b output as PGO_FINAL_BIN so the success log doesn't misleadingly reference the instrumented binary. Co-Authored-By: Claude Fable 5 --- Cross.toml | 13 ++++++++++--- scripts/pgo.bash | 8 ++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Cross.toml b/Cross.toml index 5b835101064..b4eab159e3a 100644 --- a/Cross.toml +++ b/Cross.toml @@ -8,15 +8,19 @@ passthrough = [ "CARGO_TERM_COLOR", "AWS_LC_SYS_EXTERNAL_BINDGEN", "BINDGEN_EXTRA_CLANG_ARGS", - # scripts/pgo.bash passes -Cprofile-generate / -Cprofile-use through - # RUSTFLAGS; without this the flags never reach cargo in the container. - "RUSTFLAGS", ] # Explicitly use 0.2.5 images for GNU targets (Ubuntu 16.04, glibc 2.23) +# The x86_64 targets are the PGO targets (see scripts/pgo.bash): RUSTFLAGS +# passthrough is scoped to them so -Cprofile-generate/-Cprofile-use reach +# cargo inside the container without silently injecting host RUSTFLAGS +# into every other cross build. [target.x86_64-unknown-linux-gnu] image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:0.2.5" +[target.x86_64-unknown-linux-gnu.env] +passthrough = ["RUSTFLAGS"] + [target.aarch64-unknown-linux-gnu] # Keep this pinned so the release binary does not inherit a newer glibc # floor when cross-rs updates its floating main image. @@ -40,6 +44,9 @@ pre-build = [ [target.x86_64-unknown-linux-musl] image = "ghcr.io/cross-rs/x86_64-unknown-linux-musl:0.2.5" +[target.x86_64-unknown-linux-musl.env] +passthrough = ["RUSTFLAGS"] + [target.aarch64-unknown-linux-musl] image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:0.2.5" diff --git a/scripts/pgo.bash b/scripts/pgo.bash index d1c2cfcf5e3..c97db5452cc 100755 --- a/scripts/pgo.bash +++ b/scripts/pgo.bash @@ -215,5 +215,9 @@ echo ">>> Rebuilding with -Cprofile-use" RUSTFLAGS="${RUSTFLAGS:-} -Cprofile-use=$PGO_MERGED -Cllvm-args=-pgo-warn-missing-function=false" \ "$PGO_BUILD_TOOL" build --profile="$PGO_PROFILE" $target_arg --bin mise "$@" -echo ">>> PGO build complete: $INSTRUMENTED_BIN" -ls -lh "$INSTRUMENTED_BIN" +# Phase 3b wrote to the same path as phase 1, so the file at +# $INSTRUMENTED_BIN is now the PGO-optimized build, not the instrumented +# one. Alias for clarity. +PGO_FINAL_BIN="$INSTRUMENTED_BIN" +echo ">>> PGO build complete: $PGO_FINAL_BIN" +ls -lh "$PGO_FINAL_BIN"