test: added perf test

.github/workflows/hyperfine.yml

@@ -4,6 +4,9 @@ on:
     branches: ["main"]
   pull_request:
     branches: ["main"]
+    paths:
+      - ".github/workflows/hyperfine.yml"
+      - "Cargo.toml"
   workflow_dispatch:
 
 concurrency:
@@ -22,7 +25,6 @@ jobs:
   benchmark:
     runs-on: ubuntu-latest
     timeout-minutes: 20
-    if: github.head_ref == 'release' || github.head_ref == 'hyperfine'
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
         with:
@@ -56,28 +58,40 @@ jobs:
             ~/.cache/mise
       - run: mise x wait-for-gh-rate-limit -- wait-for-gh-rate-limit
       - run: mise install
+      - run: mise run test:perf
+        env:
+          NUM_TOOLS: 200
+          NUM_TASKS: 2000
+          MISE_ALT: mise-${{ steps.versions.outputs.release }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: flamegraphs
+          path: flamegraphs
       - run: |
           CMDS=(
             "x -- echo"
             "env"
             "hook-env"
             "ls"
           )
-          echo "# Hyperfine Performance" >> comment.md
+          echo "## Hyperfine Performance" >> comment.md
           for cmd in "${CMDS[@]}"; do
-             #mise x hyperfine -- hyperfine -N -w 10 -r 500 --export-markdown out.md --reference "mise-main $cmd" "mise-${{ steps.versions.outputs.release }} $cmd" "mise $cmd"
-             mise x hyperfine -- hyperfine -N -w 10 -r 500 --export-markdown out.md --reference "mise-${{ steps.versions.outputs.release }} $cmd" "mise $cmd"
+            if [ -n "${MISE_ALT:-}" ]; then
+              mise x hyperfine -- hyperfine -N -w 10 -r 500 --export-markdown out.md --reference "$MISE_ALT $cmd" "mise $cmd"
+            else
+              mise x hyperfine -- hyperfine -N -w 10 -r 500 --export-markdown out.md --reference "mise-${{ steps.versions.outputs.release }} $cmd" "mise $cmd"
+            fi
             echo "### \`mise $cmd\`" >> comment.md
             cat out.md >> comment.md
           done
-          cat comment.md >> "$GITHUB_STEP_SUMMARY"
         env:
           SHELL: zsh
+      - run: cp comment.md "$GITHUB_STEP_SUMMARY"
+        if: always() && github.event_name == 'pull_request'
       - name: Comment on PR
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3
-        if: github.event_name == 'pull_request'
+        if: always() && github.event_name == 'pull_request'
         continue-on-error: true
-        #if: "startsWith(github.event.pull_request.title, 'perf:') || startsWith(github.event.pull_request.title, 'chore: release')"
         with:
           file-path: comment.md
           comment-tag: hyperfine

.gitignore

@@ -4,10 +4,13 @@
 package-lock.json
 .mise.lock
 /mise.local.toml
+/perf-workspace
 
 *.log
 *.profraw
 *.lcov
+flamegraph.svg
+cargo-flamegraph.trace
 
 **/snapshots/*.snap.new
 

.shellcheckrc

@@ -1,4 +1,5 @@
 disable=SC1008
 disable=SC2088
+disable=SC2129
 disable=SC2164
 disable=SC2317

e2e/run_all_tests

@@ -52,6 +52,9 @@ for index in "${!FILES[@]}"; do
   test_count=$((test_count + 1))
 done
 
+if [[ -v GITHUB_STEP_SUMMARY ]] && [[ -f "$GITHUB_STEP_SUMMARY-extra" ]]; then
+  cat "$GITHUB_STEP_SUMMARY-extra" >>"$GITHUB_STEP_SUMMARY"
+fi
 echo "E2E: ran $test_count tests, skipped $skipped_count tests" >&2
 
 exit "$status"

e2e/run_test

@@ -55,6 +55,7 @@ within_isolated_env() {
     CARGO_LLVM_COV_SHOW_ENV="${CARGO_LLVM_COV_SHOW_ENV:-}" \
     CARGO_LLVM_COV_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR:-}" \
     GITHUB_ACTION="${GITHUB_ACTION:-}" \
+    GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \
     GITHUB_TOKEN="${GITHUB_TOKEN:-}" \
     GOPROXY="${GOPROXY:-}" \
     HOME="$TEST_HOME" \

tasks.md

@@ -90,6 +90,10 @@ User to run as
 
 - **Usage**: `filetask.bat`
 
+## `flamegraph`
+
+- **Usage**: `flamegraph`
+
 ## `install-dev`
 
 - **Usage**: `install-dev`
@@ -213,6 +217,12 @@ update test snapshots
 
 run all tests
 
+## `test:build-perf-workspace`
+
+- **Usage**: `test:build-perf-workspace`
+
+task description
+
 ## `test:coverage`
 
 - **Usage**: `test:coverage`
@@ -228,6 +238,12 @@ run all tests with coverage report
 
 run end-to-end tests
 
+## `test:perf`
+
+- Depends: test:build-perf-workspace
+
+- **Usage**: `test:perf`
+
 ## `test:shuffle`
 
 - **Usage**: `test:shuffle`

tasks.toml

@@ -163,3 +163,8 @@ description = "a task for testing"
 [pre-commit]
 env = { PRE_COMMIT = 1 }
 run = ["mise run lint"]
+
+[flamegraph]
+tools = { "cargo:flamegraph" = "latest" }
+env = { CARGO_PROFILE_RELEASE_DEBUG = "true" }
+run = "cargo flamegraph"

xtasks/test/build-perf-workspace

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mkdir -p perf-workspace
+cd perf-workspace
+MISE_DATA_DIR="${MISE_DATA_DIR:-$HOME/.local/share/mise}"
+mkdir -p .mise-tasks "$MISE_DATA_DIR/plugins"
+
+num_tasks=${NUM_TASKS:-1000}
+num_tools=${NUM_TOOLS:-100}
+
+create_tasks() {
+	cat <<EOF >.mise-tasks/perf-0
+#!/bin/sh
+#MISE description="task description"
+echo running task
+EOF
+	chmod +x .mise-tasks/perf-0
+	for i in $(seq 1 "$num_tasks"); do
+		ln -sf ./perf-0 ".mise-tasks/perf-$i"
+	done
+}
+
+create_tools() {
+	# shellcheck disable=SC2207
+	tools=($(seq 0 "$num_tools"))
+	if [ ! -d "$MISE_DATA_DIR/plugins/tiny" ]; then
+		git clone https://github.com/jdx/mise-tiny.git "$MISE_DATA_DIR/plugins/tiny"
+	fi
+	{
+		echo "[tools]"
+		for i in "${tools[@]}"; do
+			echo "\"perf-$i\" = \"$i\""
+		done
+		echo "[alias]"
+		for i in "${tools[@]}"; do
+			echo "\"perf-$i\" = \"asdf:jdx/mise-tiny\""
+		done
+	} >mise.toml
+
+	for i in "${tools[@]}"; do
+		ln -sf ./tiny "$MISE_DATA_DIR/plugins/perf-$i"
+	done
+}
+
+create_tasks
+create_tools

xtasks/test/perf

@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+#MISE depends=["test:build-perf-workspace"]
+# shellcheck disable=SC2086,SC2129
+set -xeuo pipefail
+
+num_tools="${NUM_TOOLS:-100}"
+num_tasks="${NUM_TASKS:-1000}"
+cd perf-workspace
+mkdir -p flamegraphs
+MISE_DATA_DIR="${MISE_DATA_DIR:-$HOME/.local/share/mise}"
+declare -A benchmarks
+declare -A recent_benchmarks
+declare -A maximums
+declare -A minimums
+declare -A alt_benchmarks
+names=()
+errors=()
+
+if [ -v MISE_ALT ]; then
+	which mise
+	which "$MISE_ALT"
+fi
+
+recent_benchmarks["install-uncached"]=678
+recent_benchmarks["install-cached"]=186
+recent_benchmarks["ls-uncached"]=372
+recent_benchmarks["ls-cached"]=57
+recent_benchmarks["bin-paths-uncached"]=490
+recent_benchmarks["bin-paths-cached"]=61
+recent_benchmarks["task-ls-uncached"]=10000
+recent_benchmarks["task-ls-cached"]=10000
+
+for name in "${!recent_benchmarks[@]}"; do
+	maximums["$name"]=$(("${recent_benchmarks["$name"]}" * 12 / 10))
+	minimums["$name"]=$(("${recent_benchmarks["$name"]}" * 8 / 10))
+done
+
+benchmark_error() {
+	local name="$1"
+	local error="$2"
+	local cmd="$3"
+	mise cache clear
+	CLICOLOR_FORCE=1 MISE_TIMINGS=1 timeout -v 10 mise $4 >/dev/null || true
+	CLICOLOR_FORCE=1 MISE_TIMINGS=1 timeout -v 10 mise $4 >/dev/null || true
+	# TODO: mise task
+	if [ "$name" = "*-uncached" ]; then
+		mise cache clear
+	fi
+	CARGO_PROFILE_RELEASE_DEBUG=true timeout -v 120 \
+		mise x cargo:flamegraph -- \
+		cargo flamegraph --verbose -o "flamegraphs/$1.svg" --title "$1" --notes "$error" -- $cmd \
+		>/dev/null || true
+	errors+=("::error file=xtasks/test/perf,title=$1::$2")
+}
+
+time_command() {
+	local cmd="$1"
+	local start_time
+	local end_time
+	local duration
+	shift
+	start_time=$(date +%s%N)
+	echo "running $cmd $*..." >&2
+	timeout -v 10 $cmd "$@" >/dev/null || true
+	end_time=$(date +%s%N)
+	duration=$(((end_time - start_time) / 1000000))
+	echo "$duration"
+}
+
+benchmark() {
+	local name="$1"
+	local uncached_duration
+	local cached_duration
+	shift
+	mise cache clear
+	uncached_duration=$(time_command mise "$@")
+	cached_duration=$(time_command mise "$@")
+	benchmarks["$name-uncached"]=$uncached_duration
+	benchmarks["$name-cached"]=$cached_duration
+
+	if [ -n "${MISE_ALT:-}" ]; then
+		mise cache clear
+		alt_uncached_duration=$(time_command "$MISE_ALT" "$@")
+		alt_cached_duration=$(time_command "$MISE_ALT" "$@")
+		alt_benchmarks["$name-uncached"]=$alt_uncached_duration
+		alt_benchmarks["$name-cached"]=$alt_cached_duration
+	fi
+
+	check_maximum "$name" "$uncached_duration" "$cached_duration" "$@"
+	names+=("$name")
+}
+
+check_maximum() {
+	local name="$1"
+	local uncached_duration="$2"
+	local cached_duration="$3"
+	local cmd="$4"
+	if [[ ${maximums["$name-uncached"]} -lt $uncached_duration ]]; then
+		benchmark_error "$name-uncached" "maximum for $name-uncached is ${maximums["$name-uncached"]}, got $uncached_duration" "$name" "$cmd"
+	elif [[ ${maximums["$name-cached"]} -lt $cached_duration ]]; then
+		benchmark_error "$name-cached" "maximum for $name-cached is ${maximums["$name-cached"]}, got $cached_duration" "$name" "$cmd"
+	elif [[ ${minimums["$name-uncached"]} -gt $uncached_duration ]]; then
+		benchmark_error "$name-uncached" "(yay!) minimum for $name-uncached is ${minimums["$name-uncached"]}, got $uncached_duration" "$name" "$cmd"
+	elif [[ ${minimums["$name-cached"]} -gt $cached_duration ]]; then
+		benchmark_error "$name-cached" "(yay!) minimum for $name-cached is ${minimums["$name-cached"]}, got $cached_duration" "$name" "$cmd"
+	fi
+}
+
+mise install
+benchmark install install
+benchmark ls ls
+benchmark bin-paths bin-paths
+benchmark task-ls task ls
+
+if [ -n "${MISE_ALT:-}" ]; then
+	echo "| Command    | Uncached mise | Cached mise | Uncached $MISE_ALT | Cached $MISE_ALT |"
+	echo "|------------|---------------|-------------|--------------------|------------------|"
+	for name in "${names[@]}"; do
+		printf "| %-10s | %11dms | %10dms | %15dms | %13dms |\n" \
+			"$name" \
+			"${benchmarks["$name-uncached"]}" \
+			"${benchmarks["$name-cached"]}" \
+			"${alt_benchmarks["$name-uncached"]}" \
+			"${alt_benchmarks["$name-cached"]}"
+	done
+else
+	echo "| Command    | Uncached | Cached   |"
+	echo "|------------|----------|----------|"
+	for name in "${names[@]}"; do
+		printf "| %-10s | %6dms | %6dms |\n" "$name" "${benchmarks["$name-uncached"]}" "${benchmarks["$name-cached"]}"
+	done
+fi
+
+if [ -v GITHUB_STEP_SUMMARY ]; then
+	echo "## e2e/perf/test_many_tasks" >>../comment.md
+	echo "" >>../comment.md
+	echo "- NUM_TASKS: $num_tasks" >>../comment.md
+	echo "- NUM_TOOLS: $num_tools" >>../comment.md
+	echo "" >>../comment.md
+
+	if [ -n "${MISE_ALT:-}" ]; then
+		echo "| Command    | Uncached mise | Cached mise | Uncached $MISE_ALT | Cached $MISE_ALT |" >>../comment.md
+		echo "|------------|---------------|-------------|--------------------|------------------|" >>../comment.md
+		for name in "${names[@]}"; do
+			printf "| %-10s | %11dms | %10dms | %15dms | %13dms |\n" \
+				"$name" \
+				"${benchmarks["$name-uncached"]}" \
+				"${benchmarks["$name-cached"]}" \
+				"${alt_benchmarks["$name-uncached"]}" \
+				"${alt_benchmarks["$name-cached"]}" >>../comment.md
+		done
+	else
+		echo "| Command    | Uncached | Cached   |" >>../comment.md
+		echo "|------------|----------|----------|" >>../comment.md
+		for name in "${names[@]}"; do
+			printf "| %-10s | %6dms | %6dms |\n" "$name" "${benchmarks["$name-uncached"]}" "${benchmarks["$name-cached"]}" >>../comment.md
+		done
+	fi
+fi
+
+if [ ${#errors[@]} -gt 0 ]; then
+	for error in "${errors[@]}"; do
+		echo "$error" >&2
+	done
+	exit 1
+fi