Performance benchmarks using external projects with Foundry

.gitignore

@@ -10,7 +10,7 @@
 *.o
 *.obj
 *.pyc
-__pycache__
+__pycache__/
 
 # Precompiled Headers
 *.gch
@@ -36,12 +36,15 @@ __pycache__
 *.app
 
 # Build directory
-/build*
+/build*/
 emscripten_build/
-/docs/_build
+/docs/_build/
 /docs/_static/robots.txt
-/deps
-/reports
+/deps/
+
+# Reports and benchmarks
+/reports/
+/benchmarks/
 
 # vim stuff
 [._]*.sw[a-p]

scripts/common_cmdline.sh

@@ -169,3 +169,11 @@ function stripEmptyLines
 {
     sed -e '/^\s*$/d'
 }
+
+# Calculates the total size of bytecode of one or more contracts in bytes.
+# Expects the output from `solc --bin` on standard input.
+function bytecode_size {
+    local bytecode_chars
+    bytecode_chars=$(stripCLIDecorations | stripEmptyLines | wc --chars)
+    echo $(( bytecode_chars / 2 ))
+}

test/benchmarks/external-setup.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+#------------------------------------------------------------------------------
+# Downloads and configures external projects used for benchmarking by external.sh.
+#
+# By default the download location is the benchmarks/ dir at the repository root.
+# A different directory can be provided via the BENCHMARK_DIR variable.
+#
+# Dependencies: foundry, git.
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2024 solidity contributors.
+#------------------------------------------------------------------------------
+
+set -euo pipefail
+
+repo_root=$(cd "$(dirname "$0")/../../" && pwd)
+BENCHMARK_DIR="${BENCHMARK_DIR:-${repo_root}/benchmarks}"
+
+function neutralize_version_pragmas {
+    find . -name '*.sol' -type f -print0 | xargs -0 \
+        sed -i -E -e 's/pragma solidity [^;]+;/pragma solidity *;/'
+}
+
+function neutralize_via_ir {
+    sed -i '/^via_ir\s*=.*$/d' foundry.toml
+}
+
+mkdir -p "$BENCHMARK_DIR"
+cd "$BENCHMARK_DIR"
+
+if [[ ! -e openzeppelin/ ]]; then
+    git clone --depth=1 https://github.com/OpenZeppelin/openzeppelin-contracts openzeppelin/ --branch v5.0.2
+    pushd openzeppelin/
+    forge install
+    neutralize_via_ir
+    popd
+else
+    echo "Skipped openzeppelin/. Already exists."
+fi
+
+if [[ ! -e uniswap-v4/ ]]; then
+    git clone --single-branch https://github.com/Uniswap/v4-core uniswap-v4/
+    pushd uniswap-v4/
+    git checkout ae86975b058d386c9be24e8994236f662affacdb # branch main as of 2024-06-06
+    forge install
+    neutralize_via_ir
+    popd
+else
+    echo "Skipped uniswap-v4/. Already exists."
+fi
+
+if [[ ! -e seaport/ ]]; then
+    git clone --single-branch https://github.com/ProjectOpenSea/seaport
+    pushd seaport/
+    # NOTE: Can't select the tag with `git clone` because a branch of the same name exists.
+    git checkout tags/1.6
+    forge install
+    neutralize_via_ir
+    neutralize_version_pragmas
+    popd
+else
+    echo "Skipped seaport/. Already exists."
+fi
+
+if [[ ! -e eigenlayer/ ]]; then
+    git clone --depth=1 https://github.com/Layr-Labs/eigenlayer-contracts eigenlayer/ --branch v0.3.0-holesky-rewards
+    pushd eigenlayer/
+    neutralize_via_ir
+    forge install
+    popd
+else
+    echo "Skipped eigenlayer/. Already exists."
+fi
+
+if [[ ! -e sablier-v2/ ]]; then
+    git clone --depth=1 https://github.com/sablier-labs/v2-core sablier-v2/ --branch v1.1.2
+    pushd sablier-v2/
+    # NOTE: To avoid hard-coding dependency versions here we'd have to install them from npm
+    forge install --no-commit \
+        foundry-rs/forge-std@v1.5.6 \
+        OpenZeppelin/openzeppelin-contracts@v4.9.2 \
+        PaulRBerg/prb-math@v4.0.2 \
+        PaulRBerg/prb-test@v0.6.4 \
+        evmcheb/solarray@a547630 \
+        Vectorized/solady@v0.0.129
+   cat <<EOF > remappings.txt
+@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/
+forge-std/=lib/forge-std/
+@prb/math/=lib/prb-math/
+@prb/test/=lib/prb-test/
+solarray/=lib/solarray/
+solady/=lib/solady/
+EOF
+    neutralize_via_ir
+    popd
+else
+    echo "Skipped sablier-v2/. Already exists."
+fi

test/benchmarks/external.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+
+#------------------------------------------------------------------------------
+# Benchmarks a solc binary by compiling several external projects, with and without IR.
+#
+# The script expects each project to be already downloaded and set up by external-setup.sh.
+# A different directory can be provided via the BENCHMARK_DIR variable.
+#
+# The script will by default attempt to use a solc from the default build directory,
+# relative to the script directory. To use a different binary you can provide a different
+# location of the build directory (via SOLIDITY_BUILD_DIR variable) or simply specify
+# the full path to the binary as the script argument.
+#
+# Dependencies: foundry, time.
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2024 solidity contributors.
+#------------------------------------------------------------------------------
+
+set -euo pipefail
+
+repo_root=$(cd "$(dirname "$0")/../../" && pwd)
+SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${repo_root}/build}
+BENCHMARK_DIR="${BENCHMARK_DIR:-${repo_root}/benchmarks}"
+
+# shellcheck source=scripts/common.sh
+source "${repo_root}/scripts/common.sh"
+# shellcheck source=scripts/common_cmdline.sh
+source "${repo_root}/scripts/common_cmdline.sh"
+
+(( $# <= 1 )) || fail "Too many arguments. Usage: external.sh [<solc-path>]"
+
+solc="${1:-${SOLIDITY_BUILD_DIR}/solc/solc}"
+command_available "$solc" --version
+
+function benchmark_project {
+    local pipeline="$1"
+    local project="$2"
+    [[ $pipeline == legacy || $pipeline == ir ]] || assertFail
+
+    cd "$project"
+    local foundry_command=(forge build --use "$solc" --optimize --offline --no-cache)
+    [[ $pipeline == ir ]] && foundry_command+=(--via-ir)
+    local time_file="../time-and-status-${project}-${pipeline}.txt"
+
+    # NOTE: The pipeline may fail with "Stack too deep" in some cases. That's fine.
+    # We note the exit code and will later show full output.
+    "$time_bin_path" \
+        --output "$time_file" \
+        --quiet \
+        --format '%e s |         %x' \
+            "${foundry_command[@]}" \
+            > /dev/null \
+            2> "../stderr-${project}-${pipeline}.log" || true
+
+    printf '| %-20s | %8s | %21s |\n' \
+        "$project" \
+        "$pipeline" \
+        "$(cat "$time_file")"
+    cd ..
+}
+
+benchmarks=(
+    # Fastest ones first so that we get *some* output quickly
+    openzeppelin
+    uniswap-v4
+    eigenlayer
+    seaport
+    sablier-v2
+)
+time_bin_path=$(type -P time)
+
+mkdir -p "$BENCHMARK_DIR"
+cd "$BENCHMARK_DIR"
+
+echo "| Project              | Pipeline | Time      | Exit code |"
+echo "|----------------------|----------|----------:|----------:|"
+
+for project in "${benchmarks[@]}"; do
+    benchmark_project legacy "$project"
+    benchmark_project ir     "$project"
+done
+
+for project in "${benchmarks[@]}"; do
+    for pipeline in legacy ir; do
+        if [[ -s stderr-${project}-${pipeline}.log ]]; then
+            echo
+            echo "=================================="
+            echo "stderr for ${project} via ${pipeline}"
+            echo "=================================="
+            cat "stderr-${project}-${pipeline}.log"
+        fi
+    done
+done

test/benchmarks/run.sh → test/benchmarks/local.sh

@@ -31,7 +31,7 @@ source "${REPO_ROOT}/scripts/common.sh"
 # shellcheck source=scripts/common_cmdline.sh
 source "${REPO_ROOT}/scripts/common_cmdline.sh"
 
-(( $# <= 1 )) || fail "Too many arguments. Usage: run.sh [<solc-path>]"
+(( $# <= 1 )) || fail "Too many arguments. Usage: local.sh [<solc-path>]"
 
 solc="${1:-${SOLIDITY_BUILD_DIR}/solc/solc}"
 command_available "$solc" --version
@@ -45,12 +45,6 @@ function cleanup() {
 
 trap cleanup SIGINT SIGTERM
 
-function bytecode_size {
-    local bytecode_chars
-    bytecode_chars=$(stripCLIDecorations | stripEmptyLines | wc --chars)
-    echo $(( bytecode_chars / 2 ))
-}
-
 function benchmark_contract {
     local pipeline="$1"
     local input_path="$2"