Benchmarking compiler's peak memory use

scripts/common.sh

@@ -327,7 +327,7 @@ function gnu_grep
 
 function time_to_json_file
 {
-    local output_file="$1"
+    local time_file="$1"
     local cmd=("${@:2}")
     (( $# >= 2 )) || assertFail
 
@@ -340,9 +340,25 @@ function time_to_json_file
     {
         {
             time { "${cmd[@]}" 1>&3 2>&4; }
-        } 2> "$output_file"
+        } 2> "$time_file"
     } 3>&1 4>&2
 
     # Restore original format so that it does not spill outside of the function.
     TIMEFORMAT="$original_timeformat"
 }
+
+function gnu_time_to_json_file
+{
+    local time_file="$1"
+    local cmd=("${@:2}")
+    (( $# >= 2 )) || assertFail
+
+    local gnu_time_path
+    gnu_time_path=$(type -P time)
+
+    "$gnu_time_path" \
+        --output "$time_file" \
+        --quiet \
+        --format '{"real": %e, "user": %U, "sys": %S, "mem": %M, "exit": %x}' \
+            "${cmd[@]}"
+}

test/benchmarks/external.sh

@@ -46,6 +46,7 @@ source "${repo_root}/scripts/common_cmdline.sh"
 
 solc="${1:-${SOLIDITY_BUILD_DIR}/solc/solc}"
 command_available "$solc" --version
+command_available "$(type -P time)" --version
 
 function benchmark_project {
     local pipeline="$1"
@@ -59,18 +60,17 @@ function benchmark_project {
 
     # NOTE: The pipeline may fail with "Stack too deep" in some cases. That's fine.
     # We note the exit code and will later show full output.
-    "$time_bin_path" \
-        --output "$time_file" \
-        --quiet \
-        --format '%e s |         %x' \
-            "${foundry_command[@]}" \
-            > /dev/null \
-            2> "../stderr-${project}-${pipeline}.log" || true
+    gnu_time_to_json_file "$time_file" \
+        "${foundry_command[@]}" \
+        > /dev/null \
+        2> "../stderr-${project}-${pipeline}.log" || true
 
-    printf '| %-20s | %8s | %21s |\n' \
+    printf '| %-20s | %8s | %6d s | %9d MiB | %9d |\n' \
         "$project" \
         "$pipeline" \
-        "$(cat "$time_file")"
+        "$(jq '(.user + .sys) | round' "$time_file")" \
+        "$(jq '.mem / 1024 | round' "$time_file")" \
+        "$(jq '.exit' "$time_file")"
     cd ..
 }
 
@@ -82,13 +82,12 @@ benchmarks=(
     seaport
     sablier-v2
 )
-time_bin_path=$(type -P time)
 
 mkdir -p "$BENCHMARK_DIR"
 cd "$BENCHMARK_DIR"
 
-echo "| Project              | Pipeline | Time      | Exit code |"
-echo "|----------------------|----------|----------:|----------:|"
+echo "|         File         | Pipeline |   Time   | Memory (peak) | Exit code |"
+echo "|----------------------|----------|---------:|--------------:|----------:|"
 
 for project in "${benchmarks[@]}"; do
     benchmark_project legacy "$project"

test/benchmarks/local.sh

@@ -35,6 +35,7 @@ source "${REPO_ROOT}/scripts/common_cmdline.sh"
 
 solc="${1:-${SOLIDITY_BUILD_DIR}/solc/solc}"
 command_available "$solc" --version
+command_available "$(type -P time)" --version
 
 output_dir=$(mktemp -d -t solc-benchmark-XXXXXX)
 
@@ -50,33 +51,33 @@ function benchmark_contract {
     local input_path="$2"
 
     local solc_command=("${solc}" --optimize --bin --color "${input_path}")
-    [[ $pipeline == via-ir ]] && solc_command+=(--via-ir)
-    local time_args=(--output "${output_dir}/time-and-status-${pipeline}.txt" --quiet --format '%e s |         %x')
+    [[ $pipeline == ir ]] && solc_command+=(--via-ir)
+    local time_file="${output_dir}/time-and-status-${pipeline}.txt"
 
     # NOTE: Legacy pipeline may fail with "Stack too deep" in some cases. That's fine.
-    "$time_bin_path" \
-        "${time_args[@]}" \
+    gnu_time_to_json_file "$time_file" \
         "${solc_command[@]}" \
         > "${output_dir}/bytecode-${pipeline}.bin" \
         2>> "${output_dir}/benchmark-warn-err.txt" || [[ $pipeline == legacy ]]
 
-    printf '| %-20s | %s   | %7d bytes | %20s |\n' \
+    printf '| %-20s | %8s | %7d bytes | %6.2f s | %9d MiB | %9d |\n' \
         '`'"$input_file"'`' \
         "$pipeline" \
         "$(bytecode_size < "${output_dir}/bytecode-${pipeline}.bin")" \
-        "$(< "${output_dir}/time-and-status-${pipeline}.txt")"
+        "$(jq '(.user + .sys) * 100 | round / 100' "$time_file")" \
+        "$(jq '.mem / 1024 | round' "$time_file")" \
+        "$(jq '.exit' "$time_file")"
 }
 
 benchmarks=("verifier.sol" "OptimizorClub.sol" "chains.sol")
-time_bin_path=$(type -P time)
 
-echo "| File                 | Pipeline | Bytecode size | Time     | Exit code |"
-echo "|----------------------|----------|--------------:|---------:|----------:|"
+echo "|         File         | Pipeline | Bytecode size |   Time   | Memory (peak) | Exit code |"
+echo "|----------------------|----------|--------------:|---------:|--------------:|----------:|"
 
 for input_file in "${benchmarks[@]}"
 do
     benchmark_contract legacy "${REPO_ROOT}/test/benchmarks/${input_file}"
-    benchmark_contract via-ir "${REPO_ROOT}/test/benchmarks/${input_file}"
+    benchmark_contract ir     "${REPO_ROOT}/test/benchmarks/${input_file}"
 done
 
 echo