chore: Refactor benchmarks for feature handling (#365)

Helps with argumentcomputer/ci-workflows#49

- Renamed several environmental variables in various benchmarking scripts and configuration files for consistency.
- Introduced a new section for performing comparative benchmarks based on respective features and architecture.
- Added a new variable to the default benchmark configuration, `BENCH_OUTPUT`, set to `commit-comment`.

.github/workflows/bench-pr-comment.yml

@@ -20,4 +20,4 @@ jobs:
     with:
       default-runner: "self-hosted,gpu-bench"
       default-benches: "supernova-ci"
-      default-env: "ARECIBO_BENCH_OUTPUT=pr-comment ARECIBO_BENCH_NUM_CONS=16384,524288"
+      default-env: "BENCH_OUTPUT=pr-comment BENCH_NUM_CONS=16384,524288"

.github/workflows/gpu-bench.yml

@@ -38,8 +38,8 @@ jobs:
           cargo install criterion-table
       - name: Set bench output format and base SHA
         run: |
-          echo "ARECIBO_BENCH_OUTPUT=commit-comment" | tee -a $GITHUB_ENV
-          echo "ARECIBO_BENCH_NUM_CONS=16384,1038732" | tee -a $GITHUB_ENV
+          echo "BENCH_OUTPUT=commit-comment" | tee -a $GITHUB_ENV
+          echo "BENCH_NUM_CONS=16384,1038732" | tee -a $GITHUB_ENV
           echo "BASE_COMMIT=${{ github.event.merge_group.base_sha }}" | tee -a $GITHUB_ENV
           GPU_NAME=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader,nounits | tail -n1)
           echo "GPU_ID=$(echo $GPU_NAME | awk '{ print $NF }')" | tee -a $GITHUB_ENV
@@ -91,17 +91,17 @@ jobs:
         uses: peter-evans/commit-comment@v3
         with:
           body-path: BENCHMARKS.md
-      # TODO: Set `$ARECIBO_BENCH_NOISE_THRESHOLD` via `cardinalby/export-env-action` or hardcode to 1.3
-      # Check for a slowdown >= `$ARECIBO_BENCH_NOISE_THRESHOLD` (fallback is 30%/1.3x). If so, open an issue but don't block merge
+      # TODO: Set `$BENCH_NOISE_THRESHOLD` via `cardinalby/export-env-action` or hardcode to 1.3
+      # Check for a slowdown >= `$BENCH_NOISE_THRESHOLD` (fallback is 30%/1.3x). If so, open an issue but don't block merge
       # Since we are parsing for slowdowns, we simply add 1 to the noise threshold decimal to get the regression factor
       - name: Check for perf regression
         id: regression-check
         run: |
           REGRESSIONS=$(grep -o '[0-9.]*x slower' BENCHMARKS.md | cut -d 'x' -f1)
           echo $REGRESSIONS
 
-          if [ ! -z "${{ env.ARECIBO_BENCH_NOISE_THRESHOLD}}" ]; then
-            REGRESSION_FACTOR=$(echo "${{ env.ARECIBO_BENCH_NOISE_THRESHOLD }}+1" | bc)
+          if [ ! -z "${{ env.BENCH_NOISE_THRESHOLD}}" ]; then
+            REGRESSION_FACTOR=$(echo "${{ env.BENCH_NOISE_THRESHOLD }}+1" | bc)
           else
             REGRESSION_FACTOR=1.3
           fi

benches/bench.env

@@ -1,2 +1,3 @@
 # Arecibo config, used only in `justfile` by default
-ARECIBO_BENCH_NUM_CONS=16384,1048576
+BENCH_NUM_CONS=16384,1048576
+BENCH_OUTPUT=commit-comment

benches/common/mod.rs

@@ -36,13 +36,12 @@ impl BenchParams {
 }
 
 fn output_type_env() -> anyhow::Result<String> {
-  std::env::var("ARECIBO_BENCH_OUTPUT")
-    .map_err(|e| anyhow!("ARECIBO_BENCH_OUTPUT env var isn't set: {e}"))
+  std::env::var("BENCH_OUTPUT").map_err(|e| anyhow!("BENCH_OUTPUT env var isn't set: {e}"))
 }
 
 pub(crate) fn noise_threshold_env() -> anyhow::Result<f64> {
-  std::env::var("ARECIBO_BENCH_NOISE_THRESHOLD")
-    .map_err(|e| anyhow!("ARECIBO_BENCH_NOISE_THRESHOLD env var isn't set: {e}"))
+  std::env::var("BENCH_NOISE_THRESHOLD")
+    .map_err(|e| anyhow!("BENCH_NOISE_THRESHOLD env var isn't set: {e}"))
     .and_then(|nt| {
       nt.parse::<f64>()
         .map_err(|e| anyhow!("Failed to parse noise threshold: {e}"))

benches/common/supernova/mod.rs

@@ -54,8 +54,8 @@ pub fn num_cons() -> Vec<usize> {
 }
 
 fn num_cons_env() -> anyhow::Result<Vec<usize>> {
-  std::env::var("ARECIBO_BENCH_NUM_CONS")
-    .map_err(|e| anyhow!("ARECIBO_BENCH_NUM_CONS env var not set: {e}"))
+  std::env::var("BENCH_NUM_CONS")
+    .map_err(|e| anyhow!("BENCH_NUM_CONS env var not set: {e}"))
     .and_then(|rc| {
       let vec: anyhow::Result<Vec<usize>> = rc
         .split(',')

benches/justfile

@@ -9,8 +9,10 @@ commit := `git rev-parse HEAD`
 # Run CPU benchmarks
 bench +benches:
   #!/bin/sh
+  [ $(uname -m) = "x86_64" ] && FEATURES="asm" || FEATURES="default"
+
   for bench in {{benches}}; do
-    cargo criterion --bench $bench
+    cargo criterion --bench $bench --features $FEATURES
   done
 
 # Run CUDA benchmarks on GPU
@@ -22,6 +24,7 @@ gpu-bench +benches:
   export CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | sed 's/\.//g')
   export EC_GPU_CUDA_NVCC_ARGS="--fatbin --gpu-architecture=sm_$CUDA_ARCH --generate-code=arch=compute_$CUDA_ARCH,code=sm_$CUDA_ARCH"
   export EC_GPU_FRAMEWORK="cuda"
+  [ $(uname -m) = "x86_64" ] && FEATURES="cuda, asm" || FEATURES="cuda"
 
   for bench in {{benches}}; do
     cargo criterion --bench $bench --features "cuda"
@@ -31,12 +34,23 @@ gpu-bench +benches:
 gpu-bench-ci +benches:
   #!/bin/sh
   printenv PATH
-  if [ $(uname -m) = "x86_64" ]; then
-    FEATURES="cuda,asm"
-  else
-    FEATURES="cuda"
+  [ $(uname -m) = "x86_64" ] && FEATURES="cuda, asm" || FEATURES="cuda"
+
+  for bench in {{benches}}; do
+    cargo criterion --bench $bench --features $FEATURES --message-format=json > "$bench-{{commit}}".json
+  done
+
+comparative-bench +benches:
+  #!/bin/sh
+  # Initialize FEATURES based on architecture
+  [ $(uname -m) = "x86_64" ] && FEATURES="asm" || FEATURES=""
+  # Append cuda to FEATURES if nvcc is found
+  if which nvcc > /dev/null; then
+    FEATURES="${FEATURES:+$FEATURES,}cuda"
   fi
+  # add default if FEATURES is empty
+  FEATURES="${FEATURES:-default}"
 
   for bench in {{benches}}; do
     cargo criterion --bench $bench --features $FEATURES --message-format=json > "$bench-{{commit}}".json
-  done
+  done