chore: Refactor benchmarks for feature handling

Helps with argumentcomputer/ci-workflows#49

- Renamed several environmental variables in various benchmarking scripts and configuration files for consistency.
- Introduced a new section for performing comparative benchmarks based on respective features and architecture.
- Added a new variable to the default benchmark configuration, `BENCH_OUTPUT`, set to `commit-comment`.

benches/bench.env

@@ -1,2 +1,3 @@
 # Arecibo config, used only in `justfile` by default
-ARECIBO_BENCH_NUM_CONS=16384,1048576
+BENCH_NUM_CONS=16384,1048576
+BENCH_OUTPUT=commit-comment

benches/common/mod.rs

@@ -36,13 +36,13 @@ impl BenchParams {
 }
 
 fn output_type_env() -> anyhow::Result<String> {
-  std::env::var("ARECIBO_BENCH_OUTPUT")
-    .map_err(|e| anyhow!("ARECIBO_BENCH_OUTPUT env var isn't set: {e}"))
+  std::env::var("BENCH_OUTPUT")
+    .map_err(|e| anyhow!("BENCH_OUTPUT env var isn't set: {e}"))
 }
 
 pub(crate) fn noise_threshold_env() -> anyhow::Result<f64> {
-  std::env::var("ARECIBO_BENCH_NOISE_THRESHOLD")
-    .map_err(|e| anyhow!("ARECIBO_BENCH_NOISE_THRESHOLD env var isn't set: {e}"))
+  std::env::var("BENCH_NOISE_THRESHOLD")
+    .map_err(|e| anyhow!("BENCH_NOISE_THRESHOLD env var isn't set: {e}"))
     .and_then(|nt| {
       nt.parse::<f64>()
         .map_err(|e| anyhow!("Failed to parse noise threshold: {e}"))

benches/common/supernova/mod.rs

@@ -54,8 +54,8 @@ pub fn num_cons() -> Vec<usize> {
 }
 
 fn num_cons_env() -> anyhow::Result<Vec<usize>> {
-  std::env::var("ARECIBO_BENCH_NUM_CONS")
-    .map_err(|e| anyhow!("ARECIBO_BENCH_NUM_CONS env var not set: {e}"))
+  std::env::var("BENCH_NUM_CONS")
+    .map_err(|e| anyhow!("BENCH_NUM_CONS env var not set: {e}"))
     .and_then(|rc| {
       let vec: anyhow::Result<Vec<usize>> = rc
         .split(',')

benches/justfile

@@ -9,8 +9,10 @@ commit := `git rev-parse HEAD`
 # Run CPU benchmarks
 bench +benches:
   #!/bin/sh
+  [ $(uname -m) = "x86_64" ] && FEATURES="asm" || FEATURES="default"
+
   for bench in {{benches}}; do
-    cargo criterion --bench $bench
+    cargo criterion --bench $bench --features $FEATURES
   done
 
 # Run CUDA benchmarks on GPU
@@ -22,6 +24,7 @@ gpu-bench +benches:
   export CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | sed 's/\.//g')
   export EC_GPU_CUDA_NVCC_ARGS="--fatbin --gpu-architecture=sm_$CUDA_ARCH --generate-code=arch=compute_$CUDA_ARCH,code=sm_$CUDA_ARCH"
   export EC_GPU_FRAMEWORK="cuda"
+  [ $(uname -m) = "x86_64" ] && FEATURES="cuda, asm" || FEATURES="cuda"
 
   for bench in {{benches}}; do
     cargo criterion --bench $bench --features "cuda"
@@ -31,12 +34,23 @@ gpu-bench +benches:
 gpu-bench-ci +benches:
   #!/bin/sh
   printenv PATH
-  if [ $(uname -m) = "x86_64" ]; then
-    FEATURES="cuda,asm"
-  else
-    FEATURES="cuda"
+  [ $(uname -m) = "x86_64" ] && FEATURES="cuda, asm" || FEATURES="cuda"
+
+  for bench in {{benches}}; do
+    cargo criterion --bench $bench --features $FEATURES --message-format=json > "$bench-{{commit}}".json
+  done
+
+comparative-bench +benches:
+  #!/bin/sh
+  # Initialize FEATURES based on architecture
+  [ $(uname -m) = "x86_64" ] && FEATURES="asm" || FEATURES=""
+  # Append cuda to FEATURES if nvcc is found
+  if which nvcc > /dev/null; then
+    FEATURES="${FEATURES:+$FEATURES,}cuda"
   fi
+  # add default if FEATURES is empty
+  FEATURES="${FEATURES:-default}"
 
   for bench in {{benches}}; do
     cargo criterion --bench $bench --features $FEATURES --message-format=json > "$bench-{{commit}}".json
-  done
+  done