chore: add benchmark for ACVM arithmetic solver

.github/workflows/reports.yml

@@ -7,6 +7,52 @@ on:
   pull_request:
 
 jobs:
+  rust_benchmarks:
+    name: Rust Benchmarks
+    runs-on: ubuntu-22.04
+
+    timeout-minutes: 15
+    permissions:
+      pull-requests: write
+      # deployments permission to deploy GitHub pages website
+      deployments: write
+      # contents permission to update benchmark contents in gh-pages branch
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup toolchain
+        uses: dtolnay/rust-toolchain@1.85.0
+        with:
+          targets: x86_64-unknown-linux-gnu
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: x86_64-unknown-linux-gnu-bench
+          cache-on-failure: true
+          save-if: ${{ github.event_name != 'merge_group' }}
+
+      - name: Run ACVM benchmarks
+        run: |
+          cargo bench -p acvm -- --output-format bencher | tee acvm-bench.txt
+
+      - name: Store ACVM benchmarks result
+        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
+        with:
+          name: "ACVM Benchmarks"
+          tool: "cargo"
+          output-file-path: ./acvm-bench.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # We want this to only run on master to avoid garbage data from PRs being added.
+          auto-push: ${{ github.ref == 'refs/heads/master' }}
+          alert-threshold: "120%"
+          comment-on-alert: true
+          comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
+          fail-on-alert: false
+          alert-comment-cc-users: "@TomAFrench"
+          max-items-in-chart: 50
+
   benchmark-projects-list:
     name: Load benchmark projects list
     runs-on: ubuntu-22.04
@@ -729,6 +775,7 @@ jobs:
     # We want this job to always run (even if the dependant jobs fail) as we want this job to fail rather than skipping.
     if: ${{ always() }}
     needs:
+      - rust_benchmarks
       - upload_compilation_report
       - upload_compilation_memory_report
       - upload_execution_report

Cargo.toml

@@ -138,7 +138,7 @@ codespan-lsp = "0.11.1"
 codespan-reporting = "0.11.1"
 
 # Benchmarking
-criterion = "0.5.0"
+criterion = "0.5.1"
 # Note that using the "frame-pointer" feature breaks framegraphs on linux
 # https://github.com/tikv/pprof-rs/pull/172
 pprof = { version = "0.14", features = ["flamegraph", "criterion"] }
@@ -223,3 +223,6 @@ strip = true
 lto = true
 panic = "abort"
 opt-level = "z"
+
+[profile.bench]
+lto = true

acvm-repo/acir/Cargo.toml

@@ -57,6 +57,10 @@ bn254 = ["acir_field/bn254"]
 bls12_381 = ["acir_field/bls12_381"]
 arb = ["proptest", "proptest-derive", "brillig/arb"]
 
+[lib]
+# See https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
+
 [[bench]]
 name = "serialization"
 harness = false

acvm-repo/acvm/Cargo.toml

@@ -47,3 +47,13 @@ ark-bn254-v04 = { package = "ark-bn254", version = "^0.4.0", default-features =
     "curve",
 ] }
 ark-ff-v04 = {  package = "ark-ff", version = "^0.4.0", default-features = false }
+criterion.workspace = true
+pprof.workspace = true
+
+[lib]
+# See https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
+
+[[bench]]
+name = "arithmetic_solver"
+harness = false

acvm-repo/acvm/benches/arithmetic_solver.rs

@@ -0,0 +1,72 @@
+use std::{collections::BTreeMap, time::Duration};
+
+use acir::{
+    AcirField, FieldElement,
+    circuit::Opcode,
+    native_types::{Expression, Witness, WitnessMap},
+};
+use acvm::pwg::{ACVM, ACVMStatus};
+use acvm_blackbox_solver::StubbedBlackBoxSolver;
+use criterion::{Criterion, criterion_group, criterion_main};
+use pprof::criterion::{Output, PProfProfiler};
+
+fn purely_sequential_opcodes(c: &mut Criterion) {
+    // This bytecode defines a stack of constraints such that `w_{i+1} = w_i + 1`.
+    // This prevents any batch inversions as all opcodes require the result of the previous opcode as an input.
+    let bytecode: Vec<Opcode<FieldElement>> = (0..1000)
+        .map(|witness_index| {
+            Opcode::AssertZero(Expression {
+                mul_terms: Vec::new(),
+                linear_combinations: vec![
+                    (FieldElement::one(), Witness(witness_index)),
+                    (-FieldElement::one(), Witness(witness_index + 1)),
+                ],
+                q_c: FieldElement::one(),
+            })
+        })
+        .collect();
+
+    bench_bytecode(c, "purely_sequential_opcodes", &bytecode);
+}
+
+fn perfectly_parallel_opcodes(c: &mut Criterion) {
+    // This bytecode defines a set of constraints such that `w_{i+1} = w_0 + i`.
+    // This allows all opcodes to be solved with a single field inversion assuming perfect batching.
+    let bytecode: Vec<Opcode<FieldElement>> = (1..1000)
+        .map(|witness_index| {
+            Opcode::AssertZero(Expression {
+                mul_terms: Vec::new(),
+                linear_combinations: vec![
+                    (FieldElement::one(), Witness(0)),
+                    (-FieldElement::one(), Witness(witness_index)),
+                ],
+                q_c: FieldElement::from(witness_index),
+            })
+        })
+        .collect();
+
+    bench_bytecode(c, "purely_sequential_opcodes", &bytecode);
+}
+
+fn bench_bytecode<F: AcirField>(c: &mut Criterion, benchmark_name: &str, bytecode: &[Opcode<F>]) {
+    c.bench_function(benchmark_name, |b| {
+        b.iter_batched(
+            || {
+                let initial_witness = WitnessMap::from(BTreeMap::from([(Witness(0), F::one())]));
+                ACVM::new(&StubbedBlackBoxSolver(true), bytecode, initial_witness, &[], &[])
+            },
+            |mut vm| {
+                let status = vm.solve();
+                assert!(matches!(status, ACVMStatus::Solved))
+            },
+            criterion::BatchSize::SmallInput,
+        );
+    });
+}
+
+criterion_group! {
+    name = execution_benches;
+    config = Criterion::default().sample_size(20).measurement_time(Duration::from_secs(20)).with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+    targets = purely_sequential_opcodes, perfectly_parallel_opcodes
+}
+criterion_main!(execution_benches);

acvm-repo/bn254_blackbox_solver/Cargo.toml

@@ -29,7 +29,7 @@ num-bigint.workspace = true
 
 [dev-dependencies]
 ark-std.workspace = true
-criterion = "0.5.0"
+criterion.workspace = true
 pprof = { version = "0.14", features = [
     "flamegraph",
     "frame-pointer",