diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 86117863800..94e723cc183 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -7,6 +7,52 @@ on: pull_request: jobs: + rust_benchmarks: + name: Rust Benchmarks + runs-on: ubuntu-22.04 + + timeout-minutes: 15 + permissions: + pull-requests: write + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write + + steps: + - uses: actions/checkout@v4 + + - name: Setup toolchain + uses: dtolnay/rust-toolchain@1.85.0 + with: + targets: x86_64-unknown-linux-gnu + + - uses: Swatinem/rust-cache@v2 + with: + key: x86_64-unknown-linux-gnu-bench + cache-on-failure: true + save-if: ${{ github.event_name != 'merge_group' }} + + - name: Run ACVM benchmarks + run: | + cargo bench -p acvm -- --output-format bencher | tee acvm-bench.txt + + - name: Store ACVM benchmarks result + uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 + with: + name: "ACVM Benchmarks" + tool: "cargo" + output-file-path: ./acvm-bench.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + # We want this to only run on master to avoid garbage data from PRs being added. + auto-push: ${{ github.ref == 'refs/heads/master' }} + alert-threshold: "120%" + comment-on-alert: true + comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }} + fail-on-alert: false + alert-comment-cc-users: "@TomAFrench" + max-items-in-chart: 50 + benchmark-projects-list: name: Load benchmark projects list runs-on: ubuntu-22.04 @@ -729,6 +775,7 @@ jobs: # We want this job to always run (even if the dependant jobs fail) as we want this job to fail rather than skipping. if: ${{ always() }} needs: + - rust_benchmarks - upload_compilation_report - upload_compilation_memory_report - upload_execution_report diff --git a/Cargo.lock b/Cargo.lock index 75a87241d8f..9bc70629521 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,9 +64,11 @@ dependencies = [ "ark-ff 0.5.0", "bn254_blackbox_solver", "brillig_vm", + "criterion", "fxhash", "indexmap 1.9.3", "num-bigint", + "pprof", "proptest", "serde", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index 146c95da675..1077dcfd12b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -138,7 +138,7 @@ codespan-lsp = "0.11.1" codespan-reporting = "0.11.1" # Benchmarking -criterion = "0.5.0" +criterion = "0.5.1" # Note that using the "frame-pointer" feature breaks framegraphs on linux # https://github.com/tikv/pprof-rs/pull/172 pprof = { version = "0.14", features = ["flamegraph", "criterion"] } @@ -223,3 +223,6 @@ strip = true lto = true panic = "abort" opt-level = "z" + +[profile.bench] +lto = true \ No newline at end of file diff --git a/acvm-repo/acir/Cargo.toml b/acvm-repo/acir/Cargo.toml index f757ea4d7f7..7f7700442f3 100644 --- a/acvm-repo/acir/Cargo.toml +++ b/acvm-repo/acir/Cargo.toml @@ -57,6 +57,10 @@ bn254 = ["acir_field/bn254"] bls12_381 = ["acir_field/bls12_381"] arb = ["proptest", "proptest-derive", "brillig/arb"] +[lib] +# See https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options +bench = false + [[bench]] name = "serialization" harness = false diff --git a/acvm-repo/acvm/Cargo.toml b/acvm-repo/acvm/Cargo.toml index 1538e9c9925..3196d193f17 100644 --- a/acvm-repo/acvm/Cargo.toml +++ b/acvm-repo/acvm/Cargo.toml @@ -47,3 +47,13 @@ ark-bn254-v04 = { package = "ark-bn254", version = "^0.4.0", default-features = "curve", ] } ark-ff-v04 = { package = "ark-ff", version = "^0.4.0", default-features = false } +criterion.workspace = true +pprof.workspace = true + +[lib] +# See https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options +bench = false + +[[bench]] +name = "arithmetic_solver" +harness = false diff --git a/acvm-repo/acvm/benches/arithmetic_solver.rs b/acvm-repo/acvm/benches/arithmetic_solver.rs new file mode 100644 index 00000000000..2a6b5122ce3 --- /dev/null +++ b/acvm-repo/acvm/benches/arithmetic_solver.rs @@ -0,0 +1,72 @@ +use std::{collections::BTreeMap, time::Duration}; + +use acir::{ + AcirField, FieldElement, + circuit::Opcode, + native_types::{Expression, Witness, WitnessMap}, +}; +use acvm::pwg::{ACVM, ACVMStatus}; +use acvm_blackbox_solver::StubbedBlackBoxSolver; +use criterion::{Criterion, criterion_group, criterion_main}; +use pprof::criterion::{Output, PProfProfiler}; + +fn purely_sequential_opcodes(c: &mut Criterion) { + // This bytecode defines a stack of constraints such that `w_{i+1} = w_i + 1`. + // This prevents any batch inversions as all opcodes require the result of the previous opcode as an input. + let bytecode: Vec> = (0..1000) + .map(|witness_index| { + Opcode::AssertZero(Expression { + mul_terms: Vec::new(), + linear_combinations: vec![ + (FieldElement::one(), Witness(witness_index)), + (-FieldElement::one(), Witness(witness_index + 1)), + ], + q_c: FieldElement::one(), + }) + }) + .collect(); + + bench_bytecode(c, "purely_sequential_opcodes", &bytecode); +} + +fn perfectly_parallel_opcodes(c: &mut Criterion) { + // This bytecode defines a set of constraints such that `w_{i+1} = w_0 + i`. + // This allows all opcodes to be solved with a single field inversion assuming perfect batching. + let bytecode: Vec> = (1..1000) + .map(|witness_index| { + Opcode::AssertZero(Expression { + mul_terms: Vec::new(), + linear_combinations: vec![ + (FieldElement::one(), Witness(0)), + (-FieldElement::one(), Witness(witness_index)), + ], + q_c: FieldElement::from(witness_index), + }) + }) + .collect(); + + bench_bytecode(c, "purely_sequential_opcodes", &bytecode); +} + +fn bench_bytecode(c: &mut Criterion, benchmark_name: &str, bytecode: &[Opcode]) { + c.bench_function(benchmark_name, |b| { + b.iter_batched( + || { + let initial_witness = WitnessMap::from(BTreeMap::from([(Witness(0), F::one())])); + ACVM::new(&StubbedBlackBoxSolver(true), bytecode, initial_witness, &[], &[]) + }, + |mut vm| { + let status = vm.solve(); + assert!(matches!(status, ACVMStatus::Solved)) + }, + criterion::BatchSize::SmallInput, + ); + }); +} + +criterion_group! { + name = execution_benches; + config = Criterion::default().sample_size(20).measurement_time(Duration::from_secs(20)).with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + targets = purely_sequential_opcodes, perfectly_parallel_opcodes +} +criterion_main!(execution_benches); diff --git a/acvm-repo/bn254_blackbox_solver/Cargo.toml b/acvm-repo/bn254_blackbox_solver/Cargo.toml index 6986ffc05c5..0575a0f1a81 100644 --- a/acvm-repo/bn254_blackbox_solver/Cargo.toml +++ b/acvm-repo/bn254_blackbox_solver/Cargo.toml @@ -29,7 +29,7 @@ num-bigint.workspace = true [dev-dependencies] ark-std.workspace = true -criterion = "0.5.0" +criterion.workspace = true pprof = { version = "0.14", features = [ "flamegraph", "frame-pointer",