From 060b7297b6aa53f500dd3bd238df5e8b37edd7be Mon Sep 17 00:00:00 2001 From: Tom French Date: Tue, 16 Apr 2024 14:04:06 +0100 Subject: [PATCH 1/3] chore: add benchmarks for serialization --- Cargo.lock | 30 +++++- Cargo.toml | 8 ++ acvm-repo/acir/Cargo.toml | 6 ++ acvm-repo/acir/benches/serialization.rs | 123 ++++++++++++++++++++++++ acvm-repo/acvm/Cargo.toml | 6 ++ tooling/nargo_cli/Cargo.toml | 8 +- 6 files changed, 173 insertions(+), 8 deletions(-) create mode 100644 acvm-repo/acir/benches/serialization.rs diff --git a/Cargo.lock b/Cargo.lock index 0f2c1894e15..9b1ee546d71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,8 +10,10 @@ dependencies = [ "base64 0.21.2", "bincode", "brillig", + "criterion", "flate2", "fxhash", + "pprof 0.13.0", "serde", "serde-generate", "serde-reflection", @@ -42,9 +44,11 @@ dependencies = [ "acir", "acvm_blackbox_solver", "brillig_vm", + "criterion", "indexmap 1.9.3", "num-bigint", "paste", + "pprof 0.13.0", "proptest", "rand 0.8.5", "thiserror", @@ -622,7 +626,7 @@ dependencies = [ "lazy_static", "noir_grumpkin", "num-bigint", - "pprof", + "pprof 0.12.1", "thiserror", "wasm-bindgen-futures", "wasmer", @@ -2864,7 +2868,7 @@ dependencies = [ "notify", "notify-debouncer-full", "paste", - "pprof", + "pprof 0.13.0", "predicates 2.1.5", "prettytable-rs", "rayon", @@ -3545,6 +3549,28 @@ dependencies = [ "thiserror", ] +[[package]] +name = "pprof" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb" +dependencies = [ + "backtrace", + "cfg-if 1.0.0", + "criterion", + "findshlibs", + "inferno", + "libc", + "log", + "nix 0.26.4", + "once_cell", + "parking_lot 0.12.1", + "smallvec", + "symbolic-demangle", + "tempfile", + "thiserror", +] + [[package]] name = "ppv-lite86" version = "0.2.17" diff --git a/Cargo.toml b/Cargo.toml index 5dd453415aa..6a939878f9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,6 +104,14 @@ chumsky = { git = "https://github.com/jfecher/chumsky", rev = "ad9d312", default "ahash", "std", ] } + +# Benchmarking +criterion = "0.5.0" +# Note that using the "frame-pointer" feature breaks framegraphs on linux +# https://github.com/tikv/pprof-rs/pull/172 +pprof = { version = "0.13", features = ["flamegraph","criterion"] } + + dirs = "4" serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0" diff --git a/acvm-repo/acir/Cargo.toml b/acvm-repo/acir/Cargo.toml index 99095ad3f61..1702780ba9b 100644 --- a/acvm-repo/acir/Cargo.toml +++ b/acvm-repo/acir/Cargo.toml @@ -28,8 +28,14 @@ strum_macros = "0.24" serde-reflection = "0.3.6" serde-generate = "0.25.1" fxhash.workspace = true +criterion.workspace = true +pprof.workspace = true [features] default = ["bn254"] bn254 = ["acir_field/bn254", "brillig/bn254"] bls12_381 = ["acir_field/bls12_381", "brillig/bls12_381"] + +[[bench]] +name = "serialization" +harness = false diff --git a/acvm-repo/acir/benches/serialization.rs b/acvm-repo/acir/benches/serialization.rs new file mode 100644 index 00000000000..73e3916a73b --- /dev/null +++ b/acvm-repo/acir/benches/serialization.rs @@ -0,0 +1,123 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use std::{collections::BTreeSet, time::Duration}; + +use acir::{ + circuit::{Circuit, ExpressionWidth, Opcode, Program, PublicInputs}, + native_types::{Expression, Witness}, + FieldElement, +}; + +use pprof::criterion::{Output, PProfProfiler}; + +const SIZES: [usize; 9] = [10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000]; + +fn sample_program(num_opcodes: usize) -> Program { + let assert_zero_opcodes: Vec = (0..num_opcodes) + .map(|i| { + Opcode::AssertZero(Expression { + mul_terms: vec![( + FieldElement::from(2 * i), + Witness(i as u32), + Witness(i as u32 + 10), + )], + linear_combinations: vec![ + (FieldElement::from(2 * i), Witness(i as u32)), + (FieldElement::from(3 * i), Witness(i as u32 + 1)), + ], + q_c: FieldElement::from(i), + }) + }) + .collect(); + + Program { + functions: vec![Circuit { + current_witness_index: 4000, + opcodes: assert_zero_opcodes.to_vec(), + expression_width: ExpressionWidth::Bounded { width: 3 }, + private_parameters: BTreeSet::from([Witness(1), Witness(2), Witness(3), Witness(4)]), + public_parameters: PublicInputs(BTreeSet::from([Witness(5)])), + return_values: PublicInputs(BTreeSet::from([Witness(6)])), + assert_messages: Vec::new(), + recursive: false, + }], + } +} + +fn bench_serialization(c: &mut Criterion) { + let mut group = c.benchmark_group("serialize_program"); + for size in SIZES.iter() { + let program = sample_program(*size); + + group.throughput(Throughput::Elements(*size as u64)); + group.bench_with_input(BenchmarkId::from_parameter(size), &program, |b, program| { + b.iter(|| Program::serialize_program(program)); + }); + } + group.finish(); + + let mut group = c.benchmark_group("serialize_program_json"); + for size in SIZES.iter() { + let program = sample_program(*size); + + group.throughput(Throughput::Elements(*size as u64)); + group.bench_with_input(BenchmarkId::from_parameter(size), &program, |b, program| { + b.iter(|| { + let mut bytes = Vec::new(); + let mut serializer = serde_json::Serializer::new(&mut bytes); + Program::serialize_program_base64(program, &mut serializer) + }); + }); + } + group.finish(); +} + +fn bench_deserialization(c: &mut Criterion) { + let mut group = c.benchmark_group("deserialize_program"); + for size in SIZES.iter() { + let program = sample_program(*size); + let serialized_program = Program::serialize_program(&program); + + group.throughput(Throughput::Elements(*size as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(size), + &serialized_program, + |b, program| { + b.iter(|| Program::deserialize_program(program)); + }, + ); + } + group.finish(); + + let mut group = c.benchmark_group("deserialize_program_json"); + for size in SIZES.iter() { + let program = sample_program(*size); + + let serialized_program = { + let mut bytes = Vec::new(); + let mut serializer = serde_json::Serializer::new(&mut bytes); + Program::serialize_program_base64(&program, &mut serializer).expect("should succeed"); + bytes + }; + + group.throughput(Throughput::Elements(*size as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(size), + &serialized_program, + |b, program| { + b.iter(|| { + let mut deserializer = serde_json::Deserializer::from_slice(program); + Program::deserialize_program_base64(&mut deserializer) + }); + }, + ); + } + group.finish(); +} + +criterion_group!( + name = benches; + config = Criterion::default().sample_size(40).measurement_time(Duration::from_secs(20)).with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + targets = bench_serialization, bench_deserialization +); + +criterion_main!(benches); diff --git a/acvm-repo/acvm/Cargo.toml b/acvm-repo/acvm/Cargo.toml index e6554d3f773..b19ce6d614f 100644 --- a/acvm-repo/acvm/Cargo.toml +++ b/acvm-repo/acvm/Cargo.toml @@ -40,3 +40,9 @@ bls12_381 = [ rand = "0.8.5" proptest = "1.2.0" paste = "1.0.14" +criterion.workspace = true +pprof.workspace = true + +[[bench]] +name = "criterion" +harness = false diff --git a/tooling/nargo_cli/Cargo.toml b/tooling/nargo_cli/Cargo.toml index 1629ae86edb..111caaa9c92 100644 --- a/tooling/nargo_cli/Cargo.toml +++ b/tooling/nargo_cli/Cargo.toml @@ -72,13 +72,9 @@ assert_cmd = "2.0.8" assert_fs = "1.0.10" predicates = "2.1.5" fm.workspace = true -criterion = "0.5.0" +criterion.workspace = true +pprof.workspace = true paste = "1.0.14" -pprof = { version = "0.12", features = [ - "flamegraph", - "frame-pointer", - "criterion", -] } iai = "0.1.1" test-binary = "3.0.2" From d79a02872a0b8110f55c79eed3a4fc17b1f0cbc6 Mon Sep 17 00:00:00 2001 From: Tom French Date: Tue, 16 Apr 2024 14:54:38 +0100 Subject: [PATCH 2/3] chore: add scripts to set `perf_event_paranoid` --- scripts/benchmark_start.sh | 3 +++ scripts/benchmark_stop.sh | 3 +++ 2 files changed, 6 insertions(+) create mode 100755 scripts/benchmark_start.sh create mode 100755 scripts/benchmark_stop.sh diff --git a/scripts/benchmark_start.sh b/scripts/benchmark_start.sh new file mode 100755 index 00000000000..3e69b3d2c65 --- /dev/null +++ b/scripts/benchmark_start.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid diff --git a/scripts/benchmark_stop.sh b/scripts/benchmark_stop.sh new file mode 100755 index 00000000000..964e5291817 --- /dev/null +++ b/scripts/benchmark_stop.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo 4 | sudo tee /proc/sys/kernel/perf_event_paranoid From 4ab6f654b338635efea21c04129dcec0a0ba8c6f Mon Sep 17 00:00:00 2001 From: Tom French Date: Tue, 16 Apr 2024 14:57:30 +0100 Subject: [PATCH 3/3] chore: remove leftover config --- Cargo.lock | 2 -- acvm-repo/acvm/Cargo.toml | 6 ------ 2 files changed, 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b1ee546d71..4c039ccdeae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,11 +44,9 @@ dependencies = [ "acir", "acvm_blackbox_solver", "brillig_vm", - "criterion", "indexmap 1.9.3", "num-bigint", "paste", - "pprof 0.13.0", "proptest", "rand 0.8.5", "thiserror", diff --git a/acvm-repo/acvm/Cargo.toml b/acvm-repo/acvm/Cargo.toml index b19ce6d614f..e6554d3f773 100644 --- a/acvm-repo/acvm/Cargo.toml +++ b/acvm-repo/acvm/Cargo.toml @@ -40,9 +40,3 @@ bls12_381 = [ rand = "0.8.5" proptest = "1.2.0" paste = "1.0.14" -criterion.workspace = true -pprof.workspace = true - -[[bench]] -name = "criterion" -harness = false