diff --git a/.gitignore b/.gitignore index 0085848c..7fbd3439 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ Cargo.lock *.pdb *.env -*.docker-compose.yml +cb.docker-compose.yml targets.json .idea/ logs \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index e29ade9a..a180dec6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/metrics", "tests", + "benches/*", "examples/*", ] resolver = "2" diff --git a/benches/pbs/Cargo.toml b/benches/pbs/Cargo.toml new file mode 100644 index 00000000..257bd72b --- /dev/null +++ b/benches/pbs/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "cb-bench-pbs" +version.workspace = true +edition.workspace = true +rust-version.workspace = true + +[dependencies] +cb-common.workspace = true +cb-pbs.workspace = true +cb-tests = { path = "../../tests" } + +tokio.workspace = true + +axum.workspace = true + +alloy.workspace = true + +serde.workspace = true +toml.workspace = true +serde_json.workspace = true + +reqwest.workspace = true + +tracing.workspace = true +tracing-subscriber.workspace = true + +tree_hash.workspace = true +eyre.workspace = true + +rand.workspace = true +histogram = "0.11.0" +comfy-table = "7.1.1" diff --git a/benches/pbs/README.md b/benches/pbs/README.md new file mode 100644 index 00000000..be10c6d7 --- /dev/null +++ b/benches/pbs/README.md @@ -0,0 +1,124 @@ + +# PBS Benchmark + +Benchmark of the PBS module and the [MEV-Boost go-client](https://github.com/flashbots/mev-boost). + +## Benchmark info +Last updated: `02-Sep-2024` +- MEV-Boost: docker image `flashbots/mev-boost:1.8.1`, digest: `sha256:1ce07514249dbd9648773cf5ddfd75f74344c7e49ba8bbc38cec2531e26751a1` +- Commit-Boost: docker image `ghcr.io/commit-boost/pbs:v0.1.0`, digest: `sha256:7d90fc816470f9dd37640bc89b3fdb021a9023e9e3c43ea4426dd1145984ddde` + +
Runtime info + +### `rustc` version +``` +rustc 1.79.0 (129f3b996 2024-06-10) +binary: rustc +commit-hash: 129f3b9964af4d4a709d1383930ade12dfe7c081 +commit-date: 2024-06-10 +host: x86_64-unknown-linux-gnu +release: 1.79.0 +LLVM version: 18.1.7 +``` + +### CPU info +``` +Architecture: x86_64 + CPU op-mode(s): 32-bit, 64-bit + Address sizes: 46 bits physical, 48 bits virtual + Byte Order: Little Endian +CPU(s): 20 + On-line CPU(s) list: 0-19 +Vendor ID: GenuineIntel + Model name: 13th Gen Intel(R) Core(TM) i7-1370P + CPU family: 6 + Model: 186 + Thread(s) per core: 2 + Core(s) per socket: 14 + Socket(s): 1 + Stepping: 2 + CPU(s) scaling MHz: 26% + CPU max MHz: 5200.0000 + CPU min MHz: 400.0000 + BogoMIPS: 4377.60 + Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dt + es64 monitor ds_cpl smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid r + dseed adx smap clflushopt clwb intel_pt sha_ni xsaveopt xsavec xgetbv1 xsaves split_lock_detect avx_vnni dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp hwp_pkg_req hfi umip pku ospke waitpkg gfni vaes vpclmulqdq tme rdpid movdiri movdir64b fsrm md_clear ser + ialize pconfig arch_lbr ibt flush_l1d arch_capabilities +Caches (sum of all): + L1d: 544 KiB (14 instances) + L1i: 704 KiB (14 instances) + L2: 11.5 MiB (8 instances) + L3: 24 MiB (1 instance) +NUMA: + NUMA node(s): 1 + NUMA node0 CPU(s): 0-19 +Vulnerabilities: + Gather data sampling: Not affected + Itlb multihit: Not affected + L1tf: Not affected + Mds: Not affected + Meltdown: Not affected + Mmio stale data: Not affected + Retbleed: Not affected + Spec rstack overflow: Not affected + Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl + Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization + Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI BHI_DIS_S + Srbds: Not affected + Tsx async abort: Not affected +``` +
+ +## Setup +To isolate the performance of the sidecar, we create a mock validator that will trigger the sidecar, and a mock relay that will answer calls from the sidecar. Currently we support a single mock relay. + +### Setup sidecars +Setup the sidecars and fill the `bench-config.toml` file accordingly. + +#### MEV-Boost +Follow [these instructions](https://github.com/flashbots/mev-boost?tab=readme-ov-file#installing). To launch the docker image use this command: + +```bash +sudo docker run -d --network host --name mev_boost_bench flashbots/mev-boost:1.8.1 -addr 0.0.0.0:18650 -holesky -relay http://0xb060572f535ba5615b874ebfef757fbe6825352ad257e31d724e57fe25a067a13cfddd0f00cb17bf3a3d2e901a380c17@172.17.0.1:18450 +``` +After the benchmark, clean up the container: +```bash +docker rm --force mev_boost_bench +``` + +#### Commit-Boost +You can run the provided `docker-compose` file: +```bash +commit-boost start --docker benches/pbs/bench.docker-compose.yml +``` +or regenerate it using `commit-boost init`. + +To clean up after then benchmark, run: +```bash +commit-boost stop --docker benches/pbs/bench.docker-compose.yml +``` + +### Running the benchmark +Run the benchmark with +```bash +cargo run --release --bin cb-bench-pbs -- benches/pbs/bench-config.toml +``` +Based on the `bench-config.toml` file, this will simulate multiple calls to each sidecar and measure the latency. + +## Results +### Get Header +For each `get_header` call we measure the latency. Note that this latency also includes some small network overhead, and the internal overhead of the mock relay. The assumption is these overheads are ~constants across test cases. This also means that a single latency measurement is not significative, but only useful to be compared across test cases. + + +```bash +Bench results (lower is better) +Lowest is indicated with *, percentages are relative to lowest ++--------------+-------------------+------------------+------------------+------------------+ +| ID | p50 | p90 | p95 | p99 | ++===========================================================================================+ +| mev_boost | 5.22ms (+152.54%) | 6.87ms (+53.30%) | 7.63ms (+52.79%) | 8.98ms (+37.67%) | +|--------------+-------------------+------------------+------------------+------------------| +| commit_boost | 2.07ms (*) | 4.48ms (*) | 4.99ms (*) | 6.52ms (*) | ++--------------+-------------------+------------------+------------------+------------------+ +``` \ No newline at end of file diff --git a/benches/pbs/bench-config.toml b/benches/pbs/bench-config.toml new file mode 100644 index 00000000..0e10e08f --- /dev/null +++ b/benches/pbs/bench-config.toml @@ -0,0 +1,22 @@ +chain = "Holesky" + +[pbs] +port = 18750 +late_in_slot_time_ms = 1000000000000 # skip late in slot checks + +[[relays]] +id = "bench_mock_relay" +url = "http://0xb060572f535ba5615b874ebfef757fbe6825352ad257e31d724e57fe25a067a13cfddd0f00cb17bf3a3d2e901a380c17@172.17.0.1:18450" # do not change this + + +[benchmark] +n_slots = 5 +headers_per_slot = 1000 + +[[bench]] +id = "mev_boost" +url = "http://0.0.0.0:18650" + +[[bench]] +id = "commit_boost" +url = "http://0.0.0.0:18750" diff --git a/benches/pbs/bench.docker-compose.yml b/benches/pbs/bench.docker-compose.yml new file mode 100644 index 00000000..6f611b92 --- /dev/null +++ b/benches/pbs/bench.docker-compose.yml @@ -0,0 +1,10 @@ +services: + cb_pbs: + image: ghcr.io/commit-boost/pbs:v0.1.0 + container_name: cb_pbs + ports: + - 18750:18750 + environment: + CB_CONFIG: /cb-config.toml + volumes: + - ./bench-config.toml:/cb-config.toml:ro diff --git a/benches/pbs/src/config.rs b/benches/pbs/src/config.rs new file mode 100644 index 00000000..ea903b96 --- /dev/null +++ b/benches/pbs/src/config.rs @@ -0,0 +1,35 @@ +use std::fs; + +use cb_common::config::CommitBoostConfig; +use reqwest::Url; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Config { + #[serde(flatten)] + pub commit_boost: CommitBoostConfig, + pub benchmark: BenchmarkConfig, + pub bench: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct BenchmarkConfig { + pub n_slots: u64, + pub headers_per_slot: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct BenchConfig { + pub id: String, + pub url: Url, +} + +pub fn load_static_config() -> Config { + let path = + std::env::args().nth(1).expect("missing config path. Add config eg. `bench-config.toml'"); + let config_file = fs::read_to_string(&path) + .unwrap_or_else(|_| panic!("Unable to find config file: '{}'", path)); + let config: Config = toml::from_str(&config_file).expect("failed to parse toml"); + + config +} diff --git a/benches/pbs/src/main.rs b/benches/pbs/src/main.rs new file mode 100644 index 00000000..abb6ee3d --- /dev/null +++ b/benches/pbs/src/main.rs @@ -0,0 +1,172 @@ +use std::time::{Duration, Instant}; + +use alloy::{primitives::B256, rpc::types::beacon::BlsPublicKey}; +use cb_common::{ + config::RelayConfig, + pbs::{GetHeaderResponse, RelayClient, RelayEntry}, + signer::{BlsSecretKey, BlsSigner}, + types::Chain, +}; +use cb_tests::mock_relay::{start_mock_relay_service, MockRelayState}; +use comfy_table::Table; +use config::{load_static_config, BenchConfig}; +use histogram::Histogram; + +mod config; + +fn get_random_hash() -> B256 { + B256::from(rand::random::<[u8; 32]>()) +} +fn get_random_pubkey() -> BlsPublicKey { + BlsPublicKey::ZERO +} + +#[tokio::main] +async fn main() { + let config = load_static_config(); + + // start mock relay + let relay = config.commit_boost.relays.first().expect("missing relay config"); + tokio::spawn(start_mock_relay(config.commit_boost.chain, relay.clone())); + tokio::time::sleep(Duration::from_millis(100)).await; + + let mut bench_results = Vec::with_capacity(config.bench.len()); + + // get_header benchmark + for bench in config.bench { + print!("Benching {}...", bench.id); + let total_start = Instant::now(); + + let mock_validator = get_mock_validator(bench); + + // max ~1s + let mut histo = Histogram::new(12, 20).unwrap(); + + // bench + for slot in 0..config.benchmark.n_slots { + let parent_hash = get_random_hash(); + let validator_pubkey = get_random_pubkey(); + let url = mock_validator.get_header_url(slot, parent_hash, validator_pubkey).unwrap(); + + for _ in 0..config.benchmark.headers_per_slot { + let url = url.clone(); + + let start = Instant::now(); + let res = mock_validator.client.get(url).send().await; + let end = start.elapsed(); + + let res = res + .expect("failed to get header") + .bytes() + .await + .expect("failed to decode response"); + + assert!( + serde_json::from_slice::(&res).is_ok(), + "invalid header returned" + ); + + histo.increment(end.as_micros() as u64).unwrap(); + } + } + + println!("took {:?}", total_start.elapsed()); + + let p50 = histo.percentile(50.).expect("failed to get p50").unwrap().end(); + let p90 = histo.percentile(90.).expect("failed to get p90").unwrap().end(); + let p95 = histo.percentile(95.).expect("failed to get p95").unwrap().end(); + let p99 = histo.percentile(99.).expect("failed to get p99").unwrap().end(); + + bench_results.push(BenchResults { id: mock_validator.id.to_string(), p50, p90, p95, p99 }); + } + + let best_p50 = bench_results.iter().min_by_key(|b| b.p50).unwrap().p50; + let best_p90 = bench_results.iter().min_by_key(|b| b.p90).unwrap().p90; + let best_p95 = bench_results.iter().min_by_key(|b| b.p95).unwrap().p95; + let best_p99 = bench_results.iter().min_by_key(|b| b.p99).unwrap().p99; + + let mut table = Table::new(); + table.set_header(vec!["ID", "p50", "p90", "p95", "p99"]); + + for result in bench_results { + let p50_ms = result.p50 as f64 / 1000.; + let r50 = if result.p50 == best_p50 { + format!("{p50_ms:.2}ms (*)") + } else { + let slow_pct = (result.p50 as f64 / best_p50 as f64 - 1.0) * 100.; + format!("{p50_ms:.2}ms (+{slow_pct:.2}%)") + }; + + let p90_ms = result.p90 as f64 / 1000.; + let r90 = if result.p90 == best_p90 { + format!("{p90_ms:.2}ms (*)") + } else { + let slow_pct = (result.p90 as f64 / best_p90 as f64 - 1.0) * 100.; + format!("{p90_ms:.2}ms (+{slow_pct:.2}%)") + }; + + let p95_ms = result.p95 as f64 / 1000.; + let r95 = if result.p95 == best_p95 { + format!("{p95_ms:.2}ms (*)") + } else { + let slow_pct = (result.p95 as f64 / best_p95 as f64 - 1.0) * 100.; + format!("{p95_ms:.2}ms (+{slow_pct:.2}%)") + }; + + let p99_ms = result.p99 as f64 / 1000.; + let r99 = if result.p99 == best_p99 { + format!("{p99_ms:.2}ms (*)") + } else { + let slow_pct = (result.p99 as f64 / best_p99 as f64 - 1.0) * 100.; + format!("{p99_ms:.2}ms (+{slow_pct:.2}%)") + }; + + table.add_row(vec![result.id, r50, r90, r95, r99]); + } + + println!(); + + println!("Bench results (lower is better)"); + println!("Lowest is indicated with *, percentages are relative to lowest"); + println!("{table}"); +} + +// mock relay +const MOCK_RELAY_SECRET: [u8; 32] = [ + 131, 231, 162, 159, 42, 4, 109, 144, 166, 131, 12, 91, 185, 48, 106, 219, 55, 145, 120, 57, 51, + 152, 98, 59, 240, 181, 131, 47, 1, 180, 255, 245, +]; +async fn start_mock_relay(chain: Chain, relay_config: RelayConfig) { + let signer = BlsSigner::Local(BlsSecretKey::key_gen(&MOCK_RELAY_SECRET, &[]).unwrap()); + + assert_eq!(relay_config.entry.pubkey, *signer.pubkey(), "Expected relay pubkey to be 0xb060572f535ba5615b874ebfef757fbe6825352ad257e31d724e57fe25a067a13cfddd0f00cb17bf3a3d2e901a380c17"); + + let relay_port = relay_config.entry.url.port().expect("missing port"); + + let mock_relay = MockRelayState::new(chain, signer); + start_mock_relay_service(mock_relay.into(), relay_port) + .await + .expect("failed to start mock relay"); +} + +fn get_mock_validator(bench: BenchConfig) -> RelayClient { + let entry = RelayEntry { id: bench.id, pubkey: BlsPublicKey::default(), url: bench.url }; + let config = RelayConfig { + entry, + id: None, + headers: None, + enable_timing_games: false, + target_first_request_ms: None, + frequency_get_header_ms: None, + }; + + RelayClient::new(config).unwrap() +} + +struct BenchResults { + id: String, + p50: u64, + p90: u64, + p95: u64, + p99: u64, +} diff --git a/config.example.toml b/config.example.toml index 5b12a24b..01e2102a 100644 --- a/config.example.toml +++ b/config.example.toml @@ -33,7 +33,6 @@ skip_sigverify = false # Minimum bid in ETH that will be accepted from `get_header` # OPTIONAL, DEFAULT: 0.0 min_bid_eth = 0.0 -# How late in milliseconds in the slot is "late". This impacts the `get_header` requests, by shortening timeouts for `get_header` calls to # List of URLs of relay monitors to send registrations to # OPTIONAL relay_monitors = [] diff --git a/tests/src/mock_relay.rs b/tests/src/mock_relay.rs index 3f0aecec..4ceceef2 100644 --- a/tests/src/mock_relay.rs +++ b/tests/src/mock_relay.rs @@ -1,6 +1,9 @@ -use std::sync::{ - atomic::{AtomicU64, Ordering}, - Arc, +use std::{ + net::SocketAddr, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, }; use alloy::{primitives::U256, rpc::types::beacon::relay::ValidatorRegistration}; @@ -19,12 +22,22 @@ use cb_common::{ signer::ConsensusSigner, types::Chain, }; +use tokio::net::TcpListener; use tracing::debug; use tree_hash::TreeHash; +pub async fn start_mock_relay_service(state: Arc, port: u16) -> eyre::Result<()> { + let app = mock_relay_app_router(state); + + let socket = SocketAddr::new("0.0.0.0".parse()?, port); + let listener = TcpListener::bind(socket).await?; + + axum::serve(listener, app).await?; + Ok(()) +} + pub struct MockRelayState { pub chain: Chain, - pub get_header_delay_ms: u64, pub signer: ConsensusSigner, received_get_header: Arc, received_get_status: Arc, @@ -48,11 +61,11 @@ impl MockRelayState { } impl MockRelayState { - pub fn new(chain: Chain, signer: ConsensusSigner, get_header_delay_ms: u64) -> Self { + pub fn new(chain: Chain, signer: ConsensusSigner) -> Self { Self { chain, signer, - get_header_delay_ms, + received_get_header: Default::default(), received_get_status: Default::default(), received_register_validator: Default::default(), diff --git a/tests/tests/pbs_integration.rs b/tests/tests/pbs_integration.rs index fd0d7eea..13300854 100644 --- a/tests/tests/pbs_integration.rs +++ b/tests/tests/pbs_integration.rs @@ -1,4 +1,4 @@ -use std::{net::SocketAddr, sync::Arc, time::Duration, u64}; +use std::{sync::Arc, time::Duration, u64}; use alloy::primitives::U256; use cb_common::{ @@ -9,25 +9,13 @@ use cb_common::{ }; use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; use cb_tests::{ - mock_relay::{mock_relay_app_router, MockRelayState}, + mock_relay::{start_mock_relay_service, MockRelayState}, mock_validator::MockValidator, utils::{generate_mock_relay, setup_test_env}, }; use eyre::Result; -use tokio::net::TcpListener; use tracing::info; -async fn start_mock_relay_service(state: Arc, port: u16) -> Result<()> { - let app = mock_relay_app_router(state); - - let socket = SocketAddr::new("0.0.0.0".parse()?, port); - let listener = TcpListener::bind(socket).await?; - - info!("Starting mock relay on {socket:?}"); - axum::serve(listener, app).await?; - Ok(()) -} - fn get_pbs_static_config(port: u16) -> PbsConfig { PbsConfig { port, @@ -61,7 +49,7 @@ async fn test_get_header() -> Result<()> { let port = 3000; let mock_relay = generate_mock_relay(port + 1, *signer.pubkey())?; - let mock_state = Arc::new(MockRelayState::new(chain, signer, 0)); + let mock_state = Arc::new(MockRelayState::new(chain, signer)); tokio::spawn(start_mock_relay_service(mock_state.clone(), port + 1)); let config = to_pbs_config(chain, get_pbs_static_config(port), vec![mock_relay]); @@ -92,7 +80,7 @@ async fn test_get_status() -> Result<()> { generate_mock_relay(port + 1, *signer.pubkey())?, generate_mock_relay(port + 2, *signer.pubkey())?, ]; - let mock_state = Arc::new(MockRelayState::new(chain, signer, 0)); + let mock_state = Arc::new(MockRelayState::new(chain, signer)); tokio::spawn(start_mock_relay_service(mock_state.clone(), port + 1)); tokio::spawn(start_mock_relay_service(mock_state.clone(), port + 2)); @@ -121,7 +109,7 @@ async fn test_register_validators() -> Result<()> { let port = 3300; let relays = vec![generate_mock_relay(port + 1, *signer.pubkey())?]; - let mock_state = Arc::new(MockRelayState::new(chain, signer, 0)); + let mock_state = Arc::new(MockRelayState::new(chain, signer)); tokio::spawn(start_mock_relay_service(mock_state.clone(), port + 1)); let config = to_pbs_config(chain, get_pbs_static_config(port), relays); @@ -149,7 +137,7 @@ async fn test_submit_block() -> Result<()> { let port = 3400; let relays = vec![generate_mock_relay(port + 1, *signer.pubkey())?]; - let mock_state = Arc::new(MockRelayState::new(chain, signer, 0)); + let mock_state = Arc::new(MockRelayState::new(chain, signer)); tokio::spawn(start_mock_relay_service(mock_state.clone(), port + 1)); let config = to_pbs_config(chain, get_pbs_static_config(port), relays);