From 466059561b2bee6877c9c2c1416ef594a586c3be Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Sat, 9 Apr 2022 15:05:38 +0200 Subject: [PATCH 01/26] Move new_rng to shared code Signed-off-by: Oliver Tale-Yazdi --- utils/frame/benchmarking-cli/src/shared/mod.rs | 10 ++++++++++ utils/frame/benchmarking-cli/src/storage/cmd.rs | 11 ++--------- utils/frame/benchmarking-cli/src/storage/read.rs | 4 ++-- utils/frame/benchmarking-cli/src/storage/write.rs | 4 ++-- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/utils/frame/benchmarking-cli/src/shared/mod.rs b/utils/frame/benchmarking-cli/src/shared/mod.rs index f08d79b9aafca..a4da8cada0398 100644 --- a/utils/frame/benchmarking-cli/src/shared/mod.rs +++ b/utils/frame/benchmarking-cli/src/shared/mod.rs @@ -25,6 +25,8 @@ pub use record::BenchRecord; pub use stats::{StatSelect, Stats}; pub use weight_params::WeightParams; +use rand::prelude::*; + /// A Handlebars helper to add an underscore after every 3rd character, /// i.e. a separator for large numbers. #[derive(Clone, Copy)] @@ -63,3 +65,11 @@ where } s } + +/// Returns an rng and the seed that was used to create it. +/// +/// Uses a random seed if none is provided. +pub fn new_rng(seed: Option) -> (impl rand::Rng, u64) { + let seed = seed.unwrap_or(rand::thread_rng().gen::()); + (StdRng::seed_from_u64(seed), seed) +} diff --git a/utils/frame/benchmarking-cli/src/storage/cmd.rs b/utils/frame/benchmarking-cli/src/storage/cmd.rs index c2cc219ef1528..23222dbd120ab 100644 --- a/utils/frame/benchmarking-cli/src/storage/cmd.rs +++ b/utils/frame/benchmarking-cli/src/storage/cmd.rs @@ -34,7 +34,7 @@ use sp_runtime::generic::BlockId; use std::{fmt::Debug, path::PathBuf, sync::Arc}; use super::template::TemplateData; -use crate::shared::WeightParams; +use crate::shared::{new_rng, WeightParams}; /// Benchmark the storage speed of a chain snapshot. #[derive(Debug, Parser)] @@ -151,13 +151,6 @@ impl StorageCmd { } } - /// Creates an rng from a random seed. - pub(crate) fn setup_rng() -> impl rand::Rng { - let seed = rand::thread_rng().gen::(); - info!("Using seed {}", seed); - StdRng::seed_from_u64(seed) - } - /// Run some rounds of the (read) benchmark as warmup. /// See `frame_benchmarking_cli::storage::read::bench_read` for detailed comments. fn bench_warmup(&self, client: &Arc) -> Result<()> @@ -169,7 +162,7 @@ impl StorageCmd { let block = BlockId::Number(client.usage_info().chain.best_number); let empty_prefix = StorageKey(Vec::new()); let mut keys = client.storage_keys(&block, &empty_prefix)?; - let mut rng = Self::setup_rng(); + let (mut rng, _) = new_rng(None); keys.shuffle(&mut rng); for i in 0..self.params.warmups { diff --git a/utils/frame/benchmarking-cli/src/storage/read.rs b/utils/frame/benchmarking-cli/src/storage/read.rs index f58f3c3de0c19..c1dc6daba0953 100644 --- a/utils/frame/benchmarking-cli/src/storage/read.rs +++ b/utils/frame/benchmarking-cli/src/storage/read.rs @@ -28,7 +28,7 @@ use rand::prelude::*; use std::{fmt::Debug, sync::Arc, time::Instant}; use super::cmd::StorageCmd; -use crate::shared::BenchRecord; +use crate::shared::{new_rng, BenchRecord}; impl StorageCmd { /// Benchmarks the time it takes to read a single Storage item. @@ -47,7 +47,7 @@ impl StorageCmd { // Load all keys and randomly shuffle them. let empty_prefix = StorageKey(Vec::new()); let mut keys = client.storage_keys(&block, &empty_prefix)?; - let mut rng = Self::setup_rng(); + let (mut rng, _) = new_rng(None); keys.shuffle(&mut rng); // Interesting part here: diff --git a/utils/frame/benchmarking-cli/src/storage/write.rs b/utils/frame/benchmarking-cli/src/storage/write.rs index d5d5bc2fffa5b..ab25109a35d49 100644 --- a/utils/frame/benchmarking-cli/src/storage/write.rs +++ b/utils/frame/benchmarking-cli/src/storage/write.rs @@ -32,7 +32,7 @@ use rand::prelude::*; use std::{fmt::Debug, sync::Arc, time::Instant}; use super::cmd::StorageCmd; -use crate::shared::BenchRecord; +use crate::shared::{new_rng, BenchRecord}; impl StorageCmd { /// Benchmarks the time it takes to write a single Storage item. @@ -59,7 +59,7 @@ impl StorageCmd { info!("Preparing keys from block {}", block); // Load all KV pairs and randomly shuffle them. let mut kvs = trie.pairs(); - let mut rng = Self::setup_rng(); + let (mut rng, _) = new_rng(None); kvs.shuffle(&mut rng); // Generate all random values first; Make sure there are no collisions with existing From e3241adfd4913efb528f99a490d7bd62a49ba5e7 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Sat, 9 Apr 2022 15:05:53 +0200 Subject: [PATCH 02/26] Add bechmark machine command Signed-off-by: Oliver Tale-Yazdi --- Cargo.lock | 71 ++++++++- bin/node-template/node/src/command.rs | 1 + bin/node/cli/src/command.rs | 1 + bin/node/cli/tests/benchmark_machine_works.rs | 32 ++++ utils/frame/benchmarking-cli/Cargo.toml | 2 + utils/frame/benchmarking-cli/src/lib.rs | 4 + .../frame/benchmarking-cli/src/machine/mod.rs | 145 ++++++++++++++++++ 7 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 bin/node/cli/tests/benchmark_machine_works.rs create mode 100644 utils/frame/benchmarking-cli/src/machine/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 8c9227ea80ef4..b8a4a1e7f64ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1688,6 +1688,17 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users 0.3.5", + "winapi 0.3.9", +] + [[package]] name = "dirs-sys" version = "0.3.6" @@ -1695,7 +1706,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.0", "winapi 0.3.9", ] @@ -1706,7 +1717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.0", "winapi 0.3.9", ] @@ -1855,6 +1866,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "enum-as-inner" version = "0.3.3" @@ -2184,6 +2201,7 @@ dependencies = [ "log 0.4.14", "memory-db", "parity-scale-codec", + "prettytable-rs", "rand 0.8.4", "sc-block-builder", "sc-cli", @@ -2200,6 +2218,7 @@ dependencies = [ "sp-database", "sp-externalities", "sp-inherents", + "sp-io", "sp-keystore", "sp-runtime", "sp-state-machine", @@ -7205,6 +7224,20 @@ dependencies = [ "output_vt100", ] +[[package]] +name = "prettytable-rs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd04b170004fa2daccf418a7f8253aaf033c27760b5f225889024cf66d7ac2e" +dependencies = [ + "atty", + "csv", + "encode_unicode", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "primitive-types" version = "0.11.1" @@ -7692,6 +7725,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom 0.1.16", + "redox_syscall 0.1.57", + "rust-argon2", +] + [[package]] name = "redox_users" version = "0.4.0" @@ -7908,6 +7952,18 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rust-argon2" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" +dependencies = [ + "base64 0.13.0", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils 0.8.5", +] + [[package]] name = "rustc-demangle" version = "0.1.18" @@ -10931,6 +10987,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "term" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" +dependencies = [ + "byteorder", + "dirs", + "winapi 0.3.9", +] + [[package]] name = "termcolor" version = "1.1.2" diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index ede969b3572c8..1aa191e69abd7 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -137,6 +137,7 @@ pub fn run() -> sc_cli::Result<()> { cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) }, + BenchmarkCmd::Machine(cmd) => cmd.run(), } }) }, diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index c752b1c30ae26..0954a93897802 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -126,6 +126,7 @@ pub fn run() -> Result<()> { cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) }, + BenchmarkCmd::Machine(cmd) => cmd.run(), } }) }, diff --git a/bin/node/cli/tests/benchmark_machine_works.rs b/bin/node/cli/tests/benchmark_machine_works.rs new file mode 100644 index 0000000000000..95c0ac8132099 --- /dev/null +++ b/bin/node/cli/tests/benchmark_machine_works.rs @@ -0,0 +1,32 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; + +/// Tests that the `benchmark machine` command works for the substrate dev runtime. +#[test] +fn benchmark_machine_works() { + let status = Command::new(cargo_bin("substrate")) + .args(["benchmark", "machine", "--dev"]) + .args(["--hash-reps", "1", "--verify-reps", "1"]) + .status() + .unwrap(); + + assert!(status.success()); +} diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 5cb81232085a4..7832b78c7fa2a 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -29,6 +29,7 @@ sp-externalities = { version = "0.12.0", path = "../../../primitives/externaliti sp-database = { version = "4.0.0-dev", path = "../../../primitives/database" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } +sp-io = { version = "6.0.0", path = "../../../primitives/io" } sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } sp-storage = { version = "6.0.0", path = "../../../primitives/storage" } sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } @@ -52,6 +53,7 @@ hex = "0.4.3" memory-db = "0.29.0" rand = { version = "0.8.4", features = ["small_rng"] } thousands = "0.2.0" +prettytable-rs = "0.8.0" [features] default = ["db", "sc-client-db/runtime-benchmarks"] diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 2543a63b2f159..f585b6c7ec65b 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -18,12 +18,14 @@ //! Contains the root [`BenchmarkCmd`] command and exports its sub-commands. mod block; +mod machine; mod overhead; mod pallet; mod shared; mod storage; pub use block::BlockCmd; +pub use machine::MachineCmd; pub use overhead::{ExtrinsicBuilder, OverheadCmd}; pub use pallet::PalletCmd; pub use storage::StorageCmd; @@ -39,6 +41,7 @@ pub enum BenchmarkCmd { Storage(StorageCmd), Overhead(OverheadCmd), Block(BlockCmd), + Machine(MachineCmd), } /// Unwraps a [`BenchmarkCmd`] into its concrete sub-command. @@ -53,6 +56,7 @@ macro_rules! unwrap_cmd { BenchmarkCmd::Storage($cmd) => $code, BenchmarkCmd::Overhead($cmd) => $code, BenchmarkCmd::Block($cmd) => $code, + BenchmarkCmd::Machine($cmd) => $code, } } } diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs new file mode 100644 index 0000000000000..bd61fbd0af596 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -0,0 +1,145 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Contains [`MachineCmd`] as entry point for the node +//! and the core benchmarking logic. + +use sc_cli::{CliConfiguration, Result, SharedParams}; +use sp_core::{blake2_256, sr25519, Pair}; +use sp_io::crypto::sr25519_verify; +use sp_std::prelude::*; + +use clap::Parser; +use log::info; +use prettytable::{cell, row, table, Table}; +use rand::RngCore; +use std::{ + fmt::Debug, + time::{Duration, Instant}, +}; +use thousands::Separable; + +use crate::shared::new_rng; + +/// Command to benchmark the hardware. +/// +/// The implementation of this command is currently a place holder until +/// is merged. +/// +/// The output or `benchmark machine --dev` looks like this: +/// ```text +/// +----------------+------------------+------------+----------+------------+ +/// | Function | Input len [byte] | Iterations | Time [s] | Per second | +/// +----------------+------------------+------------+----------+------------+ +/// | BLAKE2-256 | 32 | 10,000,000 | 2.061 | 4,851,958 | +/// +----------------+------------------+------------+----------+------------+ +/// | SR25519 verify | 32 | 20,000 | 1.489 | 13,429 | +/// +----------------+------------------+------------+----------+------------+ +/// ``` +#[derive(Debug, Parser)] +pub struct MachineCmd { + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: SharedParams, + + /// Iterations of the hash function. + #[clap(long, default_value = "10000000")] + pub hash_reps: u64, + + /// Iterations of the verification function. + #[clap(long, default_value = "20000")] + pub verify_reps: u64, +} + +impl MachineCmd { + /// Execute the benchmark and log the results. + pub fn run(&self) -> Result<()> { + // Use a table for nicer console output. + let mut table = + table!(["Function", "Input len [byte]", "Iterations", "Time [s]", "Per second"]); + + let input_len = 32_usize; + // Benchmark the BLAKE2-256 hashing speed. + let name = "BLAKE2-256"; + let duration = Self::blake2_speed(self.hash_reps, input_len); + Self::put_col(&mut table, name, self.hash_reps, input_len, duration); + + // Benchmark the speed of SR2519 verification. + let name = "SR25519 verify"; + let duration = Self::sr_verify_speed(self.verify_reps, input_len); + Self::put_col(&mut table, name, self.verify_reps, input_len, duration); + + info!("\n{}", table); + Ok(()) + } + + /// Recursively hashes a value `reps` many times. + fn blake2_speed(reps: u64, input_len: usize) -> Duration { + let mut data = vec![0u8; input_len]; + + let start = Instant::now(); + for _ in 0..reps { + data = blake2_256(&data).into(); + } + start.elapsed() + } + + /// Verify the signature of different random data `reps` many times. + fn sr_verify_speed(reps: u64, input_len: usize) -> Duration { + let pair = sr25519::Pair::from_string("//Alice", None).unwrap(); + + let (mut rng, _) = new_rng(None); + let mut msgs = Vec::new(); + let mut sigs = Vec::new(); + + for _ in 0..reps { + let mut msg = vec![0u8; input_len]; + rng.fill_bytes(&mut msg[..]); + + sigs.push(pair.sign(&msg)); + msgs.push(msg); + } + + // Measure the time to verify all. + let start = Instant::now(); + for (sig, msg) in sigs.into_iter().zip(msgs.iter()) { + sr25519_verify(&sig, &msg[..], &pair.public()); + } + + start.elapsed() + } + + /// Format the data and put it as column into the table. + fn put_col(table: &mut Table, name: &str, reps: u64, size: usize, d: Duration) { + let d = d.as_secs_f32(); + let s = (reps as f32 / d) as u64; + table.add_row(row![ + name, + size, + reps.separate_with_commas(), + format!("{:.3}", d), + s.separate_with_commas() + ]); + } +} + +// Boilerplate +impl CliConfiguration for MachineCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } +} From 0e8bfa2cb766e823c9e3b006a6db8f212b784049 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 11 Apr 2022 13:54:25 +0200 Subject: [PATCH 03/26] Use sc-sysinfo Signed-off-by: Oliver Tale-Yazdi --- utils/frame/benchmarking-cli/Cargo.toml | 2 + .../frame/benchmarking-cli/src/machine/mod.rs | 97 ++++++++----------- 2 files changed, 43 insertions(+), 56 deletions(-) diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 7832b78c7fa2a..db0890a9312cb 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -23,6 +23,7 @@ sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" } sc-client-db = { version = "0.10.0-dev", path = "../../../client/db" } sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" } +sc-sysinfo = { version = "6.0.0-dev", path = "../../../client/sysinfo" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-externalities = { version = "0.12.0", path = "../../../primitives/externalities" } @@ -54,6 +55,7 @@ memory-db = "0.29.0" rand = { version = "0.8.4", features = ["small_rng"] } thousands = "0.2.0" prettytable-rs = "0.8.0" +tempfile = "3.3.0" [features] default = ["db", "sc-client-db/runtime-benchmarks"] diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index bd61fbd0af596..dc3738ebc85de 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -15,40 +15,49 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Contains [`MachineCmd`] as entry point for the node +//! Contains the [`MachineCmd`] as entry point for the node //! and the core benchmarking logic. use sc_cli::{CliConfiguration, Result, SharedParams}; -use sp_core::{blake2_256, sr25519, Pair}; +use sc_sysinfo::gather_hwbench; +use sp_core::{sr25519, Pair}; use sp_io::crypto::sr25519_verify; use sp_std::prelude::*; use clap::Parser; use log::info; -use prettytable::{cell, row, table, Table}; +use prettytable::{cell, row, table}; use rand::RngCore; use std::{ fmt::Debug, time::{Duration, Instant}, }; -use thousands::Separable; +use tempfile::tempdir; use crate::shared::new_rng; /// Command to benchmark the hardware. /// -/// The implementation of this command is currently a place holder until -/// is merged. +/// Runs multiple benchmarks and prints their output to console. +/// Can be used to gauge if the hardware is fast enough to keep up with a chain's requirements. +/// This command must be integrated by the client since the client can set compiler flags +/// which influence the results. /// /// The output or `benchmark machine --dev` looks like this: /// ```text -/// +----------------+------------------+------------+----------+------------+ -/// | Function | Input len [byte] | Iterations | Time [s] | Per second | -/// +----------------+------------------+------------+----------+------------+ -/// | BLAKE2-256 | 32 | 10,000,000 | 2.061 | 4,851,958 | -/// +----------------+------------------+------------+----------+------------+ -/// | SR25519 verify | 32 | 20,000 | 1.489 | 13,429 | -/// +----------------+------------------+------------+----------+------------+ +/// +----------+----------------+-------+------+ +/// | Category | Function | Score | Unit | +/// +----------+----------------+-------+------+ +/// | Memory | Copy | 9082 | MB/s | +/// +----------+----------------+-------+------+ +/// | Disk | Seq Write | 724 | MB/s | +/// +----------+----------------+-------+------+ +/// | Disk | Rnd Write | 362 | MB/s | +/// +----------+----------------+-------+------+ +/// | CPU | BLAKE2-256 | 816 | MB/s | +/// +----------+----------------+-------+------+ +/// | CPU | SR25519 Verify | 13.52 | K/s | +/// +----------+----------------+-------+------+ /// ``` #[derive(Debug, Parser)] pub struct MachineCmd { @@ -56,49 +65,38 @@ pub struct MachineCmd { #[clap(flatten)] pub shared_params: SharedParams, - /// Iterations of the hash function. - #[clap(long, default_value = "10000000")] - pub hash_reps: u64, - /// Iterations of the verification function. #[clap(long, default_value = "20000")] pub verify_reps: u64, } impl MachineCmd { - /// Execute the benchmark and log the results. + /// Execute the benchmark and print the results. pub fn run(&self) -> Result<()> { - // Use a table for nicer console output. - let mut table = - table!(["Function", "Input len [byte]", "Iterations", "Time [s]", "Per second"]); + info!("Running machine benchmarks..."); + let tmp_dir = tempdir()?; + let info = gather_hwbench(Some(tmp_dir.path())); - let input_len = 32_usize; - // Benchmark the BLAKE2-256 hashing speed. - let name = "BLAKE2-256"; - let duration = Self::blake2_speed(self.hash_reps, input_len); - Self::put_col(&mut table, name, self.hash_reps, input_len, duration); - - // Benchmark the speed of SR2519 verification. - let name = "SR25519 verify"; - let duration = Self::sr_verify_speed(self.verify_reps, input_len); - Self::put_col(&mut table, name, self.verify_reps, input_len, duration); + // Use a table for nicer console output. + let mut table = table!( + ["Category", "Function", "Score", "Unit"], + ["Memory", "Copy", info.memory_memcpy_score, "MB/s"], + ["Disk", "Seq Write", info.disk_sequential_write_score.unwrap_or_default(), "MB/s"], + ["Disk", "Rnd Write", info.disk_random_write_score.unwrap_or_default(), "MB/s"], + ["CPU", "BLAKE2-256", info.cpu_hashrate_score, "MB/s"] + ); + + // Additionally measure the speed of SR25519::Verify for 32 byte input. + let input_len = 32; + let took = Self::sr_verify_speed(self.verify_reps, input_len as usize); + let speed = (self.verify_reps as f64 / took.as_secs_f64()) / 1000.0; + table.add_row(row!["CPU", "SR25519 Verify", format!("{:.2}", speed), "K/s"]); info!("\n{}", table); Ok(()) } - /// Recursively hashes a value `reps` many times. - fn blake2_speed(reps: u64, input_len: usize) -> Duration { - let mut data = vec![0u8; input_len]; - - let start = Instant::now(); - for _ in 0..reps { - data = blake2_256(&data).into(); - } - start.elapsed() - } - - /// Verify the signature of different random data `reps` many times. + /// Verify signatures of different random data `reps` many times. fn sr_verify_speed(reps: u64, input_len: usize) -> Duration { let pair = sr25519::Pair::from_string("//Alice", None).unwrap(); @@ -122,19 +120,6 @@ impl MachineCmd { start.elapsed() } - - /// Format the data and put it as column into the table. - fn put_col(table: &mut Table, name: &str, reps: u64, size: usize, d: Duration) { - let d = d.as_secs_f32(); - let s = (reps as f32 / d) as u64; - table.add_row(row![ - name, - size, - reps.separate_with_commas(), - format!("{:.3}", d), - s.separate_with_commas() - ]); - } } // Boilerplate From d4ee98222bf1a5ea62ac60dd7d5c62070e2d7f70 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 11 Apr 2022 13:55:34 +0200 Subject: [PATCH 04/26] Add --no-hardware-benchmarks Signed-off-by: Oliver Tale-Yazdi --- bin/node/cli/tests/benchmark_block_works.rs | 2 +- bin/node/cli/tests/benchmark_machine_works.rs | 5 ++++- bin/node/cli/tests/benchmark_overhead_works.rs | 4 ++-- bin/node/cli/tests/benchmark_storage_works.rs | 2 +- bin/node/cli/tests/build_spec_works.rs | 4 ++-- bin/node/cli/tests/check_block_works.rs | 5 +++-- bin/node/cli/tests/export_import_flow.rs | 3 ++- bin/node/cli/tests/inspect_works.rs | 2 +- bin/node/cli/tests/purge_chain_works.rs | 2 +- bin/node/cli/tests/running_the_node_and_interrupt.rs | 2 +- 10 files changed, 18 insertions(+), 13 deletions(-) diff --git a/bin/node/cli/tests/benchmark_block_works.rs b/bin/node/cli/tests/benchmark_block_works.rs index 37a4db25f363b..422d2ecdae88e 100644 --- a/bin/node/cli/tests/benchmark_block_works.rs +++ b/bin/node/cli/tests/benchmark_block_works.rs @@ -34,7 +34,7 @@ async fn benchmark_block_works() { // Invoke `benchmark block` with all options to make sure that they are valid. let status = Command::new(cargo_bin("substrate")) - .args(["benchmark", "block", "--dev"]) + .args(["benchmark", "block", "--dev", "--no-hardware-benchmarks"]) .arg("-d") .arg(base_dir.path()) .args(["--pruning", "archive"]) diff --git a/bin/node/cli/tests/benchmark_machine_works.rs b/bin/node/cli/tests/benchmark_machine_works.rs index 95c0ac8132099..f5c7237f05316 100644 --- a/bin/node/cli/tests/benchmark_machine_works.rs +++ b/bin/node/cli/tests/benchmark_machine_works.rs @@ -20,11 +20,14 @@ use assert_cmd::cargo::cargo_bin; use std::process::Command; /// Tests that the `benchmark machine` command works for the substrate dev runtime. +/// +/// Additionally run the hardware benchmarks on startup since +/// `"--no-hardware-benchmarks"` is not passed, #[test] fn benchmark_machine_works() { let status = Command::new(cargo_bin("substrate")) .args(["benchmark", "machine", "--dev"]) - .args(["--hash-reps", "1", "--verify-reps", "1"]) + .args(["--verify-reps", "1"]) .status() .unwrap(); diff --git a/bin/node/cli/tests/benchmark_overhead_works.rs b/bin/node/cli/tests/benchmark_overhead_works.rs index 44dcebfbc0c35..6434bd6ddf486 100644 --- a/bin/node/cli/tests/benchmark_overhead_works.rs +++ b/bin/node/cli/tests/benchmark_overhead_works.rs @@ -29,8 +29,8 @@ fn benchmark_overhead_works() { // Only put 10 extrinsics into the block otherwise it takes forever to build it // especially for a non-release build. let status = Command::new(cargo_bin("substrate")) - .args(&["benchmark", "overhead", "--dev", "-d"]) - .arg(base_path) + .args(&["benchmark", "overhead", "--dev", "--no-hardware-benchmarks"]) + .args(["-d", base_path]) .arg("--weight-path") .arg(base_path) .args(["--warmup", "10", "--repeat", "10"]) diff --git a/bin/node/cli/tests/benchmark_storage_works.rs b/bin/node/cli/tests/benchmark_storage_works.rs index 30f860e48459f..9647ce7952468 100644 --- a/bin/node/cli/tests/benchmark_storage_works.rs +++ b/bin/node/cli/tests/benchmark_storage_works.rs @@ -39,7 +39,7 @@ fn benchmark_storage_works() { fn benchmark_storage(db: &str, base_path: &Path) -> ExitStatus { Command::new(cargo_bin("substrate")) - .args(&["benchmark", "storage", "--dev"]) + .args(&["benchmark", "storage", "--dev", "--no-hardware-benchmarks"]) .arg("--db") .arg(db) .arg("--weight-path") diff --git a/bin/node/cli/tests/build_spec_works.rs b/bin/node/cli/tests/build_spec_works.rs index aecabed60c849..73ff5b36e8afe 100644 --- a/bin/node/cli/tests/build_spec_works.rs +++ b/bin/node/cli/tests/build_spec_works.rs @@ -25,8 +25,8 @@ fn build_spec_works() { let base_path = tempdir().expect("could not create a temp dir"); let output = Command::new(cargo_bin("substrate")) - .args(&["build-spec", "--dev", "-d"]) - .arg(base_path.path()) + .args(&["build-spec", "--dev", "--no-hardware-benchmarks"]) + .args(["-d", base_path.path()]) .output() .unwrap(); assert!(output.status.success()); diff --git a/bin/node/cli/tests/check_block_works.rs b/bin/node/cli/tests/check_block_works.rs index c5447fd2311c6..fd4e7da4aea15 100644 --- a/bin/node/cli/tests/check_block_works.rs +++ b/bin/node/cli/tests/check_block_works.rs @@ -31,8 +31,9 @@ async fn check_block_works() { common::run_node_for_a_while(base_path.path(), &["--dev", "--no-hardware-benchmarks"]).await; let status = Command::new(cargo_bin("substrate")) - .args(&["check-block", "--dev", "--pruning", "archive", "-d"]) - .arg(base_path.path()) + .args(&["check-block", "--dev", "--no-hardware-benchmarks"]) + .args(["--pruning", "archive"]) + .args(["-d", base_path.path()]) .arg("1") .status() .unwrap(); diff --git a/bin/node/cli/tests/export_import_flow.rs b/bin/node/cli/tests/export_import_flow.rs index 48fccc8ca0293..2f28ea18ee98c 100644 --- a/bin/node/cli/tests/export_import_flow.rs +++ b/bin/node/cli/tests/export_import_flow.rs @@ -99,6 +99,7 @@ impl<'a> ExportImportRevertExecutor<'a> { let output = Command::new(cargo_bin("substrate")) .args(&arguments) .arg(&base_path) + .arg("--no-hardware-benchmarks") .arg(&self.exported_blocks_file) .output() .unwrap(); @@ -161,7 +162,7 @@ impl<'a> ExportImportRevertExecutor<'a> { /// Runs the `revert` command. fn run_revert(&self) { let output = Command::new(cargo_bin("substrate")) - .args(&["revert", "--dev", "--pruning", "archive", "-d"]) + .args(&["revert", "--dev", "--no-hardware-benchmarks", "--pruning", "archive", "-d"]) .arg(&self.base_path.path()) .output() .unwrap(); diff --git a/bin/node/cli/tests/inspect_works.rs b/bin/node/cli/tests/inspect_works.rs index 6f73cc69582a9..1897c6d0731c4 100644 --- a/bin/node/cli/tests/inspect_works.rs +++ b/bin/node/cli/tests/inspect_works.rs @@ -31,7 +31,7 @@ async fn inspect_works() { common::run_node_for_a_while(base_path.path(), &["--dev", "--no-hardware-benchmarks"]).await; let status = Command::new(cargo_bin("substrate")) - .args(&["inspect", "--dev", "--pruning", "archive", "-d"]) + .args(&["inspect", "--dev", "--no-hardware-benchmarks", "--pruning", "archive", "-d"]) .arg(base_path.path()) .args(&["block", "1"]) .status() diff --git a/bin/node/cli/tests/purge_chain_works.rs b/bin/node/cli/tests/purge_chain_works.rs index 811762a714a9d..63ba6349f51a9 100644 --- a/bin/node/cli/tests/purge_chain_works.rs +++ b/bin/node/cli/tests/purge_chain_works.rs @@ -30,7 +30,7 @@ async fn purge_chain_works() { common::run_node_for_a_while(base_path.path(), &["--dev", "--no-hardware-benchmarks"]).await; let status = Command::new(cargo_bin("substrate")) - .args(&["purge-chain", "--dev", "-d"]) + .args(&["purge-chain", "--dev", "--no-hardware-benchmarks", "-d"]) .arg(base_path.path()) .arg("-y") .status() diff --git a/bin/node/cli/tests/running_the_node_and_interrupt.rs b/bin/node/cli/tests/running_the_node_and_interrupt.rs index ecdb4d7671b01..7129becc40b78 100644 --- a/bin/node/cli/tests/running_the_node_and_interrupt.rs +++ b/bin/node/cli/tests/running_the_node_and_interrupt.rs @@ -36,7 +36,7 @@ async fn running_the_node_works_and_can_be_interrupted() { let base_path = tempdir().expect("could not create a temp dir"); let mut cmd = common::KillChildOnDrop( Command::new(cargo_bin("substrate")) - .args(&["--dev", "-d"]) + .args(&["--dev", "--no-hardware-benchmarks", "-d"]) .arg(base_path.path()) .arg("--no-hardware-benchmarks") .spawn() From b49dc41167c491974ef18a8d1280dcfd5777a8ba Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 11 Apr 2022 14:05:28 +0200 Subject: [PATCH 05/26] Lockfile Signed-off-by: Oliver Tale-Yazdi --- Cargo.lock | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 45f94e20433b7..72837deddd5f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2037,9 +2037,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "1.4.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca5faf057445ce5c9d4329e382b2ce7ca38550ef3b73a5348362d5f24e0c7fe3" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" dependencies = [ "instant", ] @@ -2209,6 +2209,7 @@ dependencies = [ "sc-client-db", "sc-executor", "sc-service", + "sc-sysinfo", "serde", "serde_json", "serde_nanos", @@ -2225,6 +2226,7 @@ dependencies = [ "sp-std", "sp-storage", "sp-trie", + "tempfile", "thousands", ] @@ -9162,7 +9164,7 @@ dependencies = [ name = "sc-sysinfo" version = "6.0.0-dev" dependencies = [ - "futures 0.3.19", + "futures 0.3.21", "libc", "log 0.4.14", "rand 0.7.3", @@ -10987,13 +10989,13 @@ checksum = "d7fa7e55043acb85fca6b3c01485a2eeb6b69c5d21002e273c79e465f43b7ac1" [[package]] name = "tempfile" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ "cfg-if 1.0.0", + "fastrand", "libc", - "rand 0.8.4", "redox_syscall 0.2.10", "remove_dir_all", "winapi 0.3.9", From dbb1e998c9e715a114ce9ba746230469f015bbbf Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 11 Apr 2022 14:17:03 +0200 Subject: [PATCH 06/26] Do not create components if not needed Signed-off-by: Oliver Tale-Yazdi --- bin/node-template/node/src/command.rs | 10 +++++++--- bin/node/cli/src/command.rs | 9 ++++++--- utils/frame/benchmarking-cli/src/machine/mod.rs | 10 +++++----- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index a0fcd8f995ca5..a74636628ada5 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -109,8 +109,6 @@ pub fn run() -> sc_cli::Result<()> { let runner = cli.create_runner(cmd)?; runner.sync_run(|config| { - let PartialComponents { client, backend, .. } = service::new_partial(&config)?; - // This switch needs to be in the client, since the client decides // which sub-commands it wants to support. match cmd { @@ -125,14 +123,20 @@ pub fn run() -> sc_cli::Result<()> { cmd.run::(config) }, - BenchmarkCmd::Block(cmd) => cmd.run(client), + BenchmarkCmd::Block(cmd) => { + let PartialComponents { client, .. } = service::new_partial(&config)?; + cmd.run(client) + }, BenchmarkCmd::Storage(cmd) => { + let PartialComponents { client, backend, .. } = + service::new_partial(&config)?; let db = backend.expose_db(); let storage = backend.expose_storage(); cmd.run(config, client, db, storage) }, BenchmarkCmd::Overhead(cmd) => { + let PartialComponents { client, .. } = service::new_partial(&config)?; let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index f6187e8ece6b4..ecd8b2eb0c780 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -99,8 +99,6 @@ pub fn run() -> Result<()> { let runner = cli.create_runner(cmd)?; runner.sync_run(|config| { - let PartialComponents { client, backend, .. } = new_partial(&config)?; - // This switch needs to be in the client, since the client decides // which sub-commands it wants to support. match cmd { @@ -115,14 +113,19 @@ pub fn run() -> Result<()> { cmd.run::(config) }, - BenchmarkCmd::Block(cmd) => cmd.run(client), + BenchmarkCmd::Block(cmd) => { + let PartialComponents { client, .. } = new_partial(&config)?; + cmd.run(client) + }, BenchmarkCmd::Storage(cmd) => { + let PartialComponents { client, backend, .. } = new_partial(&config)?; let db = backend.expose_db(); let storage = backend.expose_storage(); cmd.run(config, client, db, storage) }, BenchmarkCmd::Overhead(cmd) => { + let PartialComponents { client, .. } = new_partial(&config)?; let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index dc3738ebc85de..341248f7259b4 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -48,15 +48,15 @@ use crate::shared::new_rng; /// +----------+----------------+-------+------+ /// | Category | Function | Score | Unit | /// +----------+----------------+-------+------+ -/// | Memory | Copy | 9082 | MB/s | +/// | Memory | Copy | 19983 | MB/s | /// +----------+----------------+-------+------+ -/// | Disk | Seq Write | 724 | MB/s | +/// | Disk | Seq Write | 2699 | MB/s | /// +----------+----------------+-------+------+ -/// | Disk | Rnd Write | 362 | MB/s | +/// | Disk | Rnd Write | 295 | MB/s | /// +----------+----------------+-------+------+ -/// | CPU | BLAKE2-256 | 816 | MB/s | +/// | CPU | BLAKE2-256 | 841 | MB/s | /// +----------+----------------+-------+------+ -/// | CPU | SR25519 Verify | 13.52 | K/s | +/// | CPU | SR25519 Verify | 21.10 | K/s | /// +----------+----------------+-------+------+ /// ``` #[derive(Debug, Parser)] From dc8a89ff8b120e360e129c48d42e2e8cf4fd659c Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 11 Apr 2022 14:19:39 +0200 Subject: [PATCH 07/26] Fix tests Signed-off-by: Oliver Tale-Yazdi --- bin/node/cli/tests/benchmark_overhead_works.rs | 4 ++-- bin/node/cli/tests/build_spec_works.rs | 4 ++-- bin/node/cli/tests/check_block_works.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/node/cli/tests/benchmark_overhead_works.rs b/bin/node/cli/tests/benchmark_overhead_works.rs index 6434bd6ddf486..719c28308bcda 100644 --- a/bin/node/cli/tests/benchmark_overhead_works.rs +++ b/bin/node/cli/tests/benchmark_overhead_works.rs @@ -29,8 +29,8 @@ fn benchmark_overhead_works() { // Only put 10 extrinsics into the block otherwise it takes forever to build it // especially for a non-release build. let status = Command::new(cargo_bin("substrate")) - .args(&["benchmark", "overhead", "--dev", "--no-hardware-benchmarks"]) - .args(["-d", base_path]) + .args(&["benchmark", "overhead", "--dev", "--no-hardware-benchmarks", "-d"]) + .args(base_path) .arg("--weight-path") .arg(base_path) .args(["--warmup", "10", "--repeat", "10"]) diff --git a/bin/node/cli/tests/build_spec_works.rs b/bin/node/cli/tests/build_spec_works.rs index 73ff5b36e8afe..3e2c065990b6e 100644 --- a/bin/node/cli/tests/build_spec_works.rs +++ b/bin/node/cli/tests/build_spec_works.rs @@ -25,8 +25,8 @@ fn build_spec_works() { let base_path = tempdir().expect("could not create a temp dir"); let output = Command::new(cargo_bin("substrate")) - .args(&["build-spec", "--dev", "--no-hardware-benchmarks"]) - .args(["-d", base_path.path()]) + .args(&["build-spec", "--dev", "--no-hardware-benchmarks", "-d"]) + .args(base_path.path()) .output() .unwrap(); assert!(output.status.success()); diff --git a/bin/node/cli/tests/check_block_works.rs b/bin/node/cli/tests/check_block_works.rs index fd4e7da4aea15..157188b944bcd 100644 --- a/bin/node/cli/tests/check_block_works.rs +++ b/bin/node/cli/tests/check_block_works.rs @@ -31,9 +31,9 @@ async fn check_block_works() { common::run_node_for_a_while(base_path.path(), &["--dev", "--no-hardware-benchmarks"]).await; let status = Command::new(cargo_bin("substrate")) - .args(&["check-block", "--dev", "--no-hardware-benchmarks"]) + .args(&["check-block", "--dev", "--no-hardware-benchmarks", "-d"]) + .args(base_path.path()) .args(["--pruning", "archive"]) - .args(["-d", base_path.path()]) .arg("1") .status() .unwrap(); From 279789e304a105ae009f26b4053e4cb6995d227c Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 11 Apr 2022 15:24:36 +0200 Subject: [PATCH 08/26] Revert "Add --no-hardware-benchmarks" This reverts commit d4ee98222bf1a5ea62ac60dd7d5c62070e2d7f70. --- bin/node/cli/tests/benchmark_block_works.rs | 2 +- bin/node/cli/tests/benchmark_machine_works.rs | 5 +---- bin/node/cli/tests/benchmark_overhead_works.rs | 4 ++-- bin/node/cli/tests/benchmark_storage_works.rs | 2 +- bin/node/cli/tests/build_spec_works.rs | 4 ++-- bin/node/cli/tests/check_block_works.rs | 5 ++--- bin/node/cli/tests/export_import_flow.rs | 3 +-- bin/node/cli/tests/inspect_works.rs | 2 +- bin/node/cli/tests/purge_chain_works.rs | 2 +- bin/node/cli/tests/running_the_node_and_interrupt.rs | 2 +- 10 files changed, 13 insertions(+), 18 deletions(-) diff --git a/bin/node/cli/tests/benchmark_block_works.rs b/bin/node/cli/tests/benchmark_block_works.rs index 422d2ecdae88e..37a4db25f363b 100644 --- a/bin/node/cli/tests/benchmark_block_works.rs +++ b/bin/node/cli/tests/benchmark_block_works.rs @@ -34,7 +34,7 @@ async fn benchmark_block_works() { // Invoke `benchmark block` with all options to make sure that they are valid. let status = Command::new(cargo_bin("substrate")) - .args(["benchmark", "block", "--dev", "--no-hardware-benchmarks"]) + .args(["benchmark", "block", "--dev"]) .arg("-d") .arg(base_dir.path()) .args(["--pruning", "archive"]) diff --git a/bin/node/cli/tests/benchmark_machine_works.rs b/bin/node/cli/tests/benchmark_machine_works.rs index f5c7237f05316..95c0ac8132099 100644 --- a/bin/node/cli/tests/benchmark_machine_works.rs +++ b/bin/node/cli/tests/benchmark_machine_works.rs @@ -20,14 +20,11 @@ use assert_cmd::cargo::cargo_bin; use std::process::Command; /// Tests that the `benchmark machine` command works for the substrate dev runtime. -/// -/// Additionally run the hardware benchmarks on startup since -/// `"--no-hardware-benchmarks"` is not passed, #[test] fn benchmark_machine_works() { let status = Command::new(cargo_bin("substrate")) .args(["benchmark", "machine", "--dev"]) - .args(["--verify-reps", "1"]) + .args(["--hash-reps", "1", "--verify-reps", "1"]) .status() .unwrap(); diff --git a/bin/node/cli/tests/benchmark_overhead_works.rs b/bin/node/cli/tests/benchmark_overhead_works.rs index 719c28308bcda..44dcebfbc0c35 100644 --- a/bin/node/cli/tests/benchmark_overhead_works.rs +++ b/bin/node/cli/tests/benchmark_overhead_works.rs @@ -29,8 +29,8 @@ fn benchmark_overhead_works() { // Only put 10 extrinsics into the block otherwise it takes forever to build it // especially for a non-release build. let status = Command::new(cargo_bin("substrate")) - .args(&["benchmark", "overhead", "--dev", "--no-hardware-benchmarks", "-d"]) - .args(base_path) + .args(&["benchmark", "overhead", "--dev", "-d"]) + .arg(base_path) .arg("--weight-path") .arg(base_path) .args(["--warmup", "10", "--repeat", "10"]) diff --git a/bin/node/cli/tests/benchmark_storage_works.rs b/bin/node/cli/tests/benchmark_storage_works.rs index 9647ce7952468..30f860e48459f 100644 --- a/bin/node/cli/tests/benchmark_storage_works.rs +++ b/bin/node/cli/tests/benchmark_storage_works.rs @@ -39,7 +39,7 @@ fn benchmark_storage_works() { fn benchmark_storage(db: &str, base_path: &Path) -> ExitStatus { Command::new(cargo_bin("substrate")) - .args(&["benchmark", "storage", "--dev", "--no-hardware-benchmarks"]) + .args(&["benchmark", "storage", "--dev"]) .arg("--db") .arg(db) .arg("--weight-path") diff --git a/bin/node/cli/tests/build_spec_works.rs b/bin/node/cli/tests/build_spec_works.rs index 3e2c065990b6e..aecabed60c849 100644 --- a/bin/node/cli/tests/build_spec_works.rs +++ b/bin/node/cli/tests/build_spec_works.rs @@ -25,8 +25,8 @@ fn build_spec_works() { let base_path = tempdir().expect("could not create a temp dir"); let output = Command::new(cargo_bin("substrate")) - .args(&["build-spec", "--dev", "--no-hardware-benchmarks", "-d"]) - .args(base_path.path()) + .args(&["build-spec", "--dev", "-d"]) + .arg(base_path.path()) .output() .unwrap(); assert!(output.status.success()); diff --git a/bin/node/cli/tests/check_block_works.rs b/bin/node/cli/tests/check_block_works.rs index 157188b944bcd..c5447fd2311c6 100644 --- a/bin/node/cli/tests/check_block_works.rs +++ b/bin/node/cli/tests/check_block_works.rs @@ -31,9 +31,8 @@ async fn check_block_works() { common::run_node_for_a_while(base_path.path(), &["--dev", "--no-hardware-benchmarks"]).await; let status = Command::new(cargo_bin("substrate")) - .args(&["check-block", "--dev", "--no-hardware-benchmarks", "-d"]) - .args(base_path.path()) - .args(["--pruning", "archive"]) + .args(&["check-block", "--dev", "--pruning", "archive", "-d"]) + .arg(base_path.path()) .arg("1") .status() .unwrap(); diff --git a/bin/node/cli/tests/export_import_flow.rs b/bin/node/cli/tests/export_import_flow.rs index 2f28ea18ee98c..48fccc8ca0293 100644 --- a/bin/node/cli/tests/export_import_flow.rs +++ b/bin/node/cli/tests/export_import_flow.rs @@ -99,7 +99,6 @@ impl<'a> ExportImportRevertExecutor<'a> { let output = Command::new(cargo_bin("substrate")) .args(&arguments) .arg(&base_path) - .arg("--no-hardware-benchmarks") .arg(&self.exported_blocks_file) .output() .unwrap(); @@ -162,7 +161,7 @@ impl<'a> ExportImportRevertExecutor<'a> { /// Runs the `revert` command. fn run_revert(&self) { let output = Command::new(cargo_bin("substrate")) - .args(&["revert", "--dev", "--no-hardware-benchmarks", "--pruning", "archive", "-d"]) + .args(&["revert", "--dev", "--pruning", "archive", "-d"]) .arg(&self.base_path.path()) .output() .unwrap(); diff --git a/bin/node/cli/tests/inspect_works.rs b/bin/node/cli/tests/inspect_works.rs index 1897c6d0731c4..6f73cc69582a9 100644 --- a/bin/node/cli/tests/inspect_works.rs +++ b/bin/node/cli/tests/inspect_works.rs @@ -31,7 +31,7 @@ async fn inspect_works() { common::run_node_for_a_while(base_path.path(), &["--dev", "--no-hardware-benchmarks"]).await; let status = Command::new(cargo_bin("substrate")) - .args(&["inspect", "--dev", "--no-hardware-benchmarks", "--pruning", "archive", "-d"]) + .args(&["inspect", "--dev", "--pruning", "archive", "-d"]) .arg(base_path.path()) .args(&["block", "1"]) .status() diff --git a/bin/node/cli/tests/purge_chain_works.rs b/bin/node/cli/tests/purge_chain_works.rs index 63ba6349f51a9..811762a714a9d 100644 --- a/bin/node/cli/tests/purge_chain_works.rs +++ b/bin/node/cli/tests/purge_chain_works.rs @@ -30,7 +30,7 @@ async fn purge_chain_works() { common::run_node_for_a_while(base_path.path(), &["--dev", "--no-hardware-benchmarks"]).await; let status = Command::new(cargo_bin("substrate")) - .args(&["purge-chain", "--dev", "--no-hardware-benchmarks", "-d"]) + .args(&["purge-chain", "--dev", "-d"]) .arg(base_path.path()) .arg("-y") .status() diff --git a/bin/node/cli/tests/running_the_node_and_interrupt.rs b/bin/node/cli/tests/running_the_node_and_interrupt.rs index 7129becc40b78..ecdb4d7671b01 100644 --- a/bin/node/cli/tests/running_the_node_and_interrupt.rs +++ b/bin/node/cli/tests/running_the_node_and_interrupt.rs @@ -36,7 +36,7 @@ async fn running_the_node_works_and_can_be_interrupted() { let base_path = tempdir().expect("could not create a temp dir"); let mut cmd = common::KillChildOnDrop( Command::new(cargo_bin("substrate")) - .args(&["--dev", "--no-hardware-benchmarks", "-d"]) + .args(&["--dev", "-d"]) .arg(base_path.path()) .arg("--no-hardware-benchmarks") .spawn() From 2cb3bab1fc7648c0c87851b96197213720f24407 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 11 Apr 2022 16:33:12 +0200 Subject: [PATCH 09/26] Fix tests Signed-off-by: Oliver Tale-Yazdi --- bin/node/cli/tests/benchmark_block_works.rs | 2 +- bin/node/cli/tests/benchmark_machine_works.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/node/cli/tests/benchmark_block_works.rs b/bin/node/cli/tests/benchmark_block_works.rs index 37a4db25f363b..359abf3e4265f 100644 --- a/bin/node/cli/tests/benchmark_block_works.rs +++ b/bin/node/cli/tests/benchmark_block_works.rs @@ -30,7 +30,7 @@ pub mod common; async fn benchmark_block_works() { let base_dir = tempdir().expect("could not create a temp dir"); - common::run_node_for_a_while(base_dir.path(), &["--dev"]).await; + common::run_node_for_a_while(base_dir.path(), &["--dev", "--no-hardware-benchmarks"]).await; // Invoke `benchmark block` with all options to make sure that they are valid. let status = Command::new(cargo_bin("substrate")) diff --git a/bin/node/cli/tests/benchmark_machine_works.rs b/bin/node/cli/tests/benchmark_machine_works.rs index 95c0ac8132099..c127c7dff502e 100644 --- a/bin/node/cli/tests/benchmark_machine_works.rs +++ b/bin/node/cli/tests/benchmark_machine_works.rs @@ -24,7 +24,7 @@ use std::process::Command; fn benchmark_machine_works() { let status = Command::new(cargo_bin("substrate")) .args(["benchmark", "machine", "--dev"]) - .args(["--hash-reps", "1", "--verify-reps", "1"]) + .args(["--verify-reps", "1"]) .status() .unwrap(); From d625bc87f0c267485a773d3f08868d8f69e95bda Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Tue, 12 Apr 2022 14:42:48 +0200 Subject: [PATCH 10/26] Update Cargo deps Signed-off-by: Oliver Tale-Yazdi --- Cargo.lock | 4 ++-- client/sysinfo/Cargo.toml | 2 ++ utils/frame/benchmarking-cli/Cargo.toml | 2 -- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 72837deddd5f3..12f6c532af0b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2219,11 +2219,9 @@ dependencies = [ "sp-database", "sp-externalities", "sp-inherents", - "sp-io", "sp-keystore", "sp-runtime", "sp-state-machine", - "sp-std", "sp-storage", "sp-trie", "tempfile", @@ -9174,6 +9172,8 @@ dependencies = [ "serde", "serde_json", "sp-core", + "sp-io", + "sp-std", ] [[package]] diff --git a/client/sysinfo/Cargo.toml b/client/sysinfo/Cargo.toml index 8efe583fb9335..540918cb37c01 100644 --- a/client/sysinfo/Cargo.toml +++ b/client/sysinfo/Cargo.toml @@ -23,4 +23,6 @@ libc = "0.2" serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.79" sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } +sp-std = { version = "4.0.0", path = "../../primitives/std" } sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" } diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index db0890a9312cb..6ac6c623155b6 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -30,11 +30,9 @@ sp-externalities = { version = "0.12.0", path = "../../../primitives/externaliti sp-database = { version = "4.0.0-dev", path = "../../../primitives/database" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } -sp-io = { version = "6.0.0", path = "../../../primitives/io" } sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } sp-storage = { version = "6.0.0", path = "../../../primitives/storage" } sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } -sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } sp-state-machine = { version = "0.12.0", path = "../../../primitives/state-machine" } sp-trie = { version = "6.0.0", path = "../../../primitives/trie" } codec = { version = "3.0.0", package = "parity-scale-codec" } From 38d7679e9999e8d696c3272ae66cbc39d2195e56 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Tue, 12 Apr 2022 14:48:25 +0200 Subject: [PATCH 11/26] Move sr255119::verify bench to sc-sysinfo Signed-off-by: Oliver Tale-Yazdi --- client/sysinfo/src/sysinfo.rs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index ceb28447002cf..c318ee036473c 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -319,6 +319,41 @@ pub fn benchmark_disk_random_writes(directory: &Path) -> Result { // We only wrote half of the bytes hence `SIZE / 2`. benchmark("disk random write score", SIZE / 2, MAX_ITERATIONS, MAX_DURATION, run) +/// Verify signatures of different random data `reps` many times. +pub fn benchmark_sr25519_verify( + reps: usize, + duration: Duration, + input_len: usize, +) -> Result { + let pair = sr25519::Pair::from_string("//Alice", None).unwrap(); + + let mut rng = rng(); + let mut msgs = Vec::new(); + let mut sigs = Vec::new(); + + for _ in 0..reps { + let mut msg = vec![0u8; input_len]; + rng.fill_bytes(&mut msg[..]); + + sigs.push(pair.sign(&msg)); + msgs.push(msg); + } + + let run = || { + for (sig, msg) in sigs.iter().zip(msgs.iter()) { + let mut ok = sr25519_verify(&sig, &msg[..], &pair.public()); + // Clobber the result. + unsafe { + let mut raw = std::slice::from_raw_parts_mut( + &mut ok as *mut bool as *mut u8, + core::mem::size_of::(), + ); + clobber(&mut raw); + } + } + Ok(()) + }; + benchmark("sr25519 verification score", reps * input_len, 1, duration, run) } /// Benchmarks the hardware and returns the results of those benchmarks. From 1b9eb131a8a3dd7e2419b2d329cbafb03bcbbaec Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Tue, 12 Apr 2022 14:49:09 +0200 Subject: [PATCH 12/26] Move sr255119::verify bench to sc-sysinfo Signed-off-by: Oliver Tale-Yazdi --- client/sysinfo/src/sysinfo.rs | 7 +++- .../frame/benchmarking-cli/src/machine/mod.rs | 37 ------------------- 2 files changed, 6 insertions(+), 38 deletions(-) diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index c318ee036473c..42e361111d1ef 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -17,8 +17,13 @@ // along with this program. If not, see . use crate::HwBench; -use rand::{seq::SliceRandom, Rng}; + use sc_telemetry::SysInfo; +use sp_core::{sr25519, Pair}; +use sp_io::crypto::sr25519_verify; +use sp_std::prelude::*; + +use rand::{seq::SliceRandom, Rng, RngCore}; use std::{ fs::File, io::{Seek, SeekFrom, Write}, diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index 341248f7259b4..d20ad93760236 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -19,22 +19,10 @@ //! and the core benchmarking logic. use sc_cli::{CliConfiguration, Result, SharedParams}; -use sc_sysinfo::gather_hwbench; -use sp_core::{sr25519, Pair}; -use sp_io::crypto::sr25519_verify; -use sp_std::prelude::*; use clap::Parser; use log::info; use prettytable::{cell, row, table}; -use rand::RngCore; -use std::{ - fmt::Debug, - time::{Duration, Instant}, -}; -use tempfile::tempdir; - -use crate::shared::new_rng; /// Command to benchmark the hardware. /// @@ -95,31 +83,6 @@ impl MachineCmd { info!("\n{}", table); Ok(()) } - - /// Verify signatures of different random data `reps` many times. - fn sr_verify_speed(reps: u64, input_len: usize) -> Duration { - let pair = sr25519::Pair::from_string("//Alice", None).unwrap(); - - let (mut rng, _) = new_rng(None); - let mut msgs = Vec::new(); - let mut sigs = Vec::new(); - - for _ in 0..reps { - let mut msg = vec![0u8; input_len]; - rng.fill_bytes(&mut msg[..]); - - sigs.push(pair.sign(&msg)); - msgs.push(msg); - } - - // Measure the time to verify all. - let start = Instant::now(); - for (sig, msg) in sigs.into_iter().zip(msgs.iter()) { - sr25519_verify(&sig, &msg[..], &pair.public()); - } - - start.elapsed() - } } // Boilerplate From ea4b8fef0a07dcf9a3ee63ad86d9e5bc57252e07 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Tue, 12 Apr 2022 14:49:34 +0200 Subject: [PATCH 13/26] Switch benchmarks to return f64 Signed-off-by: Oliver Tale-Yazdi --- client/sysinfo/src/lib.rs | 5 ++++- client/sysinfo/src/sysinfo.rs | 18 +++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/client/sysinfo/src/lib.rs b/client/sysinfo/src/lib.rs index be13efb82c66f..9f4bcc38af25d 100644 --- a/client/sysinfo/src/lib.rs +++ b/client/sysinfo/src/lib.rs @@ -25,7 +25,10 @@ mod sysinfo; #[cfg(target_os = "linux")] mod sysinfo_linux; -pub use sysinfo::{gather_hwbench, gather_sysinfo}; +pub use sysinfo::{ + benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, + benchmark_memory, benchmark_sr25519_verify, gather_hwbench, gather_sysinfo, +}; /// The operating system part of the current target triplet. pub const TARGET_OS: &str = include_str!(concat!(env!("OUT_DIR"), "/target_os.txt")); diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index 42e361111d1ef..88bcf2d86a4db 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -39,7 +39,7 @@ pub(crate) fn benchmark( max_iterations: usize, max_duration: Duration, mut run: impl FnMut() -> Result<(), E>, -) -> Result { +) -> Result { // Run the benchmark once as a warmup to get the code into the L1 cache. run()?; @@ -58,11 +58,11 @@ pub(crate) fn benchmark( } } - let score = (((size * count) as f64 / elapsed.as_secs_f64()) / (1024.0 * 1024.0)) as u64; + let score = ((size * count) as f64 / elapsed.as_secs_f64()) / (1024.0 * 1024.0); log::trace!( "Calculated {} of {}MB/s in {} iterations in {}ms", name, - score, + score as u64, count, elapsed.as_millis() ); @@ -108,7 +108,7 @@ fn clobber(slice: &mut [u8]) { } // This benchmarks the CPU speed as measured by calculating BLAKE2b-256 hashes, in MB/s. -fn benchmark_cpu() -> u64 { +pub fn benchmark_cpu() -> u64 { // In general the results of this benchmark are somewhat sensitive to how much // data we hash at the time. The smaller this is the *less* MB/s we can hash, // the bigger this is the *more* MB/s we can hash, up until a certain point @@ -138,7 +138,7 @@ fn benchmark_cpu() -> u64 { }; benchmark("CPU score", SIZE, MAX_ITERATIONS, MAX_DURATION, run) - .expect("benchmark cannot fail; qed") + .expect("benchmark cannot fail; qed") as u64 } // This benchmarks the effective `memcpy` memory bandwidth available in MB/s. @@ -146,7 +146,7 @@ fn benchmark_cpu() -> u64 { // It doesn't technically measure the absolute maximum memory bandwidth available, // but that's fine, because real code most of the time isn't optimized to take // advantage of the full memory bandwidth either. -fn benchmark_memory() -> u64 { +pub fn benchmark_memory() -> u64 { // Ideally this should be at least as big as the CPU's L3 cache, // and it should be big enough so that the `memcpy` takes enough // time to be actually measurable. @@ -184,7 +184,7 @@ fn benchmark_memory() -> u64 { }; benchmark("memory score", SIZE, MAX_ITERATIONS, MAX_DURATION, run) - .expect("benchmark cannot fail; qed") + .expect("benchmark cannot fail; qed") as u64 } struct TemporaryFile { @@ -265,6 +265,7 @@ pub fn benchmark_disk_sequential_writes(directory: &Path) -> Result }; benchmark("disk sequential write score", SIZE, MAX_ITERATIONS, MAX_DURATION, run) + .map(|s| s as u64) } pub fn benchmark_disk_random_writes(directory: &Path) -> Result { @@ -324,6 +325,9 @@ pub fn benchmark_disk_random_writes(directory: &Path) -> Result { // We only wrote half of the bytes hence `SIZE / 2`. benchmark("disk random write score", SIZE / 2, MAX_ITERATIONS, MAX_DURATION, run) + .map(|s| s as u64) +} + /// Verify signatures of different random data `reps` many times. pub fn benchmark_sr25519_verify( reps: usize, From 025fbb14f38f59f5a039204e4b880cedda277858 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Tue, 12 Apr 2022 14:49:50 +0200 Subject: [PATCH 14/26] Review fixes Signed-off-by: Oliver Tale-Yazdi --- bin/node-template/node/src/command.rs | 2 +- bin/node/cli/src/command.rs | 2 +- .../frame/benchmarking-cli/src/machine/mod.rs | 52 +++++++++++-------- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index a74636628ada5..afa4612f1ee4a 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -141,7 +141,7 @@ pub fn run() -> sc_cli::Result<()> { cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) }, - BenchmarkCmd::Machine(cmd) => cmd.run(), + BenchmarkCmd::Machine(cmd) => cmd.run(&config), } }) }, diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index ecd8b2eb0c780..e2c772e809200 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -130,7 +130,7 @@ pub fn run() -> Result<()> { cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) }, - BenchmarkCmd::Machine(cmd) => cmd.run(), + BenchmarkCmd::Machine(cmd) => cmd.run(&config), } }) }, diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index d20ad93760236..595148ea7a804 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -19,10 +19,16 @@ //! and the core benchmarking logic. use sc_cli::{CliConfiguration, Result, SharedParams}; +use sc_service::Configuration; +use sc_sysinfo::{ + benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, + benchmark_memory, benchmark_sr25519_verify, +}; use clap::Parser; use log::info; use prettytable::{cell, row, table}; +use std::{fmt::Debug, fs, time::Duration}; /// Command to benchmark the hardware. /// @@ -31,20 +37,21 @@ use prettytable::{cell, row, table}; /// This command must be integrated by the client since the client can set compiler flags /// which influence the results. /// -/// The output or `benchmark machine --dev` looks like this: +/// You can use the `--base-path` flag to set a location for the disk benchmarks. +/// The output of `benchmark machine --dev` looks like this: /// ```text /// +----------+----------------+-------+------+ /// | Category | Function | Score | Unit | /// +----------+----------------+-------+------+ -/// | Memory | Copy | 19983 | MB/s | +/// | CPU | BLAKE2-256 | 1056 | MB/s | /// +----------+----------------+-------+------+ -/// | Disk | Seq Write | 2699 | MB/s | +/// | CPU | SR25519 Verify | 666.0 | KB/s | /// +----------+----------------+-------+------+ -/// | Disk | Rnd Write | 295 | MB/s | +/// | Memory | Copy | 21205 | MB/s | /// +----------+----------------+-------+------+ -/// | CPU | BLAKE2-256 | 841 | MB/s | +/// | Disk | Seq Write | 2500 | MB/s | /// +----------+----------------+-------+------+ -/// | CPU | SR25519 Verify | 21.10 | K/s | +/// | Disk | Rnd Write | 311 | MB/s | /// +----------+----------------+-------+------+ /// ``` #[derive(Debug, Parser)] @@ -55,31 +62,32 @@ pub struct MachineCmd { /// Iterations of the verification function. #[clap(long, default_value = "20000")] - pub verify_reps: u64, + pub verify_reps: usize, } impl MachineCmd { /// Execute the benchmark and print the results. - pub fn run(&self) -> Result<()> { + pub fn run(&self, cfg: &Configuration) -> Result<()> { + // Ensure that the dir exists since the node is not started to take care of it. + let dir = cfg.database.path().ok_or("No DB directory provided")?; + fs::create_dir_all(dir)?; + info!("Running machine benchmarks..."); - let tmp_dir = tempdir()?; - let info = gather_hwbench(Some(tmp_dir.path())); + let write = benchmark_disk_sequential_writes(dir)?; + let read = benchmark_disk_random_writes(dir)?; + let verify = + benchmark_sr25519_verify(self.verify_reps, Duration::from_secs(2), 32)? * 1024.0; // Use a table for nicer console output. - let mut table = table!( + let table = table!( ["Category", "Function", "Score", "Unit"], - ["Memory", "Copy", info.memory_memcpy_score, "MB/s"], - ["Disk", "Seq Write", info.disk_sequential_write_score.unwrap_or_default(), "MB/s"], - ["Disk", "Rnd Write", info.disk_random_write_score.unwrap_or_default(), "MB/s"], - ["CPU", "BLAKE2-256", info.cpu_hashrate_score, "MB/s"] + ["CPU", "BLAKE2-256", benchmark_cpu(), "MB/s"], + ["CPU", "SR25519 Verify", format!("{:.1}", verify), "KB/s"], + ["Memory", "Copy", benchmark_memory(), "MB/s"], + ["Disk", "Seq Write", write, "MB/s"], + ["Disk", "Rnd Write", read, "MB/s"] ); - - // Additionally measure the speed of SR25519::Verify for 32 byte input. - let input_len = 32; - let took = Self::sr_verify_speed(self.verify_reps, input_len as usize); - let speed = (self.verify_reps as f64 / took.as_secs_f64()) / 1000.0; - table.add_row(row!["CPU", "SR25519 Verify", format!("{:.2}", speed), "K/s"]); - + info!("\n{}", table); Ok(()) } From 0330f5242ec5693ef4fa1eed7c7b003964bb4cab Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Tue, 12 Apr 2022 15:06:29 +0200 Subject: [PATCH 15/26] fmt Signed-off-by: Oliver Tale-Yazdi --- utils/frame/benchmarking-cli/src/machine/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index 595148ea7a804..c755b1a5d6f89 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -87,7 +87,7 @@ impl MachineCmd { ["Disk", "Seq Write", write, "MB/s"], ["Disk", "Rnd Write", read, "MB/s"] ); - + info!("\n{}", table); Ok(()) } From 74988b2ce39c63d69aa17a739ecef529d719379e Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Tue, 12 Apr 2022 15:39:48 +0200 Subject: [PATCH 16/26] Hide command until completed Signed-off-by: Oliver Tale-Yazdi --- utils/frame/benchmarking-cli/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index f585b6c7ec65b..75e2edc042a26 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -41,6 +41,7 @@ pub enum BenchmarkCmd { Storage(StorageCmd), Overhead(OverheadCmd), Block(BlockCmd), + #[clap(hide = true)] // Hidden until fully completed. Machine(MachineCmd), } From 94480887be831fec37c4a7c3d22adc15ff1ac219 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 14 Apr 2022 11:41:18 +0200 Subject: [PATCH 17/26] Use concrete rand implementation Signed-off-by: Oliver Tale-Yazdi --- Cargo.lock | 1 + utils/frame/benchmarking-cli/Cargo.toml | 1 + utils/frame/benchmarking-cli/src/shared/mod.rs | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 85d1224ff60d4..034311ed65abc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2192,6 +2192,7 @@ dependencies = [ "parity-scale-codec", "prettytable-rs", "rand 0.8.4", + "rand_pcg 0.3.1", "sc-block-builder", "sc-cli", "sc-client-api", diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 6ac6c623155b6..b77d614cf2840 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -54,6 +54,7 @@ rand = { version = "0.8.4", features = ["small_rng"] } thousands = "0.2.0" prettytable-rs = "0.8.0" tempfile = "3.3.0" +rand_pcg = "0.3.1" [features] default = ["db", "sc-client-db/runtime-benchmarks"] diff --git a/utils/frame/benchmarking-cli/src/shared/mod.rs b/utils/frame/benchmarking-cli/src/shared/mod.rs index a4da8cada0398..853fbdef8e87f 100644 --- a/utils/frame/benchmarking-cli/src/shared/mod.rs +++ b/utils/frame/benchmarking-cli/src/shared/mod.rs @@ -71,5 +71,5 @@ where /// Uses a random seed if none is provided. pub fn new_rng(seed: Option) -> (impl rand::Rng, u64) { let seed = seed.unwrap_or(rand::thread_rng().gen::()); - (StdRng::seed_from_u64(seed), seed) + (rand_pcg::Pcg64::seed_from_u64(seed), seed) } From 124fb32976bcc01b522fda40fd0a526bb0ddd708 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 14 Apr 2022 11:59:07 +0200 Subject: [PATCH 18/26] Put clobber into a function Signed-off-by: Oliver Tale-Yazdi --- client/sysinfo/src/sysinfo.rs | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index 88bcf2d86a4db..9b7ea2e15d70a 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -88,7 +88,7 @@ pub fn gather_sysinfo() -> SysInfo { } #[inline(never)] -fn clobber(slice: &mut [u8]) { +fn clobber_slice(slice: &mut [T]) { assert!(!slice.is_empty()); // Discourage the compiler from optimizing out our benchmarks. @@ -107,6 +107,14 @@ fn clobber(slice: &mut [u8]) { } } +#[inline(never)] +fn clobber_value(input: &mut T) { + unsafe { + let value = std::ptr::read_volatile(input); + std::ptr::write_volatile(input, value); + } +} + // This benchmarks the CPU speed as measured by calculating BLAKE2b-256 hashes, in MB/s. pub fn benchmark_cpu() -> u64 { // In general the results of this benchmark are somewhat sensitive to how much @@ -130,9 +138,9 @@ pub fn benchmark_cpu() -> u64 { let mut hash = Default::default(); let run = || -> Result<(), ()> { - clobber(&mut buffer); + clobber_slice(&mut buffer); hash = sp_core::hashing::blake2_256(&buffer); - clobber(&mut hash); + clobber_slice(&mut hash); Ok(()) }; @@ -166,8 +174,8 @@ pub fn benchmark_memory() -> u64 { dst.resize(SIZE, 0x77); let run = || -> Result<(), ()> { - clobber(&mut src); - clobber(&mut dst); + clobber_slice(&mut src); + clobber_slice(&mut dst); // SAFETY: Both vectors are of the same type and of the same size, // so copying data between them is safe. @@ -177,8 +185,8 @@ pub fn benchmark_memory() -> u64 { libc::memcpy(dst.as_mut_ptr().cast(), src.as_ptr().cast(), SIZE); } - clobber(&mut dst); - clobber(&mut src); + clobber_slice(&mut dst); + clobber_slice(&mut src); Ok(()) }; @@ -351,14 +359,7 @@ pub fn benchmark_sr25519_verify( let run = || { for (sig, msg) in sigs.iter().zip(msgs.iter()) { let mut ok = sr25519_verify(&sig, &msg[..], &pair.public()); - // Clobber the result. - unsafe { - let mut raw = std::slice::from_raw_parts_mut( - &mut ok as *mut bool as *mut u8, - core::mem::size_of::(), - ); - clobber(&mut raw); - } + clobber_value(&mut ok); } Ok(()) }; From aa89d52ad75ff404ac2d900cd3d278580a596682 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 14 Apr 2022 11:59:14 +0200 Subject: [PATCH 19/26] Add test Signed-off-by: Oliver Tale-Yazdi --- client/sysinfo/src/sysinfo.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index 9b7ea2e15d70a..2b6fbb41bdee4 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -435,4 +435,9 @@ mod tests { fn test_benchmark_disk_random_writes() { assert!(benchmark_disk_random_writes("./".as_ref()).unwrap() > 0); } + + #[test] + fn test_benchmark_sr25519_verify() { + assert!(benchmark_sr25519_verify(10, Duration::from_secs(1), 32).unwrap() > 0.0); + } } From 7bc308307e516c61001537f4e3d008464358cb81 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 14 Apr 2022 12:00:17 +0200 Subject: [PATCH 20/26] Add comment Signed-off-by: Oliver Tale-Yazdi --- client/sysinfo/src/sysinfo.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index 2b6fbb41bdee4..3e437bf1778fd 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -109,6 +109,7 @@ fn clobber_slice(slice: &mut [T]) { #[inline(never)] fn clobber_value(input: &mut T) { + // Look into `clobber_slice` for a comment. unsafe { let value = std::ptr::read_volatile(input); std::ptr::write_volatile(input, value); From 2c4ba758e6c9431f2be97d8d5f0d69df2f932686 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 14 Apr 2022 12:01:50 +0200 Subject: [PATCH 21/26] Update cargo to match polkadot Signed-off-by: Oliver Tale-Yazdi --- utils/frame/benchmarking-cli/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index b77d614cf2840..858d16558e821 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -53,7 +53,7 @@ memory-db = "0.29.0" rand = { version = "0.8.4", features = ["small_rng"] } thousands = "0.2.0" prettytable-rs = "0.8.0" -tempfile = "3.3.0" +tempfile = "3.2.0" rand_pcg = "0.3.1" [features] From 69ab6a5898b407b46835d9eb1eae71919c247511 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 14 Apr 2022 12:45:45 +0200 Subject: [PATCH 22/26] Remove doc that does not format in the console Signed-off-by: Oliver Tale-Yazdi --- utils/frame/benchmarking-cli/src/machine/mod.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index c755b1a5d6f89..03d13237eab8e 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -38,22 +38,6 @@ use std::{fmt::Debug, fs, time::Duration}; /// which influence the results. /// /// You can use the `--base-path` flag to set a location for the disk benchmarks. -/// The output of `benchmark machine --dev` looks like this: -/// ```text -/// +----------+----------------+-------+------+ -/// | Category | Function | Score | Unit | -/// +----------+----------------+-------+------+ -/// | CPU | BLAKE2-256 | 1056 | MB/s | -/// +----------+----------------+-------+------+ -/// | CPU | SR25519 Verify | 666.0 | KB/s | -/// +----------+----------------+-------+------+ -/// | Memory | Copy | 21205 | MB/s | -/// +----------+----------------+-------+------+ -/// | Disk | Seq Write | 2500 | MB/s | -/// +----------+----------------+-------+------+ -/// | Disk | Rnd Write | 311 | MB/s | -/// +----------+----------------+-------+------+ -/// ``` #[derive(Debug, Parser)] pub struct MachineCmd { #[allow(missing_docs)] From b53f8f9402e3e18d4b2b5e662f0c56ba0eb76958 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 14 Apr 2022 12:46:41 +0200 Subject: [PATCH 23/26] Limit benchmark by time Signed-off-by: Oliver Tale-Yazdi --- bin/node/cli/tests/benchmark_machine_works.rs | 2 +- client/sysinfo/src/sysinfo.rs | 26 +++++++++++++------ .../frame/benchmarking-cli/src/machine/mod.rs | 9 ++++--- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/bin/node/cli/tests/benchmark_machine_works.rs b/bin/node/cli/tests/benchmark_machine_works.rs index c127c7dff502e..df407e988f636 100644 --- a/bin/node/cli/tests/benchmark_machine_works.rs +++ b/bin/node/cli/tests/benchmark_machine_works.rs @@ -24,7 +24,7 @@ use std::process::Command; fn benchmark_machine_works() { let status = Command::new(cargo_bin("substrate")) .args(["benchmark", "machine", "--dev"]) - .args(["--verify-reps", "1"]) + .args(["--verify-duration", "0.1"]) .status() .unwrap(); diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index 3e437bf1778fd..be4ff1c276196 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -337,20 +337,23 @@ pub fn benchmark_disk_random_writes(directory: &Path) -> Result { .map(|s| s as u64) } -/// Verify signatures of different random data `reps` many times. pub fn benchmark_sr25519_verify( - reps: usize, - duration: Duration, - input_len: usize, + max_iterations: Option, + max_duration: Option, ) -> Result { + if max_iterations.is_none() && max_duration.is_none() { + return Err("max_iterations and max_duration cannot both be None".into()) + } + const INPUT_SIZE: usize = 32; + const ITERATION_SIZE: usize = 2048; let pair = sr25519::Pair::from_string("//Alice", None).unwrap(); let mut rng = rng(); let mut msgs = Vec::new(); let mut sigs = Vec::new(); - for _ in 0..reps { - let mut msg = vec![0u8; input_len]; + for _ in 0..ITERATION_SIZE { + let mut msg = vec![0u8; INPUT_SIZE]; rng.fill_bytes(&mut msg[..]); sigs.push(pair.sign(&msg)); @@ -364,7 +367,13 @@ pub fn benchmark_sr25519_verify( } Ok(()) }; - benchmark("sr25519 verification score", reps * input_len, 1, duration, run) + benchmark( + "sr25519 verification score", + INPUT_SIZE * ITERATION_SIZE, + max_iterations.unwrap_or(usize::MAX), + max_duration.unwrap_or(Duration::MAX), + run, + ) } /// Benchmarks the hardware and returns the results of those benchmarks. @@ -439,6 +448,7 @@ mod tests { #[test] fn test_benchmark_sr25519_verify() { - assert!(benchmark_sr25519_verify(10, Duration::from_secs(1), 32).unwrap() > 0.0); + assert!(benchmark_sr25519_verify(Some(1), None).unwrap() > 0.0); + assert!(benchmark_sr25519_verify(None, None).is_err()); } } diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index 03d13237eab8e..03b5400bc4849 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -44,9 +44,9 @@ pub struct MachineCmd { #[clap(flatten)] pub shared_params: SharedParams, - /// Iterations of the verification function. - #[clap(long, default_value = "20000")] - pub verify_reps: usize, + /// Time limit for the verification benchmark. + #[clap(long, default_value = "2.0", value_name = "SECONDS")] + pub verify_duration: f32, } impl MachineCmd { @@ -60,7 +60,8 @@ impl MachineCmd { let write = benchmark_disk_sequential_writes(dir)?; let read = benchmark_disk_random_writes(dir)?; let verify = - benchmark_sr25519_verify(self.verify_reps, Duration::from_secs(2), 32)? * 1024.0; + benchmark_sr25519_verify(None, Some(Duration::from_secs_f32(self.verify_duration)))? * + 1024.0; // Use a table for nicer console output. let table = table!( From b478436f8a5b930c0b25de76b3db0b1b6fab3e77 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 14 Apr 2022 13:28:09 +0200 Subject: [PATCH 24/26] Add ExecutionLimit and make function infallible Signed-off-by: Oliver Tale-Yazdi --- client/sysinfo/src/lib.rs | 33 +++++++++++++++++++ client/sysinfo/src/sysinfo.rs | 20 ++++------- .../frame/benchmarking-cli/src/machine/mod.rs | 8 ++--- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/client/sysinfo/src/lib.rs b/client/sysinfo/src/lib.rs index 9f4bcc38af25d..911e725dcdd4e 100644 --- a/client/sysinfo/src/lib.rs +++ b/client/sysinfo/src/lib.rs @@ -20,6 +20,7 @@ //! and software telemetry information about the node on which we're running. use futures::prelude::*; +use std::time::Duration; mod sysinfo; #[cfg(target_os = "linux")] @@ -52,6 +53,38 @@ pub struct HwBench { pub disk_random_write_score: Option, } +/// Limit the execution time of a benchmark. +pub enum ExecutionLimit { + /// Limit by the maximal duration. + MaxDuration(Duration), + + /// Limit by the maximal number of iterations. + MaxIterations(usize), + + /// Limit by the maximal duration and maximal number of iterations. + Both { max_iterations: usize, max_duration: Duration }, +} + +impl ExecutionLimit { + /// Returns the duration limit or `MAX` if none is present. + pub fn max_duration(&self) -> Duration { + match self { + Self::MaxDuration(d) => *d, + Self::Both { max_duration, .. } => *max_duration, + _ => Duration::from_secs(u64::MAX), + } + } + + /// Returns the iterations limit or `MAX` if none is present. + pub fn max_iterations(&self) -> usize { + match self { + Self::MaxIterations(d) => *d, + Self::Both { max_iterations, .. } => *max_iterations, + _ => usize::MAX, + } + } +} + /// Prints out the system software/hardware information in the logs. pub fn print_sysinfo(sysinfo: &sc_telemetry::SysInfo) { log::info!("💻 Operating system: {}", TARGET_OS); diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index be4ff1c276196..2696e76a91494 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -16,7 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::HwBench; +use crate::{ExecutionLimit, HwBench}; use sc_telemetry::SysInfo; use sp_core::{sr25519, Pair}; @@ -337,13 +337,7 @@ pub fn benchmark_disk_random_writes(directory: &Path) -> Result { .map(|s| s as u64) } -pub fn benchmark_sr25519_verify( - max_iterations: Option, - max_duration: Option, -) -> Result { - if max_iterations.is_none() && max_duration.is_none() { - return Err("max_iterations and max_duration cannot both be None".into()) - } +pub fn benchmark_sr25519_verify(limit: ExecutionLimit) -> f64 { const INPUT_SIZE: usize = 32; const ITERATION_SIZE: usize = 2048; let pair = sr25519::Pair::from_string("//Alice", None).unwrap(); @@ -360,7 +354,7 @@ pub fn benchmark_sr25519_verify( msgs.push(msg); } - let run = || { + let run = || -> Result<(), String> { for (sig, msg) in sigs.iter().zip(msgs.iter()) { let mut ok = sr25519_verify(&sig, &msg[..], &pair.public()); clobber_value(&mut ok); @@ -370,10 +364,11 @@ pub fn benchmark_sr25519_verify( benchmark( "sr25519 verification score", INPUT_SIZE * ITERATION_SIZE, - max_iterations.unwrap_or(usize::MAX), - max_duration.unwrap_or(Duration::MAX), + limit.max_iterations(), + limit.max_duration(), run, ) + .expect("sr25519 verification cannot fail; qed") } /// Benchmarks the hardware and returns the results of those benchmarks. @@ -448,7 +443,6 @@ mod tests { #[test] fn test_benchmark_sr25519_verify() { - assert!(benchmark_sr25519_verify(Some(1), None).unwrap() > 0.0); - assert!(benchmark_sr25519_verify(None, None).is_err()); + assert!(benchmark_sr25519_verify(ExecutionLimit::MaxIterations(1)) > 0.0); } } diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index 03b5400bc4849..acd9d15b901e0 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -22,7 +22,7 @@ use sc_cli::{CliConfiguration, Result, SharedParams}; use sc_service::Configuration; use sc_sysinfo::{ benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, - benchmark_memory, benchmark_sr25519_verify, + benchmark_memory, benchmark_sr25519_verify, ExecutionLimit, }; use clap::Parser; @@ -59,9 +59,9 @@ impl MachineCmd { info!("Running machine benchmarks..."); let write = benchmark_disk_sequential_writes(dir)?; let read = benchmark_disk_random_writes(dir)?; - let verify = - benchmark_sr25519_verify(None, Some(Duration::from_secs_f32(self.verify_duration)))? * - 1024.0; + let verify_limit = + ExecutionLimit::MaxDuration(Duration::from_secs_f32(self.verify_duration)); + let verify = benchmark_sr25519_verify(verify_limit)? * 1024.0; // Use a table for nicer console output. let table = table!( From 14b36a5dcdf183a223daded91ede81a7c3657438 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 14 Apr 2022 13:38:22 +0200 Subject: [PATCH 25/26] CI Signed-off-by: Oliver Tale-Yazdi --- utils/frame/benchmarking-cli/src/machine/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index acd9d15b901e0..ee6bf765d01c4 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -61,7 +61,7 @@ impl MachineCmd { let read = benchmark_disk_random_writes(dir)?; let verify_limit = ExecutionLimit::MaxDuration(Duration::from_secs_f32(self.verify_duration)); - let verify = benchmark_sr25519_verify(verify_limit)? * 1024.0; + let verify = benchmark_sr25519_verify(verify_limit) * 1024.0; // Use a table for nicer console output. let table = table!( From 1ad85ea8f95b6ad3bf937b51d1176b178600d76f Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 14 Apr 2022 21:56:32 +0200 Subject: [PATCH 26/26] Add doc Signed-off-by: Oliver Tale-Yazdi --- client/sysinfo/src/sysinfo.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index 2696e76a91494..65d7a9e41b406 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -337,6 +337,10 @@ pub fn benchmark_disk_random_writes(directory: &Path) -> Result { .map(|s| s as u64) } +/// Benchmarks the verification speed of sr25519 signatures. +/// +/// Returns the throughput in MB/s by convention. +/// The values are rather small (0.4-0.8) so it is advised to convert them into KB/s. pub fn benchmark_sr25519_verify(limit: ExecutionLimit) -> f64 { const INPUT_SIZE: usize = 32; const ITERATION_SIZE: usize = 2048;