diff --git a/Cargo.lock b/Cargo.lock index 5731874d8ca81..034311ed65abc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1677,6 +1677,17 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users 0.3.5", + "winapi 0.3.9", +] + [[package]] name = "dirs-sys" version = "0.3.6" @@ -1684,7 +1695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.0", "winapi 0.3.9", ] @@ -1695,7 +1706,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.0", "winapi 0.3.9", ] @@ -1844,6 +1855,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "enum-as-inner" version = "0.3.3" @@ -2009,9 +2026,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "1.4.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca5faf057445ce5c9d4329e382b2ce7ca38550ef3b73a5348362d5f24e0c7fe3" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" dependencies = [ "instant", ] @@ -2173,13 +2190,16 @@ dependencies = [ "log 0.4.14", "memory-db", "parity-scale-codec", + "prettytable-rs", "rand 0.8.4", + "rand_pcg 0.3.1", "sc-block-builder", "sc-cli", "sc-client-api", "sc-client-db", "sc-executor", "sc-service", + "sc-sysinfo", "serde", "serde_json", "serde_nanos", @@ -2192,9 +2212,9 @@ dependencies = [ "sp-keystore", "sp-runtime", "sp-state-machine", - "sp-std", "sp-storage", "sp-trie", + "tempfile", "thousands", ] @@ -7172,6 +7192,20 @@ dependencies = [ "output_vt100", ] +[[package]] +name = "prettytable-rs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd04b170004fa2daccf418a7f8253aaf033c27760b5f225889024cf66d7ac2e" +dependencies = [ + "atty", + "csv", + "encode_unicode", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "primitive-types" version = "0.11.1" @@ -7659,6 +7693,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom 0.1.16", + "redox_syscall 0.1.57", + "rust-argon2", +] + [[package]] name = "redox_users" version = "0.4.0" @@ -7875,6 +7920,18 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rust-argon2" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" +dependencies = [ + "base64 0.13.0", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils 0.8.5", +] + [[package]] name = "rustc-demangle" version = "0.1.18" @@ -9088,6 +9145,8 @@ dependencies = [ "serde", "serde_json", "sp-core", + "sp-io", + "sp-std", ] [[package]] @@ -10903,18 +10962,29 @@ checksum = "d7fa7e55043acb85fca6b3c01485a2eeb6b69c5d21002e273c79e465f43b7ac1" [[package]] name = "tempfile" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ "cfg-if 1.0.0", + "fastrand", "libc", - "rand 0.8.4", "redox_syscall 0.2.10", "remove_dir_all", "winapi 0.3.9", ] +[[package]] +name = "term" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" +dependencies = [ + "byteorder", + "dirs", + "winapi 0.3.9", +] + [[package]] name = "termcolor" version = "1.1.2" diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index 0c850b322a7d2..afa4612f1ee4a 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -109,8 +109,6 @@ pub fn run() -> sc_cli::Result<()> { let runner = cli.create_runner(cmd)?; runner.sync_run(|config| { - let PartialComponents { client, backend, .. } = service::new_partial(&config)?; - // This switch needs to be in the client, since the client decides // which sub-commands it wants to support. match cmd { @@ -125,18 +123,25 @@ pub fn run() -> sc_cli::Result<()> { cmd.run::(config) }, - BenchmarkCmd::Block(cmd) => cmd.run(client), + BenchmarkCmd::Block(cmd) => { + let PartialComponents { client, .. } = service::new_partial(&config)?; + cmd.run(client) + }, BenchmarkCmd::Storage(cmd) => { + let PartialComponents { client, backend, .. } = + service::new_partial(&config)?; let db = backend.expose_db(); let storage = backend.expose_storage(); cmd.run(config, client, db, storage) }, BenchmarkCmd::Overhead(cmd) => { + let PartialComponents { client, .. } = service::new_partial(&config)?; let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) }, + BenchmarkCmd::Machine(cmd) => cmd.run(&config), } }) }, diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 880b7d7b7ecb7..e2c772e809200 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -99,8 +99,6 @@ pub fn run() -> Result<()> { let runner = cli.create_runner(cmd)?; runner.sync_run(|config| { - let PartialComponents { client, backend, .. } = new_partial(&config)?; - // This switch needs to be in the client, since the client decides // which sub-commands it wants to support. match cmd { @@ -115,18 +113,24 @@ pub fn run() -> Result<()> { cmd.run::(config) }, - BenchmarkCmd::Block(cmd) => cmd.run(client), + BenchmarkCmd::Block(cmd) => { + let PartialComponents { client, .. } = new_partial(&config)?; + cmd.run(client) + }, BenchmarkCmd::Storage(cmd) => { + let PartialComponents { client, backend, .. } = new_partial(&config)?; let db = backend.expose_db(); let storage = backend.expose_storage(); cmd.run(config, client, db, storage) }, BenchmarkCmd::Overhead(cmd) => { + let PartialComponents { client, .. } = new_partial(&config)?; let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) }, + BenchmarkCmd::Machine(cmd) => cmd.run(&config), } }) }, diff --git a/bin/node/cli/tests/benchmark_block_works.rs b/bin/node/cli/tests/benchmark_block_works.rs index 37a4db25f363b..359abf3e4265f 100644 --- a/bin/node/cli/tests/benchmark_block_works.rs +++ b/bin/node/cli/tests/benchmark_block_works.rs @@ -30,7 +30,7 @@ pub mod common; async fn benchmark_block_works() { let base_dir = tempdir().expect("could not create a temp dir"); - common::run_node_for_a_while(base_dir.path(), &["--dev"]).await; + common::run_node_for_a_while(base_dir.path(), &["--dev", "--no-hardware-benchmarks"]).await; // Invoke `benchmark block` with all options to make sure that they are valid. let status = Command::new(cargo_bin("substrate")) diff --git a/bin/node/cli/tests/benchmark_machine_works.rs b/bin/node/cli/tests/benchmark_machine_works.rs new file mode 100644 index 0000000000000..df407e988f636 --- /dev/null +++ b/bin/node/cli/tests/benchmark_machine_works.rs @@ -0,0 +1,32 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; + +/// Tests that the `benchmark machine` command works for the substrate dev runtime. +#[test] +fn benchmark_machine_works() { + let status = Command::new(cargo_bin("substrate")) + .args(["benchmark", "machine", "--dev"]) + .args(["--verify-duration", "0.1"]) + .status() + .unwrap(); + + assert!(status.success()); +} diff --git a/client/sysinfo/Cargo.toml b/client/sysinfo/Cargo.toml index 8efe583fb9335..540918cb37c01 100644 --- a/client/sysinfo/Cargo.toml +++ b/client/sysinfo/Cargo.toml @@ -23,4 +23,6 @@ libc = "0.2" serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.79" sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } +sp-std = { version = "4.0.0", path = "../../primitives/std" } sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" } diff --git a/client/sysinfo/src/lib.rs b/client/sysinfo/src/lib.rs index be13efb82c66f..911e725dcdd4e 100644 --- a/client/sysinfo/src/lib.rs +++ b/client/sysinfo/src/lib.rs @@ -20,12 +20,16 @@ //! and software telemetry information about the node on which we're running. use futures::prelude::*; +use std::time::Duration; mod sysinfo; #[cfg(target_os = "linux")] mod sysinfo_linux; -pub use sysinfo::{gather_hwbench, gather_sysinfo}; +pub use sysinfo::{ + benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, + benchmark_memory, benchmark_sr25519_verify, gather_hwbench, gather_sysinfo, +}; /// The operating system part of the current target triplet. pub const TARGET_OS: &str = include_str!(concat!(env!("OUT_DIR"), "/target_os.txt")); @@ -49,6 +53,38 @@ pub struct HwBench { pub disk_random_write_score: Option, } +/// Limit the execution time of a benchmark. +pub enum ExecutionLimit { + /// Limit by the maximal duration. + MaxDuration(Duration), + + /// Limit by the maximal number of iterations. + MaxIterations(usize), + + /// Limit by the maximal duration and maximal number of iterations. + Both { max_iterations: usize, max_duration: Duration }, +} + +impl ExecutionLimit { + /// Returns the duration limit or `MAX` if none is present. + pub fn max_duration(&self) -> Duration { + match self { + Self::MaxDuration(d) => *d, + Self::Both { max_duration, .. } => *max_duration, + _ => Duration::from_secs(u64::MAX), + } + } + + /// Returns the iterations limit or `MAX` if none is present. + pub fn max_iterations(&self) -> usize { + match self { + Self::MaxIterations(d) => *d, + Self::Both { max_iterations, .. } => *max_iterations, + _ => usize::MAX, + } + } +} + /// Prints out the system software/hardware information in the logs. pub fn print_sysinfo(sysinfo: &sc_telemetry::SysInfo) { log::info!("💻 Operating system: {}", TARGET_OS); diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index ceb28447002cf..65d7a9e41b406 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -16,9 +16,14 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::HwBench; -use rand::{seq::SliceRandom, Rng}; +use crate::{ExecutionLimit, HwBench}; + use sc_telemetry::SysInfo; +use sp_core::{sr25519, Pair}; +use sp_io::crypto::sr25519_verify; +use sp_std::prelude::*; + +use rand::{seq::SliceRandom, Rng, RngCore}; use std::{ fs::File, io::{Seek, SeekFrom, Write}, @@ -34,7 +39,7 @@ pub(crate) fn benchmark( max_iterations: usize, max_duration: Duration, mut run: impl FnMut() -> Result<(), E>, -) -> Result { +) -> Result { // Run the benchmark once as a warmup to get the code into the L1 cache. run()?; @@ -53,11 +58,11 @@ pub(crate) fn benchmark( } } - let score = (((size * count) as f64 / elapsed.as_secs_f64()) / (1024.0 * 1024.0)) as u64; + let score = ((size * count) as f64 / elapsed.as_secs_f64()) / (1024.0 * 1024.0); log::trace!( "Calculated {} of {}MB/s in {} iterations in {}ms", name, - score, + score as u64, count, elapsed.as_millis() ); @@ -83,7 +88,7 @@ pub fn gather_sysinfo() -> SysInfo { } #[inline(never)] -fn clobber(slice: &mut [u8]) { +fn clobber_slice(slice: &mut [T]) { assert!(!slice.is_empty()); // Discourage the compiler from optimizing out our benchmarks. @@ -102,8 +107,17 @@ fn clobber(slice: &mut [u8]) { } } +#[inline(never)] +fn clobber_value(input: &mut T) { + // Look into `clobber_slice` for a comment. + unsafe { + let value = std::ptr::read_volatile(input); + std::ptr::write_volatile(input, value); + } +} + // This benchmarks the CPU speed as measured by calculating BLAKE2b-256 hashes, in MB/s. -fn benchmark_cpu() -> u64 { +pub fn benchmark_cpu() -> u64 { // In general the results of this benchmark are somewhat sensitive to how much // data we hash at the time. The smaller this is the *less* MB/s we can hash, // the bigger this is the *more* MB/s we can hash, up until a certain point @@ -125,15 +139,15 @@ fn benchmark_cpu() -> u64 { let mut hash = Default::default(); let run = || -> Result<(), ()> { - clobber(&mut buffer); + clobber_slice(&mut buffer); hash = sp_core::hashing::blake2_256(&buffer); - clobber(&mut hash); + clobber_slice(&mut hash); Ok(()) }; benchmark("CPU score", SIZE, MAX_ITERATIONS, MAX_DURATION, run) - .expect("benchmark cannot fail; qed") + .expect("benchmark cannot fail; qed") as u64 } // This benchmarks the effective `memcpy` memory bandwidth available in MB/s. @@ -141,7 +155,7 @@ fn benchmark_cpu() -> u64 { // It doesn't technically measure the absolute maximum memory bandwidth available, // but that's fine, because real code most of the time isn't optimized to take // advantage of the full memory bandwidth either. -fn benchmark_memory() -> u64 { +pub fn benchmark_memory() -> u64 { // Ideally this should be at least as big as the CPU's L3 cache, // and it should be big enough so that the `memcpy` takes enough // time to be actually measurable. @@ -161,8 +175,8 @@ fn benchmark_memory() -> u64 { dst.resize(SIZE, 0x77); let run = || -> Result<(), ()> { - clobber(&mut src); - clobber(&mut dst); + clobber_slice(&mut src); + clobber_slice(&mut dst); // SAFETY: Both vectors are of the same type and of the same size, // so copying data between them is safe. @@ -172,14 +186,14 @@ fn benchmark_memory() -> u64 { libc::memcpy(dst.as_mut_ptr().cast(), src.as_ptr().cast(), SIZE); } - clobber(&mut dst); - clobber(&mut src); + clobber_slice(&mut dst); + clobber_slice(&mut src); Ok(()) }; benchmark("memory score", SIZE, MAX_ITERATIONS, MAX_DURATION, run) - .expect("benchmark cannot fail; qed") + .expect("benchmark cannot fail; qed") as u64 } struct TemporaryFile { @@ -260,6 +274,7 @@ pub fn benchmark_disk_sequential_writes(directory: &Path) -> Result }; benchmark("disk sequential write score", SIZE, MAX_ITERATIONS, MAX_DURATION, run) + .map(|s| s as u64) } pub fn benchmark_disk_random_writes(directory: &Path) -> Result { @@ -319,6 +334,45 @@ pub fn benchmark_disk_random_writes(directory: &Path) -> Result { // We only wrote half of the bytes hence `SIZE / 2`. benchmark("disk random write score", SIZE / 2, MAX_ITERATIONS, MAX_DURATION, run) + .map(|s| s as u64) +} + +/// Benchmarks the verification speed of sr25519 signatures. +/// +/// Returns the throughput in MB/s by convention. +/// The values are rather small (0.4-0.8) so it is advised to convert them into KB/s. +pub fn benchmark_sr25519_verify(limit: ExecutionLimit) -> f64 { + const INPUT_SIZE: usize = 32; + const ITERATION_SIZE: usize = 2048; + let pair = sr25519::Pair::from_string("//Alice", None).unwrap(); + + let mut rng = rng(); + let mut msgs = Vec::new(); + let mut sigs = Vec::new(); + + for _ in 0..ITERATION_SIZE { + let mut msg = vec![0u8; INPUT_SIZE]; + rng.fill_bytes(&mut msg[..]); + + sigs.push(pair.sign(&msg)); + msgs.push(msg); + } + + let run = || -> Result<(), String> { + for (sig, msg) in sigs.iter().zip(msgs.iter()) { + let mut ok = sr25519_verify(&sig, &msg[..], &pair.public()); + clobber_value(&mut ok); + } + Ok(()) + }; + benchmark( + "sr25519 verification score", + INPUT_SIZE * ITERATION_SIZE, + limit.max_iterations(), + limit.max_duration(), + run, + ) + .expect("sr25519 verification cannot fail; qed") } /// Benchmarks the hardware and returns the results of those benchmarks. @@ -390,4 +444,9 @@ mod tests { fn test_benchmark_disk_random_writes() { assert!(benchmark_disk_random_writes("./".as_ref()).unwrap() > 0); } + + #[test] + fn test_benchmark_sr25519_verify() { + assert!(benchmark_sr25519_verify(ExecutionLimit::MaxIterations(1)) > 0.0); + } } diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 5cb81232085a4..858d16558e821 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -23,6 +23,7 @@ sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" } sc-client-db = { version = "0.10.0-dev", path = "../../../client/db" } sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" } +sc-sysinfo = { version = "6.0.0-dev", path = "../../../client/sysinfo" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-externalities = { version = "0.12.0", path = "../../../primitives/externalities" } @@ -32,7 +33,6 @@ sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } sp-storage = { version = "6.0.0", path = "../../../primitives/storage" } sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } -sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } sp-state-machine = { version = "0.12.0", path = "../../../primitives/state-machine" } sp-trie = { version = "6.0.0", path = "../../../primitives/trie" } codec = { version = "3.0.0", package = "parity-scale-codec" } @@ -52,6 +52,9 @@ hex = "0.4.3" memory-db = "0.29.0" rand = { version = "0.8.4", features = ["small_rng"] } thousands = "0.2.0" +prettytable-rs = "0.8.0" +tempfile = "3.2.0" +rand_pcg = "0.3.1" [features] default = ["db", "sc-client-db/runtime-benchmarks"] diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 2543a63b2f159..75e2edc042a26 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -18,12 +18,14 @@ //! Contains the root [`BenchmarkCmd`] command and exports its sub-commands. mod block; +mod machine; mod overhead; mod pallet; mod shared; mod storage; pub use block::BlockCmd; +pub use machine::MachineCmd; pub use overhead::{ExtrinsicBuilder, OverheadCmd}; pub use pallet::PalletCmd; pub use storage::StorageCmd; @@ -39,6 +41,8 @@ pub enum BenchmarkCmd { Storage(StorageCmd), Overhead(OverheadCmd), Block(BlockCmd), + #[clap(hide = true)] // Hidden until fully completed. + Machine(MachineCmd), } /// Unwraps a [`BenchmarkCmd`] into its concrete sub-command. @@ -53,6 +57,7 @@ macro_rules! unwrap_cmd { BenchmarkCmd::Storage($cmd) => $code, BenchmarkCmd::Overhead($cmd) => $code, BenchmarkCmd::Block($cmd) => $code, + BenchmarkCmd::Machine($cmd) => $code, } } } diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs new file mode 100644 index 0000000000000..ee6bf765d01c4 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -0,0 +1,86 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Contains the [`MachineCmd`] as entry point for the node +//! and the core benchmarking logic. + +use sc_cli::{CliConfiguration, Result, SharedParams}; +use sc_service::Configuration; +use sc_sysinfo::{ + benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, + benchmark_memory, benchmark_sr25519_verify, ExecutionLimit, +}; + +use clap::Parser; +use log::info; +use prettytable::{cell, row, table}; +use std::{fmt::Debug, fs, time::Duration}; + +/// Command to benchmark the hardware. +/// +/// Runs multiple benchmarks and prints their output to console. +/// Can be used to gauge if the hardware is fast enough to keep up with a chain's requirements. +/// This command must be integrated by the client since the client can set compiler flags +/// which influence the results. +/// +/// You can use the `--base-path` flag to set a location for the disk benchmarks. +#[derive(Debug, Parser)] +pub struct MachineCmd { + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: SharedParams, + + /// Time limit for the verification benchmark. + #[clap(long, default_value = "2.0", value_name = "SECONDS")] + pub verify_duration: f32, +} + +impl MachineCmd { + /// Execute the benchmark and print the results. + pub fn run(&self, cfg: &Configuration) -> Result<()> { + // Ensure that the dir exists since the node is not started to take care of it. + let dir = cfg.database.path().ok_or("No DB directory provided")?; + fs::create_dir_all(dir)?; + + info!("Running machine benchmarks..."); + let write = benchmark_disk_sequential_writes(dir)?; + let read = benchmark_disk_random_writes(dir)?; + let verify_limit = + ExecutionLimit::MaxDuration(Duration::from_secs_f32(self.verify_duration)); + let verify = benchmark_sr25519_verify(verify_limit) * 1024.0; + + // Use a table for nicer console output. + let table = table!( + ["Category", "Function", "Score", "Unit"], + ["CPU", "BLAKE2-256", benchmark_cpu(), "MB/s"], + ["CPU", "SR25519 Verify", format!("{:.1}", verify), "KB/s"], + ["Memory", "Copy", benchmark_memory(), "MB/s"], + ["Disk", "Seq Write", write, "MB/s"], + ["Disk", "Rnd Write", read, "MB/s"] + ); + + info!("\n{}", table); + Ok(()) + } +} + +// Boilerplate +impl CliConfiguration for MachineCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } +} diff --git a/utils/frame/benchmarking-cli/src/shared/mod.rs b/utils/frame/benchmarking-cli/src/shared/mod.rs index f08d79b9aafca..853fbdef8e87f 100644 --- a/utils/frame/benchmarking-cli/src/shared/mod.rs +++ b/utils/frame/benchmarking-cli/src/shared/mod.rs @@ -25,6 +25,8 @@ pub use record::BenchRecord; pub use stats::{StatSelect, Stats}; pub use weight_params::WeightParams; +use rand::prelude::*; + /// A Handlebars helper to add an underscore after every 3rd character, /// i.e. a separator for large numbers. #[derive(Clone, Copy)] @@ -63,3 +65,11 @@ where } s } + +/// Returns an rng and the seed that was used to create it. +/// +/// Uses a random seed if none is provided. +pub fn new_rng(seed: Option) -> (impl rand::Rng, u64) { + let seed = seed.unwrap_or(rand::thread_rng().gen::()); + (rand_pcg::Pcg64::seed_from_u64(seed), seed) +} diff --git a/utils/frame/benchmarking-cli/src/storage/cmd.rs b/utils/frame/benchmarking-cli/src/storage/cmd.rs index c2cc219ef1528..23222dbd120ab 100644 --- a/utils/frame/benchmarking-cli/src/storage/cmd.rs +++ b/utils/frame/benchmarking-cli/src/storage/cmd.rs @@ -34,7 +34,7 @@ use sp_runtime::generic::BlockId; use std::{fmt::Debug, path::PathBuf, sync::Arc}; use super::template::TemplateData; -use crate::shared::WeightParams; +use crate::shared::{new_rng, WeightParams}; /// Benchmark the storage speed of a chain snapshot. #[derive(Debug, Parser)] @@ -151,13 +151,6 @@ impl StorageCmd { } } - /// Creates an rng from a random seed. - pub(crate) fn setup_rng() -> impl rand::Rng { - let seed = rand::thread_rng().gen::(); - info!("Using seed {}", seed); - StdRng::seed_from_u64(seed) - } - /// Run some rounds of the (read) benchmark as warmup. /// See `frame_benchmarking_cli::storage::read::bench_read` for detailed comments. fn bench_warmup(&self, client: &Arc) -> Result<()> @@ -169,7 +162,7 @@ impl StorageCmd { let block = BlockId::Number(client.usage_info().chain.best_number); let empty_prefix = StorageKey(Vec::new()); let mut keys = client.storage_keys(&block, &empty_prefix)?; - let mut rng = Self::setup_rng(); + let (mut rng, _) = new_rng(None); keys.shuffle(&mut rng); for i in 0..self.params.warmups { diff --git a/utils/frame/benchmarking-cli/src/storage/read.rs b/utils/frame/benchmarking-cli/src/storage/read.rs index f58f3c3de0c19..c1dc6daba0953 100644 --- a/utils/frame/benchmarking-cli/src/storage/read.rs +++ b/utils/frame/benchmarking-cli/src/storage/read.rs @@ -28,7 +28,7 @@ use rand::prelude::*; use std::{fmt::Debug, sync::Arc, time::Instant}; use super::cmd::StorageCmd; -use crate::shared::BenchRecord; +use crate::shared::{new_rng, BenchRecord}; impl StorageCmd { /// Benchmarks the time it takes to read a single Storage item. @@ -47,7 +47,7 @@ impl StorageCmd { // Load all keys and randomly shuffle them. let empty_prefix = StorageKey(Vec::new()); let mut keys = client.storage_keys(&block, &empty_prefix)?; - let mut rng = Self::setup_rng(); + let (mut rng, _) = new_rng(None); keys.shuffle(&mut rng); // Interesting part here: diff --git a/utils/frame/benchmarking-cli/src/storage/write.rs b/utils/frame/benchmarking-cli/src/storage/write.rs index d5d5bc2fffa5b..ab25109a35d49 100644 --- a/utils/frame/benchmarking-cli/src/storage/write.rs +++ b/utils/frame/benchmarking-cli/src/storage/write.rs @@ -32,7 +32,7 @@ use rand::prelude::*; use std::{fmt::Debug, sync::Arc, time::Instant}; use super::cmd::StorageCmd; -use crate::shared::BenchRecord; +use crate::shared::{new_rng, BenchRecord}; impl StorageCmd { /// Benchmarks the time it takes to write a single Storage item. @@ -59,7 +59,7 @@ impl StorageCmd { info!("Preparing keys from block {}", block); // Load all KV pairs and randomly shuffle them. let mut kvs = trie.pairs(); - let mut rng = Self::setup_rng(); + let (mut rng, _) = new_rng(None); kvs.shuffle(&mut rng); // Generate all random values first; Make sure there are no collisions with existing