Batch benchmarks together with * notation.

bin/node/runtime/src/lib.rs

@@ -858,95 +858,66 @@ impl_runtime_apis! {
 	#[cfg(feature = "runtime-benchmarks")]
 	impl frame_benchmarking::Benchmark<Block> for Runtime {
 		fn dispatch_benchmark(
-			module: Vec<u8>,
-			extrinsic: Vec<u8>,
+			pallet: Vec<u8>,
+			benchmark: Vec<u8>,
 			lowest_range_values: Vec<u32>,
 			highest_range_values: Vec<u32>,
 			steps: Vec<u32>,
 			repeat: u32,
-		) -> Result<Vec<frame_benchmarking::BenchmarkResults>, sp_runtime::RuntimeString> {
+		) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, sp_runtime::RuntimeString> {
 			use frame_benchmarking::Benchmarking;
 			// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues.
 			// To get around that, we separated the Session benchmarks into its own crate, which is why
 			// we need these two lines below.
 			use pallet_session_benchmarking::Module as SessionBench;
 			impl pallet_session_benchmarking::Trait for Runtime {}
 
-			let result = match module.as_slice() {
-				b"pallet-balances" | b"balances" => Balances::run_benchmark(
-					extrinsic,
-					lowest_range_values,
-					highest_range_values,
-					steps,
-					repeat,
-				),
-				b"pallet-im-online" | b"im-online" => ImOnline::run_benchmark(
-					extrinsic,
-					lowest_range_values,
-					highest_range_values,
-					steps,
-					repeat,
-				),
-				b"pallet-identity" | b"identity" => Identity::run_benchmark(
-					extrinsic,
-					lowest_range_values,
-					highest_range_values,
-					steps,
-					repeat,
-				),
-				b"pallet-session" | b"session" => SessionBench::<Runtime>::run_benchmark(
-					extrinsic,
-					lowest_range_values,
-					highest_range_values,
-					steps,
-					repeat,
-				),
-				b"pallet-staking" | b"staking" => Staking::run_benchmark(
-					extrinsic,
-					lowest_range_values,
-					highest_range_values,
-					steps,
-					repeat,
-				),
-				b"pallet-timestamp" | b"timestamp" => Timestamp::run_benchmark(
-					extrinsic,
-					lowest_range_values,
-					highest_range_values,
-					steps,
-					repeat,
-				),
-				b"pallet-treasury" | b"treasury" => Treasury::run_benchmark(
-					extrinsic,
-					lowest_range_values,
-					highest_range_values,
-					steps,
-					repeat,
-				),
-				b"pallet-vesting" | b"vesting" => Vesting::run_benchmark(
-					extrinsic,
-					lowest_range_values,
-					highest_range_values,
-					steps,
-					repeat,
-				),
-				b"pallet-democracy" | b"democracy" => Democracy::run_benchmark(
-					extrinsic,
-					lowest_range_values,
-					highest_range_values,
-					steps,
-					repeat,
-				),
-				b"pallet-collective" | b"collective" => Council::run_benchmark(
-					extrinsic,
-					lowest_range_values,
-					highest_range_values,
-					steps,
-					repeat,
-				),
-				_ => Err("Benchmark not found for this pallet."),
-			};
-
-			result.map_err(|e| e.into())
+			let mut batches = Vec::<frame_benchmarking::BenchmarkBatch>::new();
+			macro_rules! pallet {
+				( $name:literal, $( $location:tt )* ) => (
+					if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] {
+						if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] {
+							for benchmark in $( $location )*::benchmarks().into_iter() {
+								batches.push(frame_benchmarking::BenchmarkBatch {
+									results: $( $location )*::run_benchmark(
+										benchmark,
+										&lowest_range_values[..],
+										&highest_range_values[..],
+										&steps[..],
+										repeat,
+									)?,
+									pallet: pallet.to_vec(),
+									benchmark: benchmark.to_vec(),
+								});
+							}
+						} else {
+							batches.push(frame_benchmarking::BenchmarkBatch {
+								results: $( $location )*::run_benchmark(
+									&benchmark[..],
+									&lowest_range_values[..],
+									&highest_range_values[..],
+									&steps[..],
+									repeat,
+								)?,
+								pallet: pallet.to_vec(),
+								benchmark: benchmark.clone(),
+							});
+						}
+					}
+				)
+			}
+
+			pallet!(b"balances", Balances);
+			pallet!(b"im-online", ImOnline);
+			pallet!(b"identity", Identity);
+			pallet!(b"session", SessionBench::<Runtime>);
+			pallet!(b"staking", Staking);
+			pallet!(b"timestamp", Timestamp);
+			pallet!(b"treasury", Treasury);
+			pallet!(b"vesting", Vesting);
+			pallet!(b"democracy", Democracy);
+			pallet!(b"collective", Council);
+			Ok(batches)
 		}
 	}
 }

frame/benchmarking/src/lib.rs

@@ -505,15 +505,19 @@ macro_rules! impl_benchmark {
 		NO_INSTANCE $( $name:ident ),*
 	) => {
 		impl<T: Trait> $crate::Benchmarking<$crate::BenchmarkResults> for Module<T> {
+			fn benchmarks() -> Vec<&'static [u8]> {
+				vec![ $( stringify!($name).as_ref() ),* ]
+			}
+
 			fn run_benchmark(
-				extrinsic: Vec<u8>,
-				lowest_range_values: Vec<u32>,
-				highest_range_values: Vec<u32>,
-				steps: Vec<u32>,
+				extrinsic: &[u8],
+				lowest_range_values: &[u32],
+				highest_range_values: &[u32],
+				steps: &[u32],
 				repeat: u32,
 			) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
 				// Map the input to the selected benchmark.
-				let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice())
+				let extrinsic = sp_std::str::from_utf8(extrinsic)
 					.map_err(|_| "`extrinsic` is not a valid utf8 string!")?;
 				let selected_benchmark = match extrinsic {
 					$( stringify!($name) => SelectedBenchmark::$name, )*
@@ -597,15 +601,19 @@ macro_rules! impl_benchmark {
 		INSTANCE $( $name:ident ),*
 	) => {
 		impl<T: Trait<I>, I: Instance> $crate::Benchmarking<$crate::BenchmarkResults> for Module<T, I> {
+			fn benchmarks() -> Vec<&'static [u8]> {
+				vec![ $( stringify!($name).as_ref() ),* ]
+			}
+
 			fn run_benchmark(
-				extrinsic: Vec<u8>,
-				lowest_range_values: Vec<u32>,
-				highest_range_values: Vec<u32>,
-				steps: Vec<u32>,
+				extrinsic: &[u8],
+				lowest_range_values: &[u32],
+				highest_range_values: &[u32],
+				steps: &[u32],
 				repeat: u32,
 			) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
 				// Map the input to the selected benchmark.
-				let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice())
+				let extrinsic = sp_std::str::from_utf8(extrinsic)
 					.map_err(|_| "`extrinsic` is not a valid utf8 string!")?;
 				let selected_benchmark = match extrinsic {
 					$( stringify!($name) => SelectedBenchmark::$name, )*

frame/benchmarking/src/utils.rs

@@ -22,13 +22,24 @@ use sp_io::hashing::blake2_256;
 use sp_runtime::RuntimeString;
 
 /// An alphabet of possible parameters to use for benchmarking.
-#[derive(codec::Encode, codec::Decode, Clone, Copy, PartialEq, Debug)]
+#[derive(Encode, Decode, Clone, Copy, PartialEq, Debug)]
 #[allow(missing_docs)]
 #[allow(non_camel_case_types)]
 pub enum BenchmarkParameter {
 	a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z,
 }
 
+/// The results of a single of benchmark.
+#[derive(Encode, Decode, Clone, PartialEq, Debug)]
+pub struct BenchmarkBatch {
+	/// The pallet containing this benchmark.
+	pub pallet: Vec<u8>,
+	/// The extrinsic (or benchmark name) of this benchmark.
+	pub benchmark: Vec<u8>,
+	/// The results from this benchmark.
+	pub results: Vec<BenchmarkResults>,
+}
+
 /// Results from running benchmarks on a FRAME pallet.
 /// Contains duration of the function call in nanoseconds along with the benchmark parameters
 /// used for that benchmark result.
@@ -39,13 +50,13 @@ sp_api::decl_runtime_apis! {
 	pub trait Benchmark {
 		/// Dispatch the given benchmark.
 		fn dispatch_benchmark(
-			module: Vec<u8>,
-			extrinsic: Vec<u8>,
+			pallet: Vec<u8>,
+			benchmark: Vec<u8>,
 			lowest_range_values: Vec<u32>,
 			highest_range_values: Vec<u32>,
 			steps: Vec<u32>,
 			repeat: u32,
-		) -> Result<Vec<BenchmarkResults>, RuntimeString>;
+		) -> Result<Vec<BenchmarkBatch>, RuntimeString>;
 	}
 }
 
@@ -75,19 +86,24 @@ pub trait Benchmarking {
 
 /// The pallet benchmarking trait.
 pub trait Benchmarking<T> {
+	/// Get the benchmarks available for this pallet. Generally there is one benchmark per
+	/// extrinsic, so these are sometimes just called "extrinsics".
+	fn benchmarks() -> Vec<&'static [u8]>;
+
 	/// Run the benchmarks for this pallet.
 	///
 	/// Parameters
-	/// - `extrinsic`: The name of extrinsic function you want to benchmark encoded as bytes.
+	/// - `name`: The name of extrinsic function or benchmark you want to benchmark encoded as
+	///   bytes.
 	/// - `steps`: The number of sample points you want to take across the range of parameters.
 	/// - `lowest_range_values`: The lowest number for each range of parameters.
 	/// - `highest_range_values`: The highest number for each range of parameters.
 	/// - `repeat`: The number of times you want to repeat a benchmark.
 	fn run_benchmark(
-		extrinsic: Vec<u8>,
-		lowest_range_values: Vec<u32>,
-		highest_range_values: Vec<u32>,
-		steps: Vec<u32>,
+		name: &[u8],
+		lowest_range_values: &[u32],
+		highest_range_values: &[u32],
+		steps: &[u32],
 		repeat: u32,
 	) -> Result<Vec<T>, &'static str>;
 }

utils/frame/benchmarking-cli/src/lib.rs

@@ -22,7 +22,7 @@ use sc_client_db::BenchmarkingState;
 use sc_service::{Configuration, ChainSpec};
 use sc_executor::{NativeExecutor, NativeExecutionDispatch};
 use codec::{Encode, Decode};
-use frame_benchmarking::{BenchmarkResults, Analysis};
+use frame_benchmarking::{BenchmarkBatch, Analysis};
 use sp_core::{
 	tasks,
 	traits::KeystoreExt,
@@ -33,11 +33,11 @@ use sp_externalities::Extensions;
 /// The `benchmark` command used to benchmark FRAME Pallets.
 #[derive(Debug, structopt::StructOpt, Clone)]
 pub struct BenchmarkCmd {
-	/// Select a FRAME Pallet to benchmark.
+	/// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`).
 	#[structopt(short, long)]
 	pub pallet: String,
 
-	/// Select an extrinsic to benchmark.
+	/// Select an extrinsic inside the pallet to benchmark, or `*` for all.
 	#[structopt(short, long)]
 	pub extrinsic: String,
 
@@ -57,6 +57,18 @@ pub struct BenchmarkCmd {
 	#[structopt(short, long, default_value = "1")]
 	pub repeat: u32,
 
+	/// Print the raw results.
+	#[structopt(short, long)]
+	pub raw_data: bool,
+
+	/// Don't print the median-slopes linear regression analysis.
+	#[structopt(short, long)]
+	pub no_median_slopes: bool,
+
+	/// Don't print the min-squares linear regression analysis.
+	#[structopt(short, long)]
+	pub no_min_squares: bool,
+
 	#[allow(missing_docs)]
 	#[structopt(flatten)]
 	pub shared_params: sc_cli::SharedParams,
@@ -135,47 +147,50 @@ impl BenchmarkCmd {
 		.execute(strategy.into())
 		.map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?;
 
-		let results = <Result<Vec<BenchmarkResults>, String> as Decode>::decode(&mut &result[..])
+		let results = <Result<Vec<BenchmarkBatch>, String> as Decode>::decode(&mut &result[..])
 			.map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?;
 
 		match results {
-			Ok(results) => {
+			Ok(batches) => for batch in batches.into_iter() {
 				// Print benchmark metadata
 				println!(
 					"Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}",
-					self.pallet,
-					self.extrinsic,
+					String::from_utf8(batch.pallet).expect("Encoded from String; qed"),
+					String::from_utf8(batch.benchmark).expect("Encoded from String; qed"),
 					self.lowest_range_values,
 					self.highest_range_values,
 					self.steps,
 					self.repeat,
 				);
 
-				// Print the table header
-				results[0].0.iter().for_each(|param| print!("{:?},", param.0));
+				if self.raw_data {
+					// Print the table header
+					batch.results[0].0.iter().for_each(|param| print!("{:?},", param.0));
 
-				print!("extrinsic_time,storage_root_time\n");
-				// Print the values
-				results.iter().for_each(|result| {
-					let parameters = &result.0;
-					parameters.iter().for_each(|param| print!("{:?},", param.1));
-					// Print extrinsic time and storage root time
-					print!("{:?},{:?}\n", result.1, result.2);
-				});
+					print!("extrinsic_time,storage_root_time\n");
+					// Print the values
+					batch.results.iter().for_each(|result| {
+						let parameters = &result.0;
+						parameters.iter().for_each(|param| print!("{:?},", param.1));
+						// Print extrinsic time and storage root time
+						print!("{:?},{:?}\n", result.1, result.2);
+					});
 
-				print!("\n");
+					print!("\n");
+				}
 
 				// Conduct analysis.
-				if let Some(analysis) = Analysis::median_slopes(&results) {
-					println!("Median Slopes Analysis\n========\n{}", analysis);
+				if !self.no_median_slopes {
+					if let Some(analysis) = Analysis::median_slopes(&batch.results) {
+						println!("Median Slopes Analysis\n========\n{}", analysis);
+					}
 				}
-
-				if let Some(analysis) = Analysis::min_squares_iqr(&results) {
-					println!("Min Squares Analysis\n========\n{}", analysis);
+				if !self.no_min_squares {
+					if let Some(analysis) = Analysis::min_squares_iqr(&batch.results) {
+						println!("Min Squares Analysis\n========\n{}", analysis);
+					}
 				}
-
-				eprintln!("Done.");
-			}
+			},
 			Err(error) => eprintln!("Error: {:?}", error),
 		}