EVM benchmark utilities

ethcore/Cargo.toml

@@ -67,7 +67,7 @@ keccak-hash = { path = "../util/hash" }
 triehash = { path = "../util/triehash" }
 unexpected = { path = "../util/unexpected" }
 journaldb = { path = "../util/journaldb" }
-tempdir = "0.3"
+tempdir = { version = "0.3", optional = true }
 kvdb-rocksdb = { path = "../util/kvdb-rocksdb" }
 
 [dev-dependencies]
@@ -84,10 +84,10 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
 # EVM debug traces are printed.
 slow-blocks = []
 # Run JSON consensus tests.
-json-tests = ["ethcore-transaction/json-tests"]
+json-tests = ["ethcore-transaction/json-tests", "test-helpers", "tempdir"]
 # Run memory/cpu heavy tests.
 test-heavy = []
 # Compile benches
 benches = []
 # Compile test helpers
-test-helpers = []
+test-helpers = ["tempdir"]

ethcore/src/client/evm_test_client.rs

@@ -64,15 +64,6 @@ impl fmt::Display for EvmTestError {
 use ethereum;
 use ethjson::state::test::ForkSpec;
 
-lazy_static! {
-	pub static ref FRONTIER: spec::Spec = ethereum::new_frontier_test();
-	pub static ref HOMESTEAD: spec::Spec = ethereum::new_homestead_test();
-	pub static ref EIP150: spec::Spec = ethereum::new_eip150_test();
-	pub static ref EIP161: spec::Spec = ethereum::new_eip161_test();
-	pub static ref BYZANTIUM: spec::Spec = ethereum::new_byzantium_test();
-	pub static ref BYZANTIUM_TRANSITION: spec::Spec = ethereum::new_transition_test();
-}
-
 /// Simplified, single-block EVM test client.
 pub struct EvmTestClient<'a> {
 	state: state::State<state_db::StateDB>,
@@ -90,14 +81,14 @@ impl<'a> fmt::Debug for EvmTestClient<'a> {
 
 impl<'a> EvmTestClient<'a> {
 	/// Converts a json spec definition into spec.
-	pub fn spec_from_json(spec: &ForkSpec) -> Option<&'static spec::Spec> {
+	pub fn spec_from_json(spec: &ForkSpec) -> Option<spec::Spec> {
 		match *spec {
-			ForkSpec::Frontier => Some(&*FRONTIER),
-			ForkSpec::Homestead => Some(&*HOMESTEAD),
-			ForkSpec::EIP150 => Some(&*EIP150),
-			ForkSpec::EIP158 => Some(&*EIP161),
-			ForkSpec::Byzantium => Some(&*BYZANTIUM),
-			ForkSpec::EIP158ToByzantiumAt5 => Some(&BYZANTIUM_TRANSITION),
+			ForkSpec::Frontier => Some(ethereum::new_frontier_test()),
+			ForkSpec::Homestead => Some(ethereum::new_homestead_test()),
+			ForkSpec::EIP150 => Some(ethereum::new_eip150_test()),
+			ForkSpec::EIP158 => Some(ethereum::new_eip161_test()),
+			ForkSpec::Byzantium => Some(ethereum::new_byzantium_test()),
+			ForkSpec::EIP158ToByzantiumAt5 => Some(ethereum::new_transition_test()),
 			ForkSpec::FrontierToHomesteadAt5 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => None,
 			_ => None,
 		}

ethcore/src/json_tests/chain.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::path::Path;
 use std::sync::Arc;
 use client::{EvmTestClient, Client, ClientConfig, ChainInfo, ImportBlock};
 use block::Block;
@@ -23,12 +24,26 @@ use miner::Miner;
 use io::IoChannel;
 use test_helpers;
 
-pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
+use super::HookType;
+
+/// Run chain jsontests on a given folder.
+pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+	::json_tests::test_common::run_test_path(p, skip, json_chain_test, h)
+}
+
+/// Run chain jsontests on a given file.
+pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
+	::json_tests::test_common::run_test_file(p, json_chain_test, h)
+}
+
+pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::blockchain::Test::load(json_data).unwrap();
 	let mut failed = Vec::new();
 
 	for (name, blockchain) in tests.into_iter() {
+		start_stop_hook(&name, HookType::OnStart);
+
 		let mut fail = false;
 		{
 			let mut fail_unless = |cond: bool| if !cond && !fail {
@@ -42,7 +57,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 
 			let spec = {
 				let mut spec = match EvmTestClient::spec_from_json(&blockchain.network) {
-					Some(spec) => (*spec).clone(),
+					Some(spec) => spec,
 					None => {
 						println!("   - {} | {:?} Ignoring tests because of missing spec", name, blockchain.network);
 						continue;
@@ -82,17 +97,21 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 		if !fail {
 			flushln!("ok");
 		}
+
+		start_stop_hook(&name, HookType::OnStop);
 	}
 
 	println!("!!! {:?} tests from failed.", failed.len());
 	failed
 }
 
+#[cfg(test)]
 mod block_tests {
 	use super::json_chain_test;
+	use json_tests::HookType;
 
-	fn do_json_test(json_data: &[u8]) -> Vec<String> {
-		json_chain_test(json_data)
+	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
+		json_chain_test(json_data, h)
 	}
 
 	declare_test!{BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"}

ethcore/src/json_tests/difficulty.rs

@@ -19,12 +19,16 @@ use header::Header;
 use ethereum_types::U256;
 use spec::Spec;
 
-pub fn json_difficulty_test(json_data: &[u8], spec: Spec) -> Vec<String> {
+use super::HookType;
+
+pub fn json_difficulty_test<H: FnMut(&str, HookType)>(json_data: &[u8], spec: Spec, start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::test::DifficultyTest::load(json_data).unwrap();
 	let engine = &spec.engine;
 
 	for (name, test) in tests.into_iter() {
+		start_stop_hook(&name, HookType::OnStart);
+
 		flush!("   - {}...", name);
 		println!("   - {}...", name);
 
@@ -42,15 +46,18 @@ pub fn json_difficulty_test(json_data: &[u8], spec: Spec) -> Vec<String> {
 		let expected_difficulty: U256 = test.current_difficulty.into();
 		assert_eq!(header.difficulty(), &expected_difficulty);
 		flushln!("ok");
+
+		start_stop_hook(&name, HookType::OnStop);
 	}
 	vec![]
 }
 
 mod difficulty_test_byzantium {
 	use super::json_difficulty_test;
+	use json_tests::HookType;
 
-	fn do_json_test(json_data: &[u8]) -> Vec<String> {
-		json_difficulty_test(json_data, ::ethereum::new_byzantium_test())
+	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
+		json_difficulty_test(json_data, ::ethereum::new_byzantium_test(), h)
 	}
 
 	declare_test!{DifficultyTests_difficultyByzantium, "BasicTests/difficultyByzantium.json"}
@@ -59,10 +66,11 @@ mod difficulty_test_byzantium {
 mod difficulty_test_foundation {
 	use super::json_difficulty_test;
 	use tempdir::TempDir;
+	use json_tests::HookType;
 
-	fn do_json_test(json_data: &[u8]) -> Vec<String> {
+	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
 		let tempdir = TempDir::new("").unwrap();
-		json_difficulty_test(json_data, ::ethereum::new_foundation(&tempdir.path()))
+		json_difficulty_test(json_data, ::ethereum::new_foundation(&tempdir.path()), h)
 	}
 
 	declare_test!{DifficultyTests_difficultyMainNetwork, "BasicTests/difficultyMainNetwork.json"}

ethcore/src/json_tests/executive.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::path::Path;
 use std::sync::Arc;
 use super::test_common::*;
 use state::{Backend as StateBackend, State, Substate};
@@ -35,6 +36,18 @@ use rlp::RlpStream;
 use hash::keccak;
 use machine::EthereumMachine as Machine;
 
+use super::HookType;
+
+/// Run executive jsontests on a given folder.
+pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+	::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
+}
+
+/// Run executive jsontests on a given file.
+pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
+	::json_tests::test_common::run_test_file(p, do_json_test, h)
+}
+
 #[derive(Debug, PartialEq, Clone)]
 struct CallCreate {
 	data: Bytes,
@@ -193,20 +206,22 @@ impl<'a, T: 'a, V: 'a, B: 'a> Ext for TestExt<'a, T, V, B>
 	}
 }
 
-fn do_json_test(json_data: &[u8]) -> Vec<String> {
+fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
 	let vms = VMType::all();
 	vms
 		.iter()
-		.flat_map(|vm| do_json_test_for(vm, json_data))
+		.flat_map(|vm| do_json_test_for(vm, json_data, h))
 		.collect()
 }
 
-fn do_json_test_for(vm_type: &VMType, json_data: &[u8]) -> Vec<String> {
+fn do_json_test_for<H: FnMut(&str, HookType)>(vm_type: &VMType, json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	let tests = ethjson::vm::Test::load(json_data).unwrap();
 	let mut failed = Vec::new();
 
 	for (name, vm) in tests.into_iter() {
-		println!("name: {:?}", name);
+		start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStart);
+
+		info!(target: "jsontests", "name: {:?}", name);
 		let mut fail = false;
 
 		let mut fail_unless = |cond: bool, s: &str | if !cond && !fail {
@@ -305,10 +320,12 @@ fn do_json_test_for(vm_type: &VMType, json_data: &[u8]) -> Vec<String> {
 				fail_unless(Some(callcreates) == calls, "callcreates does not match");
 			}
 		};
+
+		start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStop);
 	}
 
 	for f in &failed {
-		println!("FAILED: {:?}", f);
+		error!("FAILED: {:?}", f);
 	}
 
 	failed

ethcore/src/json_tests/mod.rs

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+//! Helpers and tests for operating on jsontests.
+
 #[macro_use]
 mod test_common;
 
@@ -22,4 +24,21 @@ mod executive;
 mod state;
 mod chain;
 mod trie;
+
+#[cfg(test)]
 mod difficulty;
+
+pub use self::test_common::HookType;
+
+pub use self::transaction::run_test_path as run_transaction_test_path;
+pub use self::transaction::run_test_file as run_transaction_test_file;
+pub use self::executive::run_test_path as run_executive_test_path;
+pub use self::executive::run_test_file as run_executive_test_file;
+pub use self::state::run_test_path as run_state_test_path;
+pub use self::state::run_test_file as run_state_test_file;
+pub use self::chain::run_test_path as run_chain_test_path;
+pub use self::chain::run_test_file as run_chain_test_file;
+pub use self::trie::run_generic_test_path as run_generic_trie_test_path;
+pub use self::trie::run_generic_test_file as run_generic_trie_test_file;
+pub use self::trie::run_secure_test_path as run_secure_trie_test_path;
+pub use self::trie::run_secure_test_file as run_secure_trie_test_file;

ethcore/src/json_tests/state.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::path::Path;
 use super::test_common::*;
 use pod_state::PodState;
 use trace;
@@ -22,12 +23,26 @@ use ethjson;
 use transaction::SignedTransaction;
 use vm::EnvInfo;
 
-pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
+use super::HookType;
+
+/// Run state jsontests on a given folder.
+pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+	::json_tests::test_common::run_test_path(p, skip, json_chain_test, h)
+}
+
+/// Run state jsontests on a given file.
+pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
+	::json_tests::test_common::run_test_file(p, json_chain_test, h)
+}
+
+pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::state::test::Test::load(json_data).unwrap();
 	let mut failed = Vec::new();
 
 	for (name, test) in tests.into_iter() {
+		start_stop_hook(&name, HookType::OnStart);
+
 		{
 			let multitransaction = test.transaction;
 			let env: EnvInfo = test.env.into();
@@ -50,7 +65,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 					let transaction: SignedTransaction = multitransaction.select(&state.indexes).into();
 
 					let result = || -> Result<_, EvmTestError> {
-						Ok(EvmTestClient::from_pod_state(spec, pre.clone())?
+						Ok(EvmTestClient::from_pod_state(&spec, pre.clone())?
 							.transact(&env, transaction, trace::NoopTracer, trace::NoopVMTracer))
 					};
 					match result() {
@@ -81,6 +96,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 			}
 		}
 
+		start_stop_hook(&name, HookType::OnStop);
 	}
 
 	if !failed.is_empty() {
@@ -89,11 +105,13 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 	failed
 }
 
+#[cfg(test)]
 mod state_tests {
 	use super::json_chain_test;
+	use json_tests::HookType;
 
-	fn do_json_test(json_data: &[u8]) -> Vec<String> {
-		json_chain_test(json_data)
+	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
+		json_chain_test(json_data, h)
 	}
 
 	declare_test!{GeneralStateTest_stAttackTest, "GeneralStateTests/stAttackTest/"}