diff --git a/Cargo.lock b/Cargo.lock index 34309759af206..9aa31fa9a3e48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4134,9 +4134,11 @@ dependencies = [ name = "substrate-test-client" version = "0.1.0" dependencies = [ + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "parity-codec 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "sr-primitives 0.1.0", "substrate-client 0.1.0", + "substrate-client-db 0.1.0", "substrate-consensus-common 0.1.0", "substrate-executor 0.1.0", "substrate-keyring 0.1.0", diff --git a/core/client/db/Cargo.toml b/core/client/db/Cargo.toml index 9c4b3dfffac56..6408fa0dc80b3 100644 --- a/core/client/db/Cargo.toml +++ b/core/client/db/Cargo.toml @@ -10,6 +10,7 @@ log = "0.4" kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" } # FIXME replace with release as soon as our rocksdb changes are released upstream https://github.com/paritytech/parity-common/issues/88 kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" } +kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d", optional = true } lru-cache = "0.1.1" hash-db = { version = "0.11" } primitives = { package = "substrate-primitives", path = "../../primitives" } @@ -27,3 +28,7 @@ kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b031 substrate-keyring = { path = "../../keyring" } test-client = { package = "substrate-test-client", path = "../../test-client" } env_logger = { version = "0.6" } + +[features] +default = [] +test-helpers = ["kvdb-memorydb"] diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index d64708a86b3ba..b39fbb9f06adc 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -57,6 +57,9 @@ use crate::storage_cache::{CachingState, SharedCache, new_shared_cache}; use log::{trace, debug, warn}; pub use state_db::PruningMode; +#[cfg(feature = "test-helpers")] +use client::in_mem::Backend as InMemoryBackend; + const CANONICALIZATION_DELAY: u64 = 4096; const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u64 = 32768; const STATE_CACHE_SIZE_BYTES: usize = 16 * 1024 * 1024; @@ -463,8 +466,10 @@ impl client::backend::PrunableStateChangesTrieStorage state_machine::ChangesTrieRootsStorage for DbChangesTrieStorage { fn root(&self, anchor: &state_machine::ChangesTrieAnchorBlockId, block: u64) -> Result, String> { - // check API requirement - assert!(block <= anchor.number, "API requirement"); + // check API requirement: we can't get NEXT block(s) based on anchor + if block > anchor.number { + return Err(format!("Can't get changes trie root at {} using anchor at {}", block, anchor.number)); + } // we need to get hash of the block to resolve changes trie root let block_id = if block <= self.meta.read().finalized_number.as_() { @@ -531,8 +536,8 @@ impl> Backend { Backend::from_kvdb(db as Arc<_>, config.pruning, canonicalization_delay) } - #[cfg(test)] - fn new_test(keep_blocks: u32, canonicalization_delay: u64) -> Self { + #[cfg(any(test, feature = "test-helpers"))] + pub fn new_test(keep_blocks: u32, canonicalization_delay: u64) -> Self { use utils::NUM_COLUMNS; let db = Arc::new(::kvdb_memorydb::create(NUM_COLUMNS)); @@ -570,6 +575,54 @@ impl> Backend { }) } + /// Returns in-memory blockchain that contains the same set of blocks that the self. + #[cfg(feature = "test-helpers")] + pub fn as_in_memory(&self) -> InMemoryBackend { + use client::backend::{Backend as ClientBackend, BlockImportOperation}; + use client::blockchain::Backend as BlockchainBackend; + + let inmem = InMemoryBackend::::new(); + + // get all headers hashes && sort them by number (could be duplicate) + let mut headers: Vec<(NumberFor, Block::Hash, Block::Header)> = Vec::new(); + for (_, header) in self.blockchain.db.iter(columns::HEADER) { + let header = Block::Header::decode(&mut &header[..]).unwrap(); + let hash = header.hash(); + let number = *header.number(); + let pos = headers.binary_search_by(|item| item.0.cmp(&number)); + match pos { + Ok(pos) => headers.insert(pos, (number, hash, header)), + Err(pos) => headers.insert(pos, (number, hash, header)), + } + } + + // insert all other headers + bodies + justifications + let info = self.blockchain.info().unwrap(); + for (number, hash, header) in headers { + let id = BlockId::Hash(hash); + let justification = self.blockchain.justification(id).unwrap(); + let body = self.blockchain.body(id).unwrap(); + let state = self.state_at(id).unwrap().pairs(); + + let new_block_state = if number.is_zero() { + NewBlockState::Final + } else if hash == info.best_hash { + NewBlockState::Best + } else { + NewBlockState::Normal + }; + let mut op = inmem.begin_operation().unwrap(); + op.set_block_data(header, body, justification, new_block_state).unwrap(); + op.update_db_storage(state.into_iter().map(|(k, v)| (None, k, Some(v))).collect()).unwrap(); + inmem.commit_operation(op).unwrap(); + } + + // and now finalize the best block we have + inmem.finalize_block(BlockId::Hash(info.finalized_hash), None).unwrap(); + + inmem + } + /// Handle setting head within a transaction. `route_to` should be the last /// block that existed in the database. `best_to` should be the best block /// to be set. @@ -712,8 +765,8 @@ impl> Backend { operation.apply_aux(&mut transaction); let mut meta_updates = Vec::new(); + let mut last_finalized_hash = self.blockchain.meta.read().finalized_hash; if !operation.finalized_blocks.is_empty() { - let mut last_finalized_hash = self.blockchain.meta.read().finalized_hash; for (block, justification) in operation.finalized_blocks { let block_hash = self.blockchain.expect_block_hash_from_id(&block)?; let block_header = self.blockchain.expect_header(BlockId::Hash(block_hash))?; @@ -787,7 +840,7 @@ impl> Backend { if finalized { // TODO: ensure best chain contains this block. - self.ensure_sequential_finalization(header, None)?; + self.ensure_sequential_finalization(header, Some(last_finalized_hash))?; self.note_finalized(&mut transaction, header, hash)?; } else { // canonicalize blocks which are old enough, regardless of finality. diff --git a/core/client/db/src/light.rs b/core/client/db/src/light.rs index 6a0c9646b0bd6..b86ebb93ef70d 100644 --- a/core/client/db/src/light.rs +++ b/core/client/db/src/light.rs @@ -72,8 +72,8 @@ impl LightStorage Self::from_kvdb(db as Arc<_>) } - #[cfg(test)] - pub(crate) fn new_test() -> Self { + #[cfg(any(test, feature = "test-helpers"))] + pub fn new_test() -> Self { use utils::NUM_COLUMNS; let db = Arc::new(::kvdb_memorydb::create(NUM_COLUMNS)); @@ -493,7 +493,7 @@ impl LightBlockchainStorage for LightStorage } fn cache(&self) -> Option<&BlockchainCache> { - None + Some(&self.cache) } } diff --git a/core/client/src/client.rs b/core/client/src/client.rs index 9d55b24e98087..375e36e81f545 100644 --- a/core/client/src/client.rs +++ b/core/client/src/client.rs @@ -1628,7 +1628,7 @@ pub(crate) mod tests { client.import(BlockOrigin::Own, builder.bake().unwrap()).unwrap(); assert_eq!(client.info().unwrap().chain.best_number, 1); - assert!(client.state_at(&BlockId::Number(1)).unwrap() != client.state_at(&BlockId::Number(0)).unwrap()); + assert!(client.state_at(&BlockId::Number(1)).unwrap().pairs() != client.state_at(&BlockId::Number(0)).unwrap().pairs()); assert_eq!( client.runtime_api().balance_of( &BlockId::Number(client.info().unwrap().chain.best_number), @@ -1647,14 +1647,11 @@ pub(crate) mod tests { #[test] fn client_uses_authorities_from_blockchain_cache() { - let client = test_client::new(); - test_client::client::in_mem::cache_authorities_at( - client.backend().blockchain(), - Default::default(), - Some(vec![[1u8; 32].into()])); - assert_eq!(client.authorities_at( - &BlockId::Hash(Default::default())).unwrap(), - vec![[1u8; 32].into()]); + let client = test_client::new_light(); + let genesis_hash = client.header(&BlockId::Number(0)).unwrap().unwrap().hash(); + // authorities cache is first filled in genesis block + // => should be read from cache here (remote request will fail in this test) + assert!(!client.authorities_at(&BlockId::Hash(genesis_hash)).unwrap().is_empty()); } #[test] @@ -1680,7 +1677,7 @@ pub(crate) mod tests { client.import(BlockOrigin::Own, builder.bake().unwrap()).unwrap(); assert_eq!(client.info().unwrap().chain.best_number, 1); - assert!(client.state_at(&BlockId::Number(1)).unwrap() != client.state_at(&BlockId::Number(0)).unwrap()); + assert!(client.state_at(&BlockId::Number(1)).unwrap().pairs() != client.state_at(&BlockId::Number(0)).unwrap().pairs()); assert_eq!(client.body(&BlockId::Number(1)).unwrap().unwrap().len(), 1) } diff --git a/core/network/src/test/sync.rs b/core/network/src/test/sync.rs index b4b2b2078f437..edf670e82646d 100644 --- a/core/network/src/test/sync.rs +++ b/core/network/src/test/sync.rs @@ -31,7 +31,8 @@ fn sync_from_two_peers_works() { net.peer(1).push_blocks(100, false); net.peer(2).push_blocks(100, false); net.sync(); - assert!(net.peer(0).client.backend().blockchain().equals_to(net.peer(1).client.backend().blockchain())); + assert!(net.peer(0).client.backend().as_in_memory().blockchain() + .equals_to(net.peer(1).client.backend().as_in_memory().blockchain())); let status = net.peer(0).status(); assert_eq!(status.sync.state, SyncState::Idle); } @@ -45,7 +46,8 @@ fn sync_from_two_peers_with_ancestry_search_works() { net.peer(2).push_blocks(100, false); net.restart_peer(0); net.sync(); - assert!(net.peer(0).client.backend().blockchain().canon_equals_to(net.peer(1).client.backend().blockchain())); + assert!(net.peer(0).client.backend().as_in_memory().blockchain() + .canon_equals_to(net.peer(1).client.backend().as_in_memory().blockchain())); } #[test] @@ -58,7 +60,8 @@ fn sync_long_chain_works() { net.sync(); // Wait for peers to get up to speed. thread::sleep(time::Duration::from_millis(1000)); - assert!(net.peer(0).client.backend().blockchain().equals_to(net.peer(1).client.backend().blockchain())); + assert!(net.peer(0).client.backend().as_in_memory().blockchain() + .equals_to(net.peer(1).client.backend().as_in_memory().blockchain())); } #[test] @@ -68,7 +71,8 @@ fn sync_no_common_longer_chain_fails() { net.peer(0).push_blocks(20, true); net.peer(1).push_blocks(20, false); net.sync(); - assert!(!net.peer(0).client.backend().blockchain().canon_equals_to(net.peer(1).client.backend().blockchain())); + assert!(!net.peer(0).client.backend().as_in_memory().blockchain() + .canon_equals_to(net.peer(1).client.backend().as_in_memory().blockchain())); } #[test] @@ -111,11 +115,11 @@ fn sync_after_fork_works() { net.peer(2).push_blocks(1, false); // peer 1 has the best chain - let peer1_chain = net.peer(1).client.backend().blockchain().clone(); + let peer1_chain = net.peer(1).client.backend().as_in_memory().blockchain().clone(); net.sync(); - assert!(net.peer(0).client.backend().blockchain().canon_equals_to(&peer1_chain)); - assert!(net.peer(1).client.backend().blockchain().canon_equals_to(&peer1_chain)); - assert!(net.peer(2).client.backend().blockchain().canon_equals_to(&peer1_chain)); + assert!(net.peer(0).client.backend().as_in_memory().blockchain().canon_equals_to(&peer1_chain)); + assert!(net.peer(1).client.backend().as_in_memory().blockchain().canon_equals_to(&peer1_chain)); + assert!(net.peer(2).client.backend().as_in_memory().blockchain().canon_equals_to(&peer1_chain)); } #[test] @@ -131,8 +135,8 @@ fn syncs_all_forks() { net.sync(); // Check that all peers have all of the blocks. - assert_eq!(9, net.peer(0).client.backend().blockchain().blocks_count()); - assert_eq!(9, net.peer(1).client.backend().blockchain().blocks_count()); + assert_eq!(9, net.peer(0).client.backend().as_in_memory().blockchain().blocks_count()); + assert_eq!(9, net.peer(1).client.backend().as_in_memory().blockchain().blocks_count()); } #[test] @@ -147,9 +151,9 @@ fn own_blocks_are_announced() { net.sync(); assert_eq!(net.peer(0).client.backend().blockchain().info().unwrap().best_number, 1); assert_eq!(net.peer(1).client.backend().blockchain().info().unwrap().best_number, 1); - let peer0_chain = net.peer(0).client.backend().blockchain().clone(); - assert!(net.peer(1).client.backend().blockchain().canon_equals_to(&peer0_chain)); - assert!(net.peer(2).client.backend().blockchain().canon_equals_to(&peer0_chain)); + let peer0_chain = net.peer(0).client.backend().as_in_memory().blockchain().clone(); + assert!(net.peer(1).client.backend().as_in_memory().blockchain().canon_equals_to(&peer0_chain)); + assert!(net.peer(2).client.backend().as_in_memory().blockchain().canon_equals_to(&peer0_chain)); } #[test] diff --git a/core/state-machine/src/changes_trie/changes_iterator.rs b/core/state-machine/src/changes_trie/changes_iterator.rs index 4245c62e51fe0..5805427870c86 100644 --- a/core/state-machine/src/changes_trie/changes_iterator.rs +++ b/core/state-machine/src/changes_trie/changes_iterator.rs @@ -40,6 +40,9 @@ pub fn key_changes<'a, S: Storage, H: Hasher>( max: u64, key: &'a [u8], ) -> Result, String> where H::Out: HeapSizeOf { + // we can't query any roots before root + let max = ::std::cmp::min(max, end.number); + Ok(DrilldownIterator { essence: DrilldownIteratorEssence { key, @@ -67,6 +70,9 @@ pub fn key_changes_proof, H: Hasher>( max: u64, key: &[u8], ) -> Result>, String> where H::Out: HeapSizeOf { + // we can't query any roots before root + let max = ::std::cmp::min(max, end.number); + let mut iter = ProvingDrilldownIterator { essence: DrilldownIteratorEssence { key, @@ -104,6 +110,9 @@ pub fn key_changes_proof_check, H: Hasher>( max: u64, key: &[u8] ) -> Result, String> where H::Out: HeapSizeOf { + // we can't query any roots before root + let max = ::std::cmp::min(max, end.number); + let mut proof_db = MemoryDB::::default(); for item in proof { proof_db.insert(&item); diff --git a/core/test-client/Cargo.toml b/core/test-client/Cargo.toml index 786d61cd2d695..467a715de1033 100644 --- a/core/test-client/Cargo.toml +++ b/core/test-client/Cargo.toml @@ -6,6 +6,8 @@ edition = "2018" [dependencies] client = { package = "substrate-client", path = "../client" } +client-db = { package = "substrate-client-db", path = "../client/db", features = ["test-helpers"] } +futures = { version = "0.1.17" } parity-codec = "3.0" executor = { package = "substrate-executor", path = "../executor" } consensus = { package = "substrate-consensus-common", path = "../consensus/common" } diff --git a/core/test-client/src/lib.rs b/core/test-client/src/lib.rs index de730dce30911..bd2d3a298a044 100644 --- a/core/test-client/src/lib.rs +++ b/core/test-client/src/lib.rs @@ -34,9 +34,10 @@ pub use runtime; pub use consensus; use std::sync::Arc; +use futures::future::FutureResult; use primitives::Blake2Hasher; use runtime_primitives::StorageOverlay; -use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash as HashT}; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash as HashT, NumberFor}; use runtime::genesismap::{GenesisConfig, additional_storage_with_genesis}; use keyring::Keyring; use state_machine::ExecutionStrategy; @@ -59,7 +60,7 @@ mod local_executor { pub use local_executor::LocalExecutor; /// Test client database backend. -pub type Backend = client::in_mem::Backend; +pub type Backend = client_db::Backend; /// Test client executor. pub type Executor = client::LocalCallExecutor< @@ -67,16 +68,60 @@ pub type Executor = client::LocalCallExecutor< executor::NativeExecutor, >; +/// Test client light database backend. +pub type LightBackend = client::light::backend::Backend< + client_db::light::LightStorage, + LightFetcher, + Blake2Hasher, +>; + +/// Test client light fetcher. +pub struct LightFetcher; + +/// Test client light executor. +pub type LightExecutor = client::light::call_executor::RemoteOrLocalCallExecutor< + runtime::Block, + LightBackend, + client::light::call_executor::RemoteCallExecutor< + client::light::blockchain::Blockchain< + client_db::light::LightStorage, + LightFetcher + >, + LightFetcher + >, + client::LocalCallExecutor< + client::light::backend::Backend< + client_db::light::LightStorage, + LightFetcher, + Blake2Hasher + >, + executor::NativeExecutor + > +>; + /// Creates new client instance used for tests. pub fn new() -> client::Client { - new_with_backend(Arc::new(Backend::new()), false) + new_with_backend(Arc::new(Backend::new_test(::std::u32::MAX, ::std::u64::MAX)), false) +} + +/// Creates new light client instance used for tests. +pub fn new_light() -> client::Client { + let storage = client_db::light::LightStorage::new_test(); + let blockchain = Arc::new(client::light::blockchain::Blockchain::new(storage)); + let backend = Arc::new(LightBackend::new(blockchain.clone())); + let executor = NativeExecutor::new(None); + let fetcher = Arc::new(LightFetcher); + let remote_call_executor = client::light::call_executor::RemoteCallExecutor::new(blockchain.clone(), fetcher); + let local_call_executor = client::LocalCallExecutor::new(backend.clone(), executor); + let call_executor = LightExecutor::new(backend.clone(), remote_call_executor, local_call_executor); + client::Client::new(backend, call_executor, genesis_storage(false), Default::default()).unwrap() } /// Creates new client instance used for tests with the given api execution strategy. pub fn new_with_execution_strategy( execution_strategy: ExecutionStrategy ) -> client::Client { - let backend = Arc::new(Backend::new()); + let backend = Arc::new(Backend::new_test(::std::u32::MAX, ::std::u64::MAX)); let executor = NativeExecutor::new(None); let executor = LocalCallExecutor::new(backend.clone(), executor); @@ -99,7 +144,7 @@ pub fn new_with_execution_strategy( pub fn new_with_changes_trie() -> client::Client { - new_with_backend(Arc::new(Backend::new()), true) + new_with_backend(Arc::new(Backend::new_test(::std::u32::MAX, ::std::u64::MAX)), true) } /// Creates new client instance used for tests with an explicitly provided backend. @@ -133,3 +178,38 @@ fn genesis_storage(support_changes_trie: bool) -> StorageOverlay { storage.extend(additional_storage_with_genesis(&block)); storage } + +impl client::light::fetcher::Fetcher for LightFetcher { + type RemoteHeaderResult = FutureResult; + type RemoteReadResult = FutureResult>, client::error::Error>; + type RemoteCallResult = FutureResult, client::error::Error>; + type RemoteChangesResult = FutureResult, u32)>, client::error::Error>; + + fn remote_header( + &self, + _request: client::light::fetcher::RemoteHeaderRequest, + ) -> Self::RemoteHeaderResult { + unimplemented!("not (yet) used in tests") + } + + fn remote_read( + &self, + _request: client::light::fetcher::RemoteReadRequest, + ) -> Self::RemoteReadResult { + unimplemented!("not (yet) used in tests") + } + + fn remote_call( + &self, + _request: client::light::fetcher::RemoteCallRequest, + ) -> Self::RemoteCallResult { + unimplemented!("not (yet) used in tests") + } + + fn remote_changes( + &self, + _request: client::light::fetcher::RemoteChangesRequest, + ) -> Self::RemoteChangesResult { + unimplemented!("not (yet) used in tests") + } +}