From af6a8fde7a48ae666d00cfedbee60cc1cdafde35 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 23 Sep 2019 18:55:15 +0200 Subject: [PATCH 01/68] init from other pr, have some code up to client (not client db), complete some test method by providing child access to values. --- core/client/db/src/lib.rs | 85 ++++++++-- core/client/db/src/storage_cache.rs | 24 ++- core/client/src/backend.rs | 6 +- core/client/src/call_executor.rs | 14 +- core/client/src/cht.rs | 9 +- core/client/src/client.rs | 11 +- core/client/src/in_mem.rs | 3 +- core/client/src/light/backend.rs | 39 ++++- core/client/src/light/call_executor.rs | 14 +- core/client/src/light/fetcher.rs | 6 +- core/state-db/src/lib.rs | 20 ++- core/state-machine/src/backend.rs | 158 +++++++++++++++++-- core/state-machine/src/changes_trie/build.rs | 4 +- core/state-machine/src/ext.rs | 4 +- core/state-machine/src/lib.rs | 28 ++-- core/state-machine/src/offstate_backend.rs | 93 +++++++++++ core/state-machine/src/overlayed_changes.rs | 41 ++++- core/state-machine/src/proving_backend.rs | 84 +++++++--- core/state-machine/src/testing.rs | 27 +++- core/state-machine/src/trie_backend.rs | 125 ++++++++++++--- 20 files changed, 691 insertions(+), 104 deletions(-) create mode 100644 core/state-machine/src/offstate_backend.rs diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 62cc8027c2afe..d5167012f1068 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -60,6 +60,7 @@ use executor::RuntimeInfo; use state_machine::{ DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, ChangesTrieBuildCache, backend::Backend as StateBackend, + TODO, }; use crate::utils::{Meta, db_err, meta_keys, read_db, block_id_to_lookup_key, read_meta}; use client::leaves::{LeafSet, FinalizationDisplaced}; @@ -71,6 +72,8 @@ pub use state_db::PruningMode; #[cfg(feature = "test-helpers")] use client::in_mem::Backend as InMemoryBackend; +#[cfg(feature = "test-helpers")] +use state_machine::backend::InMemoryTransaction; const CANONICALIZATION_DELAY: u64 = 4096; const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u32 = 32768; @@ -79,7 +82,11 @@ const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u32 = 32768; const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); /// DB-backed patricia trie state, transaction type is an overlay of changes to commit. -pub type DbState = state_machine::TrieBackend>, Blake2Hasher>; +pub type DbState = state_machine::TrieBackend< + Arc>, + Blake2Hasher, + Arc, +>; /// A reference tracking state. /// @@ -119,6 +126,7 @@ impl StateBackend for RefTrackingState { type Error = >::Error; type Transaction = >::Transaction; type TrieBackendStorage = >::TrieBackendStorage; + type OffstateBackendStorage = >::OffstateBackendStorage; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { self.state.storage(key) @@ -170,10 +178,29 @@ impl StateBackend for RefTrackingState { self.state.child_storage_root(storage_key, delta) } + fn offstate_transaction(&self, delta: I) -> Self::Transaction + where + I: IntoIterator, Option>)> + { + self.state.offstate_transaction(delta) + } + fn pairs(&self) -> Vec<(Vec, Vec)> { self.state.pairs() } + fn children_storage_keys(&self) -> Vec> { + self.state.children_storage_keys() + } + + fn child_pairs(&self, child_storage_key: &[u8]) -> Vec<(Vec, Vec)> { + self.state.child_pairs(child_storage_key) + } + + fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { + self.state.offstate_pairs() + } + fn keys(&self, prefix: &[u8]) -> Vec> { self.state.keys(prefix) } @@ -182,7 +209,9 @@ impl StateBackend for RefTrackingState { self.state.child_keys(child_key, prefix) } - fn as_trie_backend(&mut self) -> Option<&state_machine::TrieBackend> { + fn as_trie_backend(&mut self) -> Option< + &state_machine::TrieBackend + > { self.state.as_trie_backend() } } @@ -408,7 +437,7 @@ impl client::blockchain::ProvideCache for BlockchainDb { old_state: CachingState, Block>, - db_updates: PrefixedMemoryDB, + db_updates: (PrefixedMemoryDB, Vec<(Vec, Option>)>), storage_updates: StorageCollection, child_storage_updates: ChildStorageCollection, changes_trie_updates: MemoryDB, @@ -461,7 +490,10 @@ where Block: BlockT, // Currently cache isn't implemented on full nodes. } - fn update_db_storage(&mut self, update: PrefixedMemoryDB) -> Result<(), client::error::Error> { + fn update_db_storage( + &mut self, + update: (PrefixedMemoryDB, Vec<(Vec, Option>)>), + ) -> Result<(), client::error::Error> { self.db_updates = update; Ok(()) } @@ -488,7 +520,8 @@ where Block: BlockT, let (root, transaction) = self.old_state.full_storage_root( top.into_iter().map(|(k, v)| (k, Some(v))), - child_delta + child_delta, + None, ); self.db_updates = transaction; @@ -855,7 +888,12 @@ impl> Backend { let id = BlockId::Hash(hash); let justification = self.blockchain.justification(id).unwrap(); let body = self.blockchain.body(id).unwrap(); - let state = self.state_at(id).unwrap().pairs(); + let state = self.state_at(id).unwrap(); + let mut storage: Vec<_> = state.pairs().into_iter().map(|(k, v)| (None, k, Some(v))).collect(); + for child_key in state.children_storage_keys() { + storage.extend(state.child_pairs(child_key.as_slice()) + .into_iter().map(|(k, v)| (Some(child_key.clone()), k, Some(v)))); + } let new_block_state = if number.is_zero() { NewBlockState::Final @@ -866,7 +904,10 @@ impl> Backend { }; let mut op = inmem.begin_operation().unwrap(); op.set_block_data(header, body, justification, new_block_state).unwrap(); - op.update_db_storage(state.into_iter().map(|(k, v)| (None, k, Some(v))).collect()).unwrap(); + op.update_db_storage(InMemoryTransaction { + storage, + offstate: state.offstate_pairs().into_iter().map(|(k, v)| (k, Some(v))).collect(), + }).unwrap(); inmem.commit_operation(op).unwrap(); } @@ -1103,16 +1144,32 @@ impl> Backend { } let mut changeset: state_db::ChangeSet> = state_db::ChangeSet::default(); - for (key, (val, rc)) in operation.db_updates.drain() { + for (key, (val, rc)) in operation.db_updates.0.drain() { if rc > 0 { changeset.inserted.push((key, val.to_vec())); } else if rc < 0 { changeset.deleted.push(key); } } + // remove duplicate + let mut map_offstate: HashMap<_, _> = operation.db_updates.1.into_iter().collect(); + let mut offstate_changeset: state_db::ChangeSet> = state_db::ChangeSet::default(); + for (key, option_val) in map_offstate.drain() { + if let Some(val) = option_val { + offstate_changeset.inserted.push((key, val.to_vec())); + } else { + offstate_changeset.deleted.push(key); + } + } let number_u64 = number.saturated_into::(); - let commit = self.storage.state_db.insert_block(&hash, number_u64, &pending_block.header.parent_hash(), changeset) - .map_err(|e: state_db::Error| client::error::Error::from(format!("State database error: {:?}", e)))?; + let commit = self.storage.state_db.insert_block( + &hash, + number_u64, + &pending_block.header.parent_hash(), + changeset, + offstate_changeset, + ).map_err(|e: state_db::Error| + client::error::Error::from(format!("State database error: {:?}", e)))?; apply_state_commit(&mut transaction, commit); // Check if need to finalize. Genesis is always finalized instantly. @@ -1314,7 +1371,7 @@ impl client::backend::Backend for Backend whe Ok(BlockImportOperation { pending_block: None, old_state, - db_updates: PrefixedMemoryDB::default(), + db_updates: (PrefixedMemoryDB::default(), Default::default()), storage_updates: Default::default(), child_storage_updates: Default::default(), changes_trie_updates: MemoryDB::default(), @@ -1439,7 +1496,9 @@ impl client::backend::Backend for Backend whe BlockId::Hash(h) if h == Default::default() => { let genesis_storage = DbGenesisStorage::new(); let root = genesis_storage.0.clone(); - let db_state = DbState::new(Arc::new(genesis_storage), root); + // TODO EMCH see genesis impl: that is empty storage + let genesis_offstate = TODO; + let db_state = DbState::new(Arc::new(genesis_storage), root, Arc::new(genesis_offstate)); let state = RefTrackingState::new(db_state, self.storage.clone(), None); return Ok(CachingState::new(state, self.shared_cache.clone(), None)); }, @@ -1451,7 +1510,7 @@ impl client::backend::Backend for Backend whe let hash = hdr.hash(); if let Ok(()) = self.storage.state_db.pin(&hash) { let root = H256::from_slice(hdr.state_root().as_ref()); - let db_state = DbState::new(self.storage.clone(), root); + let db_state = DbState::new(self.storage.clone(), root, Arc::new(TODO)); let state = RefTrackingState::new(db_state, self.storage.clone(), Some(hash.clone())); Ok(CachingState::new(state, self.shared_cache.clone(), Some(hash))) } else { diff --git a/core/client/db/src/storage_cache.rs b/core/client/db/src/storage_cache.rs index af8c9e379c4c1..baed84db041f0 100644 --- a/core/client/db/src/storage_cache.rs +++ b/core/client/db/src/storage_cache.rs @@ -470,6 +470,7 @@ impl, B: BlockT> StateBackend for CachingState< type Error = S::Error; type Transaction = S::Transaction; type TrieBackendStorage = S::TrieBackendStorage; + type OffstateBackendStorage = S::OffstateBackendStorage; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { let local_cache = self.cache.local_cache.upgradable_read(); @@ -570,10 +571,29 @@ impl, B: BlockT> StateBackend for CachingState< self.state.child_storage_root(storage_key, delta) } + fn offstate_transaction(&self, delta: I) -> Self::Transaction + where + I: IntoIterator, Option>)>, + { + self.state.offstate_transaction(delta) + } + fn pairs(&self) -> Vec<(Vec, Vec)> { self.state.pairs() } + fn children_storage_keys(&self) -> Vec> { + self.state.children_storage_keys() + } + + fn child_pairs(&self, storage_key: &[u8]) -> Vec<(Vec, Vec)> { + self.state.child_pairs(storage_key) + } + + fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { + self.state.offstate_pairs() + } + fn keys(&self, prefix: &[u8]) -> Vec> { self.state.keys(prefix) } @@ -582,7 +602,9 @@ impl, B: BlockT> StateBackend for CachingState< self.state.child_keys(child_key, prefix) } - fn as_trie_backend(&mut self) -> Option<&TrieBackend> { + fn as_trie_backend(&mut self) -> Option< + &TrieBackend + > { self.state.as_trie_backend() } } diff --git a/core/client/src/backend.rs b/core/client/src/backend.rs index 9b6d9ce58fbfe..99a0edd0eab17 100644 --- a/core/client/src/backend.rs +++ b/core/client/src/backend.rs @@ -41,7 +41,11 @@ pub(crate) struct ImportSummary { pub(crate) origin: BlockOrigin, pub(crate) header: Block::Header, pub(crate) is_new_best: bool, - pub(crate) storage_changes: Option<(StorageCollection, ChildStorageCollection)>, + pub(crate) storage_changes: Option<( + StorageCollection, + ChildStorageCollection, + StorageCollection, + )>, pub(crate) retracted: Vec, } diff --git a/core/client/src/call_executor.rs b/core/client/src/call_executor.rs index b49a58a0e56a5..31a28a64dc423 100644 --- a/core/client/src/call_executor.rs +++ b/core/client/src/call_executor.rs @@ -143,9 +143,12 @@ where /// Execute a call to a contract on top of given trie state, gathering execution proof. /// /// No changes are made. - fn prove_at_trie_state>( + fn prove_at_trie_state< + S: state_machine::TrieBackendStorage, + O: state_machine::OffstateBackendStorage, + >( &self, - trie_state: &state_machine::TrieBackend, + trie_state: &state_machine::TrieBackend, overlay: &mut OverlayedChanges, method: &str, call_data: &[u8] @@ -378,9 +381,12 @@ where .map_err(Into::into) } - fn prove_at_trie_state>( + fn prove_at_trie_state< + S: state_machine::TrieBackendStorage, + O: state_machine::OffstateBackendStorage, + >( &self, - trie_state: &state_machine::TrieBackend, + trie_state: &state_machine::TrieBackend, overlay: &mut OverlayedChanges, method: &str, call_data: &[u8] diff --git a/core/client/src/cht.rs b/core/client/src/cht.rs index 63a5b39c26904..f801843d053e7 100644 --- a/core/client/src/cht.rs +++ b/core/client/src/cht.rs @@ -30,8 +30,10 @@ use trie; use primitives::{H256, convert_hash}; use sr_primitives::traits::{Header as HeaderT, SimpleArithmetic, Zero, One}; use state_machine::backend::InMemory as InMemoryState; +use state_machine::backend::InMemoryTransaction; use state_machine::{MemoryDB, TrieBackend, Backend as StateBackend, prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend}; +use state_machine::offstate_backend::TODO; use crate::error::{Error as ClientError, Result as ClientResult}; @@ -100,7 +102,10 @@ pub fn build_proof( .into_iter() .map(|(k, v)| (None, k, Some(v))) .collect::>(); - let mut storage = InMemoryState::::default().update(transaction); + let mut storage = InMemoryState::::default().update(InMemoryTransaction { + storage: transaction, + offstate: Default::default(), + }); let trie_storage = storage.as_trie_backend() .expect("InMemoryState::as_trie_backend always returns Some; qed"); prove_read_on_trie_backend( @@ -143,7 +148,7 @@ pub fn check_proof_on_proving_backend( local_root: Header::Hash, local_number: Header::Number, remote_hash: Header::Hash, - proving_backend: &TrieBackend, Hasher>, + proving_backend: &TrieBackend, Hasher, TODO>, ) -> ClientResult<()> where Header: HeaderT, diff --git a/core/client/src/client.rs b/core/client/src/client.rs index acba5fa824aab..ba79a73d815f1 100644 --- a/core/client/src/client.rs +++ b/core/client/src/client.rs @@ -1035,7 +1035,8 @@ impl Client where Option>>, Option<( Vec<(Vec, Option>)>, - Vec<(Vec, Vec<(Vec, Option>)>)> + Vec<(Vec, Vec<(Vec, Option>)>)>, + Vec<(Vec, Option>)>, )> )> where @@ -1079,13 +1080,17 @@ impl Client where overlay.commit_prospective(); - let (top, children) = overlay.into_committed(); + let (top, children, offstate) = overlay.into_committed(); let children = children.map(|(sk, it)| (sk, it.collect())).collect(); if import_headers.post().state_root() != &storage_update.1 { return Err(error::Error::InvalidStateRoot); } - Ok((Some(storage_update.0), Some(changes_update), Some((top.collect(), children)))) + Ok(( + Some(storage_update.0), + Some(changes_update), + Some((top.collect(), children, offstate.collect())), + )) }, None => Ok((None, None, None)) } diff --git a/core/client/src/in_mem.rs b/core/client/src/in_mem.rs index debfbbd6c75c0..f4d074e221356 100644 --- a/core/client/src/in_mem.rs +++ b/core/client/src/in_mem.rs @@ -497,7 +497,8 @@ where let (root, transaction) = self.old_state.full_storage_root( top.into_iter().map(|(k, v)| (k, Some(v))), - child_delta + child_delta, + None, ); self.new_state = Some(InMemory::from(transaction)); diff --git a/core/client/src/light/backend.rs b/core/client/src/light/backend.rs index 300d140630d85..e3b3f09c79cc6 100644 --- a/core/client/src/light/backend.rs +++ b/core/client/src/light/backend.rs @@ -291,7 +291,11 @@ where } let storage_update: InMemoryState = storage.into(); - let (storage_root, _) = storage_update.full_storage_root(::std::iter::empty(), child_delta); + let (storage_root, _) = storage_update.full_storage_root( + ::std::iter::empty(), + child_delta, + ::std::iter::empty(), + ); self.storage_update = Some(storage_update); Ok(storage_root) @@ -340,6 +344,7 @@ impl StateBackend for GenesisOrUnavailableState type Error = ClientError; type Transaction = (); type TrieBackendStorage = MemoryDB; + type OffstateBackendStorage = state_machine::offstate_backend::TODO; fn storage(&self, key: &[u8]) -> ClientResult>> { match *self { @@ -416,6 +421,13 @@ impl StateBackend for GenesisOrUnavailableState } } + fn offstate_transaction(&self, _delta: I) -> Self::Transaction + where + I: IntoIterator, Option>)> + { + () + } + fn pairs(&self) -> Vec<(Vec, Vec)> { match *self { GenesisOrUnavailableState::Genesis(ref state) => state.pairs(), @@ -423,6 +435,27 @@ impl StateBackend for GenesisOrUnavailableState } } + fn children_storage_keys(&self) -> Vec> { + match *self { + GenesisOrUnavailableState::Genesis(ref state) => state.children_storage_keys(), + GenesisOrUnavailableState::Unavailable => Vec::new(), + } + } + + fn child_pairs(&self, child_storage_key: &[u8]) -> Vec<(Vec, Vec)> { + match *self { + GenesisOrUnavailableState::Genesis(ref state) => state.child_pairs(child_storage_key), + GenesisOrUnavailableState::Unavailable => Vec::new(), + } + } + + fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { + match *self { + GenesisOrUnavailableState::Genesis(ref state) => state.offstate_pairs(), + GenesisOrUnavailableState::Unavailable => Vec::new(), + } + } + fn keys(&self, prefix: &[u8]) -> Vec> { match *self { GenesisOrUnavailableState::Genesis(ref state) => state.keys(prefix), @@ -430,7 +463,9 @@ impl StateBackend for GenesisOrUnavailableState } } - fn as_trie_backend(&mut self) -> Option<&TrieBackend> { + fn as_trie_backend(&mut self) -> Option< + &TrieBackend + > { match self { GenesisOrUnavailableState::Genesis(ref mut state) => state.as_trie_backend(), GenesisOrUnavailableState::Unavailable => None, diff --git a/core/client/src/light/call_executor.rs b/core/client/src/light/call_executor.rs index 2c9c1f299579e..21efd1c860b78 100644 --- a/core/client/src/light/call_executor.rs +++ b/core/client/src/light/call_executor.rs @@ -177,9 +177,12 @@ impl CallExecutor for Err(ClientError::NotAvailableOnLightClient) } - fn prove_at_trie_state>( + fn prove_at_trie_state< + S: state_machine::TrieBackendStorage, + O: state_machine::OffstateBackendStorage, + >( &self, - _state: &state_machine::TrieBackend, + _state: &state_machine::TrieBackend, _changes: &mut OverlayedChanges, _method: &str, _call_data: &[u8] @@ -384,9 +387,12 @@ mod tests { unreachable!() } - fn prove_at_trie_state>( + fn prove_at_trie_state< + S: state_machine::TrieBackendStorage, + O: state_machine::OffstateBackendStorage, + >( &self, - _trie_state: &state_machine::TrieBackend, + _trie_state: &state_machine::TrieBackend, _overlay: &mut OverlayedChanges, _method: &str, _call_data: &[u8] diff --git a/core/client/src/light/fetcher.rs b/core/client/src/light/fetcher.rs index 3c4387209a490..49a47cadf1be6 100644 --- a/core/client/src/light/fetcher.rs +++ b/core/client/src/light/fetcher.rs @@ -349,8 +349,12 @@ impl> LightDataChecker { return Err(ClientError::InvalidCHTProof.into()); } + // using empty offstate as light do not use offstate information + // (things being fetch proved). + let offstate = state_machine::TODO; + // check proof for single changes trie root - let proving_backend = TrieBackend::new(storage, cht_root); + let proving_backend = TrieBackend::new(storage, cht_root, offstate); let remote_changes_trie_root = remote_roots[&block]; cht::check_proof_on_proving_backend::( local_cht_root, diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index 81772e554bc57..b7972d1b5e13a 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -200,7 +200,14 @@ impl StateDbSync { }) } - pub fn insert_block(&mut self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, mut changeset: ChangeSet) -> Result, Error> { + pub fn insert_block( + &mut self, + hash: &BlockHash, + number: u64, + parent_hash: &BlockHash, + mut changeset: ChangeSet, + mut offstate_changeset: ChangeSet, + ) -> Result, Error> { match self.mode { PruningMode::ArchiveAll => { changeset.deleted.clear(); @@ -370,8 +377,15 @@ impl StateDb { } /// Add a new non-canonical block. - pub fn insert_block(&self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, changeset: ChangeSet) -> Result, Error> { - self.db.write().insert_block(hash, number, parent_hash, changeset) + pub fn insert_block( + &self, + hash: &BlockHash, + number: u64, + parent_hash: &BlockHash, + changeset: ChangeSet, + offstate_changeset: ChangeSet, + ) -> Result, Error> { + self.db.write().insert_block(hash, number, parent_hash, changeset, offstate_changeset) } /// Finalize a previously inserted block. diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index e2f398ef7ccae..09b5eb3586674 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -21,6 +21,7 @@ use log::warn; use hash_db::Hasher; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::TrieBackendStorage; +use crate::offstate_backend::{OffstateBackendStorage, TODO}; use trie::{ TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::{TrieDBMut, Layout}, @@ -40,6 +41,10 @@ pub trait Backend: std::fmt::Debug { /// Type of trie backend storage. type TrieBackendStorage: TrieBackendStorage; + /// Type of trie backend storage. TODO EMCH move to OffstateBackend + /// after implementated. + type OffstateBackendStorage: OffstateBackendStorage; + /// Get keyed storage or None if there is nothing associated. fn storage(&self, key: &[u8]) -> Result>, Self::Error>; @@ -100,9 +105,23 @@ pub trait Backend: std::fmt::Debug { I: IntoIterator, Option>)>, H::Out: Ord; + /// Produce transaction for a given offstate information deltas. + fn offstate_transaction(&self, delta: I) -> Self::Transaction + where + I: IntoIterator, Option>)>; + /// Get all key/value pairs into a Vec. fn pairs(&self) -> Vec<(Vec, Vec)>; + /// Get all children storage keys + fn children_storage_keys(&self) -> Vec>; + + /// Get all key/value pairs into a Vec for a child storage. + fn child_pairs(&self, child_storage_key: &[u8]) -> Vec<(Vec, Vec)>; + + /// Get all key/value pairs of offstate storage. + fn offstate_pairs(&self) -> Vec<(Vec, Vec)>; + /// Get all keys with given prefix fn keys(&self, prefix: &[u8]) -> Vec> { let mut all = Vec::new(); @@ -118,25 +137,30 @@ pub trait Backend: std::fmt::Debug { } /// Try convert into trie backend. - fn as_trie_backend(&mut self) -> Option<&TrieBackend> { + fn as_trie_backend(&mut self) -> Option< + &TrieBackend + > { None } /// Calculate the storage root, with given delta over what is already stored /// in the backend, and produce a "transaction" that can be used to commit. /// Does include child storage updates. - fn full_storage_root( + fn full_storage_root( &self, delta: I1, - child_deltas: I2) - -> (H::Out, Self::Transaction) + child_deltas: I2, + offstate_deltas: I3, + ) -> (H::Out, Self::Transaction) where I1: IntoIterator, Option>)>, I2i: IntoIterator, Option>)>, I2: IntoIterator, I2i)>, + I3: IntoIterator, Option>)>, ::Out: Ord, { - let mut txs: Self::Transaction = Default::default(); + let mut txs: Self::Transaction = self.offstate_transaction(offstate_deltas); + let mut child_roots: Vec<_> = Default::default(); // child first for (storage_key, child_delta) in child_deltas { @@ -161,6 +185,7 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { type Error = T::Error; type Transaction = T::Transaction; type TrieBackendStorage = T::TrieBackendStorage; + type OffstateBackendStorage = T::OffstateBackendStorage; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { (*self).storage(key) @@ -198,10 +223,30 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { (*self).child_storage_root(storage_key, delta) } + fn offstate_transaction(&self, delta: I) -> Self::Transaction + where + I: IntoIterator, Option>)> + { + (*self).offstate_transaction(delta) + } + fn pairs(&self) -> Vec<(Vec, Vec)> { (*self).pairs() } + fn children_storage_keys(&self) -> Vec> { + (*self).children_storage_keys() + } + + fn child_pairs(&self, child_storage_key: &[u8]) -> Vec<(Vec, Vec)> { + (*self).child_pairs(child_storage_key) + } + + fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { + (*self).offstate_pairs() + } + + fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { (*self).for_key_values_with_prefix(prefix, f); } @@ -219,12 +264,27 @@ impl Consolidate for () { } } -impl Consolidate for Vec<(Option>, Vec, Option>)> { +impl Consolidate for Vec { fn consolidate(&mut self, mut other: Self) { self.append(&mut other); } } +impl Consolidate for (U, V) { + fn consolidate(&mut self, mut other: Self) { + self.0.consolidate(other.0); + self.1.consolidate(other.1); + } +} + + +impl Consolidate for InMemoryTransaction { + fn consolidate(&mut self, other: Self) { + self.storage.consolidate(other.storage); + self.offstate.consolidate(other.offstate); + } +} + impl> Consolidate for trie::GenericMemoryDB { fn consolidate(&mut self, other: Self) { trie::GenericMemoryDB::consolidate(self, other) @@ -250,7 +310,8 @@ impl error::Error for Void { /// tests. pub struct InMemory { inner: HashMap>, HashMap, Vec>>, - trie: Option, H>>, + trie: Option, H, TODO>>, + offstate: HashMap, Vec>, _hasher: PhantomData, } @@ -265,6 +326,7 @@ impl Default for InMemory { InMemory { inner: Default::default(), trie: None, + offstate: Default::default(), _hasher: PhantomData, } } @@ -275,6 +337,7 @@ impl Clone for InMemory { InMemory { inner: self.inner.clone(), trie: None, + offstate: self.offstate.clone(), _hasher: PhantomData, } } @@ -283,6 +346,7 @@ impl Clone for InMemory { impl PartialEq for InMemory { fn eq(&self, other: &Self) -> bool { self.inner.eq(&other.inner) + && self.offstate.eq(&other.offstate) } } @@ -290,14 +354,21 @@ impl InMemory { /// Copy the state, with applied updates pub fn update(&self, changes: >::Transaction) -> Self { let mut inner: HashMap<_, _> = self.inner.clone(); - for (storage_key, key, val) in changes { + let mut offstate: HashMap<_, _> = self.offstate.clone(); + for (storage_key, key, val) in changes.storage { match val { Some(v) => { inner.entry(storage_key).or_default().insert(key, v); }, None => { inner.entry(storage_key).or_default().remove(&key); }, } } + for (key, val) in changes.offstate { + match val { + Some(v) => { offstate.insert(key, v); }, + None => { offstate.remove(&key); }, + } + } - inner.into() + InMemory { inner, offstate, trie: None, _hasher: PhantomData } } } @@ -306,6 +377,7 @@ impl From>, HashMap, Vec>>> for In InMemory { inner: inner, trie: None, + offstate: Default::default(), _hasher: PhantomData, } } @@ -325,6 +397,7 @@ impl From<( InMemory { inner: inner, trie: None, + offstate: Default::default(), _hasher: PhantomData, } } @@ -337,6 +410,7 @@ impl From, Vec>> for InMemory { InMemory { inner: expanded, trie: None, + offstate: Default::default(), _hasher: PhantomData, } } @@ -354,6 +428,23 @@ impl From>, Vec, Option>)>> for InMem } } +impl From for InMemory { + fn from(inner: InMemoryTransaction) -> Self { + let mut expanded: HashMap>, HashMap, Vec>> = HashMap::new(); + for (child_key, key, value) in inner.storage { + if let Some(value) = value { + expanded.entry(child_key).or_default().insert(key, value); + } + } + let mut result: InMemory = expanded.into(); + result.offstate.extend(inner.offstate.into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v)))); + result + } +} + + + impl InMemory { /// child storage key iterator pub fn child_storage_keys(&self) -> impl Iterator { @@ -361,10 +452,21 @@ impl InMemory { } } +#[derive(Default)] +/// Transaction produced by the state machine execution for +/// in memory storage. +pub struct InMemoryTransaction { + /// State trie key values changes (both top and child trie). + pub storage: Vec<(Option>, Vec, Option>)>, + /// Changes to auxilliary data. + pub offstate: Vec<(Vec, Option>)>, +} + impl Backend for InMemory { type Error = Void; - type Transaction = Vec<(Option>, Vec, Option>)>; + type Transaction = InMemoryTransaction; type TrieBackendStorage = MemoryDB; + type OffstateBackendStorage = TODO; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone))) @@ -414,7 +516,7 @@ impl Backend for InMemory { let full_transaction = transaction.into_iter().map(|(k, v)| (None, k, v)).collect(); - (root, full_transaction) + (root, InMemoryTransaction { storage: full_transaction, offstate: Default::default() }) } fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) @@ -441,7 +543,15 @@ impl Backend for InMemory { let is_default = root == default_child_trie_root::>(&storage_key); - (root, is_default, full_transaction) + (root, is_default, InMemoryTransaction { storage: full_transaction, offstate: Default::default() }) + } + + fn offstate_transaction(&self, delta: I) -> Self::Transaction + where + I: IntoIterator, Option>)> + { + let offstate = delta.into_iter().collect(); + InMemoryTransaction { storage: Default::default(), offstate} } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -451,6 +561,21 @@ impl Backend for InMemory { .collect() } + fn children_storage_keys(&self) -> Vec> { + self.inner.iter().filter_map(|(child, _)| child.clone()).collect() + } + + fn child_pairs(&self, storage_key: &[u8]) -> Vec<(Vec, Vec)> { + self.inner.get(&Some(storage_key.to_vec())) + .into_iter() + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))) + .collect() + } + + fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { + self.offstate.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + } + fn keys(&self, prefix: &[u8]) -> Vec> { self.inner.get(&None) .into_iter() @@ -465,7 +590,9 @@ impl Backend for InMemory { .collect() } - fn as_trie_backend(&mut self)-> Option<&TrieBackend> { + fn as_trie_backend(&mut self)-> Option< + &TrieBackend + > { let mut mdb = MemoryDB::default(); let mut root = None; let mut new_child_roots = Vec::new(); @@ -489,7 +616,10 @@ impl Backend for InMemory { Some(root) => root, None => insert_into_memory_db::(&mut mdb, ::std::iter::empty())?, }; - self.trie = Some(TrieBackend::new(mdb, root)); + // Since we are running on a memorydb (not a prefixed memory db), content + // is not collision free, so an empty offtrie state can be use (no need + // for keyspace). + self.trie = Some(TrieBackend::new(mdb, root, TODO)); self.trie.as_ref() } } diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 10c38a41e2650..a90bd1506f6b8 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -427,7 +427,8 @@ mod test { extrinsics: Some(vec![0, 2].into_iter().collect()) }) ].into_iter().collect()), - ].into_iter().collect() + ].into_iter().collect(), + offstate: vec![].into_iter().collect(), }, committed: OverlayedChangeSet { top: vec![ (EXTRINSIC_INDEX.to_vec(), OverlayedValue { @@ -451,6 +452,7 @@ mod test { }) ].into_iter().collect()), ].into_iter().collect(), + offstate: vec![].into_iter().collect(), }, changes_trie_config: Some(config.clone()), }; diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index c4a2bd7f63b21..b80d1e2c19bf0 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -418,7 +418,9 @@ where H: Hasher, let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone())) .chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))); - let (root, transaction) = self.backend.full_storage_root(delta, child_delta_iter); + let offstate_delta = self.overlay.committed.offstate.clone().into_iter() + .chain(self.overlay.prospective.offstate.clone().into_iter()); + let (root, transaction) = self.backend.full_storage_root(delta, child_delta_iter, offstate_delta); self.storage_transaction = Some((transaction, root)); trace!(target: "state-trace", "{:04x}: Root {}", self.id, diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 92dd813b896b2..8133b086e97a9 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -30,6 +30,7 @@ use primitives::{ traits::{BareCryptoStorePtr, CodeExecutor}, hexdisplay::HexDisplay, }; +pub use crate::offstate_backend::{OffstateStorage, OffstateBackendStorage, TODO}; pub mod backend; mod changes_trie; @@ -41,6 +42,8 @@ mod overlayed_changes; mod proving_backend; mod trie_backend; mod trie_backend_essence; +// TODO EMCH make it private +pub mod offstate_backend; use overlayed_changes::OverlayedChangeSet; pub use trie::{TrieMut, DBValue, MemoryDB}; @@ -467,8 +470,8 @@ where /// /// Note: changes to code will be in place if this call is made again. For running partial /// blocks (e.g. a transaction at a time), ensure a different method is used. -pub fn prove_execution_on_trie_backend( - trie_backend: &TrieBackend, +pub fn prove_execution_on_trie_backend( + trie_backend: &TrieBackend, overlay: &mut OverlayedChanges, exec: &Exec, method: &str, @@ -477,6 +480,7 @@ pub fn prove_execution_on_trie_backend( ) -> Result<(Vec, Vec>), Box> where S: trie_backend_essence::TrieBackendStorage, + O: OffstateBackendStorage, H: Hasher, Exec: CodeExecutor, H::Out: Ord + 'static, @@ -516,7 +520,7 @@ where /// Check execution proof on proving backend, generated by `prove_execution` call. pub fn execution_proof_check_on_trie_backend( - trie_backend: &TrieBackend, H>, + trie_backend: &TrieBackend, H, TODO>, overlay: &mut OverlayedChanges, exec: &Exec, method: &str, @@ -577,18 +581,19 @@ where } /// Generate storage read proof on pre-created trie backend. -pub fn prove_read_on_trie_backend( - trie_backend: &TrieBackend, +pub fn prove_read_on_trie_backend( + trie_backend: &TrieBackend, keys: I, ) -> Result>, Box> where S: trie_backend_essence::TrieBackendStorage, H: Hasher, H::Out: Ord, + O: OffstateBackendStorage, I: IntoIterator, I::Item: AsRef<[u8]>, { - let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend); + let proving_backend = proving_backend::ProvingBackend::<_, H, _>::new(trie_backend); for key in keys.into_iter() { proving_backend .storage(key.as_ref()) @@ -598,8 +603,8 @@ where } /// Generate storage read proof on pre-created trie backend. -pub fn prove_child_read_on_trie_backend( - trie_backend: &TrieBackend, +pub fn prove_child_read_on_trie_backend( + trie_backend: &TrieBackend, storage_key: &[u8], keys: I, ) -> Result>, Box> @@ -607,10 +612,11 @@ where S: trie_backend_essence::TrieBackendStorage, H: Hasher, H::Out: Ord, + O: OffstateBackendStorage, I: IntoIterator, I::Item: AsRef<[u8]>, { - let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend); + let proving_backend = proving_backend::ProvingBackend::<_, H, _>::new(trie_backend); for key in keys.into_iter() { proving_backend .child_storage(storage_key, key.as_ref()) @@ -668,7 +674,7 @@ where /// Check storage read proof on pre-created proving backend. pub fn read_proof_check_on_proving_backend( - proving_backend: &TrieBackend, H>, + proving_backend: &TrieBackend, H, TODO>, key: &[u8], ) -> Result>, Box> where @@ -680,7 +686,7 @@ where /// Check child storage read proof on pre-created proving backend. pub fn read_child_proof_check_on_proving_backend( - proving_backend: &TrieBackend, H>, + proving_backend: &TrieBackend, H, TODO>, storage_key: &[u8], key: &[u8], ) -> Result>, Box> diff --git a/core/state-machine/src/offstate_backend.rs b/core/state-machine/src/offstate_backend.rs new file mode 100644 index 0000000000000..087613ad435be --- /dev/null +++ b/core/state-machine/src/offstate_backend.rs @@ -0,0 +1,93 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Backend for storing data without a state. + +use std::sync::Arc; +use std::ops::Deref; + +pub trait OffstateBackendStorage { +/* /// state type for querying data + /// (similar to hash for a trie_backend). + trait ChanState;*/ + + /// Retrieve a value from storage under given key and prefix. + fn get(&self, prefix: &[u8], key: &[u8]) -> Option>; + + /// Return in memory all values for this backend, mainly for + /// tests. + fn pairs(&self) -> Vec<(Vec, Vec)>; +} + +pub trait OffstateStorage { + + /// Persist a value in storage under given key and prefix. + fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]); + + /// Retrieve a value from storage under given key and prefix. + fn get(&self, prefix: &[u8], key: &[u8]) -> Option>; + + /// Return in memory all values for this backend, mainly for + /// tests. + fn pairs(&self) -> Vec<(Vec, Vec)>; + +} + +impl OffstateBackendStorage for TODO { + + fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { + unimplemented!() + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + unimplemented!() + } + +} + +// This implementation is used by normal storage trie clients. +impl OffstateBackendStorage for Arc { + + fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { + OffstateStorage::get(self.deref(), prefix, key) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + OffstateStorage::pairs(self.deref()) + } + +} + + +impl OffstateStorage for TODO { + + fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { + unimplemented!() + } + + fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { + ::get(&self, prefix, key) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + ::pairs(&self) + } + +} + +// TODO EMCH implement, no branch ranges. +pub struct TODO; + diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index a4952ddf73790..ae87c3678c7f1 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -57,6 +57,8 @@ pub struct OverlayedChangeSet { pub top: HashMap, OverlayedValue>, /// Child storage changes. pub children: HashMap, HashMap, OverlayedValue>>, + /// Offstate storage changes. + pub offstate: HashMap, Option>>, } #[cfg(test)] @@ -65,6 +67,7 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { Self { top: iter.into_iter().collect(), children: Default::default(), + offstate: Default::default(), } } } @@ -72,13 +75,14 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { impl OverlayedChangeSet { /// Whether the change set is empty. pub fn is_empty(&self) -> bool { - self.top.is_empty() && self.children.is_empty() + self.top.is_empty() && self.children.is_empty() && self.offstate.is_empty() } /// Clear the change set. pub fn clear(&mut self) { self.top.clear(); self.children.clear(); + self.offstate.clear(); } } @@ -132,6 +136,15 @@ impl OverlayedChanges { None } + /// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered + /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose + /// value has been set. + pub fn offstate_storage(&self, key: &[u8]) -> Option> { + self.prospective.offstate.get(key) + .or_else(|| self.committed.offstate.get(key)) + .map(|x| x.as_ref().map(AsRef::as_ref)) + } + /// Inserts the given key-value pair into the prospective change set. /// /// `None` can be used to delete a value specified by the given key. @@ -161,6 +174,13 @@ impl OverlayedChanges { } } + /// Inserts the given key-value pair as an auxilliary data. + /// + /// `None` can be used to delete a value specified by the given key. + pub(crate) fn set_offstate_storage(&mut self, key: Vec, val: Option>) { + self.prospective.offstate.insert(key, val); + } + /// Clear child storage of given storage key. /// /// NOTE that this doesn't take place immediately but written into the prospective @@ -285,6 +305,9 @@ impl OverlayedChanges { .extend(prospective_extrinsics); } } + for (key, val) in self.prospective.offstate.drain() { + self.committed.offstate.insert(key, val); + } for (storage_key, mut map) in self.prospective.children.drain() { let map_dest = self.committed.children.entry(storage_key).or_default(); for (key, val) in map.drain() { @@ -307,11 +330,13 @@ impl OverlayedChanges { pub fn into_committed(self) -> ( impl Iterator, Option>)>, impl Iterator, impl Iterator, Option>)>)>, + impl Iterator, Option>)>, ){ assert!(self.prospective.is_empty()); (self.committed.top.into_iter().map(|(k, v)| (k, v.value)), self.committed.children.into_iter() - .map(|(sk, v)| (sk, v.into_iter().map(|(k, v)| (k, v.value))))) + .map(|(sk, v)| (sk, v.into_iter().map(|(k, v)| (k, v.value)))), + self.committed.offstate.into_iter()) } /// Inserts storage entry responsible for current extrinsic index. @@ -370,27 +395,39 @@ mod tests { let mut overlayed = OverlayedChanges::default(); let key = vec![42, 69, 169, 142]; + let offstate_key = vec![43, 69, 169, 142]; assert!(overlayed.storage(&key).is_none()); overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); + overlayed.set_offstate_storage(offstate_key.clone(), Some(vec![1, 2, 2])); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + assert_eq!(overlayed.offstate_storage(&offstate_key).unwrap(), Some(&[1, 2, 2][..])); overlayed.commit_prospective(); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + assert_eq!(overlayed.offstate_storage(&offstate_key).unwrap(), Some(&[1, 2, 2][..])); overlayed.set_storage(key.clone(), Some(vec![])); + overlayed.set_offstate_storage(offstate_key.clone(), Some(vec![2])); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[][..])); + assert_eq!(overlayed.offstate_storage(&offstate_key).unwrap(), Some(&[2][..])); overlayed.set_storage(key.clone(), None); + overlayed.set_offstate_storage(offstate_key.clone(), None); assert!(overlayed.storage(&key).unwrap().is_none()); + assert!(overlayed.offstate_storage(&offstate_key).unwrap().is_none()); + overlayed.discard_prospective(); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + assert_eq!(overlayed.offstate_storage(&offstate_key).unwrap(), Some(&[1, 2, 2][..])); overlayed.set_storage(key.clone(), None); + overlayed.set_offstate_storage(offstate_key.clone(), None); overlayed.commit_prospective(); assert!(overlayed.storage(&key).unwrap().is_none()); + assert!(overlayed.offstate_storage(&offstate_key).unwrap().is_none()); } #[test] diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index 3908f62eaae0a..f8bb5f659e043 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -28,6 +28,7 @@ pub use trie::trie_types::{Layout, TrieError}; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend}; +use crate::offstate_backend::{OffstateBackendStorage, TODO}; /// Patricia trie-based backend essence which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. @@ -103,14 +104,22 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> /// Patricia trie-based backend which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. -pub struct ProvingBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { - backend: &'a TrieBackend, +pub struct ProvingBackend<'a, + S: 'a + TrieBackendStorage, + H: 'a + Hasher, + O: 'a + OffstateBackendStorage, +> { + backend: &'a TrieBackend, proof_recorder: Rc>>, } -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> { +impl<'a, + S: 'a + TrieBackendStorage, + H: 'a + Hasher, + O: 'a + OffstateBackendStorage, +> ProvingBackend<'a, S, H, O> { /// Create new proving backend. - pub fn new(backend: &'a TrieBackend) -> Self { + pub fn new(backend: &'a TrieBackend) -> Self { ProvingBackend { backend, proof_recorder: Rc::new(RefCell::new(Recorder::new())), @@ -119,7 +128,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> /// Create new proving backend with the given recorder. pub fn new_with_recorder( - backend: &'a TrieBackend, + backend: &'a TrieBackend, proof_recorder: Rc>>, ) -> Self { ProvingBackend { @@ -139,21 +148,27 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> } } -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> std::fmt::Debug for ProvingBackend<'a, S, H> { +impl<'a, + S: 'a + TrieBackendStorage, + H: 'a + Hasher, + O: 'a + OffstateBackendStorage, +> std::fmt::Debug for ProvingBackend<'a, S, H, O> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "ProvingBackend") } } -impl<'a, S, H> Backend for ProvingBackend<'a, S, H> +impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> where S: 'a + TrieBackendStorage, H: 'a + Hasher, H::Out: Ord, + O: 'a + OffstateBackendStorage, { type Error = String; - type Transaction = S::Overlay; + type Transaction = (S::Overlay, Vec<(Vec, Option>)>); type TrieBackendStorage = PrefixedMemoryDB; + type OffstateBackendStorage = O; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { ProvingBackendEssence { @@ -191,6 +206,18 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> self.backend.pairs() } + fn children_storage_keys(&self) -> Vec> { + self.backend.children_storage_keys() + } + + fn child_pairs(&self, storage_key: &[u8]) -> Vec<(Vec, Vec)> { + self.backend.child_pairs(storage_key) + } + + fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { + self.backend.offstate_pairs() + } + fn keys(&self, prefix: &[u8]) -> Vec> { self.backend.keys(prefix) } @@ -212,20 +239,28 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> { self.backend.child_storage_root(storage_key, delta) } + + fn offstate_transaction(&self, delta: I) -> Self::Transaction + where + I: IntoIterator, Option>)> + { + self.backend.offstate_transaction(delta) + } + } /// Create proof check backend. pub fn create_proof_check_backend( root: H::Out, proof: Vec> -) -> Result, H>, Box> +) -> Result, H, TODO>, Box> where H: Hasher, { let db = create_proof_check_backend_storage(proof); if db.contains(&root, EMPTY_PREFIX) { - Ok(TrieBackend::new(db, root)) + Ok(TrieBackend::new(db, root, TODO)) } else { Err(Box::new(ExecutionError::InvalidProof)) } @@ -247,14 +282,18 @@ where #[cfg(test)] mod tests { - use crate::backend::{InMemory}; + use crate::backend::{InMemory, InMemoryTransaction}; use crate::trie_backend::tests::test_trie; use super::*; use primitives::{Blake2Hasher, child_storage_key::ChildStorageKey}; + // TODO this need an actual in momery with possibly content + // as the test uses a prefixed memory db. + type OffstateBackend = TODO; + fn test_proving<'a>( - trie_backend: &'a TrieBackend,Blake2Hasher>, - ) -> ProvingBackend<'a, PrefixedMemoryDB, Blake2Hasher> { + trie_backend: &'a TrieBackend, Blake2Hasher, OffstateBackend>, + ) -> ProvingBackend<'a, PrefixedMemoryDB, Blake2Hasher, OffstateBackend> { ProvingBackend::new(trie_backend) } @@ -288,14 +327,19 @@ mod tests { let (trie_root, mut trie_mdb) = trie_backend.storage_root(::std::iter::empty()); let (proving_root, mut proving_mdb) = proving_backend.storage_root(::std::iter::empty()); assert_eq!(trie_root, proving_root); - assert_eq!(trie_mdb.drain(), proving_mdb.drain()); + assert_eq!(trie_mdb.0.drain(), proving_mdb.0.drain()); + assert_eq!(trie_mdb.1, proving_mdb.1); } #[test] fn proof_recorded_and_checked() { let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))).collect::>(); let in_memory = InMemory::::default(); - let mut in_memory = in_memory.update(contents); + let mut in_memory = in_memory.update(InMemoryTransaction { + storage: contents, + offstate: Default::default(), + }); + let in_memory_root = in_memory.storage_root(::std::iter::empty()).0; (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); @@ -324,10 +368,14 @@ mod tests { .chain((10..15).map(|i| (Some(own2.clone()), vec![i], Some(vec![i])))) .collect::>(); let in_memory = InMemory::::default(); - let mut in_memory = in_memory.update(contents); - let in_memory_root = in_memory.full_storage_root::<_, Vec<_>, _>( + let mut in_memory = in_memory.update(InMemoryTransaction { + storage: contents, + offstate: Default::default(), + }); + let in_memory_root = in_memory.full_storage_root::<_, Vec<_>, _, _>( + ::std::iter::empty(), + in_memory.child_storage_keys().map(|k|(k.to_vec(), Vec::new())), ::std::iter::empty(), - in_memory.child_storage_keys().map(|k|(k.to_vec(), Vec::new())) ).0; (0..64).for_each(|i| assert_eq!( in_memory.storage(&[i]).unwrap().unwrap(), diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index 160f7d2a47ccb..33a497e5c9280 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -19,7 +19,7 @@ use std::collections::{HashMap}; use hash_db::Hasher; use crate::{ - backend::{InMemory, Backend}, OverlayedChanges, + backend::{InMemory, InMemoryTransaction, Backend}, OverlayedChanges, changes_trie::{ build_changes_trie, InMemoryStorage as ChangesTrieInMemoryStorage, BlockNumber as ChangesTrieBlockNumber, @@ -82,7 +82,18 @@ impl TestExternalities { /// Insert key/value into backend pub fn insert(&mut self, k: Vec, v: Vec) { - self.backend = self.backend.update(vec![(None, k, Some(v))]); + self.backend = self.backend.update(InMemoryTransaction { + storage: vec![(None, k, Some(v))], + offstate: Default::default(), + }); + } + + /// Insert key/value into ofstate information backend + pub fn insert_offstate(&mut self, k: Vec, v: Vec) { + self.backend = self.backend.update(InMemoryTransaction { + storage: Default::default(), + offstate: vec![(k, Some(v))], + }); } /// Set offchain externaltiies. @@ -114,7 +125,13 @@ impl TestExternalities { .collect::>() }); - self.backend.update(top.chain(children).collect()) + let offstate = self.overlay.committed.offstate.clone().into_iter() + .chain(self.overlay.prospective.offstate.clone().into_iter()); + + self.backend.update(InMemoryTransaction { + storage: top.chain(children).collect(), + offstate: offstate.collect(), + }) } } @@ -245,7 +262,9 @@ impl Externalities for TestExternalities where // compute and memoize let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone())) .chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))); - self.backend.full_storage_root(delta, child_delta_iter).0 + + // transaction not used afterward, so not using offstate. + self.backend.full_storage_root(delta, child_delta_iter, None).0 } diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index ce5773c0b7956..0d1c6dcb5ebbb 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -22,17 +22,22 @@ use trie::{Trie, delta_trie_root, default_child_trie_root, child_delta_trie_root use trie::trie_types::{TrieDB, TrieError, Layout}; use crate::trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral}; use crate::Backend; +use crate::offstate_backend::OffstateBackendStorage; +use primitives::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. -pub struct TrieBackend, H: Hasher> { +/// TODO EMCH with offstaet in backend this should be renamed eg StateBackend. +pub struct TrieBackend, H: Hasher, O: OffstateBackendStorage> { essence: TrieBackendEssence, + offstate_storage: O, } -impl, H: Hasher> TrieBackend { +impl, O: OffstateBackendStorage, H: Hasher> TrieBackend { /// Create new trie-based backend. - pub fn new(storage: S, root: H::Out) -> Self { + pub fn new(storage: S, root: H::Out, offstate_storage: O) -> Self { TrieBackend { essence: TrieBackendEssence::new(storage, root), + offstate_storage, } } @@ -55,26 +60,48 @@ impl, H: Hasher> TrieBackend { pub fn into_storage(self) -> S { self.essence.into_storage() } + + // TODO EMCH PROTO: remove before pr. + pub fn child_keyspace(&self, key: &[u8]) -> Option> { + const PREFIX_KEYSPACE: &'static[u8] = b"offstate_keyspace"; + self.offstate_storage.get(PREFIX_KEYSPACE, key) + } + } -impl, H: Hasher> std::fmt::Debug for TrieBackend { +impl< + S: TrieBackendStorage, + H: Hasher, + O: OffstateBackendStorage, +> std::fmt::Debug for TrieBackend { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "TrieBackend") } } -impl, H: Hasher> Backend for TrieBackend where +impl< + S: TrieBackendStorage, + H: Hasher, + O: OffstateBackendStorage, +> Backend for TrieBackend where H::Out: Ord, { type Error = String; - type Transaction = S::Overlay; + type Transaction = (S::Overlay, Vec<(Vec, Option>)>); type TrieBackendStorage = S; + // TODO EMCH this does not make sens : split as a OffstateBackend from trait. + type OffstateBackendStorage = O; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { self.essence.storage(key) } fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { + + // TODO EMCH PROTO: remove before pr. TODO test it when implemented!! +// let keyspace = self.child_keyspace(storage_key); + // Then change essence functions to use keyspace as input. + self.essence.child_storage(storage_key, key) } @@ -118,6 +145,47 @@ impl, H: Hasher> Backend for TrieBackend where } } + fn children_storage_keys(&self) -> Vec> { + let mut result = Vec::new(); + self.for_keys_with_prefix(CHILD_STORAGE_KEY_PREFIX, |k| result.push(k.to_vec())); + result + } + + fn child_pairs(&self, storage_key: &[u8]) -> Vec<(Vec, Vec)> { + + let root_slice = self.essence.storage(storage_key) + .unwrap_or(None) + .unwrap_or(default_child_trie_root::>(storage_key)); + let mut root = H::Out::default(); + root.as_mut().copy_from_slice(&root_slice[..]); + + let mut read_overlay = S::Overlay::default(); + let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay); + + let collect_all = || -> Result<_, Box>> { + let trie = TrieDB::::new(&eph, &root)?; + let mut v = Vec::new(); + for x in trie.iter()? { + let (key, value) = x?; + v.push((key.to_vec(), value.to_vec())); + } + + Ok(v) + }; + + match collect_all() { + Ok(v) => v, + Err(e) => { + debug!(target: "trie", "Error extracting child trie values: {}", e); + Vec::new() + } + } + } + + fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { + self.offstate_storage.pairs() + } + fn keys(&self, prefix: &[u8]) -> Vec> { let mut read_overlay = S::Overlay::default(); let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay); @@ -138,7 +206,7 @@ impl, H: Hasher> Backend for TrieBackend where collect_all().map_err(|e| debug!(target: "trie", "Error extracting trie keys: {}", e)).unwrap_or_default() } - fn storage_root(&self, delta: I) -> (H::Out, S::Overlay) + fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) where I: IntoIterator, Option>)> { let mut write_overlay = S::Overlay::default(); @@ -156,7 +224,7 @@ impl, H: Hasher> Backend for TrieBackend where } } - (root, write_overlay) + (root, (write_overlay, Default::default())) } fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) @@ -194,10 +262,19 @@ impl, H: Hasher> Backend for TrieBackend where let is_default = root == default_root; - (root, is_default, write_overlay) + (root, is_default, (write_overlay, Default::default())) } - fn as_trie_backend(&mut self) -> Option<&TrieBackend> { + fn offstate_transaction(&self, delta: I) -> Self::Transaction + where + I: IntoIterator, Option>)> + { + (Default::default(), delta.into_iter().collect()) + } + + fn as_trie_backend(&mut self) -> Option< + &TrieBackend + > { Some(self) } } @@ -210,7 +287,11 @@ pub mod tests { use trie::{TrieMut, PrefixedMemoryDB, trie_types::TrieDBMut}; use super::*; - fn test_db() -> (PrefixedMemoryDB, H256) { + // TODO this need an actual in momery with possibly content + // as the test uses a prefixed memory db. + type OffstateBackend = crate::offstate_backend::TODO; + + fn test_db() -> (PrefixedMemoryDB, H256, OffstateBackend) { let mut root = H256::default(); let mut mdb = PrefixedMemoryDB::::default(); { @@ -232,12 +313,17 @@ pub mod tests { trie.insert(&[i], &[i]).unwrap(); } } - (mdb, root) + // empty history. + let offstate = crate::offstate_backend::TODO; + // TODO EMCH add a block in offstate or use an actual implementation of + // offstate that do not use history (a test implementation most likely) + // TODO EMCH feed offstate with keyspace for roots. + (mdb, root, offstate) } - pub(crate) fn test_trie() -> TrieBackend, Blake2Hasher> { - let (mdb, root) = test_db(); - TrieBackend::new(mdb, root) + pub(crate) fn test_trie() -> TrieBackend, Blake2Hasher, OffstateBackend> { + let (mdb, root, offstate) = test_db(); + TrieBackend::new(mdb, root, offstate) } #[test] @@ -257,9 +343,10 @@ pub mod tests { #[test] fn pairs_are_empty_on_empty_storage() { - assert!(TrieBackend::, Blake2Hasher>::new( + assert!(TrieBackend::, Blake2Hasher, OffstateBackend>::new( PrefixedMemoryDB::default(), Default::default(), + crate::offstate_backend::TODO, ).pairs().is_empty()); } @@ -270,13 +357,15 @@ pub mod tests { #[test] fn storage_root_transaction_is_empty() { - assert!(test_trie().storage_root(::std::iter::empty()).1.drain().is_empty()); + let mut tx = test_trie().storage_root(::std::iter::empty()).1; + assert!(tx.0.drain().is_empty()); + assert!(tx.1.is_empty()); } #[test] fn storage_root_transaction_is_non_empty() { let (new_root, mut tx) = test_trie().storage_root(vec![(b"new-key".to_vec(), Some(b"new-value".to_vec()))]); - assert!(!tx.drain().is_empty()); + assert!(!tx.0.drain().is_empty()); assert!(new_root != test_trie().storage_root(::std::iter::empty()).0); } From 4db0a7f85b56621fa3cd037880378f6e41333fe7 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 24 Sep 2019 19:34:31 +0200 Subject: [PATCH 02/68] Dumb state_db implementation, mostly boiler plate, but cannot work as is. Need to use historied values, maybe remove pinned, and put some context as parameter for get_offstate --- core/client/db/src/lib.rs | 8 +- core/state-db/src/lib.rs | 55 +++- core/state-db/src/noncanonical.rs | 427 +++++++++++++++++++++++------- core/state-db/src/pruning.rs | 106 +++++++- core/state-db/src/test.rs | 43 ++- 5 files changed, 525 insertions(+), 114 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index d5167012f1068..1202dd571c330 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -1770,7 +1770,7 @@ mod tests { op.reset_storage(storage.iter().cloned().collect(), Default::default()).unwrap(); - key = op.db_updates.insert(EMPTY_PREFIX, b"hello"); + key = op.db_updates.0.insert(EMPTY_PREFIX, b"hello"); op.set_block_data( header, Some(vec![]), @@ -1806,8 +1806,8 @@ mod tests { ).0.into(); let hash = header.hash(); - op.db_updates.insert(EMPTY_PREFIX, b"hello"); - op.db_updates.remove(&key, EMPTY_PREFIX); + op.db_updates.0.insert(EMPTY_PREFIX, b"hello"); + op.db_updates.0.remove(&key, EMPTY_PREFIX); op.set_block_data( header, Some(vec![]), @@ -1843,7 +1843,7 @@ mod tests { ).0.into(); let hash = header.hash(); - op.db_updates.remove(&key, EMPTY_PREFIX); + op.db_updates.0.remove(&key, EMPTY_PREFIX); op.set_block_data( header, Some(vec![]), diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index b7972d1b5e13a..e94d09ce627aa 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -44,6 +44,9 @@ use log::trace; /// Database value type. pub type DBValue = Vec; +/// Offstate storage key definition. +pub type OffstateKey = Vec; + /// Basic set of requirements for the Block hash and node key types. pub trait Hash: Send + Sync + Sized + Eq + PartialEq + Clone + Default + fmt::Debug + Codec + std::hash::Hash + 'static {} impl Hash for T {} @@ -65,6 +68,14 @@ pub trait NodeDb { fn get(&self, key: &Self::Key) -> Result, Self::Error>; } +/// Backend database trait. Read-only. +pub trait OffstateDb { + type Error: fmt::Debug; + + /// Get state trie node. + fn get_offstate(&self, key: &[u8]) -> Result, Self::Error>; +} + /// Error type. pub enum Error { /// Database backend error. @@ -120,6 +131,8 @@ pub struct CommitSet { pub data: ChangeSet, /// Metadata changes. pub meta: ChangeSet>, + /// Offstate data changes. + pub offstate: ChangeSet, } /// Pruning constraints. If none are specified pruning is @@ -206,19 +219,21 @@ impl StateDbSync { number: u64, parent_hash: &BlockHash, mut changeset: ChangeSet, - mut offstate_changeset: ChangeSet, + mut offstate_changeset: ChangeSet, ) -> Result, Error> { match self.mode { PruningMode::ArchiveAll => { changeset.deleted.clear(); + offstate_changeset.deleted.clear(); // write changes immediately Ok(CommitSet { data: changeset, meta: Default::default(), + offstate: offstate_changeset, }) }, PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => { - self.non_canonical.insert(hash, number, parent_hash, changeset) + self.non_canonical.insert(hash, number, parent_hash, changeset, offstate_changeset) } } } @@ -340,6 +355,17 @@ impl StateDbSync { db.get(key.as_ref()).map_err(|e| Error::Db(e)) } + pub fn get_offstate( + &self, + key: &[u8], + db: &D, + ) -> Result, Error> { + if let Some(value) = self.non_canonical.get_offstate(key) { + return Ok(Some(value)); + } + db.get_offstate(key).map_err(|e| Error::Db(e)) + } + pub fn apply_pending(&mut self) { self.non_canonical.apply_pending(); if let Some(pruning) = &mut self.pruning { @@ -383,7 +409,7 @@ impl StateDb { number: u64, parent_hash: &BlockHash, changeset: ChangeSet, - offstate_changeset: ChangeSet, + offstate_changeset: ChangeSet, ) -> Result, Error> { self.db.write().insert_block(hash, number, parent_hash, changeset, offstate_changeset) } @@ -410,6 +436,16 @@ impl StateDb { self.db.read().get(key, db) } + /// Get a value from non-canonical/pruning overlay or the backing DB. + pub fn get_offstate( + &self, + key: &[u8], + db: &D, + ) -> Result, Error> { + self.db.read().get_offstate(key, db) + } + + /// Revert all non-canonical blocks with the best block number. /// Returns a database commit or `None` if not possible. /// For archive an empty commit set is returned. @@ -443,10 +479,11 @@ mod tests { use std::io; use primitives::H256; use crate::{StateDb, PruningMode, Constraints}; - use crate::test::{make_db, make_changeset, TestDb}; + use crate::test::{make_db, make_changeset, make_offstate_changeset, TestDb}; fn make_test_db(settings: PruningMode) -> (TestDb, StateDb) { let mut db = make_db(&[91, 921, 922, 93, 94]); + db.initialize_offstate(&[81, 821, 822, 83, 84]); let state_db = StateDb::new(settings, &db).unwrap(); db.commit( @@ -456,6 +493,7 @@ mod tests { 1, &H256::from_low_u64_be(0), make_changeset(&[1], &[91]), + make_offstate_changeset(&[1], &[81]), ) .unwrap(), ); @@ -466,6 +504,7 @@ mod tests { 2, &H256::from_low_u64_be(1), make_changeset(&[21], &[921, 1]), + make_offstate_changeset(&[21], &[821, 1]), ) .unwrap(), ); @@ -476,6 +515,7 @@ mod tests { 2, &H256::from_low_u64_be(1), make_changeset(&[22], &[922]), + make_offstate_changeset(&[22], &[822]), ) .unwrap(), ); @@ -486,6 +526,7 @@ mod tests { 3, &H256::from_low_u64_be(21), make_changeset(&[3], &[93]), + make_offstate_changeset(&[3], &[83]), ) .unwrap(), ); @@ -499,6 +540,7 @@ mod tests { 4, &H256::from_low_u64_be(3), make_changeset(&[4], &[94]), + make_offstate_changeset(&[4], &[84]), ) .unwrap(), ); @@ -515,6 +557,7 @@ mod tests { fn full_archive_keeps_everything() { let (db, sdb) = make_test_db(PruningMode::ArchiveAll); assert!(db.data_eq(&make_db(&[1, 21, 22, 3, 4, 91, 921, 922, 93, 94]))); + assert!(db.offstate_eq(&[1, 21, 22, 3, 4, 81, 821, 822, 83, 84])); assert!(!sdb.is_pruned(&H256::from_low_u64_be(0), 0)); } @@ -522,6 +565,7 @@ mod tests { fn canonical_archive_keeps_canonical() { let (db, _) = make_test_db(PruningMode::ArchiveCanonical); assert!(db.data_eq(&make_db(&[1, 21, 3, 91, 921, 922, 93, 94]))); + assert!(db.offstate_eq(&[1, 21, 3, 81, 821, 822, 83, 84])); } #[test] @@ -531,6 +575,7 @@ mod tests { max_mem: None, })); assert!(db.data_eq(&make_db(&[21, 3, 922, 94]))); + assert!(db.offstate_eq(&[21, 3, 822, 84])); } #[test] @@ -544,6 +589,7 @@ mod tests { assert!(sdb.is_pruned(&H256::from_low_u64_be(21), 2)); assert!(sdb.is_pruned(&H256::from_low_u64_be(22), 2)); assert!(db.data_eq(&make_db(&[21, 3, 922, 93, 94]))); + assert!(db.offstate_eq(&[21, 3, 822, 83, 84])); } #[test] @@ -557,5 +603,6 @@ mod tests { assert!(!sdb.is_pruned(&H256::from_low_u64_be(21), 2)); assert!(sdb.is_pruned(&H256::from_low_u64_be(22), 2)); assert!(db.data_eq(&make_db(&[1, 21, 3, 921, 922, 93, 94]))); + assert!(db.offstate_eq(&[1, 21, 3, 821, 822, 83, 84])); } } diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 58715715ccdd2..59983e74c9037 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -22,11 +22,12 @@ use std::fmt; use std::collections::{HashMap, VecDeque, hash_map::Entry}; -use super::{Error, DBValue, ChangeSet, CommitSet, MetaDb, Hash, to_meta_key}; +use super::{Error, DBValue, ChangeSet, CommitSet, MetaDb, Hash, to_meta_key, OffstateKey}; use codec::{Encode, Decode}; use log::trace; const NON_CANONICAL_JOURNAL: &[u8] = b"noncanonical_journal"; +const NON_CANONICAL_OFFSTATE_JOURNAL: &[u8] = b"offstate_noncanonical_journal"; const LAST_CANONICAL: &[u8] = b"last_canonical"; /// See module documentation. @@ -37,7 +38,10 @@ pub struct NonCanonicalOverlay { pending_canonicalizations: Vec, pending_insertions: Vec, values: HashMap, //ref counted - pinned: HashMap>, //would be deleted but kept around because block is pinned + offstate_values: HashMap, //ref counted + //would be deleted but kept around because block is pinned + // TODO EMCH sense if pinning offstate? + pinned: HashMap, HashMap)>, } #[derive(Encode, Decode)] @@ -48,16 +52,29 @@ struct JournalRecord { deleted: Vec, } +#[derive(Encode, Decode)] +struct OffstateJournalRecord { + inserted: Vec<(OffstateKey, DBValue)>, + deleted: Vec, +} + fn to_journal_key(block: u64, index: u64) -> Vec { to_meta_key(NON_CANONICAL_JOURNAL, &(block, index)) } +fn to_offstate_journal_key(block: u64, index: u64) -> Vec { + to_meta_key(NON_CANONICAL_OFFSTATE_JOURNAL, &(block, index)) +} + #[cfg_attr(test, derive(PartialEq, Debug))] struct BlockOverlay { hash: BlockHash, journal_key: Vec, + offstate_journal_key: Vec, inserted: Vec, deleted: Vec, + offstate_inserted: Vec, + offstate_deleted: Vec, } fn insert_values(values: &mut HashMap, inserted: Vec<(Key, DBValue)>) { @@ -95,9 +112,10 @@ fn discard_values( fn discard_descendants( levels: &mut VecDeque>>, mut values: &mut HashMap, + mut offstate_values: &mut HashMap, index: usize, parents: &mut HashMap, - pinned: &mut HashMap>, + pinned: &mut HashMap, HashMap)>, hash: &BlockHash, ) { let mut discarded = Vec::new(); @@ -107,7 +125,13 @@ fn discard_descendants( if parent == *hash { parents.remove(&overlay.hash); discarded.push(overlay.hash); - discard_values(&mut values, overlay.inserted, pinned.get_mut(hash)); + let mut pinned = pinned.get_mut(hash); + discard_values(&mut values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); + discard_values( + &mut offstate_values, + overlay.offstate_inserted, + pinned.as_mut().map(|p| &mut p.1), + ); None } else { Some(overlay) @@ -115,7 +139,7 @@ fn discard_descendants( }).collect(); } for hash in discarded { - discard_descendants(levels, values, index + 1, parents, pinned, &hash); + discard_descendants(levels, values, offstate_values, index + 1, parents, pinned, &hash); } } @@ -131,6 +155,7 @@ impl NonCanonicalOverlay { let mut levels = VecDeque::new(); let mut parents = HashMap::new(); let mut values = HashMap::new(); + let mut offstate_values = HashMap::new(); if let Some((ref hash, mut block)) = last_canonicalized { // read the journal trace!(target: "state-db", "Reading uncanonicalized journal. Last canonicalized #{} ({:?})", block, hash); @@ -141,18 +166,42 @@ impl NonCanonicalOverlay { let mut level = Vec::new(); loop { let journal_key = to_journal_key(block, index); + let offstate_journal_key = to_offstate_journal_key(block, index); match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? { Some(record) => { let record: JournalRecord = Decode::decode(&mut record.as_slice())?; + let (offstate_record_inserted, offstate_record_deleted) = if let Some(record) = db + .get_meta(&offstate_journal_key).map_err(|e| Error::Db(e))? { + let record = OffstateJournalRecord::decode(&mut record.as_slice())?; + (Some(record.inserted), Some(record.deleted)) + } else { (None, None) }; let inserted = record.inserted.iter().map(|(k, _)| k.clone()).collect(); + let offstate_inserted = offstate_record_inserted.as_ref() + .map(|inserted| inserted.iter().map(|(k, _)| k.clone()).collect()) + .unwrap_or(Vec::new()); let overlay = BlockOverlay { hash: record.hash.clone(), journal_key, + offstate_journal_key, inserted: inserted, deleted: record.deleted, + offstate_inserted: offstate_inserted, + offstate_deleted: offstate_record_deleted.unwrap_or(Vec::new()), }; insert_values(&mut values, record.inserted); - trace!(target: "state-db", "Uncanonicalized journal entry {}.{} ({} inserted, {} deleted)", block, index, overlay.inserted.len(), overlay.deleted.len()); + if let Some(inserted) = offstate_record_inserted { + insert_values(&mut offstate_values, inserted); + } + trace!( + target: "state-db", + "Uncanonicalized journal entry {}.{} ({} {} inserted, {} {} deleted)", + block, + index, + overlay.inserted.len(), + overlay.offstate_inserted.len(), + overlay.deleted.len(), + overlay.offstate_deleted.len(), + ); level.push(overlay); parents.insert(record.hash, record.parent_hash); index += 1; @@ -177,11 +226,19 @@ impl NonCanonicalOverlay { pending_insertions: Default::default(), pinned: Default::default(), values: values, + offstate_values: offstate_values, }) } /// Insert a new block into the overlay. If inserted on the second level or lover expects parent to be present in the window. - pub fn insert(&mut self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, changeset: ChangeSet) -> Result, Error> { + pub fn insert( + &mut self, hash: + &BlockHash, + number: u64, + parent_hash: &BlockHash, + changeset: ChangeSet, + offstate_changeset: ChangeSet, + ) -> Result, Error> { let mut commit = CommitSet::default(); let front_block_number = self.front_block_number(); if self.levels.is_empty() && self.last_canonicalized.is_none() && number > 0 { @@ -217,13 +274,18 @@ impl NonCanonicalOverlay { let index = level.len() as u64; let journal_key = to_journal_key(number, index); + let offstate_journal_key = to_offstate_journal_key(number, index); let inserted = changeset.inserted.iter().map(|(k, _)| k.clone()).collect(); + let offstate_inserted = offstate_changeset.inserted.iter().map(|(k, _)| k.clone()).collect(); let overlay = BlockOverlay { hash: hash.clone(), journal_key: journal_key.clone(), + offstate_journal_key: offstate_journal_key.clone(), inserted: inserted, deleted: changeset.deleted.clone(), + offstate_inserted: offstate_inserted, + offstate_deleted: offstate_changeset.deleted.clone(), }; level.push(overlay); self.parents.insert(hash.clone(), parent_hash.clone()); @@ -234,8 +296,24 @@ impl NonCanonicalOverlay { deleted: changeset.deleted, }; commit.meta.inserted.push((journal_key, journal_record.encode())); - trace!(target: "state-db", "Inserted uncanonicalized changeset {}.{} ({} inserted, {} deleted)", number, index, journal_record.inserted.len(), journal_record.deleted.len()); + let offstate_journal_record = OffstateJournalRecord { + inserted: offstate_changeset.inserted, + deleted: offstate_changeset.deleted, + }; + commit.meta.inserted.push((offstate_journal_key, offstate_journal_record.encode())); + + trace!( + target: "state-db", + "Inserted uncanonicalized changeset {}.{} ({} {} inserted, {} {} deleted)", + number, + index, + journal_record.inserted.len(), + offstate_journal_record.inserted.len(), + journal_record.deleted.len(), + offstate_journal_record.deleted.len(), + ); insert_values(&mut self.values, journal_record.inserted); + insert_values(&mut self.offstate_values, offstate_journal_record.inserted); self.pending_insertions.push(hash.clone()); Ok(commit) } @@ -252,6 +330,7 @@ impl NonCanonicalOverlay { let parent = self.parents.get(&overlay.hash).expect("there is a parent entry for each entry in levels; qed").clone(); if parent == *hash { discarded_journals.push(overlay.journal_key.clone()); + discarded_journals.push(overlay.offstate_journal_key.clone()); discarded_blocks.push(overlay.hash.clone()); self.discard_journals(level_index + 1, discarded_journals, discarded_blocks, &overlay.hash); } @@ -309,6 +388,7 @@ impl NonCanonicalOverlay { ); } discarded_journals.push(overlay.journal_key.clone()); + discarded_journals.push(overlay.offstate_journal_key.clone()); discarded_blocks.push(overlay.hash.clone()); } @@ -317,6 +397,10 @@ impl NonCanonicalOverlay { commit.data.inserted.extend(overlay.inserted.iter() .map(|k| (k.clone(), self.values.get(k).expect("For each key in overlays there's a value in values").1.clone()))); commit.data.deleted.extend(overlay.deleted.clone()); + commit.offstate.inserted.extend(overlay.offstate_inserted.iter() + .map(|k| (k.clone(), self.offstate_values.get(k) + .expect("For each key in overlays there's a value in values").1.clone()))); + commit.offstate.deleted.extend(overlay.offstate_deleted.clone()); commit.meta.deleted.append(&mut discarded_journals); let canonicalized = (hash.clone(), self.front_block_number() + self.pending_canonicalizations.len() as u64); @@ -344,13 +428,21 @@ impl NonCanonicalOverlay { discard_descendants( &mut self.levels, &mut self.values, + &mut self.offstate_values, 0, &mut self.parents, &mut self.pinned, &overlay.hash, ); } - discard_values(&mut self.values, overlay.inserted, self.pinned.get_mut(&overlay.hash)); + + let mut pinned = self.pinned.get_mut(&overlay.hash); + discard_values(&mut self.values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); + discard_values( + &mut self.offstate_values, + overlay.offstate_inserted, + pinned.as_mut().map(|p| &mut p.1), + ); } } if let Some(hash) = last { @@ -365,7 +457,28 @@ impl NonCanonicalOverlay { return Some(value.clone()); } for pinned in self.pinned.values() { - if let Some(value) = pinned.get(&key) { + if let Some(value) = pinned.0.get(&key) { + return Some(value.clone()); + } + } + None + } + + /// Get a value from the node overlay. This searches in every existing changeset. + /// TODO EMCH this approach does not work !!! I need the previous historied-data + /// on trie for it (put branch ix in offstate journal record) and remove + /// pinned values (or pinned btreemap>, but + /// I mostly prefer my historied data struct. + /// + /// TODO also need branch ix as parameter... (need context) + /// or maybe a Number is enough (considering the way levels + /// seems to work). + pub fn get_offstate(&self, key: &[u8]) -> Option { + if let Some((_, value)) = self.offstate_values.get(key) { + return Some(value.clone()); + } + for pinned in self.pinned.values() { + if let Some(value) = pinned.1.get(key) { return Some(value.clone()); } } @@ -386,6 +499,7 @@ impl NonCanonicalOverlay { commit.meta.deleted.push(overlay.journal_key); self.parents.remove(&overlay.hash); discard_values(&mut self.values, overlay.inserted, None); + discard_values(&mut self.offstate_values, overlay.offstate_inserted, None); } commit }) @@ -403,6 +517,7 @@ impl NonCanonicalOverlay { let overlay = self.levels[level_index].pop().expect("Empty levels are not allowed in self.levels"); discard_values(&mut self.values, overlay.inserted, None); + discard_values(&mut self.offstate_values, overlay.offstate_inserted, None); if self.levels[level_index].is_empty() { debug_assert_eq!(level_index, self.levels.len() - 1); self.levels.pop_back(); @@ -424,7 +539,7 @@ impl NonCanonicalOverlay { /// Pin state values in memory pub fn pin(&mut self, hash: &BlockHash) { - self.pinned.insert(hash.clone(), HashMap::default()); + self.pinned.insert(hash.clone(), Default::default()); } /// Discard pinned state @@ -438,13 +553,27 @@ mod tests { use std::io; use primitives::H256; use super::{NonCanonicalOverlay, to_journal_key}; - use crate::{ChangeSet, CommitSet}; - use crate::test::{make_db, make_changeset}; + use crate::{ChangeSet, CommitSet, OffstateKey}; + use crate::test::{make_db, make_changeset, make_offstate_changeset}; fn contains(overlay: &NonCanonicalOverlay, key: u64) -> bool { overlay.get(&H256::from_low_u64_be(key)) == Some(H256::from_low_u64_be(key).as_bytes().to_vec()) } + fn contains_offstate(overlay: &NonCanonicalOverlay, key: u64) -> bool { + overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec()) + == Some(H256::from_low_u64_be(key).as_bytes().to_vec()) + } + + fn contains_both(overlay: &NonCanonicalOverlay, key: u64) -> bool { + contains(overlay, key) && contains_offstate(overlay, key) + } + + fn contains_any(overlay: &NonCanonicalOverlay, key: u64) -> bool { + contains(overlay, key) || contains_offstate(overlay, key) + } + + #[test] fn created_from_empty_db() { let db = make_db(&[]); @@ -470,8 +599,14 @@ mod tests { let h1 = H256::random(); let h2 = H256::random(); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); - overlay.insert::(&h1, 2, &H256::default(), ChangeSet::default()).unwrap(); - overlay.insert::(&h2, 1, &h1, ChangeSet::default()).unwrap(); + overlay.insert::( + &h1, 2, &H256::default(), + ChangeSet::default(), ChangeSet::default(), + ).unwrap(); + overlay.insert::( + &h2, 1, &h1, + ChangeSet::default(), ChangeSet::default(), + ).unwrap(); } #[test] @@ -481,8 +616,14 @@ mod tests { let h2 = H256::random(); let db = make_db(&[]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); - overlay.insert::(&h1, 1, &H256::default(), ChangeSet::default()).unwrap(); - overlay.insert::(&h2, 3, &h1, ChangeSet::default()).unwrap(); + overlay.insert::( + &h1, 1, &H256::default(), + ChangeSet::default(), ChangeSet::default(), + ).unwrap(); + overlay.insert::( + &h2, 3, &h1, + ChangeSet::default(), ChangeSet::default(), + ).unwrap(); } #[test] @@ -492,8 +633,14 @@ mod tests { let h1 = H256::random(); let h2 = H256::random(); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); - overlay.insert::(&h1, 1, &H256::default(), ChangeSet::default()).unwrap(); - overlay.insert::(&h2, 2, &H256::default(), ChangeSet::default()).unwrap(); + overlay.insert::( + &h1, 1, &H256::default(), + ChangeSet::default(), ChangeSet::default(), + ).unwrap(); + overlay.insert::( + &h2, 2, &H256::default(), + ChangeSet::default(), ChangeSet::default(), + ).unwrap(); } #[test] @@ -503,7 +650,10 @@ mod tests { let h2 = H256::random(); let db = make_db(&[]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); - overlay.insert::(&h1, 1, &H256::default(), ChangeSet::default()).unwrap(); + overlay.insert::( + &h1, 1, &H256::default(), + ChangeSet::default(), ChangeSet::default(), + ).unwrap(); let mut commit = CommitSet::default(); overlay.canonicalize::(&h2, &mut commit).unwrap(); } @@ -514,9 +664,15 @@ mod tests { let mut db = make_db(&[1, 2]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); let changeset = make_changeset(&[3, 4], &[2]); - let insertion = overlay.insert::(&h1, 1, &H256::default(), changeset.clone()).unwrap(); + let offstate_changeset = make_offstate_changeset(&[3, 4], &[2]); + let insertion = overlay.insert::( + &h1, 1, &H256::default(), + changeset.clone(), offstate_changeset.clone(), + ).unwrap(); assert_eq!(insertion.data.inserted.len(), 0); assert_eq!(insertion.data.deleted.len(), 0); + assert_eq!(insertion.offstate.inserted.len(), 0); + assert_eq!(insertion.offstate.deleted.len(), 0); assert_eq!(insertion.meta.inserted.len(), 2); assert_eq!(insertion.meta.deleted.len(), 0); db.commit(&insertion); @@ -524,10 +680,13 @@ mod tests { overlay.canonicalize::(&h1, &mut finalization).unwrap(); assert_eq!(finalization.data.inserted.len(), changeset.inserted.len()); assert_eq!(finalization.data.deleted.len(), changeset.deleted.len()); + assert_eq!(finalization.offstate.inserted.len(), offstate_changeset.inserted.len()); + assert_eq!(finalization.offstate.deleted.len(), offstate_changeset.deleted.len()); assert_eq!(finalization.meta.inserted.len(), 1); assert_eq!(finalization.meta.deleted.len(), 1); db.commit(&finalization); assert!(db.data_eq(&make_db(&[1, 3, 4]))); + assert!(db.offstate_eq(&[1, 3, 4])); } #[test] @@ -536,8 +695,16 @@ mod tests { let h2 = H256::random(); let mut db = make_db(&[1, 2]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); - db.commit(&overlay.insert::(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2])).unwrap()); - db.commit(&overlay.insert::(&h2, 11, &h1, make_changeset(&[5], &[3])).unwrap()); + db.commit(&overlay.insert::( + &h1, 10, &H256::default(), + make_changeset(&[3, 4], &[2]), + make_offstate_changeset(&[3, 4], &[2]), + ).unwrap()); + db.commit(&overlay.insert::( + &h2, 11, &h1, + make_changeset(&[5], &[3]), + make_offstate_changeset(&[5], &[3]), + ).unwrap()); assert_eq!(db.meta.len(), 3); let overlay2 = NonCanonicalOverlay::::new(&db).unwrap(); @@ -552,8 +719,16 @@ mod tests { let h2 = H256::random(); let mut db = make_db(&[1, 2]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); - db.commit(&overlay.insert::(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2])).unwrap()); - db.commit(&overlay.insert::(&h2, 11, &h1, make_changeset(&[5], &[3])).unwrap()); + db.commit(&overlay.insert::( + &h1, 10, &H256::default(), + make_changeset(&[3, 4], &[2]), + make_offstate_changeset(&[3, 4], &[2]), + ).unwrap()); + db.commit(&overlay.insert::( + &h2,11, &h1, + make_changeset(&[5], &[3]), + make_offstate_changeset(&[5], &[3]), + ).unwrap()); let mut commit = CommitSet::default(); overlay.canonicalize::(&h1, &mut commit).unwrap(); db.commit(&commit); @@ -574,24 +749,32 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); let changeset1 = make_changeset(&[5, 6], &[2]); let changeset2 = make_changeset(&[7, 8], &[5, 3]); - db.commit(&overlay.insert::(&h1, 1, &H256::default(), changeset1).unwrap()); - assert!(contains(&overlay, 5)); - db.commit(&overlay.insert::(&h2, 2, &h1, changeset2).unwrap()); - assert!(contains(&overlay, 7)); - assert!(contains(&overlay, 5)); + let offstate_changeset1 = make_offstate_changeset(&[5, 6], &[2]); + let offstate_changeset2 = make_offstate_changeset(&[7, 8], &[5, 3]); + db.commit(&overlay.insert::( + &h1, 1, &H256::default(), + changeset1, offstate_changeset1, + ).unwrap()); + assert!(contains_both(&overlay, 5)); + db.commit(&overlay.insert::( + &h2, 2, &h1, + changeset2, offstate_changeset2, + ).unwrap()); + assert!(contains_both(&overlay, 7)); + assert!(contains_both(&overlay, 5)); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 2); let mut commit = CommitSet::default(); overlay.canonicalize::(&h1, &mut commit).unwrap(); db.commit(&commit); - assert!(contains(&overlay, 5)); + assert!(contains_both(&overlay, 5)); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 2); overlay.apply_pending(); assert_eq!(overlay.levels.len(), 1); assert_eq!(overlay.parents.len(), 1); - assert!(!contains(&overlay, 5)); - assert!(contains(&overlay, 7)); + assert!(!contains_any(&overlay, 5)); + assert!(contains_both(&overlay, 7)); let mut commit = CommitSet::default(); overlay.canonicalize::(&h2, &mut commit).unwrap(); db.commit(&commit); @@ -599,6 +782,7 @@ mod tests { assert_eq!(overlay.levels.len(), 0); assert_eq!(overlay.parents.len(), 0); assert!(db.data_eq(&make_db(&[1, 4, 6, 7, 8]))); + assert!(db.offstate_eq(&[1, 4, 6, 7, 8])); } #[test] @@ -606,17 +790,19 @@ mod tests { let mut db = make_db(&[]); let (h_1, c_1) = (H256::random(), make_changeset(&[1], &[])); let (h_2, c_2) = (H256::random(), make_changeset(&[1], &[])); + let o_c_1 = make_offstate_changeset(&[1], &[]); + let o_c_2 = make_offstate_changeset(&[1], &[]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); - db.commit(&overlay.insert::(&h_1, 1, &H256::default(), c_1).unwrap()); - db.commit(&overlay.insert::(&h_2, 1, &H256::default(), c_2).unwrap()); - assert!(contains(&overlay, 1)); + db.commit(&overlay.insert::(&h_1, 1, &H256::default(), c_1, o_c_1).unwrap()); + db.commit(&overlay.insert::(&h_2, 1, &H256::default(), c_2, o_c_2).unwrap()); + assert!(contains_both(&overlay, 1)); let mut commit = CommitSet::default(); overlay.canonicalize::(&h_1, &mut commit).unwrap(); db.commit(&commit); - assert!(contains(&overlay, 1)); + assert!(contains_both(&overlay, 1)); overlay.apply_pending(); - assert!(!contains(&overlay, 1)); + assert!(!contains_any(&overlay, 1)); } #[test] @@ -627,18 +813,40 @@ mod tests { let mut db = make_db(&[]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); let changeset = make_changeset(&[], &[]); - db.commit(&overlay.insert::(&h1, 1, &H256::default(), changeset.clone()).unwrap()); - db.commit(&overlay.insert::(&h2, 2, &h1, changeset.clone()).unwrap()); + let ochangeset = make_offstate_changeset(&[], &[]); + db.commit(&overlay.insert::( + &h1, 1, &H256::default(), + changeset.clone(), ochangeset.clone(), + ).unwrap()); + db.commit(&overlay.insert::( + &h2, 2, &h1, + changeset.clone(), ochangeset.clone(), + ).unwrap()); overlay.apply_pending(); let mut commit = CommitSet::default(); overlay.canonicalize::(&h1, &mut commit).unwrap(); overlay.canonicalize::(&h2, &mut commit).unwrap(); db.commit(&commit); - db.commit(&overlay.insert::(&h3, 3, &h2, changeset.clone()).unwrap()); + db.commit(&overlay.insert::( + &h3, 3, &h2, + changeset.clone(), ochangeset.clone(), + ).unwrap()); overlay.apply_pending(); assert_eq!(overlay.levels.len(), 1); } + fn make_both_changeset(inserted: &[u64], deleted: &[u64]) -> ( + H256, + ChangeSet, + ChangeSet, + ) { + ( + H256::random(), + make_changeset(inserted, deleted), + make_offstate_changeset(inserted, deleted), + ) + } + #[test] fn complex_tree() { use crate::MetaDb; @@ -654,43 +862,43 @@ mod tests { // // 1_2_2 is the winner - let (h_1, c_1) = (H256::random(), make_changeset(&[1], &[])); - let (h_2, c_2) = (H256::random(), make_changeset(&[2], &[])); + let (h_1, c_1, o_c_1) = make_both_changeset(&[1], &[]); + let (h_2, c_2, o_c_2) = make_both_changeset(&[2], &[]); - let (h_1_1, c_1_1) = (H256::random(), make_changeset(&[11], &[])); - let (h_1_2, c_1_2) = (H256::random(), make_changeset(&[12], &[])); - let (h_2_1, c_2_1) = (H256::random(), make_changeset(&[21], &[])); - let (h_2_2, c_2_2) = (H256::random(), make_changeset(&[22], &[])); + let (h_1_1, c_1_1, o_c_1_1) = make_both_changeset(&[11], &[]); + let (h_1_2, c_1_2, o_c_1_2) = make_both_changeset(&[12], &[]); + let (h_2_1, c_2_1, o_c_2_1) = make_both_changeset(&[21], &[]); + let (h_2_2, c_2_2, o_c_2_2) = make_both_changeset(&[22], &[]); - let (h_1_1_1, c_1_1_1) = (H256::random(), make_changeset(&[111], &[])); - let (h_1_2_1, c_1_2_1) = (H256::random(), make_changeset(&[121], &[])); - let (h_1_2_2, c_1_2_2) = (H256::random(), make_changeset(&[122], &[])); - let (h_1_2_3, c_1_2_3) = (H256::random(), make_changeset(&[123], &[])); - let (h_2_1_1, c_2_1_1) = (H256::random(), make_changeset(&[211], &[])); + let (h_1_1_1, c_1_1_1, o_c_1_1_1) = make_both_changeset(&[111], &[]); + let (h_1_2_1, c_1_2_1, o_c_1_2_1) = make_both_changeset(&[121], &[]); + let (h_1_2_2, c_1_2_2, o_c_1_2_2) = make_both_changeset(&[122], &[]); + let (h_1_2_3, c_1_2_3, o_c_1_2_3) = make_both_changeset(&[123], &[]); + let (h_2_1_1, c_2_1_1, o_c_2_1_1) = make_both_changeset(&[211], &[]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); - db.commit(&overlay.insert::(&h_1, 1, &H256::default(), c_1).unwrap()); + db.commit(&overlay.insert::(&h_1, 1, &H256::default(), c_1, o_c_1).unwrap()); - db.commit(&overlay.insert::(&h_1_1, 2, &h_1, c_1_1).unwrap()); - db.commit(&overlay.insert::(&h_1_2, 2, &h_1, c_1_2).unwrap()); + db.commit(&overlay.insert::(&h_1_1, 2, &h_1, c_1_1, o_c_1_1).unwrap()); + db.commit(&overlay.insert::(&h_1_2, 2, &h_1, c_1_2, o_c_1_2).unwrap()); - db.commit(&overlay.insert::(&h_2, 1, &H256::default(), c_2).unwrap()); + db.commit(&overlay.insert::(&h_2, 1, &H256::default(), c_2, o_c_2).unwrap()); - db.commit(&overlay.insert::(&h_2_1, 2, &h_2, c_2_1).unwrap()); - db.commit(&overlay.insert::(&h_2_2, 2, &h_2, c_2_2).unwrap()); + db.commit(&overlay.insert::(&h_2_1, 2, &h_2, c_2_1, o_c_2_1).unwrap()); + db.commit(&overlay.insert::(&h_2_2, 2, &h_2, c_2_2, o_c_2_2).unwrap()); - db.commit(&overlay.insert::(&h_1_1_1, 3, &h_1_1, c_1_1_1).unwrap()); - db.commit(&overlay.insert::(&h_1_2_1, 3, &h_1_2, c_1_2_1).unwrap()); - db.commit(&overlay.insert::(&h_1_2_2, 3, &h_1_2, c_1_2_2).unwrap()); - db.commit(&overlay.insert::(&h_1_2_3, 3, &h_1_2, c_1_2_3).unwrap()); - db.commit(&overlay.insert::(&h_2_1_1, 3, &h_2_1, c_2_1_1).unwrap()); + db.commit(&overlay.insert::(&h_1_1_1, 3, &h_1_1, c_1_1_1, o_c_1_1_1).unwrap()); + db.commit(&overlay.insert::(&h_1_2_1, 3, &h_1_2, c_1_2_1, o_c_1_2_1).unwrap()); + db.commit(&overlay.insert::(&h_1_2_2, 3, &h_1_2, c_1_2_2, o_c_1_2_2).unwrap()); + db.commit(&overlay.insert::(&h_1_2_3, 3, &h_1_2, c_1_2_3, o_c_1_2_3).unwrap()); + db.commit(&overlay.insert::(&h_2_1_1, 3, &h_2_1, c_2_1_1, o_c_2_1_1).unwrap()); - assert!(contains(&overlay, 2)); - assert!(contains(&overlay, 11)); - assert!(contains(&overlay, 21)); - assert!(contains(&overlay, 111)); - assert!(contains(&overlay, 122)); - assert!(contains(&overlay, 211)); + assert!(contains_both(&overlay, 2)); + assert!(contains_both(&overlay, 11)); + assert!(contains_both(&overlay, 21)); + assert!(contains_both(&overlay, 111)); + assert!(contains_both(&overlay, 122)); + assert!(contains_both(&overlay, 211)); assert_eq!(overlay.levels.len(), 3); assert_eq!(overlay.parents.len(), 11); assert_eq!(overlay.last_canonicalized, Some((H256::default(), 0))); @@ -708,13 +916,13 @@ mod tests { overlay.apply_pending(); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 6); - assert!(!contains(&overlay, 1)); - assert!(!contains(&overlay, 2)); - assert!(!contains(&overlay, 21)); - assert!(!contains(&overlay, 22)); - assert!(!contains(&overlay, 211)); - assert!(contains(&overlay, 111)); - assert!(!contains(&overlay, 211)); + assert!(!contains_any(&overlay, 1)); + assert!(!contains_any(&overlay, 2)); + assert!(!contains_any(&overlay, 21)); + assert!(!contains_any(&overlay, 22)); + assert!(!contains_any(&overlay, 211)); + assert!(contains_both(&overlay, 111)); + assert!(!contains_any(&overlay, 211)); // check that journals are deleted assert!(db.get_meta(&to_journal_key(1, 0)).unwrap().is_none()); assert!(db.get_meta(&to_journal_key(1, 1)).unwrap().is_none()); @@ -729,11 +937,11 @@ mod tests { overlay.apply_pending(); assert_eq!(overlay.levels.len(), 1); assert_eq!(overlay.parents.len(), 3); - assert!(!contains(&overlay, 11)); - assert!(!contains(&overlay, 111)); - assert!(contains(&overlay, 121)); - assert!(contains(&overlay, 122)); - assert!(contains(&overlay, 123)); + assert!(!contains_any(&overlay, 11)); + assert!(!contains_any(&overlay, 111)); + assert!(contains_both(&overlay, 121)); + assert!(contains_both(&overlay, 122)); + assert!(contains_both(&overlay, 123)); assert!(overlay.have_block(&h_1_2_1)); assert!(!overlay.have_block(&h_1_2)); assert!(!overlay.have_block(&h_1_1)); @@ -747,6 +955,7 @@ mod tests { assert_eq!(overlay.levels.len(), 0); assert_eq!(overlay.parents.len(), 0); assert!(db.data_eq(&make_db(&[1, 12, 122]))); + assert!(db.offstate_eq(&[1, 12, 122])); assert_eq!(overlay.last_canonicalized, Some((h_1_2_2, 3))); } @@ -758,14 +967,22 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); assert!(overlay.revert_one().is_none()); let changeset1 = make_changeset(&[5, 6], &[2]); + let ochangeset1 = make_offstate_changeset(&[5, 6], &[2]); let changeset2 = make_changeset(&[7, 8], &[5, 3]); - db.commit(&overlay.insert::(&h1, 1, &H256::default(), changeset1).unwrap()); - db.commit(&overlay.insert::(&h2, 2, &h1, changeset2).unwrap()); - assert!(contains(&overlay, 7)); + let ochangeset2 = make_offstate_changeset(&[7, 8], &[5, 3]); + db.commit(&overlay.insert::( + &h1, 1, &H256::default(), + changeset1, ochangeset1, + ).unwrap()); + db.commit(&overlay.insert::( + &h2, 2, &h1, + changeset2, ochangeset2, + ).unwrap()); + assert!(contains_both(&overlay, 7)); db.commit(&overlay.revert_one().unwrap()); assert_eq!(overlay.parents.len(), 1); - assert!(contains(&overlay, 5)); - assert!(!contains(&overlay, 7)); + assert!(contains_both(&overlay, 5)); + assert!(!contains_any(&overlay, 7)); db.commit(&overlay.revert_one().unwrap()); assert_eq!(overlay.levels.len(), 0); assert_eq!(overlay.parents.len(), 0); @@ -780,19 +997,31 @@ mod tests { let db = make_db(&[]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); let changeset1 = make_changeset(&[5, 6], &[2]); + let ochangeset1 = make_offstate_changeset(&[5, 6], &[2]); let changeset2 = make_changeset(&[7, 8], &[5, 3]); + let ochangeset2 = make_offstate_changeset(&[7, 8], &[5, 3]); let changeset3 = make_changeset(&[9], &[]); - overlay.insert::(&h1, 1, &H256::default(), changeset1).unwrap(); + let ochangeset3 = make_offstate_changeset(&[9], &[]); + overlay.insert::( + &h1, 1, &H256::default(), + changeset1, ochangeset1, + ).unwrap(); assert!(contains(&overlay, 5)); - overlay.insert::(&h2_1, 2, &h1, changeset2).unwrap(); - overlay.insert::(&h2_2, 2, &h1, changeset3).unwrap(); - assert!(contains(&overlay, 7)); - assert!(contains(&overlay, 5)); - assert!(contains(&overlay, 9)); + overlay.insert::( + &h2_1, 2, &h1, + changeset2, ochangeset2, + ).unwrap(); + overlay.insert::( + &h2_2, 2, &h1, + changeset3, ochangeset3, + ).unwrap(); + assert!(contains_both(&overlay, 7)); + assert!(contains_both(&overlay, 5)); + assert!(contains_both(&overlay, 9)); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 3); overlay.revert_pending(); - assert!(!contains(&overlay, 5)); + assert!(!contains_any(&overlay, 5)); assert_eq!(overlay.levels.len(), 0); assert_eq!(overlay.parents.len(), 0); } @@ -804,12 +1033,12 @@ mod tests { // - 1 - 1_1 // \ 1_2 - let (h_1, c_1) = (H256::random(), make_changeset(&[1], &[])); - let (h_2, c_2) = (H256::random(), make_changeset(&[2], &[])); + let (h_1, c_1, o_c_1) = make_both_changeset(&[1], &[]); + let (h_2, c_2, o_c_2) = make_both_changeset(&[2], &[]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); - db.commit(&overlay.insert::(&h_1, 1, &H256::default(), c_1).unwrap()); - db.commit(&overlay.insert::(&h_2, 1, &H256::default(), c_2).unwrap()); + db.commit(&overlay.insert::(&h_1, 1, &H256::default(), c_1, o_c_1).unwrap()); + db.commit(&overlay.insert::(&h_2, 1, &H256::default(), c_2, o_c_2).unwrap()); overlay.pin(&h_1); @@ -817,8 +1046,8 @@ mod tests { overlay.canonicalize::(&h_2, &mut commit).unwrap(); db.commit(&commit); overlay.apply_pending(); - assert!(contains(&overlay, 1)); + assert!(contains_both(&overlay, 1)); overlay.unpin(&h_1); - assert!(!contains(&overlay, 1)); + assert!(!contains_any(&overlay, 1)); } } diff --git a/core/state-db/src/pruning.rs b/core/state-db/src/pruning.rs index 21f472fe69da9..d62d80faac884 100644 --- a/core/state-db/src/pruning.rs +++ b/core/state-db/src/pruning.rs @@ -24,11 +24,12 @@ use std::collections::{HashMap, HashSet, VecDeque}; use codec::{Encode, Decode}; -use crate::{CommitSet, Error, MetaDb, to_meta_key, Hash}; +use crate::{CommitSet, Error, MetaDb, to_meta_key, Hash, OffstateKey}; use log::{trace, warn}; const LAST_PRUNED: &[u8] = b"last_pruned"; const PRUNING_JOURNAL: &[u8] = b"pruning_journal"; +const OFFSTATE_PRUNING_JOURNAL: &[u8] = b"offstate_pruning_journal"; /// See module documentation. pub struct RefWindow { @@ -36,6 +37,8 @@ pub struct RefWindow { death_rows: VecDeque>, /// An index that maps each key from `death_rows` to block number. death_index: HashMap, + /// An index that maps each key from `death_rows` to block number. + offstate_death_index: HashMap, /// Block number that corresponts to the front of `death_rows` pending_number: u64, /// Number of call of `note_canonical` after @@ -50,7 +53,9 @@ pub struct RefWindow { struct DeathRow { hash: BlockHash, journal_key: Vec, + offstate_journal_key: Vec, deleted: HashSet, + offstate_deleted: HashSet, } #[derive(Encode, Decode)] @@ -60,10 +65,20 @@ struct JournalRecord { deleted: Vec, } +#[derive(Encode, Decode)] +struct OffstateJournalRecord { + inserted: Vec, + deleted: Vec, +} + fn to_journal_key(block: u64) -> Vec { to_meta_key(PRUNING_JOURNAL, &block) } +fn to_offstate_journal_key(block: u64) -> Vec { + to_meta_key(OFFSTATE_PRUNING_JOURNAL, &block) +} + impl RefWindow { pub fn new(db: &D) -> Result, Error> { let last_pruned = db.get_meta(&to_meta_key(LAST_PRUNED, &())) @@ -76,6 +91,7 @@ impl RefWindow { let mut pruning = RefWindow { death_rows: Default::default(), death_index: Default::default(), + offstate_death_index: Default::default(), pending_number: pending_number, pending_canonicalizations: 0, pending_prunings: 0, @@ -84,11 +100,34 @@ impl RefWindow { trace!(target: "state-db", "Reading pruning journal. Pending #{}", pending_number); loop { let journal_key = to_journal_key(block); + let offstate_journal_key = to_offstate_journal_key(block); match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? { Some(record) => { let record: JournalRecord = Decode::decode(&mut record.as_slice())?; - trace!(target: "state-db", "Pruning journal entry {} ({} inserted, {} deleted)", block, record.inserted.len(), record.deleted.len()); - pruning.import(&record.hash, journal_key, record.inserted.into_iter(), record.deleted); + let (offstate_record_inserted, offstate_record_deleted) = if let Some(record) = db + .get_meta(&offstate_journal_key).map_err(|e| Error::Db(e))? { + let record = OffstateJournalRecord::decode(&mut record.as_slice())?; + (record.inserted, record.deleted) + } else { (Vec::new(), Vec::new()) }; + + trace!( + target: "state-db", + "Pruning journal entry {} ({} {} inserted, {} {} deleted)", + block, + record.inserted.len(), + offstate_record_inserted.len(), + record.deleted.len(), + offstate_record_deleted.len(), + ); + pruning.import( + &record.hash, + journal_key, + offstate_journal_key, + record.inserted.into_iter(), + record.deleted, + offstate_record_inserted.into_iter(), + offstate_record_deleted, + ); }, None => break, } @@ -97,24 +136,43 @@ impl RefWindow { Ok(pruning) } - fn import>(&mut self, hash: &BlockHash, journal_key: Vec, inserted: I, deleted: Vec) { + fn import, I2: IntoIterator>( + &mut self, + hash: &BlockHash, + journal_key: Vec, + offstate_journal_key: Vec, + inserted: I, + deleted: Vec, + offstate_inserted: I2, + offstate_deleted: Vec, + ) { // remove all re-inserted keys from death rows for k in inserted { if let Some(block) = self.death_index.remove(&k) { self.death_rows[(block - self.pending_number) as usize].deleted.remove(&k); } } + for k in offstate_inserted { + if let Some(block) = self.offstate_death_index.remove(&k) { + self.death_rows[(block - self.pending_number) as usize].offstate_deleted.remove(&k); + } + } // add new keys let imported_block = self.pending_number + self.death_rows.len() as u64; for k in deleted.iter() { self.death_index.insert(k.clone(), imported_block); } + for k in offstate_deleted.iter() { + self.offstate_death_index.insert(k.clone(), imported_block); + } self.death_rows.push_back( DeathRow { hash: hash.clone(), deleted: deleted.into_iter().collect(), - journal_key: journal_key, + offstate_deleted: offstate_deleted.into_iter().collect(), + journal_key, + offstate_journal_key, } ); } @@ -145,6 +203,7 @@ impl RefWindow { trace!(target: "state-db", "Pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len()); let index = self.pending_number + self.pending_prunings as u64; commit.data.deleted.extend(pruned.deleted.iter().cloned()); + commit.offstate.deleted.extend(pruned.offstate_deleted.iter().cloned()); commit.meta.inserted.push((to_meta_key(LAST_PRUNED, &()), index.encode())); commit.meta.deleted.push(pruned.journal_key.clone()); self.pending_prunings += 1; @@ -157,16 +216,32 @@ impl RefWindow { pub fn note_canonical(&mut self, hash: &BlockHash, commit: &mut CommitSet) { trace!(target: "state-db", "Adding to pruning window: {:?} ({} inserted, {} deleted)", hash, commit.data.inserted.len(), commit.data.deleted.len()); let inserted = commit.data.inserted.iter().map(|(k, _)| k.clone()).collect(); + let offstate_inserted = commit.offstate.inserted.iter().map(|(k, _)| k.clone()).collect(); let deleted = ::std::mem::replace(&mut commit.data.deleted, Vec::new()); + let offstate_deleted = ::std::mem::replace(&mut commit.offstate.deleted, Vec::new()); let journal_record = JournalRecord { hash: hash.clone(), inserted, deleted, }; + let offstate_journal_record = OffstateJournalRecord { + inserted: offstate_inserted, + deleted: offstate_deleted, + }; let block = self.pending_number + self.death_rows.len() as u64; let journal_key = to_journal_key(block); + let offstate_journal_key = to_offstate_journal_key(block); commit.meta.inserted.push((journal_key.clone(), journal_record.encode())); - self.import(&journal_record.hash, journal_key, journal_record.inserted.into_iter(), journal_record.deleted); + commit.meta.inserted.push((offstate_journal_key.clone(), offstate_journal_record.encode())); + self.import( + &journal_record.hash, + journal_key, + offstate_journal_key, + journal_record.inserted.into_iter(), + journal_record.deleted, + offstate_journal_record.inserted.into_iter(), + offstate_journal_record.deleted, + ); self.pending_canonicalizations += 1; } @@ -179,6 +254,9 @@ impl RefWindow { for k in pruned.deleted.iter() { self.death_index.remove(&k); } + for k in pruned.offstate_deleted.iter() { + self.offstate_death_index.remove(k); + } self.pending_number += 1; } self.pending_prunings = 0; @@ -210,6 +288,7 @@ mod tests { assert_eq!(pruning.pending_number, restored.pending_number); assert_eq!(pruning.death_rows, restored.death_rows); assert_eq!(pruning.death_index, restored.death_index); + assert_eq!(pruning.offstate_death_index, restored.offstate_death_index); } #[test] @@ -219,6 +298,7 @@ mod tests { assert_eq!(pruning.pending_number, 0); assert!(pruning.death_rows.is_empty()); assert!(pruning.death_index.is_empty()); + assert!(pruning.offstate_death_index.is_empty()); } #[test] @@ -230,6 +310,7 @@ mod tests { assert_eq!(pruning.pending_number, 0); assert!(pruning.death_rows.is_empty()); assert!(pruning.death_index.is_empty()); + assert!(pruning.offstate_death_index.is_empty()); assert!(pruning.pending_prunings == 0); assert!(pruning.pending_canonicalizations == 0); } @@ -237,8 +318,10 @@ mod tests { #[test] fn prune_one() { let mut db = make_db(&[1, 2, 3]); + db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); let mut commit = make_commit(&[4, 5], &[1, 3]); + commit.initialize_offstate(&[4, 5], &[1, 3]); let h = H256::random(); pruning.note_canonical(&h, &mut commit); db.commit(&commit); @@ -246,9 +329,12 @@ mod tests { pruning.apply_pending(); assert!(pruning.have_block(&h)); assert!(commit.data.deleted.is_empty()); + assert!(commit.offstate.deleted.is_empty()); assert_eq!(pruning.death_rows.len(), 1); assert_eq!(pruning.death_index.len(), 2); + assert_eq!(pruning.offstate_death_index.len(), 2); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); + assert!(db.offstate_eq(&[1, 2, 3, 4, 5])); check_journal(&pruning, &db); let mut commit = CommitSet::default(); @@ -258,8 +344,10 @@ mod tests { pruning.apply_pending(); assert!(!pruning.have_block(&h)); assert!(db.data_eq(&make_db(&[2, 4, 5]))); + assert!(db.offstate_eq(&[2, 4, 5])); assert!(pruning.death_rows.is_empty()); assert!(pruning.death_index.is_empty()); + assert!(pruning.offstate_death_index.is_empty()); assert_eq!(pruning.pending_number, 1); } @@ -317,8 +405,10 @@ mod tests { #[test] fn reinserted_survives() { let mut db = make_db(&[1, 2, 3]); + db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); let mut commit = make_commit(&[], &[2]); + commit.initialize_offstate(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); let mut commit = make_commit(&[2], &[]); @@ -328,6 +418,7 @@ mod tests { pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); + assert!(db.offstate_eq(&[1, 2, 3])); pruning.apply_pending(); check_journal(&pruning, &db); @@ -336,13 +427,16 @@ mod tests { pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); + assert!(db.offstate_eq(&[1, 2, 3])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); + assert!(db.offstate_eq(&[1, 2, 3])); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 3]))); + assert!(db.offstate_eq(&[1, 3])); pruning.apply_pending(); assert_eq!(pruning.pending_number, 3); } diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index d90c36990612e..2fff45c1ef514 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -18,12 +18,13 @@ use std::collections::HashMap; use primitives::H256; -use crate::{DBValue, ChangeSet, CommitSet, MetaDb, NodeDb}; +use crate::{DBValue, ChangeSet, CommitSet, MetaDb, NodeDb, OffstateDb, OffstateKey}; #[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct TestDb { pub data: HashMap, pub meta: HashMap, DBValue>, + pub offstate: HashMap, } impl MetaDb for TestDb { @@ -34,6 +35,14 @@ impl MetaDb for TestDb { } } +impl OffstateDb for TestDb { + type Error = (); + + fn get_offstate(&self, key: &[u8]) -> Result, ()> { + Ok(self.offstate.get(key).cloned()) + } +} + impl NodeDb for TestDb { type Error = (); type Key = H256; @@ -59,6 +68,16 @@ impl TestDb { pub fn data_eq(&self, other: &TestDb) -> bool { self.data == other.data } + + pub fn offstate_eq(&self, values: &[u64]) -> bool { + let data = make_offstate_changeset(values, &[]); + self.offstate == data.inserted.into_iter().collect() + } + + pub fn initialize_offstate(&mut self, inserted: &[u64]) { + let data = make_offstate_changeset(inserted, &[]); + self.offstate = data.inserted.into_iter().collect(); + } } pub fn make_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet { @@ -73,10 +92,31 @@ pub fn make_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet { } } +pub fn make_offstate_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet { + ChangeSet { + inserted: inserted + .iter() + .map(|v| {( + H256::from_low_u64_be(*v).as_bytes().to_vec(), + H256::from_low_u64_be(*v).as_bytes().to_vec(), + )}) + .collect(), + deleted: deleted.iter().map(|v| H256::from_low_u64_be(*v).as_bytes().to_vec()).collect(), + } +} + pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet { CommitSet { data: make_changeset(inserted, deleted), meta: ChangeSet::default(), + offstate: ChangeSet::default(), + } +} + +impl CommitSet { + pub fn initialize_offstate(&mut self, inserted: &[u64], deleted: &[u64]) { + let data = make_offstate_changeset(inserted, deleted); + self.offstate = data; } } @@ -89,6 +129,7 @@ pub fn make_db(inserted: &[u64]) -> TestDb { }) .collect(), meta: Default::default(), + offstate: Default::default(), } } From 19df6e9a9aa7986ca9d76a0a943234d1301fe386 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 25 Sep 2019 11:14:50 +0200 Subject: [PATCH 03/68] TODO2 variant for in memory --- core/client/db/src/lib.rs | 8 ++-- core/client/src/cht.rs | 4 +- core/client/src/light/backend.rs | 2 +- core/client/src/light/fetcher.rs | 2 +- core/state-machine/src/backend.rs | 8 ++-- core/state-machine/src/lib.rs | 8 ++-- core/state-machine/src/offstate_backend.rs | 53 ++++++++++++++++++++-- core/state-machine/src/proving_backend.rs | 8 ++-- core/state-machine/src/trie_backend.rs | 6 +-- 9 files changed, 73 insertions(+), 26 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 1202dd571c330..8fbf010dadbcb 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -60,7 +60,7 @@ use executor::RuntimeInfo; use state_machine::{ DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, ChangesTrieBuildCache, backend::Backend as StateBackend, - TODO, + TODO, TODO2, }; use crate::utils::{Meta, db_err, meta_keys, read_db, block_id_to_lookup_key, read_meta}; use client::leaves::{LeafSet, FinalizationDisplaced}; @@ -440,6 +440,7 @@ pub struct BlockImportOperation { db_updates: (PrefixedMemoryDB, Vec<(Vec, Option>)>), storage_updates: StorageCollection, child_storage_updates: ChildStorageCollection, + // TODO EMCH offstate update and offstate values in cache changes_trie_updates: MemoryDB, changes_trie_cache_update: Option>>, pending_block: Option>, @@ -1497,7 +1498,7 @@ impl client::backend::Backend for Backend whe let genesis_storage = DbGenesisStorage::new(); let root = genesis_storage.0.clone(); // TODO EMCH see genesis impl: that is empty storage - let genesis_offstate = TODO; + let genesis_offstate = TODO2; let db_state = DbState::new(Arc::new(genesis_storage), root, Arc::new(genesis_offstate)); let state = RefTrackingState::new(db_state, self.storage.clone(), None); return Ok(CachingState::new(state, self.shared_cache.clone(), None)); @@ -1510,7 +1511,8 @@ impl client::backend::Backend for Backend whe let hash = hdr.hash(); if let Ok(()) = self.storage.state_db.pin(&hash) { let root = H256::from_slice(hdr.state_root().as_ref()); - let db_state = DbState::new(self.storage.clone(), root, Arc::new(TODO)); + let block_number = hdr.number().clone(); + let db_state = DbState::new(self.storage.clone(), root, Arc::new(TODO::new(block_number))); let state = RefTrackingState::new(db_state, self.storage.clone(), Some(hash.clone())); Ok(CachingState::new(state, self.shared_cache.clone(), Some(hash))) } else { diff --git a/core/client/src/cht.rs b/core/client/src/cht.rs index f801843d053e7..b4015dd33ec7e 100644 --- a/core/client/src/cht.rs +++ b/core/client/src/cht.rs @@ -33,7 +33,7 @@ use state_machine::backend::InMemory as InMemoryState; use state_machine::backend::InMemoryTransaction; use state_machine::{MemoryDB, TrieBackend, Backend as StateBackend, prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend}; -use state_machine::offstate_backend::TODO; +use state_machine::offstate_backend::TODO2; use crate::error::{Error as ClientError, Result as ClientResult}; @@ -148,7 +148,7 @@ pub fn check_proof_on_proving_backend( local_root: Header::Hash, local_number: Header::Number, remote_hash: Header::Hash, - proving_backend: &TrieBackend, Hasher, TODO>, + proving_backend: &TrieBackend, Hasher, TODO2>, ) -> ClientResult<()> where Header: HeaderT, diff --git a/core/client/src/light/backend.rs b/core/client/src/light/backend.rs index e3b3f09c79cc6..1601608f8f67e 100644 --- a/core/client/src/light/backend.rs +++ b/core/client/src/light/backend.rs @@ -344,7 +344,7 @@ impl StateBackend for GenesisOrUnavailableState type Error = ClientError; type Transaction = (); type TrieBackendStorage = MemoryDB; - type OffstateBackendStorage = state_machine::offstate_backend::TODO; + type OffstateBackendStorage = state_machine::offstate_backend::TODO2; fn storage(&self, key: &[u8]) -> ClientResult>> { match *self { diff --git a/core/client/src/light/fetcher.rs b/core/client/src/light/fetcher.rs index 49a47cadf1be6..b59f342df6db1 100644 --- a/core/client/src/light/fetcher.rs +++ b/core/client/src/light/fetcher.rs @@ -351,7 +351,7 @@ impl> LightDataChecker { // using empty offstate as light do not use offstate information // (things being fetch proved). - let offstate = state_machine::TODO; + let offstate = state_machine::TODO2; // check proof for single changes trie root let proving_backend = TrieBackend::new(storage, cht_root, offstate); diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index 09b5eb3586674..a6bd1cdd0e63f 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -21,7 +21,7 @@ use log::warn; use hash_db::Hasher; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::TrieBackendStorage; -use crate::offstate_backend::{OffstateBackendStorage, TODO}; +use crate::offstate_backend::{OffstateBackendStorage, TODO, TODO2}; use trie::{ TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::{TrieDBMut, Layout}, @@ -310,7 +310,7 @@ impl error::Error for Void { /// tests. pub struct InMemory { inner: HashMap>, HashMap, Vec>>, - trie: Option, H, TODO>>, + trie: Option, H, TODO2>>, offstate: HashMap, Vec>, _hasher: PhantomData, } @@ -466,7 +466,7 @@ impl Backend for InMemory { type Error = Void; type Transaction = InMemoryTransaction; type TrieBackendStorage = MemoryDB; - type OffstateBackendStorage = TODO; + type OffstateBackendStorage = TODO2; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone))) @@ -619,7 +619,7 @@ impl Backend for InMemory { // Since we are running on a memorydb (not a prefixed memory db), content // is not collision free, so an empty offtrie state can be use (no need // for keyspace). - self.trie = Some(TrieBackend::new(mdb, root, TODO)); + self.trie = Some(TrieBackend::new(mdb, root, TODO2)); self.trie.as_ref() } } diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 8133b086e97a9..70e2c8fddb7ff 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -30,7 +30,7 @@ use primitives::{ traits::{BareCryptoStorePtr, CodeExecutor}, hexdisplay::HexDisplay, }; -pub use crate::offstate_backend::{OffstateStorage, OffstateBackendStorage, TODO}; +pub use crate::offstate_backend::{OffstateStorage, OffstateBackendStorage, TODO, TODO2}; pub mod backend; mod changes_trie; @@ -520,7 +520,7 @@ where /// Check execution proof on proving backend, generated by `prove_execution` call. pub fn execution_proof_check_on_trie_backend( - trie_backend: &TrieBackend, H, TODO>, + trie_backend: &TrieBackend, H, TODO2>, overlay: &mut OverlayedChanges, exec: &Exec, method: &str, @@ -674,7 +674,7 @@ where /// Check storage read proof on pre-created proving backend. pub fn read_proof_check_on_proving_backend( - proving_backend: &TrieBackend, H, TODO>, + proving_backend: &TrieBackend, H, TODO2>, key: &[u8], ) -> Result>, Box> where @@ -686,7 +686,7 @@ where /// Check child storage read proof on pre-created proving backend. pub fn read_child_proof_check_on_proving_backend( - proving_backend: &TrieBackend, H, TODO>, + proving_backend: &TrieBackend, H, TODO2>, storage_key: &[u8], key: &[u8], ) -> Result>, Box> diff --git a/core/state-machine/src/offstate_backend.rs b/core/state-machine/src/offstate_backend.rs index 087613ad435be..63f82507b14a4 100644 --- a/core/state-machine/src/offstate_backend.rs +++ b/core/state-machine/src/offstate_backend.rs @@ -46,7 +46,7 @@ pub trait OffstateStorage { } -impl OffstateBackendStorage for TODO { +impl OffstateBackendStorage for TODO { fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { unimplemented!() @@ -58,6 +58,19 @@ impl OffstateBackendStorage for TODO { } +impl OffstateBackendStorage for TODO2 { + + fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { + unimplemented!() + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + unimplemented!() + } + +} + + // This implementation is used by normal storage trie clients. impl OffstateBackendStorage for Arc { @@ -72,7 +85,7 @@ impl OffstateBackendStorage for Arc { } -impl OffstateStorage for TODO { +impl OffstateStorage for TODO { fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { unimplemented!() @@ -88,6 +101,38 @@ impl OffstateStorage for TODO { } -// TODO EMCH implement, no branch ranges. -pub struct TODO; +impl OffstateStorage for TODO2 { + + fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { + unimplemented!() + } + fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { + ::get(&self, prefix, key) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + ::pairs(&self) + } + +} + + +/// TODO EMCH implement, no branch ranges. +pub struct TODO(N); + +/// TODO EMCH variant for proof check or no +/// need to keep multiple block state. +/// TODO rename to something like SingleState +pub struct TODO2; + +impl TODO { + /// Build for a given block number. + /// TODO EMCH may or may not need a branch index to + pub fn new(block_number: N) -> Self { + TODO(block_number) + } +} + +impl TODO { +} diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index f8bb5f659e043..8be71e21263ad 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -28,7 +28,7 @@ pub use trie::trie_types::{Layout, TrieError}; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend}; -use crate::offstate_backend::{OffstateBackendStorage, TODO}; +use crate::offstate_backend::{OffstateBackendStorage, TODO, TODO2}; /// Patricia trie-based backend essence which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. @@ -253,14 +253,14 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> pub fn create_proof_check_backend( root: H::Out, proof: Vec> -) -> Result, H, TODO>, Box> +) -> Result, H, TODO2>, Box> where H: Hasher, { let db = create_proof_check_backend_storage(proof); if db.contains(&root, EMPTY_PREFIX) { - Ok(TrieBackend::new(db, root, TODO)) + Ok(TrieBackend::new(db, root, TODO2)) } else { Err(Box::new(ExecutionError::InvalidProof)) } @@ -289,7 +289,7 @@ mod tests { // TODO this need an actual in momery with possibly content // as the test uses a prefixed memory db. - type OffstateBackend = TODO; + type OffstateBackend = TODO2; fn test_proving<'a>( trie_backend: &'a TrieBackend, Blake2Hasher, OffstateBackend>, diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index 0d1c6dcb5ebbb..d1c9953a797e8 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -289,7 +289,7 @@ pub mod tests { // TODO this need an actual in momery with possibly content // as the test uses a prefixed memory db. - type OffstateBackend = crate::offstate_backend::TODO; + type OffstateBackend = crate::offstate_backend::TODO2; fn test_db() -> (PrefixedMemoryDB, H256, OffstateBackend) { let mut root = H256::default(); @@ -314,7 +314,7 @@ pub mod tests { } } // empty history. - let offstate = crate::offstate_backend::TODO; + let offstate = crate::offstate_backend::TODO2; // TODO EMCH add a block in offstate or use an actual implementation of // offstate that do not use history (a test implementation most likely) // TODO EMCH feed offstate with keyspace for roots. @@ -346,7 +346,7 @@ pub mod tests { assert!(TrieBackend::, Blake2Hasher, OffstateBackend>::new( PrefixedMemoryDB::default(), Default::default(), - crate::offstate_backend::TODO, + crate::offstate_backend::TODO2, ).pairs().is_empty()); } From 3989fea659012e9fdbe8d1964dba4df5632e6b7c Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 25 Sep 2019 15:33:03 +0200 Subject: [PATCH 04/68] traits in place, still need to parameterized with blocknb &| branchix --- core/client/db/src/lib.rs | 31 +++++++++- core/state-db/src/lib.rs | 22 +++++++- core/state-db/src/test.rs | 4 ++ core/state-machine/src/offstate_backend.rs | 66 ++++++++++++---------- core/state-machine/src/trie_backend.rs | 5 +- 5 files changed, 92 insertions(+), 36 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 8fbf010dadbcb..74d96050aa0d6 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -274,6 +274,8 @@ pub(crate) mod columns { pub const AUX: Option = Some(8); /// Offchain workers local storage pub const OFFCHAIN: Option = Some(9); + /// Offstate data + pub const OFFSTATE: Option = Some(10); } struct PendingBlock { @@ -589,6 +591,30 @@ impl state_db::NodeDb for StorageDb { } } +impl state_db::OffstateDb for StorageDb { + + type Error = io::Error; + + fn get_offstate(&self, key: &[u8]) -> Result>, Self::Error> { + self.db.get(columns::OFFSTATE, key); + unimplemented!("TODO map with history fetch at requested block") + } + + fn get_offstate_pairs(&self) -> Vec<(Vec, Vec)> { + unimplemented!("TODO need to be an iterator") + } +} + +impl state_machine::OffstateStorage for StorageDb { + fn get(&self, key: &[u8]) -> Result>, String> { + self.state_db.get_offstate(key, self) + .map_err(|e| format!("Database backend error: {:?}", e)) + } + fn pairs(&self) -> Vec<(Vec, Vec)> { + self.state_db.get_offstate_pairs(self) + } +} + struct DbGenesisStorage(pub H256); impl DbGenesisStorage { @@ -1511,8 +1537,9 @@ impl client::backend::Backend for Backend whe let hash = hdr.hash(); if let Ok(()) = self.storage.state_db.pin(&hash) { let root = H256::from_slice(hdr.state_root().as_ref()); - let block_number = hdr.number().clone(); - let db_state = DbState::new(self.storage.clone(), root, Arc::new(TODO::new(block_number))); + //let block_number = hdr.number().clone(); + //let db_state = DbState::new(self.storage.clone(), root, Arc::new(TODO::new(block_number))); + let db_state = DbState::new(self.storage.clone(), root, self.storage.clone()); let state = RefTrackingState::new(db_state, self.storage.clone(), Some(hash.clone())); Ok(CachingState::new(state, self.shared_cache.clone(), Some(hash))) } else { diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index e94d09ce627aa..55aed958d1014 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -74,6 +74,8 @@ pub trait OffstateDb { /// Get state trie node. fn get_offstate(&self, key: &[u8]) -> Result, Self::Error>; + /// Get all pairs of key at current state. + fn get_offstate_pairs(&self) -> Vec<(OffstateKey, DBValue)>; } /// Error type. @@ -366,6 +368,17 @@ impl StateDbSync { db.get_offstate(key).map_err(|e| Error::Db(e)) } + pub fn get_offstate_pairs( + &self, + db: &D, + ) -> Vec<(OffstateKey, DBValue)> { + // TODO get non_canonical optional values and then filter + // db pairs over it : TODO -> db.get_offstate would return + // an iterator. + unimplemented!() + } + + pub fn apply_pending(&mut self) { self.non_canonical.apply_pending(); if let Some(pruning) = &mut self.pruning { @@ -445,7 +458,14 @@ impl StateDb { self.db.read().get_offstate(key, db) } - + /// Get pairs values offstate. + pub fn get_offstate_pairs( + &self, + db: &D, + ) -> Vec<(OffstateKey, DBValue)> { + self.db.read().get_offstate_pairs(db) + } + /// Revert all non-canonical blocks with the best block number. /// Returns a database commit or `None` if not possible. /// For archive an empty commit set is returned. diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 2fff45c1ef514..c5801f3627d85 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -41,6 +41,10 @@ impl OffstateDb for TestDb { fn get_offstate(&self, key: &[u8]) -> Result, ()> { Ok(self.offstate.get(key).cloned()) } + + fn get_offstate_pairs(&self) -> Vec<(OffstateKey, DBValue)> { + self.offstate.iter().map(|(a, b)| (a.clone(), b.clone())).collect() + } } impl NodeDb for TestDb { diff --git a/core/state-machine/src/offstate_backend.rs b/core/state-machine/src/offstate_backend.rs index 63f82507b14a4..7ffe84c5117bf 100644 --- a/core/state-machine/src/offstate_backend.rs +++ b/core/state-machine/src/offstate_backend.rs @@ -16,39 +16,53 @@ //! Backend for storing data without a state. +/// TODO EMCH merge offstate storage and offstate backend storage ?? + use std::sync::Arc; use std::ops::Deref; -pub trait OffstateBackendStorage { +/// TODO EMCH replace ofstate storage? +pub trait OffstateStorage: Send + Sync { + /// Retrieve a value from storage under given key. + fn get(&self, key: &[u8]) -> Result>, String>; + + /// Return in memory all values for this backend, mainly for + /// tests. + fn pairs(&self) -> Vec<(Vec, Vec)>; +} + +pub trait OffstateBackendStorage: Send + Sync { /* /// state type for querying data /// (similar to hash for a trie_backend). trait ChanState;*/ - /// Retrieve a value from storage under given key and prefix. - fn get(&self, prefix: &[u8], key: &[u8]) -> Option>; + /// Retrieve a value from storage under given key. + fn get(&self, key: &[u8]) -> Result>, String>; /// Return in memory all values for this backend, mainly for /// tests. fn pairs(&self) -> Vec<(Vec, Vec)>; + } +/*// TODO EMCH is it use?? pub trait OffstateStorage { - /// Persist a value in storage under given key and prefix. - fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]); + /// Persist a value in storage under given key. + fn set(&mut self, key: &[u8], value: &[u8]); - /// Retrieve a value from storage under given key and prefix. - fn get(&self, prefix: &[u8], key: &[u8]) -> Option>; + /// Retrieve a value from storage under given key. + fn get(&self, key: &[u8]) -> Option>; /// Return in memory all values for this backend, mainly for /// tests. fn pairs(&self) -> Vec<(Vec, Vec)>; -} +}*/ -impl OffstateBackendStorage for TODO { +impl OffstateBackendStorage for TODO { - fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { + fn get(&self, key: &[u8]) -> Result>, String> { unimplemented!() } @@ -60,7 +74,7 @@ impl OffstateBackendStorage for TODO { impl OffstateBackendStorage for TODO2 { - fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { + fn get(&self, key: &[u8]) -> Result>, String> { unimplemented!() } @@ -70,49 +84,39 @@ impl OffstateBackendStorage for TODO2 { } - // This implementation is used by normal storage trie clients. impl OffstateBackendStorage for Arc { - - fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { - OffstateStorage::get(self.deref(), prefix, key) + fn get(&self, key: &[u8]) -> Result>, String> { + OffstateStorage::get(self.deref(), key) } - fn pairs(&self) -> Vec<(Vec, Vec)> { OffstateStorage::pairs(self.deref()) } - } -impl OffstateStorage for TODO { - fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { - unimplemented!() - } +impl OffstateStorage for TODO { - fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { - ::get(&self, prefix, key) + fn get(&self, key: &[u8]) -> Result>, String> { + ::get(&self, key) } fn pairs(&self) -> Vec<(Vec, Vec)> { - ::pairs(&self) + unimplemented!() } } -impl OffstateStorage for TODO2 { - fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { - unimplemented!() - } +impl OffstateStorage for TODO2 { - fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { - ::get(&self, prefix, key) + fn get(&self, key: &[u8]) -> Result>, String> { + ::get(&self, key) } fn pairs(&self) -> Vec<(Vec, Vec)> { - ::pairs(&self) + unimplemented!() } } diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index d1c9953a797e8..866812597bf11 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -62,9 +62,10 @@ impl, O: OffstateBackendStorage, H: Hasher> TrieBackend } // TODO EMCH PROTO: remove before pr. - pub fn child_keyspace(&self, key: &[u8]) -> Option> { + pub fn child_keyspace(&self, key: &[u8]) -> Result>, String> { const PREFIX_KEYSPACE: &'static[u8] = b"offstate_keyspace"; - self.offstate_storage.get(PREFIX_KEYSPACE, key) + // TODO EMCH do prefixing manually. + self.offstate_storage.get(key) } } From 6a6606de56cbe73fbf6987a2cc67aabc1169683e Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 25 Sep 2019 17:09:59 +0200 Subject: [PATCH 05/68] passing block hash as state in a convoluted way --- core/client/db/src/lib.rs | 69 +++++++++++++++++++--- core/state-db/src/lib.rs | 31 ++++++---- core/state-db/src/test.rs | 6 +- core/state-machine/src/offstate_backend.rs | 32 ++++++++-- 4 files changed, 111 insertions(+), 27 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 74d96050aa0d6..7d0d5d6beb296 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -34,6 +34,7 @@ mod storage_cache; mod utils; use std::sync::Arc; +use std::ops::Deref; use std::path::PathBuf; use std::io; use std::collections::{HashMap, HashSet}; @@ -85,7 +86,7 @@ const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); pub type DbState = state_machine::TrieBackend< Arc>, Blake2Hasher, - Arc, + Arc>, >; /// A reference tracking state. @@ -574,6 +575,18 @@ struct StorageDb { pub state_db: StateDb>, } +struct StorageDbAt { + pub storage_db: Arc>, + pub state: State, +} + +// Semantic is one block statedb with block in todo2, +// state is here for compatibility only. +struct TODO2At { + pub storage_db: TODO2, + pub state: State, +} + impl state_machine::Storage for StorageDb { fn get(&self, key: &H256, prefix: Prefix) -> Result, String> { let key = prefixed_key::(key, prefix); @@ -591,30 +604,61 @@ impl state_db::NodeDb for StorageDb { } } -impl state_db::OffstateDb for StorageDb { +// TODO EMCH try with Block::Hash but no real hope +impl state_db::OffstateDb for StorageDb { type Error = io::Error; - fn get_offstate(&self, key: &[u8]) -> Result>, Self::Error> { + fn get_offstate(&self, key: &[u8], state: &Block::Hash) -> Result>, Self::Error> { self.db.get(columns::OFFSTATE, key); unimplemented!("TODO map with history fetch at requested block") } - fn get_offstate_pairs(&self) -> Vec<(Vec, Vec)> { + fn get_offstate_pairs(&self, state: &Block::Hash) -> Vec<(Vec, Vec)> { unimplemented!("TODO need to be an iterator") } } -impl state_machine::OffstateStorage for StorageDb { +impl state_machine::OffstateStorage for StorageDbAt { + + fn state(&self) -> &Block::Hash { + &self.state + } + + fn change_state(&mut self, state: Block::Hash) -> Block::Hash { + std::mem::replace(&mut self.state, state) + } + + fn get(&self, key: &[u8]) -> Result>, String> { + self.storage_db.state_db.get_offstate(key, &self.state, self.storage_db.deref()) + .map_err(|e| format!("Database backend error: {:?}", e)) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + self.storage_db.state_db.get_offstate_pairs(&self.state, self.storage_db.deref()) + } +} + +impl state_machine::OffstateStorage for TODO2At { + + fn state(&self) -> &S { + &self.state + } + + fn change_state(&mut self, state: S) -> S { + std::mem::replace(&mut self.state, state) + } + fn get(&self, key: &[u8]) -> Result>, String> { - self.state_db.get_offstate(key, self) + self.storage_db.get(key) .map_err(|e| format!("Database backend error: {:?}", e)) } fn pairs(&self) -> Vec<(Vec, Vec)> { - self.state_db.get_offstate_pairs(self) + self.storage_db.pairs() } } + struct DbGenesisStorage(pub H256); impl DbGenesisStorage { @@ -1524,7 +1568,10 @@ impl client::backend::Backend for Backend whe let genesis_storage = DbGenesisStorage::new(); let root = genesis_storage.0.clone(); // TODO EMCH see genesis impl: that is empty storage - let genesis_offstate = TODO2; + let genesis_offstate = TODO2At { + storage_db: TODO2, + state: h, + }; let db_state = DbState::new(Arc::new(genesis_storage), root, Arc::new(genesis_offstate)); let state = RefTrackingState::new(db_state, self.storage.clone(), None); return Ok(CachingState::new(state, self.shared_cache.clone(), None)); @@ -1539,7 +1586,11 @@ impl client::backend::Backend for Backend whe let root = H256::from_slice(hdr.state_root().as_ref()); //let block_number = hdr.number().clone(); //let db_state = DbState::new(self.storage.clone(), root, Arc::new(TODO::new(block_number))); - let db_state = DbState::new(self.storage.clone(), root, self.storage.clone()); + let offstate = StorageDbAt { + storage_db: self.storage.clone(), + state: hash.clone(), + }; + let db_state = DbState::new(self.storage.clone(), root, Arc::new(offstate)); let state = RefTrackingState::new(db_state, self.storage.clone(), Some(hash.clone())); Ok(CachingState::new(state, self.shared_cache.clone(), Some(hash))) } else { diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index 55aed958d1014..3866de329bc8c 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -69,13 +69,20 @@ pub trait NodeDb { } /// Backend database trait. Read-only. -pub trait OffstateDb { +/// +/// State parameter indicate where to query offstate storage, +/// this is a Block hash, as branch index are +/// build on load, with a persistence for this +/// data it would be more direct to use a +/// tuple of block number and branchindex. +pub trait OffstateDb { type Error: fmt::Debug; /// Get state trie node. - fn get_offstate(&self, key: &[u8]) -> Result, Self::Error>; + fn get_offstate(&self, key: &[u8], state: &State) -> Result, Self::Error>; + /// Get all pairs of key at current state. - fn get_offstate_pairs(&self) -> Vec<(OffstateKey, DBValue)>; + fn get_offstate_pairs(&self, state: &State) -> Vec<(OffstateKey, DBValue)>; } /// Error type. @@ -357,19 +364,21 @@ impl StateDbSync { db.get(key.as_ref()).map_err(|e| Error::Db(e)) } - pub fn get_offstate( + pub fn get_offstate>( &self, key: &[u8], + state: &BlockHash, db: &D, ) -> Result, Error> { if let Some(value) = self.non_canonical.get_offstate(key) { return Ok(Some(value)); } - db.get_offstate(key).map_err(|e| Error::Db(e)) + db.get_offstate(key, state).map_err(|e| Error::Db(e)) } - pub fn get_offstate_pairs( + pub fn get_offstate_pairs>( &self, + state: &BlockHash, db: &D, ) -> Vec<(OffstateKey, DBValue)> { // TODO get non_canonical optional values and then filter @@ -450,20 +459,22 @@ impl StateDb { } /// Get a value from non-canonical/pruning overlay or the backing DB. - pub fn get_offstate( + pub fn get_offstate>( &self, key: &[u8], + state: &BlockHash, db: &D, ) -> Result, Error> { - self.db.read().get_offstate(key, db) + self.db.read().get_offstate(key, state, db) } /// Get pairs values offstate. - pub fn get_offstate_pairs( + pub fn get_offstate_pairs>( &self, + state: &BlockHash, db: &D, ) -> Vec<(OffstateKey, DBValue)> { - self.db.read().get_offstate_pairs(db) + self.db.read().get_offstate_pairs(state, db) } /// Revert all non-canonical blocks with the best block number. diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index c5801f3627d85..224cfc6801240 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -35,14 +35,14 @@ impl MetaDb for TestDb { } } -impl OffstateDb for TestDb { +impl OffstateDb for TestDb { type Error = (); - fn get_offstate(&self, key: &[u8]) -> Result, ()> { + fn get_offstate(&self, key: &[u8], state: &H256) -> Result, ()> { Ok(self.offstate.get(key).cloned()) } - fn get_offstate_pairs(&self) -> Vec<(OffstateKey, DBValue)> { + fn get_offstate_pairs(&self, state: &H256) -> Vec<(OffstateKey, DBValue)> { self.offstate.iter().map(|(a, b)| (a.clone(), b.clone())).collect() } } diff --git a/core/state-machine/src/offstate_backend.rs b/core/state-machine/src/offstate_backend.rs index 7ffe84c5117bf..f75ff77466fd4 100644 --- a/core/state-machine/src/offstate_backend.rs +++ b/core/state-machine/src/offstate_backend.rs @@ -21,11 +21,17 @@ use std::sync::Arc; use std::ops::Deref; -/// TODO EMCH replace ofstate storage? -pub trait OffstateStorage: Send + Sync { +pub trait OffstateStorage: Send + Sync { + /// Retrieve a value from storage under given key. fn get(&self, key: &[u8]) -> Result>, String>; + /// Access inner state to query. + fn state(&self) -> &State; + + /// Change inner state to query, return old state + fn change_state(&mut self, state: State) -> State; + /// Return in memory all values for this backend, mainly for /// tests. fn pairs(&self) -> Vec<(Vec, Vec)>; @@ -85,7 +91,7 @@ impl OffstateBackendStorage for TODO2 { } // This implementation is used by normal storage trie clients. -impl OffstateBackendStorage for Arc { +impl OffstateBackendStorage for Arc> { fn get(&self, key: &[u8]) -> Result>, String> { OffstateStorage::get(self.deref(), key) } @@ -96,7 +102,15 @@ impl OffstateBackendStorage for Arc { -impl OffstateStorage for TODO { +impl OffstateStorage for TODO { + + fn state(&self) -> &N { + &self.0 + } + + fn change_state(&mut self, state: N) -> N { + std::mem::replace(&mut self.0, state) + } fn get(&self, key: &[u8]) -> Result>, String> { ::get(&self, key) @@ -108,8 +122,16 @@ impl OffstateStorage for TODO { } +// TODO EMCH remove?? +impl OffstateStorage<()> for TODO2 { -impl OffstateStorage for TODO2 { + fn state(&self) -> &() { + &() + } + + fn change_state(&mut self, _state: ()) -> () { + () + } fn get(&self, key: &[u8]) -> Result>, String> { ::get(&self, key) From 934c80060b7ca49083c38acfa720512a0b839dc4 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 25 Sep 2019 17:49:17 +0200 Subject: [PATCH 06/68] passing block hash as state in a convoluted way --- core/state-db/src/lib.rs | 2 +- core/state-db/src/noncanonical.rs | 85 +++++++++++++++---------------- 2 files changed, 43 insertions(+), 44 deletions(-) diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index 3866de329bc8c..269636f5e4428 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -370,7 +370,7 @@ impl StateDbSync { state: &BlockHash, db: &D, ) -> Result, Error> { - if let Some(value) = self.non_canonical.get_offstate(key) { + if let Some(value) = self.non_canonical.get_offstate(key, state) { return Ok(Some(value)); } db.get_offstate(key, state).map_err(|e| Error::Db(e)) diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 59983e74c9037..d4e81ff889089 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -473,7 +473,7 @@ impl NonCanonicalOverlay { /// TODO also need branch ix as parameter... (need context) /// or maybe a Number is enough (considering the way levels /// seems to work). - pub fn get_offstate(&self, key: &[u8]) -> Option { + pub fn get_offstate(&self, key: &[u8], state: &BlockHash) -> Option { if let Some((_, value)) = self.offstate_values.get(key) { return Some(value.clone()); } @@ -560,17 +560,17 @@ mod tests { overlay.get(&H256::from_low_u64_be(key)) == Some(H256::from_low_u64_be(key).as_bytes().to_vec()) } - fn contains_offstate(overlay: &NonCanonicalOverlay, key: u64) -> bool { - overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec()) + fn contains_offstate(overlay: &NonCanonicalOverlay, key: u64, state: &H256) -> bool { + overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), state) == Some(H256::from_low_u64_be(key).as_bytes().to_vec()) } - fn contains_both(overlay: &NonCanonicalOverlay, key: u64) -> bool { - contains(overlay, key) && contains_offstate(overlay, key) + fn contains_both(overlay: &NonCanonicalOverlay, key: u64, state: &H256) -> bool { + contains(overlay, key) && contains_offstate(overlay, key, state) } - fn contains_any(overlay: &NonCanonicalOverlay, key: u64) -> bool { - contains(overlay, key) || contains_offstate(overlay, key) + fn contains_any(overlay: &NonCanonicalOverlay, key: u64, state: &H256) -> bool { + contains(overlay, key) || contains_offstate(overlay, key, state) } @@ -755,26 +755,26 @@ mod tests { &h1, 1, &H256::default(), changeset1, offstate_changeset1, ).unwrap()); - assert!(contains_both(&overlay, 5)); + assert!(contains_both(&overlay, 5, &h1)); db.commit(&overlay.insert::( &h2, 2, &h1, changeset2, offstate_changeset2, ).unwrap()); - assert!(contains_both(&overlay, 7)); - assert!(contains_both(&overlay, 5)); + assert!(contains_both(&overlay, 7, &h2)); + assert!(contains_both(&overlay, 5, &h2)); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 2); let mut commit = CommitSet::default(); overlay.canonicalize::(&h1, &mut commit).unwrap(); db.commit(&commit); - assert!(contains_both(&overlay, 5)); + assert!(contains_both(&overlay, 5, &h2)); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 2); overlay.apply_pending(); assert_eq!(overlay.levels.len(), 1); assert_eq!(overlay.parents.len(), 1); - assert!(!contains_any(&overlay, 5)); - assert!(contains_both(&overlay, 7)); + assert!(!contains_any(&overlay, 5, &h1)); + assert!(contains_both(&overlay, 7, &h2)); let mut commit = CommitSet::default(); overlay.canonicalize::(&h2, &mut commit).unwrap(); db.commit(&commit); @@ -796,13 +796,13 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); db.commit(&overlay.insert::(&h_1, 1, &H256::default(), c_1, o_c_1).unwrap()); db.commit(&overlay.insert::(&h_2, 1, &H256::default(), c_2, o_c_2).unwrap()); - assert!(contains_both(&overlay, 1)); + assert!(contains_both(&overlay, 1, &h_2)); let mut commit = CommitSet::default(); overlay.canonicalize::(&h_1, &mut commit).unwrap(); db.commit(&commit); - assert!(contains_both(&overlay, 1)); + assert!(contains_both(&overlay, 1, &h_2)); overlay.apply_pending(); - assert!(!contains_any(&overlay, 1)); + assert!(!contains_any(&overlay, 1, &h_2)); } #[test] @@ -893,12 +893,12 @@ mod tests { db.commit(&overlay.insert::(&h_1_2_3, 3, &h_1_2, c_1_2_3, o_c_1_2_3).unwrap()); db.commit(&overlay.insert::(&h_2_1_1, 3, &h_2_1, c_2_1_1, o_c_2_1_1).unwrap()); - assert!(contains_both(&overlay, 2)); - assert!(contains_both(&overlay, 11)); - assert!(contains_both(&overlay, 21)); - assert!(contains_both(&overlay, 111)); - assert!(contains_both(&overlay, 122)); - assert!(contains_both(&overlay, 211)); + assert!(contains_both(&overlay, 2, &h_2_1_1)); + assert!(contains_both(&overlay, 11, &h_1_1_1)); + assert!(contains_both(&overlay, 21, &h_2_1_1)); + assert!(contains_both(&overlay, 111, &h_1_1_1)); + assert!(contains_both(&overlay, 122, &h_1_2_2)); + assert!(contains_both(&overlay, 211, &h_2_1_1)); assert_eq!(overlay.levels.len(), 3); assert_eq!(overlay.parents.len(), 11); assert_eq!(overlay.last_canonicalized, Some((H256::default(), 0))); @@ -916,13 +916,12 @@ mod tests { overlay.apply_pending(); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 6); - assert!(!contains_any(&overlay, 1)); - assert!(!contains_any(&overlay, 2)); - assert!(!contains_any(&overlay, 21)); - assert!(!contains_any(&overlay, 22)); - assert!(!contains_any(&overlay, 211)); - assert!(contains_both(&overlay, 111)); - assert!(!contains_any(&overlay, 211)); + assert!(!contains_any(&overlay, 1, &h_1)); + assert!(!contains_any(&overlay, 2, &h_2)); + assert!(!contains_any(&overlay, 21, &h_2_1)); + assert!(!contains_any(&overlay, 22, &h_2_2)); + assert!(!contains_any(&overlay, 211, &h_2_1_1)); + assert!(contains_both(&overlay, 111, &h_1_1_1)); // check that journals are deleted assert!(db.get_meta(&to_journal_key(1, 0)).unwrap().is_none()); assert!(db.get_meta(&to_journal_key(1, 1)).unwrap().is_none()); @@ -937,11 +936,11 @@ mod tests { overlay.apply_pending(); assert_eq!(overlay.levels.len(), 1); assert_eq!(overlay.parents.len(), 3); - assert!(!contains_any(&overlay, 11)); - assert!(!contains_any(&overlay, 111)); - assert!(contains_both(&overlay, 121)); - assert!(contains_both(&overlay, 122)); - assert!(contains_both(&overlay, 123)); + assert!(!contains_any(&overlay, 11, &h_1_1)); + assert!(!contains_any(&overlay, 111, &h_1_1_1)); + assert!(contains_both(&overlay, 121, &h_1_2_1)); + assert!(contains_both(&overlay, 122, &h_1_2_2)); + assert!(contains_both(&overlay, 123, &h_1_2_3)); assert!(overlay.have_block(&h_1_2_1)); assert!(!overlay.have_block(&h_1_2)); assert!(!overlay.have_block(&h_1_1)); @@ -978,11 +977,11 @@ mod tests { &h2, 2, &h1, changeset2, ochangeset2, ).unwrap()); - assert!(contains_both(&overlay, 7)); + assert!(contains_both(&overlay, 7, &h2)); db.commit(&overlay.revert_one().unwrap()); assert_eq!(overlay.parents.len(), 1); - assert!(contains_both(&overlay, 5)); - assert!(!contains_any(&overlay, 7)); + assert!(contains_both(&overlay, 5, &h2)); + assert!(!contains_any(&overlay, 7, &h2)); db.commit(&overlay.revert_one().unwrap()); assert_eq!(overlay.levels.len(), 0); assert_eq!(overlay.parents.len(), 0); @@ -1015,13 +1014,13 @@ mod tests { &h2_2, 2, &h1, changeset3, ochangeset3, ).unwrap(); - assert!(contains_both(&overlay, 7)); - assert!(contains_both(&overlay, 5)); - assert!(contains_both(&overlay, 9)); + assert!(contains_both(&overlay, 7, &h2_1)); + assert!(contains_both(&overlay, 5, &h2_1)); + assert!(contains_both(&overlay, 9, &h2_2)); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 3); overlay.revert_pending(); - assert!(!contains_any(&overlay, 5)); + assert!(!contains_any(&overlay, 5, &h1)); assert_eq!(overlay.levels.len(), 0); assert_eq!(overlay.parents.len(), 0); } @@ -1046,8 +1045,8 @@ mod tests { overlay.canonicalize::(&h_2, &mut commit).unwrap(); db.commit(&commit); overlay.apply_pending(); - assert!(contains_both(&overlay, 1)); + assert!(contains_both(&overlay, 1, &h_1)); overlay.unpin(&h_1); - assert!(!contains_any(&overlay, 1)); + assert!(!contains_any(&overlay, 1, &h_1)); } } From 5d3d47ee8f56927c209ad74bdcede67467d29d2a Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 25 Sep 2019 19:21:41 +0200 Subject: [PATCH 07/68] Recover previous branch index code without persistence. --- core/state-db/src/branch.rs | 505 ++++++++++++++++++++++++++++++++++++ core/state-db/src/lib.rs | 4 + 2 files changed, 509 insertions(+) create mode 100644 core/state-db/src/branch.rs diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs new file mode 100644 index 0000000000000..82e772520e639 --- /dev/null +++ b/core/state-db/src/branch.rs @@ -0,0 +1,505 @@ + +// Copyright 2018-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Helper for managing the set of indexed available branches. + +use std::collections::{BTreeMap, BTreeSet, HashMap, btree_map::Entry}; +use std::cmp::Reverse; +use crate::Error; +use std::hash::Hash as StdHash; +use std::convert::TryInto; + + +/// Reference to state that is enough for query updates, but not +/// for gc. +/// Values are ordered by branch_ix, +/// and only a logic branch path should be present. +/// +/// Note that an alternative could be a pointer to a full state +/// branch for a given branch index, here we use an in memory +/// copied representation in relation to an actual use case. +pub type BranchRanges = Vec; + + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StatesBranchRef { + pub branch_index: u64, + pub state: LinearStatesRef, +} + +/// This is a simple range, end non inclusive. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LinearStatesRef { + pub start: u64, + pub end: u64, +} + +/// current branches range definition, indexed by branch +/// numbers. +/// +/// New branches index are using `last_index`. +/// `treshold` is a branch index under which branches are undefined +/// but data there is seen as finalized. +/// +/// Also acts as a cache, storage can store +/// unknown db value as `None`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RangeSet { + storage: BTreeMap>, + last_index: u64, + treshold: u64, + // change and removed concern both storage and appendable + changed: BTreeSet, + removed: BTreeSet, +} + +const DEFAULT_START_TRESHOLD: u64 = 1; + +impl Default for RangeSet { + fn default() -> Self { + RangeSet { + storage: BTreeMap::new(), + last_index: 0, + treshold: DEFAULT_START_TRESHOLD, + changed: BTreeSet::new(), + removed: BTreeSet::new(), + } + } +} + +impl RangeSet { + + #[cfg(test)] + /// test access to treshold. + pub fn range_treshold(&self) -> u64 { + self.treshold + } + + #[cfg(test)] + /// test access to strage. + pub fn inner_storage(&self) -> &BTreeMap> { + &self.storage + } + + + /// Construct a new range set + pub fn new(treshold: u64, last_index: u64) -> Self { + RangeSet { + storage: BTreeMap::new(), + last_index, + treshold, + changed: BTreeSet::new(), + removed: BTreeSet::new(), + } + } + + // TODO EMCH can rw lock over the latest accessed range (lru) and + // return an clone of it (arc of the value so it will be fine). this is call by state_at that is call a lot!!! + pub fn branch_ranges_from_cache( + &self, + mut branch_index: u64, + ) -> BranchRanges { + // TODO EMCH transform this method to an iterator!!! + // (avoid some intermediatory Vec (eg when building Hashset) + let mut result = Vec::new(); + if branch_index < self.treshold { + return result; + } + let mut previous_start = u64::max_value(); + loop { + if let Some(Some(StatesBranch{ state, parent_branch_index, .. })) = self.storage.get(&branch_index) { + // TODO EMCH consider vecdeque ?? + let state = if state.end > previous_start { + LinearStatesRef { + start: state.start, + end: previous_start, + } + } else { state.clone() }; + + previous_start = state.start; + + result.insert(0, StatesBranchRef { + state, + branch_index, + }); + + branch_index = *parent_branch_index; + if branch_index < self.treshold { + break; + } + } else { + debug_assert!(false, "inconsistent branch range cache"); + break; + } + } + result + } + + + /// Return anchor index for this branch history: + /// - same index as input if branch is not empty + /// - parent index if branch is empty + pub fn drop_state( + &mut self, + branch_index: u64, + ) -> Result> { + let mut do_remove = None; + match self.storage.get_mut(&branch_index) { + Some(Some(branch_state)) => { + if let Some(drop_index) = branch_state.drop_state() { + if drop_index == 0 { + self.removed.insert(branch_index); + do_remove = Some(branch_state.parent_branch_index); + } else { + branch_state.can_append = false; + self.changed.insert(branch_index); + } + } else { + // deleted branch, do nothing + } + }, + Some(None) => (), // already dropped. + None => // TODO not sure keeping this error (we may want to clear storage) + return Err(Error::InvalidRange), + } + + if let Some(parent_index) = do_remove { + self.storage.remove(&branch_index); + Ok(parent_index) + } else { + Ok(branch_index) + } + } + + /// Return anchor index for this branch history: + /// - same index as input if the branch was modifiable + /// - new index in case of branch range creation + pub fn add_state( + &mut self, + branch_index: u64, + number: u64, + ) -> Result> { + let mut create_new = false; + if branch_index == 0 || branch_index < self.treshold { + create_new = true; + } else { match self.storage.get_mut(&branch_index) { + Some(Some(branch_state)) => { + if branch_state.can_append && branch_state.can_add(number) { + branch_state.add_state(); + self.changed.insert(branch_index); + } else { + create_new = true; + } + }, + Some(None) => + return Err(Error::InvalidRange), + None => // TODO not sure keeping this error (we may want to clear storage) + return Err(Error::InvalidRange), + }} + + if create_new { + self.last_index += 1; + + + let state = StatesBranch::new(number, branch_index); + self.storage.insert(self.last_index, Some(state)); + self.changed.insert(self.last_index); + Ok(self.last_index) + } else { + Ok(branch_index) + } + } + + // TODO EMCH this access can be optimize at multiple places (by returning ref + // instead of an anchor_id). + pub fn state_ref(&self, branch_index: u64) -> Option { + self.storage.get(&branch_index).and_then(|v| v.as_ref().map(|v| v.state_ref())) + .map(|state| StatesBranchRef { + branch_index, + state, + }) + } + + /// update the range set on import. returns a displaced leaf if there was one. + pub fn import( + &mut self, + number: u64, + parent_branch_index: u64, + parent_branch_range: Option, + ) -> (BranchRanges, u64) { + + if let Some(mut branch_ranges) = parent_branch_range { + let anchor_index = self.add_state(parent_branch_index, number) + .expect("coherent branch index state"); // TODO EMCH fail in add_state + if anchor_index == parent_branch_index { + branch_ranges.pop(); + } + branch_ranges.push(self.state_ref(anchor_index).expect("added just above")); + (branch_ranges, anchor_index) + } else { + let anchor_index = self.add_state(parent_branch_index, number) + .expect("coherent branch index state"); // TODO EMCH fail in add_state + (vec![self.state_ref(anchor_index).expect("added just above")], anchor_index) + } + } + + /// Apply a post finalize update of treshold: requires + /// that values before new treshold are all clear (since + /// we stop maintaining branches for them). + pub fn update_finalize_treshold( + &mut self, + branch_index: u64, + full: bool, + ) { + + if branch_index <= self.treshold { + return; + } + // range set update + let old_treshold = self.treshold; + // we do not finalize current branch cause it + // can contains other blocks + self.treshold = branch_index; + let removed_ranges = if branch_index == 0 || !full { + // remove cached value under treshold only + let new_storage = self.storage.split_off(&(self.treshold)); + std::mem::replace(&mut self.storage, new_storage) + } else { + let new_storage = self.storage.split_off(&(self.treshold)); + let mut removed = std::mem::replace(&mut self.storage, new_storage); + self.finalize_full(&mut removed, branch_index); + removed + }; + self.removed.extend(removed_ranges.keys().cloned()); + } + + /// Apply a post finalize without considering treshold. + fn finalize_full( + &mut self, + output: &mut BTreeMap>, + branch_index: u64, + ) { + // TODO EMCH consider working directly on ordered vec (should be fastest in most cases) + let mut finalize_branches_map: BTreeMap<_, _> = self.branch_ranges_from_cache(branch_index) + .into_iter().map(|r| (r.branch_index, r.state)).collect(); + + for (index, state) in self.storage.iter_mut() { + if let Some(final_state) = finalize_branches_map.remove(&index) { + // update for case where end of range differs (see `branch_ranges_from_cache`). + state.as_mut().map(|state| { + if state.state != final_state { + output.insert(*index, Some(state.clone())); + state.state = final_state; + state.can_append = false; + } + }); + } else { + output.insert(*index, state.take()); + } + } + } + + #[cfg(test)] + pub fn test_finalize_full( + &mut self, + branch_index: u64, + ) { + let mut removed_ranges = BTreeMap::new(); + self.finalize_full(&mut removed_ranges, branch_index); + self.removed.extend(removed_ranges.keys().cloned()); + } + + /// Revert some ranges, without any way to revert. + /// Returning ranges for the parent index. + pub fn revert(&mut self, branch_ix: u64) -> BranchRanges { + let parent_branch_index = if branch_ix != 0 { + self.drop_state(branch_ix) + // silenced error + .unwrap_or(0) + } else { + 0 + }; + + self.branch_ranges_from_cache(parent_branch_index) + } + + #[cfg(test)] + pub fn contains_range(&self, branch_index: u64, size: u64) -> bool { + if let Some(Some(s)) = self.storage.get(&branch_index) { + (s.state_ref().end - s.state_ref().start) == size + } else { + false + } + } + +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StatesBranch { + state: LinearStatesRef, + can_append: bool, + parent_branch_index: u64, +} + + +// TODO EMCH temporary structs until merge with historied-data branch. + +type LinearStates = StatesBranch; + +impl Default for LinearStates { + // initialize with one element + fn default() -> Self { + Self::new(0, 0) + } +} + +impl LinearStates { + pub fn new(offset: u64, parent_branch_index: u64) -> Self { + let offset = offset as u64; + LinearStates { + state: LinearStatesRef { + start: offset, + end: offset + 1, + }, + can_append: true, + parent_branch_index, + } + } + + pub fn state_ref(&self) -> LinearStatesRef { + self.state.clone() + } + + pub fn has_deleted_index(&self) -> bool { + !self.can_append + } + + pub fn add_state(&mut self) -> bool { + if !self.has_deleted_index() { + self.state.end += 1; + true + } else { + false + } + } + + /// return possible dropped state + pub fn drop_state(&mut self) -> Option { + if self.state.end - self.state.start > 0 { + self.state.end -= 1; + Some(self.state.end) + } else { + None + } + } + + /// Return true if state exists. + pub fn get_state(&self, index: u64) -> bool { + index < self.state.end && index >= self.state.start + } + + /// Return true if you can add this index. + pub fn can_add(&self, index: u64) -> bool { + index == self.state.end + } + + pub fn latest_ix(&self) -> Option { + if self.state.end - self.state.start > 0 { + Some(self.state.end - 1) + } else { + None + } + } + +} + +// TODO EMCH end from historied - data +// +mod test { + use super::*; + + // set: + // 0 + // |> 1: [10,1] + // |> 2: [10,2] [11,1] + // |> 3: [11,2] + // |> 4: [11,123] [12,1] + // full finalize: 3 + // 0 + // |> 2: [10,2] + // |> 3: [11,2] + fn build_finalize_set() -> (RangeSet, u64) { + let mut set = RangeSet::default(); + set.import(10, 0, None); + let (b1, anchor_1) = set.import(10, 0, None); + set.import(11, anchor_1, Some(b1.clone())); + let (_, finalize) = set.import(11, anchor_1, Some(b1.clone())); + let (b2, anchor_2) = set.import(11, anchor_1, Some(b1)); + set.import(12, anchor_2, Some(b2)); + + (set, finalize) + } + + #[test] + fn branch_range_finalize() { + const PREFIX: &[u8] = b"prefix"; + const TRESHOLD: &[u8] = b"hijkl"; + const LAST_INDEX: &[u8] = b"mnopq"; + let with_full_finalize = | + full: bool, + ranges_valid: &[(u64, u64)], + not_ranges: &[(u64, u64)], + ranges2: Option<&[(u64, u64)]>, + | { + let (mut ranges, finalize) = build_finalize_set(); + + // test ranges changes + if let Some(range2) = ranges2 { + let mut ranges2 = ranges.clone(); + let _ = ranges2.test_finalize_full(finalize); + for (range, size) in range2 { + assert!(ranges2.contains_range(*range, *size), "{} {}", range, size); + } + } + + let _ = ranges.update_finalize_treshold(finalize, full); + + assert!(ranges.range_treshold() == 3); + + for (range, size) in ranges_valid { + assert!(ranges.contains_range(*range, *size), "contains {:?}", range); + } + for (range, size) in not_ranges { + assert!(!ranges.contains_range(*range, *size), "contains {:?}", range); + } + + }; + + with_full_finalize(false, + &[(3, 1), (4, 2)][..], + &[(1, 1), (2, 2)][..], + None, + ); + with_full_finalize(true, + &[(3, 1)][..], + &[(1, 1), (2, 2), (4, 2)][..], + // here 2, 1 test state removal from finalize branch + Some(&[(2, 1), (3, 1)][..]), + ); + } + +} diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index 269636f5e4428..d7c11e143aa30 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -31,6 +31,7 @@ mod noncanonical; mod pruning; +mod branch; #[cfg(test)] mod test; use std::fmt; @@ -97,6 +98,8 @@ pub enum Error { InvalidBlockNumber, /// Trying to insert block with unknown parent. InvalidParent, + /// branch range access error + InvalidRange, } /// Pinning error type. @@ -119,6 +122,7 @@ impl fmt::Debug for Error { Error::InvalidBlock => write!(f, "Trying to canonicalize invalid block"), Error::InvalidBlockNumber => write!(f, "Trying to insert block with invalid number"), Error::InvalidParent => write!(f, "Trying to insert block with unknown parent"), + Error::InvalidRange => write!(f, "Trying to use invalid branch range"), } } } From aeab06485ec3ce658c9c54a550d95f11749fb668 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Sep 2019 11:52:33 +0200 Subject: [PATCH 08/68] let's remove branch range set of changed and removed in next commit --- core/state-db/src/noncanonical.rs | 76 +++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index d4e81ff889089..b2b7245cd7091 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -25,22 +25,30 @@ use std::collections::{HashMap, VecDeque, hash_map::Entry}; use super::{Error, DBValue, ChangeSet, CommitSet, MetaDb, Hash, to_meta_key, OffstateKey}; use codec::{Encode, Decode}; use log::trace; +use crate::branch::RangeSet; const NON_CANONICAL_JOURNAL: &[u8] = b"noncanonical_journal"; const NON_CANONICAL_OFFSTATE_JOURNAL: &[u8] = b"offstate_noncanonical_journal"; const LAST_CANONICAL: &[u8] = b"last_canonical"; +type BranchIndex = u64; + +type BlockNumber = u64; + /// See module documentation. pub struct NonCanonicalOverlay { - last_canonicalized: Option<(BlockHash, u64)>, + last_canonicalized: Option<(BlockHash, BlockNumber)>, levels: VecDeque>>, - parents: HashMap, + parents: HashMap, pending_canonicalizations: Vec, pending_insertions: Vec, values: HashMap, //ref counted + branches: RangeSet, offstate_values: HashMap, //ref counted - //would be deleted but kept around because block is pinned - // TODO EMCH sense if pinning offstate? + // would be deleted but kept around because block is pinned + // TODO EMCH sense if pinning offstate? done while using state_at + // -> that is import processing (so we can revert) -> should be good + // to use on offstate too pinned: HashMap, HashMap)>, } @@ -58,11 +66,11 @@ struct OffstateJournalRecord { deleted: Vec, } -fn to_journal_key(block: u64, index: u64) -> Vec { +fn to_journal_key(block: BlockNumber, index: u64) -> Vec { to_meta_key(NON_CANONICAL_JOURNAL, &(block, index)) } -fn to_offstate_journal_key(block: u64, index: u64) -> Vec { +fn to_offstate_journal_key(block: BlockNumber, index: u64) -> Vec { to_meta_key(NON_CANONICAL_OFFSTATE_JOURNAL, &(block, index)) } @@ -114,14 +122,14 @@ fn discard_descendants( mut values: &mut HashMap, mut offstate_values: &mut HashMap, index: usize, - parents: &mut HashMap, + parents: &mut HashMap, pinned: &mut HashMap, HashMap)>, hash: &BlockHash, ) { let mut discarded = Vec::new(); if let Some(level) = levels.get_mut(index) { *level = level.drain(..).filter_map(|overlay| { - let parent = parents.get(&overlay.hash).expect("there is a parent entry for each entry in levels; qed").clone(); + let parent = parents.get(&overlay.hash).expect("there is a parent entry for each entry in levels; qed").0.clone(); if parent == *hash { parents.remove(&overlay.hash); discarded.push(overlay.hash); @@ -149,17 +157,20 @@ impl NonCanonicalOverlay { let last_canonicalized = db.get_meta(&to_meta_key(LAST_CANONICAL, &())) .map_err(|e| Error::Db(e))?; let last_canonicalized = match last_canonicalized { - Some(buffer) => Some(<(BlockHash, u64)>::decode(&mut buffer.as_slice())?), + Some(buffer) => Some(<(BlockHash, BlockNumber)>::decode(&mut buffer.as_slice())?), None => None, }; let mut levels = VecDeque::new(); let mut parents = HashMap::new(); let mut values = HashMap::new(); let mut offstate_values = HashMap::new(); + let mut branches = RangeSet::default(); if let Some((ref hash, mut block)) = last_canonicalized { // read the journal trace!(target: "state-db", "Reading uncanonicalized journal. Last canonicalized #{} ({:?})", block, hash); let mut total: u64 = 0; + let mut parent_branch_index = 0u64; + let mut parent_branch_range = None; block += 1; loop { let mut index: u64 = 0; @@ -170,6 +181,18 @@ impl NonCanonicalOverlay { match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? { Some(record) => { let record: JournalRecord = Decode::decode(&mut record.as_slice())?; + + // TODO EMCh following two range fetch can be optimize when parent level + // got a single element. + // fetch parent info + parent_branch_index = parents.get(&record.parent_hash).map(|(_, i)| *i).unwrap_or(0); + parent_branch_range = Some(branches.branch_ranges_from_cache(parent_branch_index)); + let (_branch_range, branch_index) = branches.import( + block, + parent_branch_index, + parent_branch_range, + ); + let (offstate_record_inserted, offstate_record_deleted) = if let Some(record) = db .get_meta(&offstate_journal_key).map_err(|e| Error::Db(e))? { let record = OffstateJournalRecord::decode(&mut record.as_slice())?; @@ -203,7 +226,7 @@ impl NonCanonicalOverlay { overlay.offstate_deleted.len(), ); level.push(overlay); - parents.insert(record.hash, record.parent_hash); + parents.insert(record.hash, (record.parent_hash, branch_index)); index += 1; total += 1; }, @@ -225,16 +248,18 @@ impl NonCanonicalOverlay { pending_canonicalizations: Default::default(), pending_insertions: Default::default(), pinned: Default::default(), - values: values, - offstate_values: offstate_values, + values, + offstate_values, + branches, }) } - /// Insert a new block into the overlay. If inserted on the second level or lover expects parent to be present in the window. + /// Insert a new block into the overlay. If inserted on the second level or lower + /// expects parent to be present in the window. pub fn insert( &mut self, hash: &BlockHash, - number: u64, + number: BlockNumber, parent_hash: &BlockHash, changeset: ChangeSet, offstate_changeset: ChangeSet, @@ -288,7 +313,15 @@ impl NonCanonicalOverlay { offstate_deleted: offstate_changeset.deleted.clone(), }; level.push(overlay); - self.parents.insert(hash.clone(), parent_hash.clone()); + let parent_branch_index = self.parents.get(&parent_hash).map(|(_, i)| *i).unwrap_or(0); + let parent_branch_range = Some(self.branches.branch_ranges_from_cache(parent_branch_index)); + let (_branch_range, branch_index) = self.branches.import( + number, + parent_branch_index, + parent_branch_range, + ); + + self.parents.insert(hash.clone(), (parent_hash.clone(), branch_index)); let journal_record = JournalRecord { hash: hash.clone(), parent_hash: parent_hash.clone(), @@ -327,8 +360,9 @@ impl NonCanonicalOverlay { ) { if let Some(level) = self.levels.get(level_index) { level.iter().for_each(|overlay| { - let parent = self.parents.get(&overlay.hash).expect("there is a parent entry for each entry in levels; qed").clone(); - if parent == *hash { + let (parent, _branch_index) = self.parents.get(&overlay.hash) + .expect("there is a parent entry for each entry in levels; qed"); + if parent == hash { discarded_journals.push(overlay.journal_key.clone()); discarded_journals.push(overlay.offstate_journal_key.clone()); discarded_blocks.push(overlay.hash.clone()); @@ -338,11 +372,11 @@ impl NonCanonicalOverlay { } } - fn front_block_number(&self) -> u64 { + fn front_block_number(&self) -> BlockNumber { self.last_canonicalized.as_ref().map(|&(_, n)| n + 1).unwrap_or(0) } - pub fn last_canonicalized_block_number(&self) -> Option { + pub fn last_canonicalized_block_number(&self) -> Option { match self.last_canonicalized.as_ref().map(|&(_, n)| n) { Some(n) => Some(n + self.pending_canonicalizations.len() as u64), None if !self.pending_canonicalizations.is_empty() => Some(self.pending_canonicalizations.len() as u64), @@ -412,6 +446,10 @@ impl NonCanonicalOverlay { fn apply_canonicalizations(&mut self) { let last = self.pending_canonicalizations.last().cloned(); + if let Some((_, branch_index_cannonicalize)) = last.as_ref().and_then(|last| self.parents.get(last)) { + // using costy finalize + self.branches.update_finalize_treshold(*branch_index_cannonicalize, true); + } let count = self.pending_canonicalizations.len() as u64; for hash in self.pending_canonicalizations.drain(..) { trace!(target: "state-db", "Post canonicalizing {:?}", hash); From 57e862e99c023752fab2e6cf313b38133b6390d5 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Sep 2019 11:58:24 +0200 Subject: [PATCH 09/68] removed delta in branch (not needed if no persistence) --- core/state-db/src/branch.rs | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index 82e772520e639..4c2f023ad0ce1 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -62,9 +62,6 @@ pub struct RangeSet { storage: BTreeMap>, last_index: u64, treshold: u64, - // change and removed concern both storage and appendable - changed: BTreeSet, - removed: BTreeSet, } const DEFAULT_START_TRESHOLD: u64 = 1; @@ -75,8 +72,6 @@ impl Default for RangeSet { storage: BTreeMap::new(), last_index: 0, treshold: DEFAULT_START_TRESHOLD, - changed: BTreeSet::new(), - removed: BTreeSet::new(), } } } @@ -102,8 +97,6 @@ impl RangeSet { storage: BTreeMap::new(), last_index, treshold, - changed: BTreeSet::new(), - removed: BTreeSet::new(), } } @@ -162,11 +155,9 @@ impl RangeSet { Some(Some(branch_state)) => { if let Some(drop_index) = branch_state.drop_state() { if drop_index == 0 { - self.removed.insert(branch_index); do_remove = Some(branch_state.parent_branch_index); } else { branch_state.can_append = false; - self.changed.insert(branch_index); } } else { // deleted branch, do nothing @@ -200,7 +191,6 @@ impl RangeSet { Some(Some(branch_state)) => { if branch_state.can_append && branch_state.can_add(number) { branch_state.add_state(); - self.changed.insert(branch_index); } else { create_new = true; } @@ -217,7 +207,6 @@ impl RangeSet { let state = StatesBranch::new(number, branch_index); self.storage.insert(self.last_index, Some(state)); - self.changed.insert(self.last_index); Ok(self.last_index) } else { Ok(branch_index) @@ -274,23 +263,20 @@ impl RangeSet { // we do not finalize current branch cause it // can contains other blocks self.treshold = branch_index; - let removed_ranges = if branch_index == 0 || !full { + if branch_index == 0 || !full { // remove cached value under treshold only let new_storage = self.storage.split_off(&(self.treshold)); - std::mem::replace(&mut self.storage, new_storage) + self.storage = new_storage; } else { let new_storage = self.storage.split_off(&(self.treshold)); - let mut removed = std::mem::replace(&mut self.storage, new_storage); - self.finalize_full(&mut removed, branch_index); - removed + self.storage = new_storage; + self.finalize_full(branch_index); }; - self.removed.extend(removed_ranges.keys().cloned()); } /// Apply a post finalize without considering treshold. fn finalize_full( &mut self, - output: &mut BTreeMap>, branch_index: u64, ) { // TODO EMCH consider working directly on ordered vec (should be fastest in most cases) @@ -302,13 +288,12 @@ impl RangeSet { // update for case where end of range differs (see `branch_ranges_from_cache`). state.as_mut().map(|state| { if state.state != final_state { - output.insert(*index, Some(state.clone())); state.state = final_state; state.can_append = false; } }); } else { - output.insert(*index, state.take()); + *state = None; } } } @@ -318,9 +303,7 @@ impl RangeSet { &mut self, branch_index: u64, ) { - let mut removed_ranges = BTreeMap::new(); - self.finalize_full(&mut removed_ranges, branch_index); - self.removed.extend(removed_ranges.keys().cloned()); + self.finalize_full(branch_index); } /// Revert some ranges, without any way to revert. From 4edbf27f11d0660ccf26cf65a169b44d71d7cbde Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Sep 2019 17:31:15 +0200 Subject: [PATCH 10/68] Before using range instead of hash (we move range to upper client on pin). --- core/client/db/src/lib.rs | 2 +- core/state-db/src/branch.rs | 24 ++++- core/state-db/src/lib.rs | 9 +- core/state-db/src/noncanonical.rs | 163 ++++++++++++++++++++++++++---- 4 files changed, 167 insertions(+), 31 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 7d0d5d6beb296..e62874765a72b 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -1582,7 +1582,7 @@ impl client::backend::Backend for Backend whe match self.blockchain.header(block) { Ok(Some(ref hdr)) => { let hash = hdr.hash(); - if let Ok(()) = self.storage.state_db.pin(&hash) { + if let Ok(Some(range)) = self.storage.state_db.pin(&hash) { let root = H256::from_slice(hdr.state_root().as_ref()); //let block_number = hdr.number().clone(); //let db_state = DbState::new(self.storage.clone(), root, Arc::new(TODO::new(block_number))); diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index 4c2f023ad0ce1..1c096ac649601 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -59,6 +59,7 @@ pub struct LinearStatesRef { /// unknown db value as `None`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RangeSet { + // TODO EMCH using a option value makes not sense when all in memory storage: BTreeMap>, last_index: u64, treshold: u64, @@ -270,14 +271,17 @@ impl RangeSet { } else { let new_storage = self.storage.split_off(&(self.treshold)); self.storage = new_storage; - self.finalize_full(branch_index); + self.finalize_full(branch_index, None); }; } /// Apply a post finalize without considering treshold. - fn finalize_full( + /// TODO EMCH rename as it is also use to prepare a gc + /// without moving the treshould first. + pub fn finalize_full( &mut self, branch_index: u64, + linear_index: Option, ) { // TODO EMCH consider working directly on ordered vec (should be fastest in most cases) let mut finalize_branches_map: BTreeMap<_, _> = self.branch_ranges_from_cache(branch_index) @@ -285,12 +289,24 @@ impl RangeSet { for (index, state) in self.storage.iter_mut() { if let Some(final_state) = finalize_branches_map.remove(&index) { - // update for case where end of range differs (see `branch_ranges_from_cache`). state.as_mut().map(|state| { + // update for case where end of range differs + // (see `branch_ranges_from_cache`). if state.state != final_state { state.state = final_state; state.can_append = false; } + if *index == branch_index { + if let Some(linear_index) = linear_index { + state.state.end = std::cmp::min(linear_index + 1, state.state.end); + } + // set this branch as non appendable (ensure it get gc + // at some point even if there is no branching). + // Also if gc and treshold happen after this call, + // ensure this branch can get remove. + state.can_append = false; + } + }); } else { *state = None; @@ -303,7 +319,7 @@ impl RangeSet { &mut self, branch_index: u64, ) { - self.finalize_full(branch_index); + self.finalize_full(branch_index, None); } /// Revert some ranges, without any way to revert. diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index d7c11e143aa30..e11ef9ea9ea35 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -41,6 +41,7 @@ use std::collections::{HashMap, hash_map::Entry}; use noncanonical::NonCanonicalOverlay; use pruning::RefWindow; use log::trace; +pub use branch::BranchRanges; /// Database value type. pub type DBValue = Vec; @@ -322,9 +323,9 @@ impl StateDbSync { } } - pub fn pin(&mut self, hash: &BlockHash) -> Result<(), PinError> { + pub fn pin(&mut self, hash: &BlockHash) -> Result, PinError> { match self.mode { - PruningMode::ArchiveAll => Ok(()), + PruningMode::ArchiveAll => Ok(self.non_canonical.get_branch_range(hash)), PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => { if self.non_canonical.have_block(hash) || self.pruning.as_ref().map_or(false, |pruning| pruning.have_block(hash)) @@ -335,7 +336,7 @@ impl StateDbSync { self.non_canonical.pin(hash); } *refs += 1; - Ok(()) + Ok(self.non_canonical.get_branch_range(hash)) } else { Err(PinError::InvalidBlock) } @@ -446,7 +447,7 @@ impl StateDb { } /// Prevents pruning of specified block and its descendants. - pub fn pin(&self, hash: &BlockHash) -> Result<(), PinError> { + pub fn pin(&self, hash: &BlockHash) -> Result, PinError> { self.db.write().pin(hash) } diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index b2b7245cd7091..90109affc61c7 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -21,11 +21,11 @@ //! `revert_pending` use std::fmt; -use std::collections::{HashMap, VecDeque, hash_map::Entry}; +use std::collections::{HashMap, VecDeque, hash_map::Entry, HashSet}; use super::{Error, DBValue, ChangeSet, CommitSet, MetaDb, Hash, to_meta_key, OffstateKey}; use codec::{Encode, Decode}; use log::trace; -use crate::branch::RangeSet; +use crate::branch::{RangeSet, BranchRanges}; const NON_CANONICAL_JOURNAL: &[u8] = b"noncanonical_journal"; const NON_CANONICAL_OFFSTATE_JOURNAL: &[u8] = b"offstate_noncanonical_journal"; @@ -49,9 +49,86 @@ pub struct NonCanonicalOverlay { // TODO EMCH sense if pinning offstate? done while using state_at // -> that is import processing (so we can revert) -> should be good // to use on offstate too - pinned: HashMap, HashMap)>, + // + /// second value is offstate pinned index: used in order to determine if the pinned + /// thread should block garbage collection. + pinned: HashMap, u64)>, + offstate_gc: OffstatePendingGC, } +#[derive(Default)] +/// offstate gc only happen when all pinned threads that where +/// running at the time of cannonicalisation are finished. +struct OffstatePendingGC { + /// Keep trace of last cannonicalization branch index height. + /// All data in state are added after this value (branch is + /// set as non modifiable on canonicalisation). + pending_canonicalisation_query: Option, + /// keys to gc that got their journal removed. + keys_pending_gc: HashSet, + /// branch index that are not garbage collected. + /// Note that it can also contain branch index created after cannonicalisation + /// query. + keep_indexes: Vec, +} + +impl OffstatePendingGC { + fn set_pending_gc(&mut self, branch_index: u64) { + self.pending_canonicalisation_query = Some(branch_index); + self.keep_indexes.clear(); + } + fn try_gc( + &mut self, + pinned: &HashMap, + ) { + if let Some(pending) = self.pending_canonicalisation_query { + if pending < self.max_pinned_index(pinned) { + + unimplemented!("TODO feed keepindexes with branch at pending then actual gc"); + + self.pending_canonicalisation_query = None; + self.keep_indexes.clear(); + } + } + } + fn pin(&mut self, branch_index: u64, set: &RangeSet) -> BranchRanges { + let range = set.branch_ranges_from_cache(branch_index); + if let Some(pending) = self.pending_canonicalisation_query { + // pending is set non modifiable before setting pending gc. + debug_assert!(pending < branch_index); + self.keep_indexes.push(range.clone()); + } + range + } + fn max_pinned_index( + &self, + pinned: &HashMap, + ) -> u64 { + let mut max = 0; + if let Some(pending) = self.pending_canonicalisation_query { + // max up to pending only + for (_, (_, index)) in pinned.iter() { + if *index > max && *index <= pending { + max = *index; + } + } + } else { + // global max + for (_, (_, index)) in pinned.iter() { + if *index > max { + max = *index; + } + } + } + max + } +} +struct OffstatePinnedThread { + /// index of branch which was pinned + branch_index: u64, + + +} #[derive(Encode, Decode)] struct JournalRecord { hash: BlockHash, @@ -98,6 +175,9 @@ fn discard_values( inserted: Vec, mut into: Option<&mut HashMap>, ) { + if into.is_none() { + return; + } for k in inserted { match values.entry(k) { Entry::Occupied(mut e) => { @@ -117,13 +197,37 @@ fn discard_values( } } +fn discard_offset_values( + values: &mut HashMap, + inserted: Vec, + into: &mut OffstatePendingGC, +) { + for k in inserted { + match values.entry(k) { + Entry::Occupied(mut e) => { + let (ref mut counter, _) = e.get_mut(); + *counter -= 1; + if *counter == 0 { + let (key, _) = e.remove_entry(); + into.keys_pending_gc.insert(key); + } + }, + Entry::Vacant(_) => { + debug_assert!(false, "Trying to discard missing value"); + } + } + } +} + + fn discard_descendants( levels: &mut VecDeque>>, mut values: &mut HashMap, mut offstate_values: &mut HashMap, index: usize, parents: &mut HashMap, - pinned: &mut HashMap, HashMap)>, + pinned: &mut HashMap, u64)>, + offstate_gc: &mut OffstatePendingGC, hash: &BlockHash, ) { let mut discarded = Vec::new(); @@ -135,10 +239,10 @@ fn discard_descendants( discarded.push(overlay.hash); let mut pinned = pinned.get_mut(hash); discard_values(&mut values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); - discard_values( + discard_offset_values( &mut offstate_values, overlay.offstate_inserted, - pinned.as_mut().map(|p| &mut p.1), + offstate_gc, ); None } else { @@ -147,7 +251,7 @@ fn discard_descendants( }).collect(); } for hash in discarded { - discard_descendants(levels, values, offstate_values, index + 1, parents, pinned, &hash); + discard_descendants(levels, values, offstate_values, index + 1, parents, pinned, offstate_gc, &hash); } } @@ -250,6 +354,7 @@ impl NonCanonicalOverlay { pinned: Default::default(), values, offstate_values, + offstate_gc: Default::default(), branches, }) } @@ -445,12 +550,18 @@ impl NonCanonicalOverlay { } fn apply_canonicalizations(&mut self) { + let count = self.pending_canonicalizations.len() as u64; + let last_block_number = self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1); let last = self.pending_canonicalizations.last().cloned(); - if let Some((_, branch_index_cannonicalize)) = last.as_ref().and_then(|last| self.parents.get(last)) { - // using costy finalize - self.branches.update_finalize_treshold(*branch_index_cannonicalize, true); + if let Some(branch_index_cannonicalize) = last.as_ref().and_then(|last| self.parents.get(last)) + .map(|(_, index)| *index) { + // set branch state synchronously + self.branches.finalize_full(branch_index_cannonicalize, Some(last_block_number)); + self.offstate_gc.set_pending_gc(branch_index_cannonicalize); + // try to run the garbage collection (can run later if there is + // pinned process). + self.offstate_gc.try_gc(&self.pinned); } - let count = self.pending_canonicalizations.len() as u64; for hash in self.pending_canonicalizations.drain(..) { trace!(target: "state-db", "Post canonicalizing {:?}", hash); let level = self.levels.pop_front().expect("Hash validity is checked in `canonicalize`"); @@ -470,21 +581,22 @@ impl NonCanonicalOverlay { 0, &mut self.parents, &mut self.pinned, + &mut self.offstate_gc, &overlay.hash, ); } let mut pinned = self.pinned.get_mut(&overlay.hash); discard_values(&mut self.values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); - discard_values( + discard_offset_values( &mut self.offstate_values, overlay.offstate_inserted, - pinned.as_mut().map(|p| &mut p.1), + &mut self.offstate_gc, ); } } if let Some(hash) = last { - let last_canonicalized = (hash, self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1)); + let last_canonicalized = (hash, last_block_number); self.last_canonicalized = Some(last_canonicalized); } } @@ -515,11 +627,6 @@ impl NonCanonicalOverlay { if let Some((_, value)) = self.offstate_values.get(key) { return Some(value.clone()); } - for pinned in self.pinned.values() { - if let Some(value) = pinned.1.get(key) { - return Some(value.clone()); - } - } None } @@ -535,7 +642,9 @@ impl NonCanonicalOverlay { let mut commit = CommitSet::default(); for overlay in level.into_iter() { commit.meta.deleted.push(overlay.journal_key); - self.parents.remove(&overlay.hash); + if let Some((_, branch_index)) = self.parents.remove(&overlay.hash) { + self.branches.revert(branch_index); + } discard_values(&mut self.values, overlay.inserted, None); discard_values(&mut self.offstate_values, overlay.offstate_inserted, None); } @@ -576,8 +685,18 @@ impl NonCanonicalOverlay { } /// Pin state values in memory - pub fn pin(&mut self, hash: &BlockHash) { - self.pinned.insert(hash.clone(), Default::default()); + pub fn pin(&mut self, hash: &BlockHash) -> Option { + self.parents.get(hash).map(|(_, branch_index)| *branch_index).map(|branch_index| { + self.pinned.insert(hash.clone(), (Default::default(), branch_index)); + self.offstate_gc.pin(branch_index, &self.branches) + }) + } + + /// TODO EMCH aka get state for hash to query offstate storage. + pub fn get_branch_range(&self, hash: &BlockHash) -> Option { + self.parents.get(hash).map(|(_, branch_index)| *branch_index).map(|branch_index| { + self.branches.branch_ranges_from_cache(branch_index) + }) } /// Discard pinned state From 91b87d76c71c529805e3ea89868d637d6f7e66a0 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Sep 2019 20:34:52 +0200 Subject: [PATCH 11/68] use also block number for query (when not in statedb we need number). --- core/client/db/src/lib.rs | 25 ++++++++++++------------- core/state-db/src/lib.rs | 23 ++++++++++++----------- core/state-db/src/noncanonical.rs | 21 +++++++++++++-------- core/state-db/src/test.rs | 6 +++--- 4 files changed, 40 insertions(+), 35 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index e62874765a72b..c17e66acaf699 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -67,6 +67,7 @@ use crate::utils::{Meta, db_err, meta_keys, read_db, block_id_to_lookup_key, rea use client::leaves::{LeafSet, FinalizationDisplaced}; use client::children; use state_db::StateDb; +use state_db::BranchRanges; use crate::storage_cache::{CachingState, SharedCache, new_shared_cache}; use log::{trace, debug, warn}; pub use state_db::PruningMode; @@ -86,7 +87,7 @@ const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); pub type DbState = state_machine::TrieBackend< Arc>, Blake2Hasher, - Arc>, + Arc>, >; /// A reference tracking state. @@ -604,28 +605,27 @@ impl state_db::NodeDb for StorageDb { } } -// TODO EMCH try with Block::Hash but no real hope -impl state_db::OffstateDb for StorageDb { +impl state_db::OffstateDb> for StorageDb { type Error = io::Error; - fn get_offstate(&self, key: &[u8], state: &Block::Hash) -> Result>, Self::Error> { + fn get_offstate(&self, key: &[u8], state: &Option) -> Result>, Self::Error> { self.db.get(columns::OFFSTATE, key); unimplemented!("TODO map with history fetch at requested block") } - fn get_offstate_pairs(&self, state: &Block::Hash) -> Vec<(Vec, Vec)> { + fn get_offstate_pairs(&self, state: &Option) -> Vec<(Vec, Vec)> { unimplemented!("TODO need to be an iterator") } } -impl state_machine::OffstateStorage for StorageDbAt { +impl state_machine::OffstateStorage<(BranchRanges, u64)> for StorageDbAt { - fn state(&self) -> &Block::Hash { + fn state(&self) -> &(BranchRanges, u64) { &self.state } - fn change_state(&mut self, state: Block::Hash) -> Block::Hash { + fn change_state(&mut self, state: (BranchRanges, u64)) -> (BranchRanges, u64) { std::mem::replace(&mut self.state, state) } @@ -1570,7 +1570,7 @@ impl client::backend::Backend for Backend whe // TODO EMCH see genesis impl: that is empty storage let genesis_offstate = TODO2At { storage_db: TODO2, - state: h, + state: Default::default(), }; let db_state = DbState::new(Arc::new(genesis_storage), root, Arc::new(genesis_offstate)); let state = RefTrackingState::new(db_state, self.storage.clone(), None); @@ -1582,13 +1582,12 @@ impl client::backend::Backend for Backend whe match self.blockchain.header(block) { Ok(Some(ref hdr)) => { let hash = hdr.hash(); - if let Ok(Some(range)) = self.storage.state_db.pin(&hash) { + if let Ok(range) = self.storage.state_db.pin(&hash) { let root = H256::from_slice(hdr.state_root().as_ref()); - //let block_number = hdr.number().clone(); - //let db_state = DbState::new(self.storage.clone(), root, Arc::new(TODO::new(block_number))); + let block_number = hdr.number().clone().saturated_into::(); let offstate = StorageDbAt { storage_db: self.storage.clone(), - state: hash.clone(), + state: (range.unwrap_or_else(Default::default), block_number), }; let db_state = DbState::new(self.storage.clone(), root, Arc::new(offstate)); let state = RefTrackingState::new(db_state, self.storage.clone(), Some(hash.clone())); diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index e11ef9ea9ea35..6c93693c44dfa 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -77,7 +77,7 @@ pub trait NodeDb { /// build on load, with a persistence for this /// data it would be more direct to use a /// tuple of block number and branchindex. -pub trait OffstateDb { +pub trait OffstateDb { type Error: fmt::Debug; /// Get state trie node. @@ -235,6 +235,7 @@ impl StateDbSync { mut changeset: ChangeSet, mut offstate_changeset: ChangeSet, ) -> Result, Error> { + match self.mode { PruningMode::ArchiveAll => { changeset.deleted.clear(); @@ -369,21 +370,21 @@ impl StateDbSync { db.get(key.as_ref()).map_err(|e| Error::Db(e)) } - pub fn get_offstate>( + pub fn get_offstate>>( &self, key: &[u8], - state: &BlockHash, + state: &(BranchRanges, u64), db: &D, ) -> Result, Error> { - if let Some(value) = self.non_canonical.get_offstate(key, state) { + if let Some(value) = self.non_canonical.get_offstate(key, &state.0) { return Ok(Some(value)); } - db.get_offstate(key, state).map_err(|e| Error::Db(e)) + db.get_offstate(key, &Some(state.1)).map_err(|e| Error::Db(e)) } - pub fn get_offstate_pairs>( + pub fn get_offstate_pairs>>( &self, - state: &BlockHash, + state: &(BranchRanges, u64), db: &D, ) -> Vec<(OffstateKey, DBValue)> { // TODO get non_canonical optional values and then filter @@ -464,19 +465,19 @@ impl StateDb { } /// Get a value from non-canonical/pruning overlay or the backing DB. - pub fn get_offstate>( + pub fn get_offstate>>( &self, key: &[u8], - state: &BlockHash, + state: &(BranchRanges, u64), db: &D, ) -> Result, Error> { self.db.read().get_offstate(key, state, db) } /// Get pairs values offstate. - pub fn get_offstate_pairs>( + pub fn get_offstate_pairs>>( &self, - state: &BlockHash, + state: &(BranchRanges, u64), db: &D, ) -> Vec<(OffstateKey, DBValue)> { self.db.read().get_offstate_pairs(state, db) diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 90109affc61c7..f4c49fec8cc5d 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -91,6 +91,7 @@ impl OffstatePendingGC { } } } + fn pin(&mut self, branch_index: u64, set: &RangeSet) -> BranchRanges { let range = set.branch_ranges_from_cache(branch_index); if let Some(pending) = self.pending_canonicalisation_query { @@ -100,6 +101,7 @@ impl OffstatePendingGC { } range } + fn max_pinned_index( &self, pinned: &HashMap, @@ -550,13 +552,13 @@ impl NonCanonicalOverlay { } fn apply_canonicalizations(&mut self) { - let count = self.pending_canonicalizations.len() as u64; - let last_block_number = self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1); let last = self.pending_canonicalizations.last().cloned(); + let count = self.pending_canonicalizations.len() as u64; if let Some(branch_index_cannonicalize) = last.as_ref().and_then(|last| self.parents.get(last)) .map(|(_, index)| *index) { // set branch state synchronously - self.branches.finalize_full(branch_index_cannonicalize, Some(last_block_number)); + let block_number = self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1); + self.branches.finalize_full(branch_index_cannonicalize, Some(block_number)); self.offstate_gc.set_pending_gc(branch_index_cannonicalize); // try to run the garbage collection (can run later if there is // pinned process). @@ -596,7 +598,7 @@ impl NonCanonicalOverlay { } } if let Some(hash) = last { - let last_canonicalized = (hash, last_block_number); + let last_canonicalized = (hash, self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1)); self.last_canonicalized = Some(last_canonicalized); } } @@ -623,11 +625,12 @@ impl NonCanonicalOverlay { /// TODO also need branch ix as parameter... (need context) /// or maybe a Number is enough (considering the way levels /// seems to work). - pub fn get_offstate(&self, key: &[u8], state: &BlockHash) -> Option { - if let Some((_, value)) = self.offstate_values.get(key) { + pub fn get_offstate(&self, key: &[u8], state: &BranchRanges) -> Option { + unimplemented!("TODO"); +/* if let Some((_, value)) = self.offstate_values.get(key) { return Some(value.clone()); } - None + None*/ } /// Check if the block is in the canonicalization queue. @@ -712,13 +715,15 @@ mod tests { use super::{NonCanonicalOverlay, to_journal_key}; use crate::{ChangeSet, CommitSet, OffstateKey}; use crate::test::{make_db, make_changeset, make_offstate_changeset}; + use crate::branch::BranchRanges; fn contains(overlay: &NonCanonicalOverlay, key: u64) -> bool { overlay.get(&H256::from_low_u64_be(key)) == Some(H256::from_low_u64_be(key).as_bytes().to_vec()) } fn contains_offstate(overlay: &NonCanonicalOverlay, key: u64, state: &H256) -> bool { - overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), state) + overlay.get_branch_range(state).and_then(|state| + overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state)) == Some(H256::from_low_u64_be(key).as_bytes().to_vec()) } diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 224cfc6801240..41768bd74926f 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -35,14 +35,14 @@ impl MetaDb for TestDb { } } -impl OffstateDb for TestDb { +impl OffstateDb> for TestDb { type Error = (); - fn get_offstate(&self, key: &[u8], state: &H256) -> Result, ()> { + fn get_offstate(&self, key: &[u8], state: &Option) -> Result, ()> { Ok(self.offstate.get(key).cloned()) } - fn get_offstate_pairs(&self, state: &H256) -> Vec<(OffstateKey, DBValue)> { + fn get_offstate_pairs(&self, state: &Option) -> Vec<(OffstateKey, DBValue)> { self.offstate.iter().map(|(a, b)| (a.clone(), b.clone())).collect() } } From 966a09fa02b87204f558de35f8cfe020a5599a23 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 27 Sep 2019 14:24:19 +0200 Subject: [PATCH 12/68] Fixing finalize state algo. --- core/client/db/src/lib.rs | 3 +- core/state-db/src/branch.rs | 81 ++++++++++------------ core/state-db/src/lib.rs | 18 +++-- core/state-db/src/noncanonical.rs | 110 +++++++++++++++--------------- 4 files changed, 107 insertions(+), 105 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index c17e66acaf699..88517c4c2ed95 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -1582,7 +1582,8 @@ impl client::backend::Backend for Backend whe match self.blockchain.header(block) { Ok(Some(ref hdr)) => { let hash = hdr.hash(); - if let Ok(range) = self.storage.state_db.pin(&hash) { + if let Ok(()) = self.storage.state_db.pin(&hash) { + let range = self.storage.state_db.get_branch_range(&hash); let root = H256::from_slice(hdr.state_root().as_ref()); let block_number = hdr.number().clone().saturated_into::(); let offstate = StorageDbAt { diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index 1c096ac649601..9dd0aac10cdef 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -52,8 +52,7 @@ pub struct LinearStatesRef { /// numbers. /// /// New branches index are using `last_index`. -/// `treshold` is a branch index under which branches are undefined -/// but data there is seen as finalized. +/// `treshold` is a branch index under which branches are undefined. /// /// Also acts as a cache, storage can store /// unknown db value as `None`. @@ -253,6 +252,7 @@ impl RangeSet { pub fn update_finalize_treshold( &mut self, branch_index: u64, + linear_index: Option, full: bool, ) { @@ -268,6 +268,13 @@ impl RangeSet { // remove cached value under treshold only let new_storage = self.storage.split_off(&(self.treshold)); self.storage = new_storage; + // set this branch as non appendable (ensure it get gc + // at some point even if there is no branching). + // Also if gc and treshold happen after this call, + // ensure this branch can get remove. + // TODO EMCH this can also make sense for full + self.storage.get_mut(&branch_index) + .map(|state| state.as_mut().map(|state| state.can_append = false)); } else { let new_storage = self.storage.split_off(&(self.treshold)); self.storage = new_storage; @@ -283,35 +290,27 @@ impl RangeSet { branch_index: u64, linear_index: Option, ) { - // TODO EMCH consider working directly on ordered vec (should be fastest in most cases) - let mut finalize_branches_map: BTreeMap<_, _> = self.branch_ranges_from_cache(branch_index) - .into_iter().map(|r| (r.branch_index, r.state)).collect(); - - for (index, state) in self.storage.iter_mut() { - if let Some(final_state) = finalize_branches_map.remove(&index) { - state.as_mut().map(|state| { - // update for case where end of range differs - // (see `branch_ranges_from_cache`). - if state.state != final_state { - state.state = final_state; - state.can_append = false; - } - if *index == branch_index { - if let Some(linear_index) = linear_index { - state.state.end = std::cmp::min(linear_index + 1, state.state.end); - } - // set this branch as non appendable (ensure it get gc - // at some point even if there is no branching). - // Also if gc and treshold happen after this call, - // ensure this branch can get remove. - state.can_append = false; + if let Some(Some(state)) = self.storage.remove(&branch_index) { + let mut child_anchor: LinearStates = state.clone(); + child_anchor.state.start = linear_index.unwrap_or(child_anchor.state.start); + let old_storage = std::mem::replace(&mut self.storage, BTreeMap::new()); + self.storage.insert(branch_index, Some(child_anchor)); + for (index, state) in old_storage.into_iter().filter_map(|(k, v)| v.map(|v| (k, v))) { + // ordered property of branch index allows to skip in depth branch search + if let Some(Some(parent_state)) = self.storage.get(&state.parent_branch_index) { + // first state is splitted + // use property that start of a branch - 1 is origin of parent + // this is obvious for parent is first branch (truncated), but also + // true for case where we revert some state. + let linear_index_parent = state.state.start - 1; + if linear_index_parent < parent_state.state.end + && linear_index_parent >= parent_state.state.start { + self.storage.insert(index, Some(state)); } - - }); - } else { - *state = None; + } } } + } #[cfg(test)] @@ -435,7 +434,8 @@ mod test { // 0 // |> 1: [10,1] // |> 2: [10,2] [11,1] - // |> 3: [11,2] + // |> 3: [11,2] [12, 2] + // | > 5: [12,1] // |> 4: [11,123] [12,1] // full finalize: 3 // 0 @@ -446,9 +446,11 @@ mod test { set.import(10, 0, None); let (b1, anchor_1) = set.import(10, 0, None); set.import(11, anchor_1, Some(b1.clone())); - let (_, finalize) = set.import(11, anchor_1, Some(b1.clone())); + let (bf, finalize) = set.import(11, anchor_1, Some(b1.clone())); let (b2, anchor_2) = set.import(11, anchor_1, Some(b1)); set.import(12, anchor_2, Some(b2)); + set.import(12, finalize, Some(bf.clone())); + set.import(12, finalize, Some(bf)); (set, finalize) } @@ -462,20 +464,10 @@ mod test { full: bool, ranges_valid: &[(u64, u64)], not_ranges: &[(u64, u64)], - ranges2: Option<&[(u64, u64)]>, | { let (mut ranges, finalize) = build_finalize_set(); - // test ranges changes - if let Some(range2) = ranges2 { - let mut ranges2 = ranges.clone(); - let _ = ranges2.test_finalize_full(finalize); - for (range, size) in range2 { - assert!(ranges2.contains_range(*range, *size), "{} {}", range, size); - } - } - - let _ = ranges.update_finalize_treshold(finalize, full); + let _ = ranges.update_finalize_treshold(finalize, None, full); assert!(ranges.range_treshold() == 3); @@ -489,15 +481,12 @@ mod test { }; with_full_finalize(false, - &[(3, 1), (4, 2)][..], + &[(3, 2), (4, 2), (5, 1)][..], &[(1, 1), (2, 2)][..], - None, ); with_full_finalize(true, - &[(3, 1)][..], + &[(3, 2), (5, 1)][..], &[(1, 1), (2, 2), (4, 2)][..], - // here 2, 1 test state removal from finalize branch - Some(&[(2, 1), (3, 1)][..]), ); } diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index 6c93693c44dfa..adf2a01aae707 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -324,9 +324,14 @@ impl StateDbSync { } } - pub fn pin(&mut self, hash: &BlockHash) -> Result, PinError> { + /// TODO EMCH + pub fn get_branch_range(&self, hash: &BlockHash) -> Option { + self.non_canonical.get_branch_range(hash) + } + + pub fn pin(&mut self, hash: &BlockHash) -> Result<(), PinError> { match self.mode { - PruningMode::ArchiveAll => Ok(self.non_canonical.get_branch_range(hash)), + PruningMode::ArchiveAll => Ok(()), PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => { if self.non_canonical.have_block(hash) || self.pruning.as_ref().map_or(false, |pruning| pruning.have_block(hash)) @@ -337,7 +342,7 @@ impl StateDbSync { self.non_canonical.pin(hash); } *refs += 1; - Ok(self.non_canonical.get_branch_range(hash)) + Ok(()) } else { Err(PinError::InvalidBlock) } @@ -447,8 +452,13 @@ impl StateDb { self.db.write().canonicalize_block(hash) } + /// TODO EMCH + pub fn get_branch_range(&self, hash: &BlockHash) -> Option { + self.db.read().get_branch_range(hash) + } + /// Prevents pruning of specified block and its descendants. - pub fn pin(&self, hash: &BlockHash) -> Result, PinError> { + pub fn pin(&self, hash: &BlockHash) -> Result<(), PinError> { self.db.write().pin(hash) } diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index f4c49fec8cc5d..10cb6662c203b 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -35,6 +35,8 @@ type BranchIndex = u64; type BlockNumber = u64; +type GcIndex = u64; + /// See module documentation. pub struct NonCanonicalOverlay { last_canonicalized: Option<(BlockHash, BlockNumber)>, @@ -52,7 +54,7 @@ pub struct NonCanonicalOverlay { // /// second value is offstate pinned index: used in order to determine if the pinned /// thread should block garbage collection. - pinned: HashMap, u64)>, + pinned: HashMap, GcIndex)>, offstate_gc: OffstatePendingGC, } @@ -60,71 +62,67 @@ pub struct NonCanonicalOverlay { /// offstate gc only happen when all pinned threads that where /// running at the time of cannonicalisation are finished. struct OffstatePendingGC { + /// Each gc call uses a new index. + gc_last_index: GcIndex, /// Keep trace of last cannonicalization branch index height. /// All data in state are added after this value (branch is /// set as non modifiable on canonicalisation). pending_canonicalisation_query: Option, /// keys to gc that got their journal removed. keys_pending_gc: HashSet, - /// branch index that are not garbage collected. - /// Note that it can also contain branch index created after cannonicalisation - /// query. - keep_indexes: Vec, + /// store keys while waiting for a gc. + next_keys_pending_gc: HashSet, } impl OffstatePendingGC { fn set_pending_gc(&mut self, branch_index: u64) { + self.gc_last_index += 1; + if self.pending_canonicalisation_query.is_some() { + // TODO EMCH some better merge + self.keys_pending_gc.extend(self.next_keys_pending_gc.drain()); + } self.pending_canonicalisation_query = Some(branch_index); - self.keep_indexes.clear(); } + fn try_gc( &mut self, pinned: &HashMap, ) { if let Some(pending) = self.pending_canonicalisation_query { - if pending < self.max_pinned_index(pinned) { + if pending < self.min_pinned_index(pinned) { unimplemented!("TODO feed keepindexes with branch at pending then actual gc"); self.pending_canonicalisation_query = None; - self.keep_indexes.clear(); } } } - fn pin(&mut self, branch_index: u64, set: &RangeSet) -> BranchRanges { - let range = set.branch_ranges_from_cache(branch_index); - if let Some(pending) = self.pending_canonicalisation_query { - // pending is set non modifiable before setting pending gc. - debug_assert!(pending < branch_index); - self.keep_indexes.push(range.clone()); - } - range - } - - fn max_pinned_index( + fn min_pinned_index( &self, pinned: &HashMap, ) -> u64 { - let mut max = 0; - if let Some(pending) = self.pending_canonicalisation_query { - // max up to pending only - for (_, (_, index)) in pinned.iter() { - if *index > max && *index <= pending { - max = *index; - } - } - } else { - // global max - for (_, (_, index)) in pinned.iter() { - if *index > max { - max = *index; - } + let mut min = u64::max_value(); + // global min + for (_, (_, index)) in pinned.iter() { + if *index < min { + min = *index; } } - max + min } + + // TODO is it of any use?? + fn revert(&mut self) { + self.pending_canonicalisation_query = None; + // TODO EMCH here reverting on a double set_pending_gc + // will fail: need to stack those offstatepending_gc?? + self.keys_pending_gc.clear(); + self.next_keys_pending_gc.clear(); + } + } + struct OffstatePinnedThread { /// index of branch which was pinned branch_index: u64, @@ -204,6 +202,11 @@ fn discard_offset_values( inserted: Vec, into: &mut OffstatePendingGC, ) { + let into = if into.pending_canonicalisation_query.is_some() { + &mut into.next_keys_pending_gc + } else { + &mut into.keys_pending_gc + }; for k in inserted { match values.entry(k) { Entry::Occupied(mut e) => { @@ -211,7 +214,7 @@ fn discard_offset_values( *counter -= 1; if *counter == 0 { let (key, _) = e.remove_entry(); - into.keys_pending_gc.insert(key); + into.insert(key); } }, Entry::Vacant(_) => { @@ -275,8 +278,6 @@ impl NonCanonicalOverlay { // read the journal trace!(target: "state-db", "Reading uncanonicalized journal. Last canonicalized #{} ({:?})", block, hash); let mut total: u64 = 0; - let mut parent_branch_index = 0u64; - let mut parent_branch_range = None; block += 1; loop { let mut index: u64 = 0; @@ -291,8 +292,8 @@ impl NonCanonicalOverlay { // TODO EMCh following two range fetch can be optimize when parent level // got a single element. // fetch parent info - parent_branch_index = parents.get(&record.parent_hash).map(|(_, i)| *i).unwrap_or(0); - parent_branch_range = Some(branches.branch_ranges_from_cache(parent_branch_index)); + let parent_branch_index = parents.get(&record.parent_hash).map(|(_, i)| *i).unwrap_or(0); + let parent_branch_range = Some(branches.branch_ranges_from_cache(parent_branch_index)); let (_branch_range, branch_index) = branches.import( block, parent_branch_index, @@ -554,16 +555,6 @@ impl NonCanonicalOverlay { fn apply_canonicalizations(&mut self) { let last = self.pending_canonicalizations.last().cloned(); let count = self.pending_canonicalizations.len() as u64; - if let Some(branch_index_cannonicalize) = last.as_ref().and_then(|last| self.parents.get(last)) - .map(|(_, index)| *index) { - // set branch state synchronously - let block_number = self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1); - self.branches.finalize_full(branch_index_cannonicalize, Some(block_number)); - self.offstate_gc.set_pending_gc(branch_index_cannonicalize); - // try to run the garbage collection (can run later if there is - // pinned process). - self.offstate_gc.try_gc(&self.pinned); - } for hash in self.pending_canonicalizations.drain(..) { trace!(target: "state-db", "Post canonicalizing {:?}", hash); let level = self.levels.pop_front().expect("Hash validity is checked in `canonicalize`"); @@ -597,6 +588,19 @@ impl NonCanonicalOverlay { ); } } + + if let Some(branch_index_cannonicalize) = last.as_ref().and_then(|last| self.parents.get(last)) + .map(|(_, index)| *index) { + // set branch state synchronously + let block_number = self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1); + // this needs to be call after parents update + self.branches.update_finalize_treshold(branch_index_cannonicalize, Some(block_number), true); + self.offstate_gc.set_pending_gc(branch_index_cannonicalize); + // try to run the garbage collection (can run later if there is + // pinned process). + self.offstate_gc.try_gc(&self.pinned); + } + if let Some(hash) = last { let last_canonicalized = (hash, self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1)); self.last_canonicalized = Some(last_canonicalized); @@ -688,11 +692,8 @@ impl NonCanonicalOverlay { } /// Pin state values in memory - pub fn pin(&mut self, hash: &BlockHash) -> Option { - self.parents.get(hash).map(|(_, branch_index)| *branch_index).map(|branch_index| { - self.pinned.insert(hash.clone(), (Default::default(), branch_index)); - self.offstate_gc.pin(branch_index, &self.branches) - }) + pub fn pin(&mut self, hash: &BlockHash) { + self.pinned.insert(hash.clone(), (Default::default(), self.offstate_gc.gc_last_index)); } /// TODO EMCH aka get state for hash to query offstate storage. @@ -705,6 +706,7 @@ impl NonCanonicalOverlay { /// Discard pinned state pub fn unpin(&mut self, hash: &BlockHash) { self.pinned.remove(hash); + self.offstate_gc.try_gc(&self.pinned); } } From a37a32fd5a1f16b9641d44d7ea9316cf53d80dc9 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 27 Sep 2019 18:15:40 +0200 Subject: [PATCH 13/68] remove keys from commit set, this likely indicates broken code, will probably need to store state, apply gc on state and depending on result restore state or actually gc historied values. --- core/state-db/src/branch.rs | 31 ++++++++++++ core/state-db/src/noncanonical.rs | 78 +++++++++++++++---------------- 2 files changed, 70 insertions(+), 39 deletions(-) diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index 9dd0aac10cdef..afc45586c35cb 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -491,3 +491,34 @@ mod test { } } + + +// TODO EMCH use from historied data on merge. (this is just a stub) + +#[derive(Debug, Clone)] +pub struct History(Vec<()>, Option); + +impl Default for History { + fn default() -> Self { + History(Vec::new(), None) + } +} + + +impl History { + + pub fn set(&mut self, state: &BranchRanges, value: V) { + unimplemented!() + } + + pub fn get(&self, state: &BranchRanges) -> Option<&V> { + unimplemented!() + } + + pub fn gc(&mut self, state: &RangeSet) { + unimplemented!() + } + +} + +// End TODO EMCH diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 10cb6662c203b..6c077cf3aabc7 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -25,7 +25,7 @@ use std::collections::{HashMap, VecDeque, hash_map::Entry, HashSet}; use super::{Error, DBValue, ChangeSet, CommitSet, MetaDb, Hash, to_meta_key, OffstateKey}; use codec::{Encode, Decode}; use log::trace; -use crate::branch::{RangeSet, BranchRanges}; +use crate::branch::{RangeSet, BranchRanges, History}; const NON_CANONICAL_JOURNAL: &[u8] = b"noncanonical_journal"; const NON_CANONICAL_OFFSTATE_JOURNAL: &[u8] = b"offstate_noncanonical_journal"; @@ -46,12 +46,7 @@ pub struct NonCanonicalOverlay { pending_insertions: Vec, values: HashMap, //ref counted branches: RangeSet, - offstate_values: HashMap, //ref counted - // would be deleted but kept around because block is pinned - // TODO EMCH sense if pinning offstate? done while using state_at - // -> that is import processing (so we can revert) -> should be good - // to use on offstate too - // + offstate_values: HashMap>, /// second value is offstate pinned index: used in order to determine if the pinned /// thread should block garbage collection. pinned: HashMap, GcIndex)>, @@ -170,6 +165,17 @@ fn insert_values(values: &mut HashMap, inserted: } } +fn insert_offstate_values( + state: &BranchRanges, + values: &mut HashMap>, + inserted: Vec<(Key, DBValue)>, +) { + for (k, v) in inserted { + let entry = values.entry(k).or_insert_with(|| Default::default()); + entry.set(state, v); + } +} + fn discard_values( values: &mut HashMap, inserted: Vec, @@ -198,7 +204,6 @@ fn discard_values( } fn discard_offset_values( - values: &mut HashMap, inserted: Vec, into: &mut OffstatePendingGC, ) { @@ -207,28 +212,13 @@ fn discard_offset_values( } else { &mut into.keys_pending_gc }; - for k in inserted { - match values.entry(k) { - Entry::Occupied(mut e) => { - let (ref mut counter, _) = e.get_mut(); - *counter -= 1; - if *counter == 0 { - let (key, _) = e.remove_entry(); - into.insert(key); - } - }, - Entry::Vacant(_) => { - debug_assert!(false, "Trying to discard missing value"); - } - } - } + into.extend(inserted); } fn discard_descendants( levels: &mut VecDeque>>, mut values: &mut HashMap, - mut offstate_values: &mut HashMap, index: usize, parents: &mut HashMap, pinned: &mut HashMap, u64)>, @@ -245,7 +235,6 @@ fn discard_descendants( let mut pinned = pinned.get_mut(hash); discard_values(&mut values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); discard_offset_values( - &mut offstate_values, overlay.offstate_inserted, offstate_gc, ); @@ -256,7 +245,7 @@ fn discard_descendants( }).collect(); } for hash in discarded { - discard_descendants(levels, values, offstate_values, index + 1, parents, pinned, offstate_gc, &hash); + discard_descendants(levels, values, index + 1, parents, pinned, offstate_gc, &hash); } } @@ -294,7 +283,7 @@ impl NonCanonicalOverlay { // fetch parent info let parent_branch_index = parents.get(&record.parent_hash).map(|(_, i)| *i).unwrap_or(0); let parent_branch_range = Some(branches.branch_ranges_from_cache(parent_branch_index)); - let (_branch_range, branch_index) = branches.import( + let (branch_range, branch_index) = branches.import( block, parent_branch_index, parent_branch_range, @@ -320,7 +309,7 @@ impl NonCanonicalOverlay { }; insert_values(&mut values, record.inserted); if let Some(inserted) = offstate_record_inserted { - insert_values(&mut offstate_values, inserted); + insert_offstate_values(&branch_range, &mut offstate_values, inserted); } trace!( target: "state-db", @@ -365,8 +354,8 @@ impl NonCanonicalOverlay { /// Insert a new block into the overlay. If inserted on the second level or lower /// expects parent to be present in the window. pub fn insert( - &mut self, hash: - &BlockHash, + &mut self, + hash: &BlockHash, number: BlockNumber, parent_hash: &BlockHash, changeset: ChangeSet, @@ -423,7 +412,7 @@ impl NonCanonicalOverlay { level.push(overlay); let parent_branch_index = self.parents.get(&parent_hash).map(|(_, i)| *i).unwrap_or(0); let parent_branch_range = Some(self.branches.branch_ranges_from_cache(parent_branch_index)); - let (_branch_range, branch_index) = self.branches.import( + let (branch_range, branch_index) = self.branches.import( number, parent_branch_index, parent_branch_range, @@ -454,7 +443,7 @@ impl NonCanonicalOverlay { offstate_journal_record.deleted.len(), ); insert_values(&mut self.values, journal_record.inserted); - insert_values(&mut self.offstate_values, offstate_journal_record.inserted); + insert_offstate_values(&branch_range, &mut self.offstate_values, offstate_journal_record.inserted); self.pending_insertions.push(hash.clone()); Ok(commit) } @@ -539,9 +528,20 @@ impl NonCanonicalOverlay { commit.data.inserted.extend(overlay.inserted.iter() .map(|k| (k.clone(), self.values.get(k).expect("For each key in overlays there's a value in values").1.clone()))); commit.data.deleted.extend(overlay.deleted.clone()); - commit.offstate.inserted.extend(overlay.offstate_inserted.iter() - .map(|k| (k.clone(), self.offstate_values.get(k) - .expect("For each key in overlays there's a value in values").1.clone()))); + if !overlay.offstate_inserted.is_empty() { + // canonicalization is not frequent enough that we pass range + // in parameter for now + // TODO EMCH it seems bad ide to maintain this commit offstate + // field as his: simply delta of state could be better?? + if let Some(range) = self.get_branch_range(hash) { + commit.offstate.inserted.extend(overlay.offstate_inserted.iter() + .map(|k| (k.clone(), self.offstate_values.get(k) + .expect("For each key in overlays there's a value in values") + .get(&range) + .expect("For each key in overlays there's a historied entry in values") + .clone()))); + } + } commit.offstate.deleted.extend(overlay.offstate_deleted.clone()); commit.meta.deleted.append(&mut discarded_journals); @@ -570,7 +570,6 @@ impl NonCanonicalOverlay { discard_descendants( &mut self.levels, &mut self.values, - &mut self.offstate_values, 0, &mut self.parents, &mut self.pinned, @@ -582,7 +581,6 @@ impl NonCanonicalOverlay { let mut pinned = self.pinned.get_mut(&overlay.hash); discard_values(&mut self.values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); discard_offset_values( - &mut self.offstate_values, overlay.offstate_inserted, &mut self.offstate_gc, ); @@ -594,7 +592,9 @@ impl NonCanonicalOverlay { // set branch state synchronously let block_number = self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1); // this needs to be call after parents update + // TODO EMCH may be needed in 'canonicalize', and restore or commit here self.branches.update_finalize_treshold(branch_index_cannonicalize, Some(block_number), true); + // gc is at the right place self.offstate_gc.set_pending_gc(branch_index_cannonicalize); // try to run the garbage collection (can run later if there is // pinned process). @@ -653,7 +653,6 @@ impl NonCanonicalOverlay { self.branches.revert(branch_index); } discard_values(&mut self.values, overlay.inserted, None); - discard_values(&mut self.offstate_values, overlay.offstate_inserted, None); } commit }) @@ -671,7 +670,8 @@ impl NonCanonicalOverlay { let overlay = self.levels[level_index].pop().expect("Empty levels are not allowed in self.levels"); discard_values(&mut self.values, overlay.inserted, None); - discard_values(&mut self.offstate_values, overlay.offstate_inserted, None); + // TODO EMCH we need discard state?? from update finalize treshold here!! see discard value + // usgage if self.levels[level_index].is_empty() { debug_assert_eq!(level_index, self.levels.len() - 1); self.levels.pop_back(); From 4714b7078cfc3ca9d0ecc942e564b14fb63acfb2 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 27 Sep 2019 21:11:55 +0200 Subject: [PATCH 14/68] put a client test, for rest of test reset_storage and genesis build should get updated. --- core/client/db/src/lib.rs | 17 ++++++++++++++--- core/client/src/backend.rs | 2 +- core/client/src/in_mem.rs | 2 +- core/client/src/light/backend.rs | 2 +- core/service/src/lib.rs | 2 +- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 88517c4c2ed95..581dd70795860 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -1510,8 +1510,8 @@ impl client::backend::Backend for Backend whe Some(&self.changes_tries_storage) } - fn offchain_storage(&self) -> Option { - Some(self.offchain_storage.clone()) + fn offchain_storage(&self) -> Option<&Self::OffchainStorage> { + Some(&self.offchain_storage) } fn revert(&self, n: NumberFor) -> Result, client::error::Error> { @@ -1766,6 +1766,7 @@ mod tests { ).0.into(); let hash = header.hash(); + // TODO EMCH implement for offstate in another pr (need also build genesis). op.reset_storage(storage.iter().cloned().collect(), Default::default()).unwrap(); op.set_block_data( header.clone(), @@ -1801,7 +1802,12 @@ mod tests { (vec![5, 5, 5], Some(vec![4, 5, 6])), ]; - let (root, overlay) = op.old_state.storage_root(storage.iter().cloned()); + let child: Option<(_, Option<_>)> = None; + let (root, overlay) = op.old_state.full_storage_root( + storage.iter().cloned(), + child, + storage.iter().cloned(), + ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); @@ -1819,6 +1825,11 @@ mod tests { assert_eq!(state.storage(&[1, 3, 5]).unwrap(), None); assert_eq!(state.storage(&[1, 2, 3]).unwrap(), Some(vec![9, 9, 9])); assert_eq!(state.storage(&[5, 5, 5]).unwrap(), Some(vec![4, 5, 6])); + // TODO EMCH have a offstate get method + assert_eq!(state.offstate_pairs(), vec![(vec![5, 5, 5], vec![4, 5, 6])]); + let state = db.state_at(BlockId::Number(0)).unwrap(); + assert_eq!(state.offstate_pairs(), vec![]); + } } diff --git a/core/client/src/backend.rs b/core/client/src/backend.rs index 99a0edd0eab17..a9ab9772dd3bd 100644 --- a/core/client/src/backend.rs +++ b/core/client/src/backend.rs @@ -224,7 +224,7 @@ pub trait Backend: AuxStore + Send + Sync where /// Returns reference to changes trie storage. fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage>; /// Returns a handle to offchain storage. - fn offchain_storage(&self) -> Option; + fn offchain_storage(&self) -> Option<&Self::OffchainStorage>; /// Returns true if state for given block is available. fn have_state_at(&self, hash: &Block::Hash, _number: NumberFor) -> bool { self.state_at(BlockId::Hash(hash.clone())).is_ok() diff --git a/core/client/src/in_mem.rs b/core/client/src/in_mem.rs index f4d074e221356..8dde6374a2f2d 100644 --- a/core/client/src/in_mem.rs +++ b/core/client/src/in_mem.rs @@ -673,7 +673,7 @@ where Some(&self.changes_trie_storage) } - fn offchain_storage(&self) -> Option { + fn offchain_storage(&self) -> Option<&Self::OffchainStorage> { None } diff --git a/core/client/src/light/backend.rs b/core/client/src/light/backend.rs index 1601608f8f67e..7f15d19328936 100644 --- a/core/client/src/light/backend.rs +++ b/core/client/src/light/backend.rs @@ -185,7 +185,7 @@ impl ClientBackend for Backend where None } - fn offchain_storage(&self) -> Option { + fn offchain_storage(&self) -> Option<&Self::OffchainStorage> { None } diff --git a/core/service/src/lib.rs b/core/service/src/lib.rs index 431776b31e9b4..0f56ef13af883 100644 --- a/core/service/src/lib.rs +++ b/core/service/src/lib.rs @@ -211,7 +211,7 @@ macro_rules! new_impl { let offchain_storage = backend.offchain_storage(); let offchain_workers = match ($config.offchain_worker, offchain_storage) { (true, Some(db)) => { - Some(Arc::new(offchain::OffchainWorkers::new(client.clone(), db))) + Some(Arc::new(offchain::OffchainWorkers::new(client.clone(), db.clone()))) }, (true, None) => { log::warn!("Offchain workers disabled, due to lack of offchain storage support in backend."); From 5e90b207c160c0a9c367d55f22376bd12b0d74b2 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 1 Oct 2019 14:54:32 +0200 Subject: [PATCH 15/68] Get needed component from branch history-data --- Cargo.lock | 8 + Cargo.toml | 1 + core/utils/historied-data/Cargo.toml | 20 + core/utils/historied-data/README.md | 18 + core/utils/historied-data/src/lib.rs | 82 ++ core/utils/historied-data/src/linear.rs | 358 +++++++++ core/utils/historied-data/src/tree.rs | 956 ++++++++++++++++++++++++ 7 files changed, 1443 insertions(+) create mode 100644 core/utils/historied-data/Cargo.toml create mode 100644 core/utils/historied-data/README.md create mode 100644 core/utils/historied-data/src/lib.rs create mode 100644 core/utils/historied-data/src/linear.rs create mode 100644 core/utils/historied-data/src/tree.rs diff --git a/Cargo.lock b/Cargo.lock index c852d480edd5d..f370bcd322028 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1211,6 +1211,14 @@ dependencies = [ "proc-macro-hack 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "historied-data" +version = "2.0.0" +dependencies = [ + "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", + "sr-std 2.0.0", +] + [[package]] name = "hmac" version = "0.4.2" diff --git a/Cargo.toml b/Cargo.toml index 684363c227cc7..118850f83617c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ members = [ "core/transaction-pool/graph", "core/trie", "core/utils/fork-tree", + "core/utils/historied-data", "core/utils/wasm-builder", "core/utils/wasm-builder-runner", "core/wasm-interface", diff --git a/core/utils/historied-data/Cargo.toml b/core/utils/historied-data/Cargo.toml new file mode 100644 index 0000000000000..79ab10dd8658b --- /dev/null +++ b/core/utils/historied-data/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "historied-data" +version = "2.0.0" +authors = ["Parity Technologies "] +description = "Data associated with its history" +edition = "2018" + +[dependencies] +rstd = { package = "sr-std", path = "../../sr-std", default-features = false } +smallvec = { version = "0.6", optional = true } + +[dev-dependencies] + +[features] +default = ["std"] +std = [ + "rstd/std", + "smallvec", +] +test = [] diff --git a/core/utils/historied-data/README.md b/core/utils/historied-data/README.md new file mode 100644 index 0000000000000..8d469fbb00b56 --- /dev/null +++ b/core/utils/historied-data/README.md @@ -0,0 +1,18 @@ +## Historied data + +Crate with methods to manage data that stores its own history. + +This covers: +- linear history driven data, eg. transactional layers for overlay. +- long term storage with multiple branch, eg. offchain storage. + +General design is container where query and update requires global +history context. + +History is serialize as a per item basis. + +This crates is be `no_std` compatible, unless feature `std` is used. + +For more information see + +License: GPL-3.0 diff --git a/core/utils/historied-data/src/lib.rs b/core/utils/historied-data/src/lib.rs new file mode 100644 index 0000000000000..0169829122d24 --- /dev/null +++ b/core/utils/historied-data/src/lib.rs @@ -0,0 +1,82 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! History driven data storage. +//! Useful to store information with history +//! on a per item basis. + +#![cfg_attr(not(feature = "std"), no_std)] + +use rstd::convert::{TryFrom, TryInto}; + +pub mod tree; +pub mod linear; + +/// An entry at a given history index. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct HistoriedValue { + /// The stored value. + pub value: V, + /// The moment in history when the value got set. + pub index: I, +} + +impl From<(V, I)> for HistoriedValue { + fn from(input: (V, I)) -> HistoriedValue { + HistoriedValue { value: input.0, index: input.1 } + } +} + +impl HistoriedValue { + pub fn as_ref(&self) -> HistoriedValue<&V, I> { + HistoriedValue { + value: &self.value, + index: self.index, + } + } + + pub fn as_mut(&mut self) -> HistoriedValue<&mut V, I> { + HistoriedValue { + value: &mut self.value, + index: self.index, + } + } + + pub fn map R>(self, f: F) -> HistoriedValue { + HistoriedValue { + value: f(self.value), + index: self.index, + } + } + +} + +// utility function for panicking cast (similar to a `as` cast for number). +fn as_usize>(i: I) -> usize { + match i.try_into() { + Ok(index) => index, + Err(_) => panic!("historied value index overflow"), + } +} + +// utility function for panicking cast (similar to a `as` cast for number). +fn as_i>(i: usize) -> I { + match i.try_into() { + Ok(index) => index, + Err(_) => panic!("historied value index underflow"), + } +} diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs new file mode 100644 index 0000000000000..654dfce62e985 --- /dev/null +++ b/core/utils/historied-data/src/linear.rs @@ -0,0 +1,358 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Linear historied data. + +#[cfg(not(feature = "std"))] +use rstd::{vec::Vec, vec}; +use rstd::marker::PhantomData; +use rstd::borrow::Cow; +use crate::HistoriedValue; + + +/// Array like buffer for in memory storage. +/// By in memory we expect that this will +/// not required persistence and is not serialized. +#[cfg(not(feature = "std"))] +pub(crate) type MemoryOnly = Vec>; + +/// Array like buffer for in memory storage. +/// By in memory we expect that this will +/// not required persistence and is not serialized. +#[cfg(feature = "std")] +pub(crate) type MemoryOnly = smallvec::SmallVec<[HistoriedValue; ALLOCATED_HISTORY]>; + +/// Size of preallocated history per element. +/// Currently at two for committed and prospective only. +/// It means that using transaction in a module got a direct allocation cost. +#[cfg(feature = "std")] +const ALLOCATED_HISTORY: usize = 2; + +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +/// Arraylike buffer with in place byte data. +/// Can be written as is in underlying +/// storage. +/// Could be use for direct access memory to. +pub struct Serialized<'a, F>(Cow<'a, [u8]>, PhantomData); + +/// Serialized specific behavior. +pub trait SerializedConfig { + /// encoded empty slice + fn empty() -> &'static [u8]; + /// size at start for encoding version. + fn version_len() -> usize; +} +/// Serialize without versioning. +pub struct NoVersion; + +/// Serialize with default verison +pub struct DefaultVersion; + +impl SerializedConfig for NoVersion { + fn empty() -> &'static [u8] { + &EMPTY_SERIALIZED + } + fn version_len() -> usize { + 0 + } +} + +impl SerializedConfig for DefaultVersion { + fn empty() -> &'static [u8] { + &DEFAULT_VERSION_EMPTY_SERIALIZED + } + fn version_len() -> usize { + 1 + } +} + +// encoding size as u64 +const SIZE_BYTE_LEN: usize = 8; + +// Basis implementation to be on par with implementation using +// vec like container. Those method could be move to a trait +// implementation. +// Those function requires checked index. +impl<'a, F: SerializedConfig> Serialized<'a, F> { + + pub fn into_owned(self) -> Serialized<'static, F> { + Serialized(Cow::from(self.0.into_owned()), PhantomData) + } + + pub(crate) fn len(&self) -> usize { + let len = self.0.len(); + self.read_le_usize(len - SIZE_BYTE_LEN) as usize + } + + fn clear(&mut self) { + self.write_le_usize(F::version_len(), 0); + self.0.to_mut().truncate(F::version_len() + SIZE_BYTE_LEN); + } + + #[cfg(test)] + fn truncate(&mut self, index: usize) { + if index == 0 { + self.clear(); + return; + } + let len = self.len(); + if index >= len { + return; + } + let start_ix = self.index_start(); + let new_start = self.index_element(index) as usize; + let len_ix = index * SIZE_BYTE_LEN; + self.slice_copy(start_ix, new_start, len_ix); + self.write_le_usize(new_start + len_ix - SIZE_BYTE_LEN, index); + self.0.to_mut().truncate(new_start + len_ix); + } + + // index stay in truncated content + pub(crate) fn truncate_start(&mut self, index: usize) { + self.remove_range(0, index); + } + + pub(crate) fn pop(&mut self) -> Option, u64>> { + let len = self.len(); + if len == 0 { + return None; + } + let start_ix = self.index_element(len - 1); + let end_ix = self.index_start(); + let state = self.read_le_u64(start_ix); + let value = self.0[start_ix + SIZE_BYTE_LEN..end_ix].to_vec(); + if len - 1 == 0 { + self.clear(); + return Some(HistoriedValue { value, index: state }) + } else { + self.write_le_usize(self.0.len() - (SIZE_BYTE_LEN * 2), len - 1); + }; + let ix_size = (len * SIZE_BYTE_LEN) - SIZE_BYTE_LEN; + self.slice_copy(end_ix, start_ix, ix_size); + self.0.to_mut().truncate(start_ix + ix_size); + Some(HistoriedValue { value, index: state }) + } + + pub(crate) fn push(&mut self, val: HistoriedValue<&[u8], u64>) { + let len = self.len(); + let start_ix = self.index_start(); + let end_ix = self.0.len(); + // A sized buffer and multiple index to avoid to big copy + // should be use here. + let mut new_ix = self.0[start_ix..end_ix].to_vec(); + // truncate here can be bad + self.0.to_mut().truncate(start_ix + SIZE_BYTE_LEN); + self.write_le_u64(start_ix, val.index); + self.0.to_mut().extend_from_slice(val.value); + self.0.to_mut().append(&mut new_ix); + if len > 0 { + self.write_le_usize(self.0.len() - SIZE_BYTE_LEN, start_ix); + self.append_le_usize(len + 1); + } else { + self.write_le_usize(self.0.len() - SIZE_BYTE_LEN, 1); + } + } + + #[cfg(test)] + fn remove(&mut self, index: usize) { + self.remove_range(index, index + 1); + } + + fn remove_range(&mut self, index: usize, end: usize) { + let len = self.len(); + if len == 1 && index == 0 { + self.clear(); + return; + } + // eager removal is costy, running some gc impl + // can be interesting. + let elt_start = self.index_element(index); + let start_ix = self.index_start(); + let elt_end = if end == len { + start_ix + } else { + self.index_element(end) + }; + let delete_size = elt_end - elt_start; + for _ in elt_start..elt_end { + let _ = self.0.to_mut().remove(elt_start); + } + let start_ix = start_ix - delete_size; + for i in 1..len - index - 1 { + let old_value = self.read_le_usize(start_ix + i * SIZE_BYTE_LEN); + self.write_le_usize(start_ix + (i - 1) * SIZE_BYTE_LEN, old_value - delete_size); + } + let len = len - (end - index); + let end_index = start_ix + len * SIZE_BYTE_LEN; + self.write_le_usize(end_index - SIZE_BYTE_LEN, len); + self.0.to_mut().truncate(end_index); + + } + + pub(crate) fn get_state(&self, index: usize) -> HistoriedValue<&[u8], u64> { + let start_ix = self.index_element(index); + let len = self.len(); + let end_ix = if index == len - 1 { + self.index_start() + } else { + self.index_element(index + 1) + }; + let state = self.read_le_u64(start_ix); + HistoriedValue { + value: &self.0[start_ix + SIZE_BYTE_LEN..end_ix], + index: state, + } + } + +} + +const EMPTY_SERIALIZED: [u8; SIZE_BYTE_LEN] = [0u8; SIZE_BYTE_LEN]; +const DEFAULT_VERSION: u8 = 1; +const DEFAULT_VERSION_EMPTY_SERIALIZED: [u8; SIZE_BYTE_LEN + 1] = { + let mut buf = [0u8; SIZE_BYTE_LEN + 1]; + buf[0] = DEFAULT_VERSION; + buf +}; + +impl<'a, F: SerializedConfig> Default for Serialized<'a, F> { + fn default() -> Self { + Serialized(Cow::Borrowed(F::empty()), PhantomData) + } +} + +// Utility function for basis implementation. +impl<'a, F: SerializedConfig> Serialized<'a, F> { + + // Index at end, also contains the encoded size + fn index_start(&self) -> usize { + let nb_ix = self.len(); + if nb_ix == 0 { return F::version_len(); } + let end = self.0.len(); + end - (nb_ix * SIZE_BYTE_LEN) + } + + fn index_element(&self, position: usize) -> usize { + if position == 0 { + return F::version_len(); + } + let i = self.index_start() + (position - 1) * SIZE_BYTE_LEN; + self.read_le_usize(i) + } + + // move part of array that can overlap + // This is a memory inefficient implementation. + fn slice_copy(&mut self, start_from: usize, start_to: usize, size: usize) { + let buffer = self.0[start_from..start_from + size].to_vec(); + self.0.to_mut()[start_to..start_to + size].copy_from_slice(&buffer[..]); + } + + // Usize encoded as le u64 (for historied value). + fn read_le_u64(&self, pos: usize) -> u64 { + let mut buffer = [0u8; SIZE_BYTE_LEN]; + buffer.copy_from_slice(&self.0[pos..pos + SIZE_BYTE_LEN]); + u64::from_le_bytes(buffer) + } + + // Usize encoded as le u64 (only for internal indexing). + // TODO EMCH change usize encoding to u32? + fn read_le_usize(&self, pos: usize) -> usize { + let mut buffer = [0u8; SIZE_BYTE_LEN]; + buffer.copy_from_slice(&self.0[pos..pos + SIZE_BYTE_LEN]); + u64::from_le_bytes(buffer) as usize + } + + // Usize encoded as le u64. + fn write_le_usize(&mut self, pos: usize, value: usize) { + let buffer = (value as u64).to_le_bytes(); + self.0.to_mut()[pos..pos + SIZE_BYTE_LEN].copy_from_slice(&buffer[..]); + } + + // Usize encoded as le u64. + fn append_le_usize(&mut self, value: usize) { + let buffer = (value as u64).to_le_bytes(); + self.0.to_mut().extend_from_slice(&buffer[..]); + } + + // Usize encoded as le u64. + fn write_le_u64(&mut self, pos: usize, value: u64) { + let buffer = (value as u64).to_le_bytes(); + self.0.to_mut()[pos..pos + SIZE_BYTE_LEN].copy_from_slice(&buffer[..]); + } + +} + +#[cfg(test)] +mod test { + use super::*; + + fn test_serialized_basis(mut ser: Serialized) { + // test basis unsafe function similar to a simple vec + // without index checking. + let v1 = &b"val1"[..]; + let v2 = &b"value_2"[..]; + let v3 = &b"a third value 3"[..]; + + assert_eq!(ser.len(), 0); + assert_eq!(ser.pop(), None); + ser.push((v1, 1).into()); + assert_eq!(ser.get_state(0), (v1, 1).into()); + assert_eq!(ser.pop(), Some((v1.to_vec(), 1).into())); + assert_eq!(ser.len(), 0); + ser.push((v1, 1).into()); + ser.push((v2, 2).into()); + ser.push((v3, 3).into()); + assert_eq!(ser.get_state(0), (v1, 1).into()); + assert_eq!(ser.get_state(1), (v2, 2).into()); + assert_eq!(ser.get_state(2), (v3, 3).into()); + assert_eq!(ser.pop(), Some((v3.to_vec(), 3).into())); + assert_eq!(ser.len(), 2); + ser.push((v3, 3).into()); + assert_eq!(ser.get_state(2), (v3, 3).into()); + ser.remove(0); + assert_eq!(ser.len(), 2); + assert_eq!(ser.get_state(0), (v2, 2).into()); + assert_eq!(ser.get_state(1), (v3, 3).into()); + ser.push((v1, 1).into()); + ser.remove(1); + assert_eq!(ser.len(), 2); + assert_eq!(ser.get_state(0), (v2, 2).into()); + assert_eq!(ser.get_state(1), (v1, 1).into()); + ser.push((v1, 1).into()); + ser.truncate(1); + assert_eq!(ser.len(), 1); + assert_eq!(ser.get_state(0), (v2, 2).into()); + ser.push((v1, 1).into()); + ser.push((v3, 3).into()); + ser.truncate_start(1); + assert_eq!(ser.len(), 2); + assert_eq!(ser.get_state(0), (v1, 1).into()); + assert_eq!(ser.get_state(1), (v3, 3).into()); + ser.push((v2, 2).into()); + ser.truncate_start(2); + assert_eq!(ser.len(), 1); + assert_eq!(ser.get_state(0), (v2, 2).into()); + + } + + #[test] + fn serialized_basis() { + let ser1: Serialized = Default::default(); + let ser2: Serialized = Default::default(); + test_serialized_basis(ser1); + test_serialized_basis(ser2); + } +} diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs new file mode 100644 index 0000000000000..bc1cf3ff49d52 --- /dev/null +++ b/core/utils/historied-data/src/tree.rs @@ -0,0 +1,956 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Data store acyclic directed graph as tree. +//! +//! General structure is an array of branch, each branch originates +//! from another branch at designated index. +//! +//! No particular state (just present or missing). + +use crate::linear::{ + MemoryOnly as BranchBackend, + Serialized as SerializedInner, + SerializedConfig, +}; +use crate::HistoriedValue; +use crate::{as_usize, as_i}; +use rstd::rc::Rc; +use rstd::vec::Vec; +use rstd::collections::btree_map::BTreeMap; +use rstd::convert::{TryFrom, TryInto}; + +pub trait TreeStateTrait { + type Branch: BranchStateTrait; + type Iter: Iterator; + + fn get_branch(self, index: I) -> Option; + + /// Inclusive. + fn last_index(self) -> I; + + /// Iterator. + fn iter(self) -> Self::Iter; +} + +pub trait BranchStateTrait { + + fn get_node(&self, i: I) -> S; + + /// Inclusive. + fn last_index(&self) -> I; +} + + +impl<'a> TreeStateTrait for &'a StatesRef { + type Branch = (&'a StatesBranchRef, Option); + type Iter = StatesRefIter<'a>; + + fn get_branch(self, i: u64) -> Option { + for (b, bi) in self.iter() { + if bi == i { + return Some(b); + } else if bi < i { + break; + } + } + None + } + + fn last_index(self) -> u64 { + let l = self.history.len(); + let l = if l > 0 { + self.history[l - 1].branch_index + } else { + 0 + }; + self.upper_branch_index.map(|u| rstd::cmp::min(u, l)).unwrap_or(l) + } + + fn iter(self) -> Self::Iter { + let mut end = self.history.len(); + let last_index = self.last_index(); + let upper_node_index = if Some(last_index) == self.upper_branch_index { + self.upper_node_index + } else { None }; + while end > 0 { + if self.history[end - 1].branch_index <= last_index { + break; + } + end -= 1; + } + + StatesRefIter(self, end, upper_node_index) + } +} + +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct BranchState { + /// Index of first element (only use for indexed access). + /// Element before offset are not in state. + offset: u64, + /// number of elements: all elements equal or bellow + /// this index are valid, over this index they are not. + len: u64, + /// Maximum index before first deletion, it indicates + /// if the state is modifiable (when an element is dropped + /// we cannot append and need to create a new branch). + max_len_ix: u64, +} + +/// This is a simple range, end non inclusive. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct BranchStateRef { + start: u64, + end: u64, +} + +impl<'a> BranchStateTrait for (&'a StatesBranchRef, Option) { + + fn get_node(&self, i: u64) -> bool { + let l = self.0.state.end; + let upper = self.1.map(|u| rstd::cmp::min(u + 1, l)).unwrap_or(l); + i >= self.0.state.start && i < upper + } + + fn last_index(&self) -> u64 { + // underflow should not happen as long as branchstateref are not allowed to be empty. + let state_end = self.0.state.end - 1; + self.1.map(|bound| rstd::cmp::min(state_end, bound)).unwrap_or(state_end) + } + +} + +impl Default for BranchState { + // initialize with one element + fn default() -> Self { + Self::new_from(0) + } +} + +impl BranchState { + pub fn new_from(offset: u64) -> Self { + BranchState { + offset, + len: 1, + max_len_ix: offset, + } + } + + pub fn state_ref(&self) -> BranchStateRef { + BranchStateRef { + start: self.offset, + end: self.offset + self.len, + } + } + + pub fn has_deleted_index(&self) -> bool { + self.max_len_ix >= self.offset + self.len + } + + pub fn add_state(&mut self) -> bool { + if !self.has_deleted_index() { + self.len += 1; + self.max_len_ix += 1; + true + } else { + false + } + } + + /// return possible dropped state + pub fn drop_state(&mut self) -> Option { + if self.len > 0 { + self.len -= 1; + Some(self.offset + self.len) + } else { + None + } + } + + /// Return true if state exists. + pub fn get_state(&self, index: u64) -> bool { + if index < self.offset { + return false; + } + self.len > index + self.offset + } + + pub fn latest_ix(&self) -> Option { + if self.len > 0 { + Some(self.offset + self.len - 1) + } else { + None + } + } + +} + +impl BranchStateRef { + /// Return true if the state exists, false otherwhise. + pub fn get_state(&self, index: u64) -> bool { + index < self.end && index >= self.start + } +} + +/// At this point this is only use for testing and as an example +/// implementation. +/// It keeps trace of dropped value, and have some costy recursive +/// deletion. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct TestStates { + branches: BTreeMap, + last_branch_index: u64, + /// a lower treshold under which every branch are seen + /// as containing only valid values. + /// This can only be updated after a full garbage collection. + valid_treshold: u64, +} + +impl StatesBranch { + pub fn branch_ref(&self) -> StatesBranchRef { + StatesBranchRef { + branch_index: self.branch_index, + state: self.state.state_ref(), + } + } +} + +impl Default for TestStates { + fn default() -> Self { + TestStates { + branches: Default::default(), + last_branch_index: 0, + valid_treshold: 0, + } + } +} + +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct StatesBranch { + // this is the key (need to growth unless full gc (can still have + // content pointing to it even if it seems safe to reuse a previously + // use ix). + branch_index: u64, + + origin_branch_index: u64, + origin_node_index: u64, + + state: BranchState, +} + +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct StatesBranchRef { + branch_index: u64, + state: BranchStateRef, +} + + +#[derive(Clone)] +/// Reference to state to use for query updates. +/// It is a single brannch path with branches ordered by branch_index. +/// +/// Note that an alternative representation could be a pointer to full +/// tree state with a defined branch index implementing an iterator. +pub struct StatesRef { + /// Oredered by branch index linear branch states. + history: Rc>, + /// Index is included, acts as length of history. + upper_branch_index: Option, + /// Index is included, acts as a branch ref end value. + upper_node_index: Option, +} + +/// Iterator, contains index of last inner struct. +pub struct StatesRefIter<'a>(&'a StatesRef, usize, Option); + +impl<'a> Iterator for StatesRefIter<'a> { + type Item = ((&'a StatesBranchRef, Option), u64); + + fn next(&mut self) -> Option { + if self.1 > 0 { + let upper_node_index = self.2.take(); + Some(( + (&self.0.history[self.1 - 1], upper_node_index), + self.0.history[self.1 - 1].branch_index, + )) + } else { + None + } + } +} + +impl StatesRef { + /// limit to a given branch (included). + /// Optionally limiting branch to a linear index (included). + pub fn limit_branch(&mut self, branch_index: u64, node_index: Option) { + debug_assert!(branch_index > 0); + self.upper_branch_index = Some(branch_index); + self.upper_node_index = node_index; + } + + /// remove any limit. + pub fn clear_limit(&mut self) { + self.upper_branch_index = None; + self.upper_node_index = None; + } + +} + +impl TestStates { + + /// clear state but keep existing branch values (can be call after a full gc: + /// enforcing no commited containing dropped values). + pub fn unsafe_clear(&mut self) { + self.branches.clear(); + self.last_branch_index = 0; + } + + /// warning it should be the index of the leaf, otherwhise the ref will be incomplete. + /// (which is fine as long as we use this state to query something that refer to this state. + pub fn state_ref(&self, mut branch_index: u64) -> StatesRef { + let mut result = Vec::new(); + let mut previous_origin_node_index = u64::max_value() - 1; + while branch_index > self.valid_treshold { + if let Some(branch) = self.branches.get(&branch_index) { + let mut branch_ref = branch.branch_ref(); + if branch_ref.state.end > previous_origin_node_index + 1 { + branch_ref.state.end = previous_origin_node_index + 1; + } + previous_origin_node_index = branch.origin_node_index; + // TODO EMCH consider reversing state_ref + result.insert(0, branch_ref); + branch_index = branch.origin_branch_index; + } else { + break; + } + } + StatesRef { history: Rc::new(result), upper_branch_index: None, upper_node_index: None } + } + + // create a branches. End current branch. + // Return first created index (next branch are sequential indexed) + // or None if origin branch does not allow branch creation (commited branch or non existing). + pub fn create_branch( + &mut self, + nb_branch: usize, + branch_index: u64, + node_index: Option, + ) -> Option { + if nb_branch == 0 { + return None; + } + + // for 0 is the first branch creation case + let node_index = if branch_index == 0 { + debug_assert!(node_index.is_none()); + 0 + } else { + if let Some(node_index) = self.get_node(branch_index, node_index) { + node_index + } else { + return None; + } + }; + + let result_ix = self.last_branch_index + 1; + for i in result_ix .. result_ix + (nb_branch as u64) { + self.branches.insert(i, StatesBranch { + branch_index: i, + origin_branch_index: branch_index, + origin_node_index: node_index, + state: Default::default(), + }); + } + self.last_branch_index += nb_branch as u64; + + Some(result_ix) + } + + /// check if node is valid for given index. + /// return node_index. + pub fn get_node( + &self, + branch_index: u64, + node_index: Option, + ) -> Option { + if let Some(branch) = self.branches.get(&branch_index) { + if let Some(node_index) = node_index { + if branch.state.get_state(node_index) { + Some(node_index) + } else { + None + } + } else { + branch.state.latest_ix() + } + } else { + None + } + } + + /// Do node exist (return state (being true or false only)). + pub fn get(&self, branch_index: u64, node_index: u64) -> bool { + self.get_node(branch_index, Some(node_index)).is_some() + } + + pub fn branch_state(&self, branch_index: u64) -> Option<&BranchState> { + self.branches.get(&branch_index) + .map(|b| &b.state) + } + + pub fn branch_state_mut(&mut self, branch_index: u64) -> Option<&mut BranchState> { + self.branches.get_mut(&branch_index) + .map(|b| &mut b.state) + } + + /// this function can go into deep recursion with full scan, it indicates + /// that the in memory model use here should only be use for small data or + /// tests. + pub fn apply_drop_state(&mut self, branch_index: u64, node_index: u64) { + let mut to_delete = Vec::new(); + for (i, s) in self.branches.iter() { + if s.origin_branch_index == branch_index && s.origin_node_index == node_index { + to_delete.push(*i); + } + } + for i in to_delete.into_iter() { + loop { + match self.branch_state_mut(i).map(|ls| ls.drop_state()) { + Some(Some(li)) => self.apply_drop_state(i, li), + Some(None) => break, // we keep empty branch + None => break, + } + } + } + } +} + +/// First field is the actual history against which we run +/// the state. +/// Second field is an optional value for the no match case. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct History(Vec>, Option); + +impl Default for History { + fn default() -> Self { + History(Vec::new(), None) + } +} + +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct HistoryBranch { + branch_index: u64, + history: BranchBackend, +} + +impl History { + + /// Set or update value for a given state. + pub fn set(&mut self, state: S, value: V) + where + S: TreeStateTrait, + I: Copy + Eq + TryFrom + TryInto, + BI: Copy + Eq + TryFrom + TryInto, + { + // TODO EMCH it does not seem stricly needed to pass + // a full state, double index looks enough. + // but this api can help using consistent state. + if let Some((state_branch, state_index)) = state.iter().next() { + if let Ok(state_index_usize) = state_index.try_into() { + let state_index_u64 = state_index_usize as u64; + let mut i = self.0.len(); + let (branch_position, new_branch) = loop { + if i == 0 { + break (0, true); + } + let branch_index = self.0[i - 1].branch_index; + if branch_index == state_index_u64 { + break (i - 1, false); + } else if branch_index < state_index_u64 { + break (i, true); + } + i -= 1; + }; + if new_branch { + if let Ok(index_usize) = state_branch.last_index().try_into() { + let index = index_usize as u64; + let mut history = BranchBackend::::default(); + history.push(HistoriedValue { + value, + index, + }); + let h_value = HistoryBranch { + branch_index: state_index_u64, + history, + }; + if branch_position == self.0.len() { + self.0.push(h_value); + } else { + self.0.insert(branch_position, h_value); + } + } + } else { + self.node_set(branch_position, &state_branch, value) + } + } + } + } + + fn node_set(&mut self, branch_index: usize, state: &S, value: V) + where + S: BranchStateTrait, + I: Copy + Eq + TryFrom + TryInto, + { + if let Ok(node_index_usize) = state.last_index().try_into() { + let node_index_u64 = node_index_usize as u64; + let history = &mut self.0[branch_index]; + let mut index = history.history.len(); + debug_assert!(index > 0); + loop { + if index == 0 || history.history[index - 1].index < node_index_u64 { + let h_value = HistoriedValue { + value, + index: node_index_u64 + }; + if index == history.history.len() { + history.history.push(h_value); + } else { + history.history.insert(index, h_value); + } + break; + } else if history.history[index - 1].index == node_index_u64 { + history.history[index - 1].value = value; + break; + } + index -= 1; + } + } + } + + /// Access to last valid value (non dropped state in history). + /// When possible please use `get_mut` as it can free some memory. + pub fn get (&self, state: S) -> Option<&V> + where + S: TreeStateTrait, + I: Copy + Eq + TryFrom + TryInto, + BI: Copy + Eq + TryFrom + TryInto, + { + let mut index = self.0.len(); + // note that we expect branch index to be linearily set + // along a branch (no state containing unordered branch_index + // and no history containing unorderd branch_index). + if index == 0 { + return self.1.as_ref(); + } + + // TODO EMCH reverse loops ? probably. + for (state_branch, state_index) in state.iter() { + while index > 0 { + index -= 1; + if let Ok(branch_index) = self.0[index].branch_index.try_into() { + if let Ok(state_index) = state_index.try_into() { + if state_index == branch_index { + if let Some(result) = self.branch_get(index, &state_branch) { + return Some(result) + } + } + } + } + } + if index == 0 { + break; + } + } + self.1.as_ref() + } + + fn branch_get(&self, index: usize, state: &S) -> Option<&V> + where + S: BranchStateTrait, + I: Copy + Eq + TryFrom + TryInto, + { + let history = &self.0[index]; + let mut index = history.history.len(); + while index > 0 { + index -= 1; + if let Some(&v) = history.history.get(index).as_ref() { + if let Ok(i) = (v.index as usize).try_into() { + if state.get_node(i) { + return Some(&v.value); + } + } + } + } + None + } + + /// Gc an historied value other its possible values. + /// Iterator need to be reversed ordered by branch index. + pub fn gc(&mut self, mut states: IT) + where + IT: Iterator, + S: BranchStateTrait, + I: Copy + Eq + TryFrom + TryInto, + { + // state is likely bigger than history. + let mut current_state = states.next(); + for branch_index in (0..self.0.len()).rev() { + let history_branch = self.0[branch_index].branch_index; + loop { + if let Some(state) = current_state.as_ref() { + if let Ok(state_index_usize) = state.1.try_into() { + let state_index_u64 = state_index_usize as u64; + if history_branch < state_index_u64 { + current_state = states.next(); + } else if history_branch == state_index_u64 { + let len = self.0[branch_index].history.len(); + for history_index in (0..len).rev() { + + let node_index = self.0[branch_index].history[history_index].index as usize; + if let Ok(node_index) = node_index.try_into() { + if !state.0.get_node(node_index) { + if history_index == len - 1 { + let _ = self.0[branch_index].history.pop(); + } else { + let _ = self.0[branch_index].history.remove(history_index); + } + } + } + } + if self.0[branch_index].history.len() == 0 { + let _ = self.0.remove(branch_index); + } + current_state = states.next(); + break; + } else { + self.0.remove(branch_index); + break; + } + } + } else { + break; + } + } + } + } + +/* + /// Access to last valid value (non dropped state in history). + /// When possible please use `get_mut` as it can free some memory. + pub fn get_mut(&mut self, state: &StatesRef) -> Option<&mut V> { + let mut index = self.0.len(); + let mut index_state = state.history.len() - 1; + + // note that we expect branch index to be linearily set + // along a branch (no state containing unordered branch_index + // and no history containing unorderd branch_index). + if index == 0 || index_state == 0 { + return self.1.as_mut(); + } + while index > 0 && index_state > 0 { + index -= 1; + let branch_index = self.0[index].branch_index; + + while index_state > 0 && state.history[index_state].branch_index > branch_index { + index_state -= 1; + } + if state.history[index_state].branch_index == branch_index { + if let Some(result) = self.branch_get_unchecked_mut(branch_index, &state.history[index_state]) { + return Some(result) + } + } + } + self.1.as_mut() + } + + fn branch_get_unchecked_mut(&mut self, branch_index: u64, state: &StatesBranchRef) -> Option<&mut V> { + let history = &mut self.0[branch_index as usize]; + let mut index = history.history.len(); + if index == 0 { + return None; + } + while index > 0 { + index -= 1; + if let Some(&mut v) = history.history.get_mut(index).as_mut() { + if v.index < state.state.end { + return Some(&mut v.value); + } + } + } + None + } +*/ +} + +impl<'a, F: SerializedConfig> Serialized<'a, F> { + pub fn get (&self, state: S) -> Option<&[u8]> + where + S: BranchStateTrait, + I: Copy + Eq + TryFrom + TryInto, + { + let mut index = self.0.len(); + if index == 0 { + return None; + } + while index > 0 { + index -= 1; + let HistoriedValue { value, index: state_index } = self.0.get_state(index); + if state.get_node(as_i(state_index as usize)) { + return Some(value) + } + } + None + } + + /// This append the value, and can only be use in an + /// orderly fashion. + pub fn push(&mut self, state: S, value: &[u8]) + where + S: BranchStateTrait, + I: Copy + Eq + TryFrom + TryInto, + { + let target_state_index = as_usize(state.last_index()) as u64; + let index = self.0.len(); + if index > 0 { + let last = self.0.get_state(index); + debug_assert!(target_state_index >= last.index); + if target_state_index == last.index { + self.0.pop(); + } + } + self.0.push(HistoriedValue {value, index: target_state_index}); + } + + /// keep an single history before the state. + pub fn prune(&mut self, index: I) + where + I: Copy + Eq + TryFrom + TryInto, + { + let from = as_usize(index) as u64; + let len = self.0.len(); + for index in 0..len { + let history = self.0.get_state(index); + if history.index >= from { + // delete all index up to index + self.0.truncate_start(index); + break; + } + } + } + +} + + +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +/// Serialized implementation when transaction support is not +/// needed. +pub struct Serialized<'a, F>(SerializedInner<'a, F>); + +impl<'a, F> Into> for SerializedInner<'a, F> { + fn into(self) -> Serialized<'a, F> { + Serialized(self) + } +} + +impl<'a, F> Into> for Serialized<'a, F> { + fn into(self) -> SerializedInner<'a, F> { + self.0 + } +} + +impl<'a, F: SerializedConfig> Default for Serialized<'a, F> { + fn default() -> Self { + SerializedInner::<'a, F>::default().into() + } +} + + +#[cfg(test)] +mod test { + use super::*; + + fn test_states() -> TestStates { + let mut states = TestStates::default(); + assert_eq!(states.create_branch(1, 0, None), Some(1)); + // root branching. + assert_eq!(states.create_branch(1, 0, None), Some(2)); + assert_eq!(Some(true), states.branch_state_mut(1).map(|ls| ls.add_state())); + assert_eq!(states.create_branch(2, 1, None), Some(3)); + assert_eq!(states.create_branch(1, 1, Some(0)), Some(5)); + assert_eq!(states.create_branch(1, 1, Some(2)), None); + assert_eq!(Some(true), states.branch_state_mut(1).map(|ls| ls.add_state())); + assert_eq!(Some(Some(2)), states.branch_state_mut(1).map(|ls| ls.drop_state())); + // cannot create when dropped happen on branch + assert_eq!(Some(false), states.branch_state_mut(1).map(|ls| ls.add_state())); + + // TODO add content through branching + assert!(states.get(1, 1)); + // 0> 1: _ _ X + // | |> 3: 1 + // | |> 4: 1 + // | |> 5: 1 + // |> 2: _ + + states + } + + #[test] + fn test_remove_attached() { + let mut states = test_states(); + assert_eq!(Some(Some(1)), states.branch_state_mut(1).map(|ls| ls.drop_state())); + assert!(states.get(3, 0)); + assert!(states.get(4, 0)); + states.apply_drop_state(1, 1); + assert!(!states.get(3, 0)); + assert!(!states.get(4, 0)); + } + + #[test] + fn test_state_refs() { + let states = test_states(); + let ref_3 = vec![ + StatesBranchRef { + branch_index: 1, + state: BranchStateRef { start: 0, end: 2 }, + }, + StatesBranchRef { + branch_index: 3, + state: BranchStateRef { start: 0, end: 1 }, + }, + ]; + assert_eq!(*states.state_ref(3).history, ref_3); + + let mut states = states; + + assert_eq!(states.create_branch(1, 1, Some(0)), Some(6)); + let ref_6 = vec![ + StatesBranchRef { + branch_index: 1, + state: BranchStateRef { start: 0, end: 1 }, + }, + StatesBranchRef { + branch_index: 6, + state: BranchStateRef { start: 0, end: 1 }, + }, + ]; + assert_eq!(*states.state_ref(6).history, ref_6); + + states.valid_treshold = 3; + let mut ref_6 = ref_6; + ref_6.remove(0); + assert_eq!(*states.state_ref(6).history, ref_6); + } + + #[test] + fn test_set_get() { + // 0> 1: _ _ X + // | |> 3: 1 + // | |> 4: 1 + // | |> 5: 1 + // |> 2: _ + let states = test_states(); + let mut item: History = Default::default(); + + for i in 0..6 { + assert_eq!(item.get(&states.state_ref(i)), None); + } + + // setting value respecting branch build order + for i in 1..6 { + item.set(&states.state_ref(i), i); + } + + for i in 1..6 { + assert_eq!(item.get(&states.state_ref(i)), Some(&i)); + } + + let mut ref_3 = states.state_ref(3); + ref_3.limit_branch(1, None); + assert_eq!(item.get(&ref_3), Some(&1)); + + let mut ref_1 = states.state_ref(1); + ref_1.limit_branch(1, Some(0)); + assert_eq!(item.get(&ref_1), None); + item.set(&ref_1, 11); + assert_eq!(item.get(&ref_1), Some(&11)); + + item = Default::default(); + + // could rand shuffle if rand get imported later. + let disordered = [ + [1,2,3,5,4], + [2,5,1,3,4], + [5,3,2,4,1], + ]; + for r in disordered.iter() { + for i in r { + item.set(&states.state_ref(*i), *i); + } + for i in r { + assert_eq!(item.get(&states.state_ref(*i)), Some(i)); + } + } + + } + + + #[test] + fn test_gc() { + // 0> 1: _ _ X + // | |> 3: 1 + // | |> 4: 1 + // | |> 5: 1 + // |> 2: _ + let states = test_states(); + let mut item: History = Default::default(); + // setting value respecting branch build order + for i in 1..6 { + item.set(&states.state_ref(i), i); + } + + let mut states1 = states.branches.clone(); + let action = [(1, true), (2, false), (3, false), (4, true), (5, false)]; + for a in action.iter() { + if !a.1 { + states1.remove(&a.0); + } + } + // makes invalid tree (detaches 4) + states1.get_mut(&1).map(|br| br.state.len = 1); + let states1: BTreeMap<_, _> = states1.iter().map(|(k,v)| (k, v.branch_ref())).collect(); + let mut item1 = item.clone(); + item1.gc(states1.iter().map(|(k, v)| ((v, None), **k)).rev()); + assert_eq!(item1.get(&states.state_ref(1)), None); + for a in action.iter().skip(1) { + if a.1 { + assert_eq!(item1.get(&states.state_ref(a.0)), Some(&a.0)); + } else { + assert_eq!(item1.get(&states.state_ref(a.0)), None); + } + } + + } + +} From f6956b1b2f860f7034fb6d54238072ce95828785 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 1 Oct 2019 16:05:37 +0200 Subject: [PATCH 16/68] use history from utils --- Cargo.lock | 1 + core/state-db/Cargo.toml | 2 + core/state-db/src/branch.rs | 111 ++++++++++++------------ core/state-db/src/noncanonical.rs | 3 +- core/utils/historied-data/src/linear.rs | 1 - core/utils/historied-data/src/tree.rs | 45 +++++++--- 6 files changed, 93 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f370bcd322028..0d84402a9755e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5361,6 +5361,7 @@ name = "substrate-state-db" version = "2.0.0" dependencies = [ "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "historied-data 2.0.0", "log 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/core/state-db/Cargo.toml b/core/state-db/Cargo.toml index 332751c927f5d..d780023d5bcb4 100644 --- a/core/state-db/Cargo.toml +++ b/core/state-db/Cargo.toml @@ -9,6 +9,8 @@ parking_lot = "0.9.0" log = "0.4" primitives = { package = "substrate-primitives", path = "../../core/primitives" } codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } +historied-data = { path = "../../core/utils/historied-data" } [dev-dependencies] env_logger = "0.6" +historied-data = { path = "../../core/utils/historied-data", features = ["test"] } diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index afc45586c35cb..7ed74215dd30b 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -22,8 +22,9 @@ use std::cmp::Reverse; use crate::Error; use std::hash::Hash as StdHash; use std::convert::TryInto; +use historied_data::tree::{TreeStateTrait, BranchStateTrait, StatesBranchRef, BranchStateRef}; - +#[derive(Clone, Default, Debug)] /// Reference to state that is enough for query updates, but not /// for gc. /// Values are ordered by branch_ix, @@ -32,20 +33,55 @@ use std::convert::TryInto; /// Note that an alternative could be a pointer to a full state /// branch for a given branch index, here we use an in memory /// copied representation in relation to an actual use case. -pub type BranchRanges = Vec; +pub struct BranchRanges(Vec); + +impl<'a> TreeStateTrait for &'a BranchRanges { + // TODO do we need parent ix field of statesbranchref?? + type Branch = &'a StatesBranchRef; + type Iter = BranchRangesIter<'a>; + + fn get_branch(self, i: u64) -> Option { + for (b, bi) in self.iter() { + if bi == i { + return Some(b); + } else if bi < i { + break; + } + } + None + } + + fn last_index(self) -> u64 { + let l = self.0.len(); + if l > 0 { + self.0[l - 1].branch_index + } else { + 0 + } + } + fn iter(self) -> Self::Iter { + BranchRangesIter(self, self.0.len()) + } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct StatesBranchRef { - pub branch_index: u64, - pub state: LinearStatesRef, } -/// This is a simple range, end non inclusive. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct LinearStatesRef { - pub start: u64, - pub end: u64, +/// Iterator, contains index of last inner struct. +pub struct BranchRangesIter<'a>(&'a BranchRanges, usize); + +impl<'a> Iterator for BranchRangesIter<'a> { + type Item = (&'a StatesBranchRef, u64); + + fn next(&mut self) -> Option { + if self.1 > 0 { + Some(( + &(self.0).0[self.1 - 1], + (self.0).0[self.1 - 1].branch_index, + )) + } else { + None + } + } } /// current branches range definition, indexed by branch @@ -110,14 +146,14 @@ impl RangeSet { // (avoid some intermediatory Vec (eg when building Hashset) let mut result = Vec::new(); if branch_index < self.treshold { - return result; + return BranchRanges(result); } let mut previous_start = u64::max_value(); loop { if let Some(Some(StatesBranch{ state, parent_branch_index, .. })) = self.storage.get(&branch_index) { // TODO EMCH consider vecdeque ?? let state = if state.end > previous_start { - LinearStatesRef { + BranchStateRef { start: state.start, end: previous_start, } @@ -139,7 +175,7 @@ impl RangeSet { break; } } - result + BranchRanges(result) } @@ -235,14 +271,14 @@ impl RangeSet { let anchor_index = self.add_state(parent_branch_index, number) .expect("coherent branch index state"); // TODO EMCH fail in add_state if anchor_index == parent_branch_index { - branch_ranges.pop(); + branch_ranges.0.pop(); } - branch_ranges.push(self.state_ref(anchor_index).expect("added just above")); + branch_ranges.0.push(self.state_ref(anchor_index).expect("added just above")); (branch_ranges, anchor_index) } else { let anchor_index = self.add_state(parent_branch_index, number) .expect("coherent branch index state"); // TODO EMCH fail in add_state - (vec![self.state_ref(anchor_index).expect("added just above")], anchor_index) + (BranchRanges(vec![self.state_ref(anchor_index).expect("added just above")]), anchor_index) } } @@ -348,7 +384,7 @@ impl RangeSet { #[derive(Debug, Clone, PartialEq, Eq)] pub struct StatesBranch { - state: LinearStatesRef, + state: BranchStateRef, can_append: bool, parent_branch_index: u64, } @@ -369,7 +405,7 @@ impl LinearStates { pub fn new(offset: u64, parent_branch_index: u64) -> Self { let offset = offset as u64; LinearStates { - state: LinearStatesRef { + state: BranchStateRef { start: offset, end: offset + 1, }, @@ -378,7 +414,7 @@ impl LinearStates { } } - pub fn state_ref(&self) -> LinearStatesRef { + pub fn state_ref(&self) -> BranchStateRef { self.state.clone() } @@ -426,7 +462,7 @@ impl LinearStates { } // TODO EMCH end from historied - data -// +#[cfg(test)] mod test { use super::*; @@ -457,9 +493,6 @@ mod test { #[test] fn branch_range_finalize() { - const PREFIX: &[u8] = b"prefix"; - const TRESHOLD: &[u8] = b"hijkl"; - const LAST_INDEX: &[u8] = b"mnopq"; let with_full_finalize = | full: bool, ranges_valid: &[(u64, u64)], @@ -492,33 +525,3 @@ mod test { } - -// TODO EMCH use from historied data on merge. (this is just a stub) - -#[derive(Debug, Clone)] -pub struct History(Vec<()>, Option); - -impl Default for History { - fn default() -> Self { - History(Vec::new(), None) - } -} - - -impl History { - - pub fn set(&mut self, state: &BranchRanges, value: V) { - unimplemented!() - } - - pub fn get(&self, state: &BranchRanges) -> Option<&V> { - unimplemented!() - } - - pub fn gc(&mut self, state: &RangeSet) { - unimplemented!() - } - -} - -// End TODO EMCH diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 6c077cf3aabc7..21d6d00979b44 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -25,7 +25,8 @@ use std::collections::{HashMap, VecDeque, hash_map::Entry, HashSet}; use super::{Error, DBValue, ChangeSet, CommitSet, MetaDb, Hash, to_meta_key, OffstateKey}; use codec::{Encode, Decode}; use log::trace; -use crate::branch::{RangeSet, BranchRanges, History}; +use crate::branch::{RangeSet, BranchRanges}; +use historied_data::tree::History; const NON_CANONICAL_JOURNAL: &[u8] = b"noncanonical_journal"; const NON_CANONICAL_OFFSTATE_JOURNAL: &[u8] = b"offstate_noncanonical_journal"; diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 654dfce62e985..e032b2c653fd9 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -268,7 +268,6 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { } // Usize encoded as le u64 (only for internal indexing). - // TODO EMCH change usize encoding to u32? fn read_le_usize(&self, pos: usize) -> usize { let mut buffer = [0u8; SIZE_BYTE_LEN]; buffer.copy_from_slice(&self.0[pos..pos + SIZE_BYTE_LEN]); diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index bc1cf3ff49d52..6ea0fccff1e00 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -33,6 +33,9 @@ use rstd::vec::Vec; use rstd::collections::btree_map::BTreeMap; use rstd::convert::{TryFrom, TryInto}; +/// Trait defining a state for querying or modifying a tree. +/// This is therefore the representation of a branch. +/// TODO EMCH rename to BranchesStateTrait? pub trait TreeStateTrait { type Branch: BranchStateTrait; type Iter: Iterator; @@ -46,6 +49,8 @@ pub trait TreeStateTrait { fn iter(self) -> Self::Iter; } +/// Trait defining a state for querying or modifying a branch. +/// This is therefore the representation of a branch state. pub trait BranchStateTrait { fn get_node(&self, i: I) -> S; @@ -54,7 +59,6 @@ pub trait BranchStateTrait { fn last_index(&self) -> I; } - impl<'a> TreeStateTrait for &'a StatesRef { type Branch = (&'a StatesBranchRef, Option); type Iter = StatesRefIter<'a>; @@ -98,7 +102,7 @@ impl<'a> TreeStateTrait for &'a StatesRef { } #[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq, Eq))] pub struct BranchState { /// Index of first element (only use for indexed access). /// Element before offset are not in state. @@ -113,11 +117,10 @@ pub struct BranchState { } /// This is a simple range, end non inclusive. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct BranchStateRef { - start: u64, - end: u64, + pub start: u64, + pub end: u64, } impl<'a> BranchStateTrait for (&'a StatesBranchRef, Option) { @@ -136,6 +139,21 @@ impl<'a> BranchStateTrait for (&'a StatesBranchRef, Option) { } +impl<'a> BranchStateTrait for &'a StatesBranchRef { + + fn get_node(&self, i: u64) -> bool { + i >= self.state.start && i < self.state.end + } + + fn last_index(&self) -> u64 { + // underflow should not happen as long as branchstateref are not allowed to be empty. + self.state.end - 1 + } + +} + + + impl Default for BranchState { // initialize with one element fn default() -> Self { @@ -243,7 +261,7 @@ impl Default for TestStates { } #[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq, Eq))] pub struct StatesBranch { // this is the key (need to growth unless full gc (can still have // content pointing to it even if it seems safe to reuse a previously @@ -256,11 +274,10 @@ pub struct StatesBranch { state: BranchState, } -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct StatesBranchRef { - branch_index: u64, - state: BranchStateRef, + pub branch_index: u64, + pub state: BranchStateRef, } @@ -336,7 +353,7 @@ impl TestStates { branch_ref.state.end = previous_origin_node_index + 1; } previous_origin_node_index = branch.origin_node_index; - // TODO EMCH consider reversing state_ref + // vecdeque would be better suited result.insert(0, branch_ref); branch_index = branch.origin_branch_index; } else { @@ -447,6 +464,7 @@ impl TestStates { /// First field is the actual history against which we run /// the state. /// Second field is an optional value for the no match case. +/// TODO EMCH consider removing second field (not use out of test) #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test"), derive(PartialEq))] pub struct History(Vec>, Option); @@ -564,7 +582,7 @@ impl History { return self.1.as_ref(); } - // TODO EMCH reverse loops ? probably. + // TODO EMCH switch loops ? probably. for (state_branch, state_index) in state.iter() { while index > 0 { index -= 1; @@ -805,7 +823,6 @@ mod test { // cannot create when dropped happen on branch assert_eq!(Some(false), states.branch_state_mut(1).map(|ls| ls.add_state())); - // TODO add content through branching assert!(states.get(1, 1)); // 0> 1: _ _ X // | |> 3: 1 From 9ba805772fb012494471ae58068a1099865d89ce Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 1 Oct 2019 21:08:27 +0200 Subject: [PATCH 17/68] Test issue is that query on remove hash do not work --- core/state-db/src/branch.rs | 1 - core/state-db/src/lib.rs | 2 +- core/state-db/src/noncanonical.rs | 39 +++++++++++++------------------ 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index 7ed74215dd30b..a4dfdc2a1e6c1 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -36,7 +36,6 @@ use historied_data::tree::{TreeStateTrait, BranchStateTrait, StatesBranchRef, Br pub struct BranchRanges(Vec); impl<'a> TreeStateTrait for &'a BranchRanges { - // TODO do we need parent ix field of statesbranchref?? type Branch = &'a StatesBranchRef; type Iter = BranchRangesIter<'a>; diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index adf2a01aae707..15e706e74b14b 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -382,7 +382,7 @@ impl StateDbSync { db: &D, ) -> Result, Error> { if let Some(value) = self.non_canonical.get_offstate(key, &state.0) { - return Ok(Some(value)); + return Ok(Some(value.clone())); } db.get_offstate(key, &Some(state.1)).map_err(|e| Error::Db(e)) } diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 21d6d00979b44..ae06ffa94ca57 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -182,9 +182,6 @@ fn discard_values( inserted: Vec, mut into: Option<&mut HashMap>, ) { - if into.is_none() { - return; - } for k in inserted { match values.entry(k) { Entry::Occupied(mut e) => { @@ -622,20 +619,11 @@ impl NonCanonicalOverlay { } /// Get a value from the node overlay. This searches in every existing changeset. - /// TODO EMCH this approach does not work !!! I need the previous historied-data - /// on trie for it (put branch ix in offstate journal record) and remove - /// pinned values (or pinned btreemap>, but - /// I mostly prefer my historied data struct. - /// - /// TODO also need branch ix as parameter... (need context) - /// or maybe a Number is enough (considering the way levels - /// seems to work). - pub fn get_offstate(&self, key: &[u8], state: &BranchRanges) -> Option { - unimplemented!("TODO"); -/* if let Some((_, value)) = self.offstate_values.get(key) { - return Some(value.clone()); + pub fn get_offstate(&self, key: &[u8], state: &BranchRanges) -> Option<&DBValue> { + if let Some(value) = self.offstate_values.get(key) { + return value.get(state); } - None*/ + None } /// Check if the block is in the canonicalization queue. @@ -650,10 +638,12 @@ impl NonCanonicalOverlay { let mut commit = CommitSet::default(); for overlay in level.into_iter() { commit.meta.deleted.push(overlay.journal_key); + commit.meta.deleted.push(overlay.offstate_journal_key); if let Some((_, branch_index)) = self.parents.remove(&overlay.hash) { self.branches.revert(branch_index); } discard_values(&mut self.values, overlay.inserted, None); + discard_offset_values(overlay.offstate_inserted, &mut self.offstate_gc); } commit }) @@ -725,9 +715,9 @@ mod tests { } fn contains_offstate(overlay: &NonCanonicalOverlay, key: u64, state: &H256) -> bool { - overlay.get_branch_range(state).and_then(|state| - overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state)) - == Some(H256::from_low_u64_be(key).as_bytes().to_vec()) + overlay.get_branch_range(state).and_then(|state| { + overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state) + }) == Some(&H256::from_low_u64_be(key).as_bytes().to_vec()) } fn contains_both(overlay: &NonCanonicalOverlay, key: u64, state: &H256) -> bool { @@ -838,7 +828,8 @@ mod tests { assert_eq!(insertion.data.deleted.len(), 0); assert_eq!(insertion.offstate.inserted.len(), 0); assert_eq!(insertion.offstate.deleted.len(), 0); - assert_eq!(insertion.meta.inserted.len(), 2); + // last cannonical, journal_record and offstate_journal_record + assert_eq!(insertion.meta.inserted.len(), 3); assert_eq!(insertion.meta.deleted.len(), 0); db.commit(&insertion); let mut finalization = CommitSet::default(); @@ -848,7 +839,8 @@ mod tests { assert_eq!(finalization.offstate.inserted.len(), offstate_changeset.inserted.len()); assert_eq!(finalization.offstate.deleted.len(), offstate_changeset.deleted.len()); assert_eq!(finalization.meta.inserted.len(), 1); - assert_eq!(finalization.meta.deleted.len(), 1); + // normal and offstate discarded journall + assert_eq!(finalization.meta.deleted.len(), 2); db.commit(&finalization); assert!(db.data_eq(&make_db(&[1, 3, 4]))); assert!(db.offstate_eq(&[1, 3, 4])); @@ -1145,8 +1137,9 @@ mod tests { assert!(contains_both(&overlay, 7, &h2)); db.commit(&overlay.revert_one().unwrap()); assert_eq!(overlay.parents.len(), 1); - assert!(contains_both(&overlay, 5, &h2)); - assert!(!contains_any(&overlay, 7, &h2)); + assert!(contains_both(&overlay, 5, &h1)); + assert!(!contains(&overlay, 7)); + assert!(!contains_any(&overlay, 7, &h1)); db.commit(&overlay.revert_one().unwrap()); assert_eq!(overlay.levels.len(), 0); assert_eq!(overlay.parents.len(), 0); From 31c1ee383fc6e25300bd1378d2e16ad49db53c9b Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 2 Oct 2019 10:49:40 +0200 Subject: [PATCH 18/68] test commit set --- core/state-db/src/noncanonical.rs | 7 ++++++- core/state-db/src/test.rs | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index ae06ffa94ca57..94398210970ce 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -704,6 +704,7 @@ impl NonCanonicalOverlay { #[cfg(test)] mod tests { use std::io; + use std::collections::BTreeMap; use primitives::H256; use super::{NonCanonicalOverlay, to_journal_key}; use crate::{ChangeSet, CommitSet, OffstateKey}; @@ -1063,7 +1064,11 @@ mod tests { // check if restoration from journal results in the same tree let overlay2 = NonCanonicalOverlay::::new(&db).unwrap(); assert_eq!(overlay.levels, overlay2.levels); - assert_eq!(overlay.parents, overlay2.parents); + let parents_no_iter: BTreeMap<_, _> = overlay.parents.iter() + .map(|(k, (v, _))| (k.clone(), v.clone())).collect(); + let parents_no_iter2: BTreeMap<_, _> = overlay2.parents.iter() + .map(|(k, (v, _))| (k.clone(), v.clone())).collect(); + assert_eq!(parents_no_iter, parents_no_iter2); assert_eq!(overlay.last_canonicalized, overlay2.last_canonicalized); // canonicalize 1. 2 and all its children should be discarded diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 41768bd74926f..595a8df35ef6f 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -59,10 +59,13 @@ impl NodeDb for TestDb { impl TestDb { pub fn commit(&mut self, commit: &CommitSet) { self.data.extend(commit.data.inserted.iter().cloned()); - self.meta.extend(commit.meta.inserted.iter().cloned()); for k in commit.data.deleted.iter() { self.data.remove(k); } + self.offstate.extend(commit.offstate.inserted.iter().cloned()); + for k in commit.offstate.deleted.iter() { + self.offstate.remove(k); + } self.meta.extend(commit.meta.inserted.iter().cloned()); for k in commit.meta.deleted.iter() { self.meta.remove(k); From 6209f4d1f00e90f2dcee50aa6f7e81c050462abd Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 2 Oct 2019 12:34:16 +0200 Subject: [PATCH 19/68] fix unpin gc --- core/state-db/src/branch.rs | 13 +++-- core/state-db/src/noncanonical.rs | 68 ++++++++++++++++++--------- core/state-db/src/test.rs | 8 +++- core/utils/historied-data/src/tree.rs | 27 ++++++----- 4 files changed, 77 insertions(+), 39 deletions(-) diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index a4dfdc2a1e6c1..e60816d993b6d 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -36,7 +36,7 @@ use historied_data::tree::{TreeStateTrait, BranchStateTrait, StatesBranchRef, Br pub struct BranchRanges(Vec); impl<'a> TreeStateTrait for &'a BranchRanges { - type Branch = &'a StatesBranchRef; + type Branch = &'a BranchStateRef; type Iter = BranchRangesIter<'a>; fn get_branch(self, i: u64) -> Option { @@ -69,12 +69,12 @@ impl<'a> TreeStateTrait for &'a BranchRanges { pub struct BranchRangesIter<'a>(&'a BranchRanges, usize); impl<'a> Iterator for BranchRangesIter<'a> { - type Item = (&'a StatesBranchRef, u64); + type Item = (&'a BranchStateRef, u64); fn next(&mut self) -> Option { if self.1 > 0 { Some(( - &(self.0).0[self.1 - 1], + &(self.0).0[self.1 - 1].state, (self.0).0[self.1 - 1].branch_index, )) } else { @@ -95,7 +95,9 @@ impl<'a> Iterator for BranchRangesIter<'a> { pub struct RangeSet { // TODO EMCH using a option value makes not sense when all in memory storage: BTreeMap>, + // TODO EMCH remove this? last_index: u64, + // TODO EMCH remove this? treshold: u64, } @@ -135,6 +137,11 @@ impl RangeSet { } } + /// Iterator over all its range sets. + pub fn reverse_iter_ranges(&self) -> impl Iterator { + self.storage.iter().rev().filter_map(|(k, v)| v.as_ref().map(|v| (&v.state, *k))) + } + // TODO EMCH can rw lock over the latest accessed range (lru) and // return an clone of it (arc of the value so it will be fine). this is call by state_at that is call a lot!!! pub fn branch_ranges_from_cache( diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 94398210970ce..36f7dd9e9d64e 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -82,12 +82,20 @@ impl OffstatePendingGC { fn try_gc( &mut self, + offstate_values: &mut HashMap>, + branches: &RangeSet, pinned: &HashMap, ) { if let Some(pending) = self.pending_canonicalisation_query { if pending < self.min_pinned_index(pinned) { - unimplemented!("TODO feed keepindexes with branch at pending then actual gc"); + // TODO EMCH double get is rather inefficient, see if it is possible + // to reuse the hash of the hashset into the hashmap + for key in self.keys_pending_gc.drain() { + offstate_values.get_mut(&key).map(|historied_value| { + historied_value.gc(branches.reverse_iter_ranges()); + }); + } self.pending_canonicalisation_query = None; } @@ -552,6 +560,8 @@ impl NonCanonicalOverlay { fn apply_canonicalizations(&mut self) { let last = self.pending_canonicalizations.last().cloned(); + let last_index = last.as_ref().and_then(|last| self.parents.get(last)) + .map(|(_, index)| *index); let count = self.pending_canonicalizations.len() as u64; for hash in self.pending_canonicalizations.drain(..) { trace!(target: "state-db", "Post canonicalizing {:?}", hash); @@ -584,25 +594,22 @@ impl NonCanonicalOverlay { ); } } - - if let Some(branch_index_cannonicalize) = last.as_ref().and_then(|last| self.parents.get(last)) - .map(|(_, index)| *index) { - // set branch state synchronously + if let Some(hash) = last { let block_number = self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1); - // this needs to be call after parents update - // TODO EMCH may be needed in 'canonicalize', and restore or commit here - self.branches.update_finalize_treshold(branch_index_cannonicalize, Some(block_number), true); - // gc is at the right place - self.offstate_gc.set_pending_gc(branch_index_cannonicalize); - // try to run the garbage collection (can run later if there is - // pinned process). - self.offstate_gc.try_gc(&self.pinned); + self.last_canonicalized = Some((hash, block_number)); + + if let Some(branch_index_cannonicalize) = last_index { + // this needs to be call after parents update + // TODO EMCH may be needed in 'canonicalize', and restore or commit here + self.branches.update_finalize_treshold(branch_index_cannonicalize, Some(block_number), true); + // gc is at the right place + self.offstate_gc.set_pending_gc(branch_index_cannonicalize); + // try to run the garbage collection (can run later if there is + // pinned process). + self.offstate_gc.try_gc(&mut self.offstate_values, &self.branches, &self.pinned); + } } - if let Some(hash) = last { - let last_canonicalized = (hash, self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1)); - self.last_canonicalized = Some(last_canonicalized); - } } /// Get a value from the node overlay. This searches in every existing changeset. @@ -697,7 +704,7 @@ impl NonCanonicalOverlay { /// Discard pinned state pub fn unpin(&mut self, hash: &BlockHash) { self.pinned.remove(hash); - self.offstate_gc.try_gc(&self.pinned); + self.offstate_gc.try_gc(&mut self.offstate_values, &self.branches, &self.pinned); } } @@ -721,6 +728,11 @@ mod tests { }) == Some(&H256::from_low_u64_be(key).as_bytes().to_vec()) } + fn contains_offstate2(overlay: &NonCanonicalOverlay, key: u64, state: &BranchRanges) -> bool { + overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state) + == Some(&H256::from_low_u64_be(key).as_bytes().to_vec()) + } + fn contains_both(overlay: &NonCanonicalOverlay, key: u64, state: &H256) -> bool { contains(overlay, key) && contains_offstate(overlay, key, state) } @@ -863,7 +875,7 @@ mod tests { make_changeset(&[5], &[3]), make_offstate_changeset(&[5], &[3]), ).unwrap()); - assert_eq!(db.meta.len(), 3); + assert_eq!(db.meta.len(), 5); let overlay2 = NonCanonicalOverlay::::new(&db).unwrap(); assert_eq!(overlay.levels, overlay2.levels); @@ -1203,13 +1215,25 @@ mod tests { db.commit(&overlay.insert::(&h_2, 1, &H256::default(), c_2, o_c_2).unwrap()); overlay.pin(&h_1); - + let h1_context = overlay.get_branch_range(&h_1).unwrap(); + let mut commit = CommitSet::default(); + println!("b1{:?}", overlay.branches); overlay.canonicalize::(&h_2, &mut commit).unwrap(); + println!("b2{:?}", overlay.branches); db.commit(&commit); + println!("b3{:?}", overlay.branches); overlay.apply_pending(); - assert!(contains_both(&overlay, 1, &h_1)); + assert!(contains(&overlay, 1)); + // we cannot use contains_offstate because offstate pining is relying on + // asumption that pinned context memoïzed its branch state. + assert!(contains_offstate2(&overlay, 1, &h1_context)); + assert!(!contains_offstate(&overlay, 1, &h_1)); + println!("b4{:?}", overlay.branches); + println!("{:?}", overlay.offstate_values); overlay.unpin(&h_1); - assert!(!contains_any(&overlay, 1, &h_1)); + println!("{:?}", overlay.offstate_values); + assert!(!contains(&overlay, 1)); + assert!(!contains_offstate2(&overlay, 1, &h1_context)); } } diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 595a8df35ef6f..6d3e8b8bb6e7c 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -136,7 +136,13 @@ pub fn make_db(inserted: &[u64]) -> TestDb { }) .collect(), meta: Default::default(), - offstate: Default::default(), + offstate: inserted + .iter() + .map(|v| ( + H256::from_low_u64_be(*v).as_bytes().to_vec(), + H256::from_low_u64_be(*v).as_bytes().to_vec(), + )) + .collect(), } } diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index 6ea0fccff1e00..45d27b3ad99a2 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -60,7 +60,7 @@ pub trait BranchStateTrait { } impl<'a> TreeStateTrait for &'a StatesRef { - type Branch = (&'a StatesBranchRef, Option); + type Branch = (&'a BranchStateRef, Option); type Iter = StatesRefIter<'a>; fn get_branch(self, i: u64) -> Option { @@ -123,31 +123,31 @@ pub struct BranchStateRef { pub end: u64, } -impl<'a> BranchStateTrait for (&'a StatesBranchRef, Option) { +impl<'a> BranchStateTrait for (&'a BranchStateRef, Option) { fn get_node(&self, i: u64) -> bool { - let l = self.0.state.end; + let l = self.0.end; let upper = self.1.map(|u| rstd::cmp::min(u + 1, l)).unwrap_or(l); - i >= self.0.state.start && i < upper + i >= self.0.start && i < upper } fn last_index(&self) -> u64 { // underflow should not happen as long as branchstateref are not allowed to be empty. - let state_end = self.0.state.end - 1; + let state_end = self.0.end - 1; self.1.map(|bound| rstd::cmp::min(state_end, bound)).unwrap_or(state_end) } } -impl<'a> BranchStateTrait for &'a StatesBranchRef { +impl<'a> BranchStateTrait for &'a BranchStateRef { fn get_node(&self, i: u64) -> bool { - i >= self.state.start && i < self.state.end + i >= self.start && i < self.end } fn last_index(&self) -> u64 { // underflow should not happen as long as branchstateref are not allowed to be empty. - self.state.end - 1 + self.end - 1 } } @@ -300,13 +300,13 @@ pub struct StatesRef { pub struct StatesRefIter<'a>(&'a StatesRef, usize, Option); impl<'a> Iterator for StatesRefIter<'a> { - type Item = ((&'a StatesBranchRef, Option), u64); + type Item = ((&'a BranchStateRef, Option), u64); fn next(&mut self) -> Option { if self.1 > 0 { let upper_node_index = self.2.take(); Some(( - (&self.0.history[self.1 - 1], upper_node_index), + (&self.0.history[self.1 - 1].state, upper_node_index), self.0.history[self.1 - 1].branch_index, )) } else { @@ -659,13 +659,14 @@ impl History { if self.0[branch_index].history.len() == 0 { let _ = self.0.remove(branch_index); } - current_state = states.next(); break; } else { - self.0.remove(branch_index); + let _ = self.0.remove(branch_index); break; } } + self.0.remove(branch_index); + break; } else { break; } @@ -958,7 +959,7 @@ mod test { states1.get_mut(&1).map(|br| br.state.len = 1); let states1: BTreeMap<_, _> = states1.iter().map(|(k,v)| (k, v.branch_ref())).collect(); let mut item1 = item.clone(); - item1.gc(states1.iter().map(|(k, v)| ((v, None), **k)).rev()); + item1.gc(states1.iter().map(|(k, v)| ((&v.state, None), **k)).rev()); assert_eq!(item1.get(&states.state_ref(1)), None); for a in action.iter().skip(1) { if a.1 { From 70f69b67244b6e7644deb9e36ad61fbc3d8c4472 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 2 Oct 2019 13:02:42 +0200 Subject: [PATCH 20/68] missing offstate prunnig handling --- core/state-db/src/lib.rs | 1 + core/state-db/src/noncanonical.rs | 19 ++++++++-------- core/state-db/src/pruning.rs | 37 +++++++++++++++++++++---------- core/state-db/src/test.rs | 16 +++++++------ 4 files changed, 44 insertions(+), 29 deletions(-) diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index 15e706e74b14b..93b2901d1a83b 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -262,6 +262,7 @@ impl StateDbSync { Ok(()) => { if self.mode == PruningMode::ArchiveCanonical { commit.data.deleted.clear(); + commit.offstate.deleted.clear(); } } Err(e) => return Err(e), diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 36f7dd9e9d64e..bd9f073ecacfc 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -209,7 +209,7 @@ fn discard_values( } } -fn discard_offset_values( +fn discard_offstate_values( inserted: Vec, into: &mut OffstatePendingGC, ) { @@ -240,7 +240,7 @@ fn discard_descendants( discarded.push(overlay.hash); let mut pinned = pinned.get_mut(hash); discard_values(&mut values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); - discard_offset_values( + discard_offstate_values( overlay.offstate_inserted, offstate_gc, ); @@ -588,7 +588,7 @@ impl NonCanonicalOverlay { let mut pinned = self.pinned.get_mut(&overlay.hash); discard_values(&mut self.values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); - discard_offset_values( + discard_offstate_values( overlay.offstate_inserted, &mut self.offstate_gc, ); @@ -650,7 +650,7 @@ impl NonCanonicalOverlay { self.branches.revert(branch_index); } discard_values(&mut self.values, overlay.inserted, None); - discard_offset_values(overlay.offstate_inserted, &mut self.offstate_gc); + discard_offstate_values(overlay.offstate_inserted, &mut self.offstate_gc); } commit }) @@ -830,6 +830,7 @@ mod tests { fn insert_canonicalize_one() { let h1 = H256::random(); let mut db = make_db(&[1, 2]); + db.initialize_offstate(&[1, 2]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); let changeset = make_changeset(&[3, 4], &[2]); let offstate_changeset = make_offstate_changeset(&[3, 4], &[2]); @@ -864,6 +865,7 @@ mod tests { let h1 = H256::random(); let h2 = H256::random(); let mut db = make_db(&[1, 2]); + db.initialize_offstate(&[1, 2]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); db.commit(&overlay.insert::( &h1, 10, &H256::default(), @@ -888,6 +890,7 @@ mod tests { let h1 = H256::random(); let h2 = H256::random(); let mut db = make_db(&[1, 2]); + db.initialize_offstate(&[1, 2]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); db.commit(&overlay.insert::( &h1, 10, &H256::default(), @@ -916,6 +919,7 @@ mod tests { let h1 = H256::random(); let h2 = H256::random(); let mut db = make_db(&[1, 2, 3, 4]); + db.initialize_offstate(&[1, 2, 3, 4]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); let changeset1 = make_changeset(&[5, 6], &[2]); let changeset2 = make_changeset(&[7, 8], &[5, 3]); @@ -1137,6 +1141,7 @@ mod tests { let h1 = H256::random(); let h2 = H256::random(); let mut db = make_db(&[1, 2, 3, 4]); + db.initialize_offstate(&[1, 2, 3, 4]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); assert!(overlay.revert_one().is_none()); let changeset1 = make_changeset(&[5, 6], &[2]); @@ -1218,21 +1223,15 @@ mod tests { let h1_context = overlay.get_branch_range(&h_1).unwrap(); let mut commit = CommitSet::default(); - println!("b1{:?}", overlay.branches); overlay.canonicalize::(&h_2, &mut commit).unwrap(); - println!("b2{:?}", overlay.branches); db.commit(&commit); - println!("b3{:?}", overlay.branches); overlay.apply_pending(); assert!(contains(&overlay, 1)); // we cannot use contains_offstate because offstate pining is relying on // asumption that pinned context memoïzed its branch state. assert!(contains_offstate2(&overlay, 1, &h1_context)); assert!(!contains_offstate(&overlay, 1, &h_1)); - println!("b4{:?}", overlay.branches); - println!("{:?}", overlay.offstate_values); overlay.unpin(&h_1); - println!("{:?}", overlay.offstate_values); assert!(!contains(&overlay, 1)); assert!(!contains_offstate2(&overlay, 1, &h1_context)); } diff --git a/core/state-db/src/pruning.rs b/core/state-db/src/pruning.rs index d62d80faac884..cb5e939e524a7 100644 --- a/core/state-db/src/pruning.rs +++ b/core/state-db/src/pruning.rs @@ -281,7 +281,7 @@ mod tests { use super::RefWindow; use primitives::H256; use crate::CommitSet; - use crate::test::{make_db, make_commit, TestDb}; + use crate::test::{make_db, make_commit_both, TestDb}; fn check_journal(pruning: &RefWindow, db: &TestDb) { let restored: RefWindow = RefWindow::new(db).unwrap(); @@ -320,7 +320,7 @@ mod tests { let mut db = make_db(&[1, 2, 3]); db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = make_commit(&[4, 5], &[1, 3]); + let mut commit = make_commit_both(&[4, 5], &[1, 3]); commit.initialize_offstate(&[4, 5], &[1, 3]); let h = H256::random(); pruning.note_canonical(&h, &mut commit); @@ -354,15 +354,17 @@ mod tests { #[test] fn prune_two() { let mut db = make_db(&[1, 2, 3]); + db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = make_commit(&[4], &[1]); + let mut commit = make_commit_both(&[4], &[1]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); - let mut commit = make_commit(&[5], &[2]); + let mut commit = make_commit_both(&[5], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); + assert!(db.offstate_eq(&[1, 2, 3, 4, 5])); check_journal(&pruning, &db); @@ -371,34 +373,40 @@ mod tests { db.commit(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[2, 3, 4, 5]))); + assert!(db.offstate_eq(&[2, 3, 4, 5])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[3, 4, 5]))); + assert!(db.offstate_eq(&[3, 4, 5])); assert_eq!(pruning.pending_number, 2); } #[test] fn prune_two_pending() { let mut db = make_db(&[1, 2, 3]); + db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = make_commit(&[4], &[1]); + let mut commit = make_commit_both(&[4], &[1]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); - let mut commit = make_commit(&[5], &[2]); + let mut commit = make_commit_both(&[5], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); + assert!(db.offstate_eq(&[1, 2, 3, 4, 5])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[2, 3, 4, 5]))); + assert!(db.offstate_eq(&[2, 3, 4, 5])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[3, 4, 5]))); + assert!(db.offstate_eq(&[3, 4, 5])); assert_eq!(pruning.pending_number, 2); } @@ -407,14 +415,14 @@ mod tests { let mut db = make_db(&[1, 2, 3]); db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = make_commit(&[], &[2]); + let mut commit = make_commit_both(&[], &[2]); commit.initialize_offstate(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); - let mut commit = make_commit(&[2], &[]); + let mut commit = make_commit_both(&[2], &[]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); - let mut commit = make_commit(&[], &[2]); + let mut commit = make_commit_both(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); @@ -444,29 +452,34 @@ mod tests { #[test] fn reinserted_survivew_pending() { let mut db = make_db(&[1, 2, 3]); + db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = make_commit(&[], &[2]); + let mut commit = make_commit_both(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); - let mut commit = make_commit(&[2], &[]); + let mut commit = make_commit_both(&[2], &[]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); - let mut commit = make_commit(&[], &[2]); + let mut commit = make_commit_both(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); + assert!(db.offstate_eq(&[1, 2, 3])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); + assert!(db.offstate_eq(&[1, 2, 3])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); + assert!(db.offstate_eq(&[1, 2, 3])); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 3]))); + assert!(db.offstate_eq(&[1, 3])); pruning.apply_pending(); assert_eq!(pruning.pending_number, 3); } diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 6d3e8b8bb6e7c..3b10e6540859b 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -120,6 +120,14 @@ pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet { } } +pub fn make_commit_both(inserted: &[u64], deleted: &[u64]) -> CommitSet { + CommitSet { + data: make_changeset(inserted, deleted), + meta: ChangeSet::default(), + offstate: make_offstate_changeset(inserted, deleted), + } +} + impl CommitSet { pub fn initialize_offstate(&mut self, inserted: &[u64], deleted: &[u64]) { let data = make_offstate_changeset(inserted, deleted); @@ -136,13 +144,7 @@ pub fn make_db(inserted: &[u64]) -> TestDb { }) .collect(), meta: Default::default(), - offstate: inserted - .iter() - .map(|v| ( - H256::from_low_u64_be(*v).as_bytes().to_vec(), - H256::from_low_u64_be(*v).as_bytes().to_vec(), - )) - .collect(), + offstate: Default::default(), } } From 6803c861a16ef646a7ba11ca69e921fb4fe55703 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 2 Oct 2019 19:25:40 +0200 Subject: [PATCH 21/68] previous pruning did not make sense, switch to only storing touched key for gc. Need to change commit set to contain an history of values and adjust test db. --- core/client/db/src/lib.rs | 19 +-- core/state-db/src/branch.rs | 6 +- core/state-db/src/lib.rs | 26 ++-- core/state-db/src/noncanonical.rs | 177 ++++++++++++------------ core/state-db/src/pruning.rs | 63 ++------- core/state-db/src/test.rs | 64 ++++++--- core/utils/historied-data/src/linear.rs | 7 + core/utils/historied-data/src/tree.rs | 20 +++ 8 files changed, 196 insertions(+), 186 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 581dd70795860..2c1f9cea7eabc 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -1222,16 +1222,10 @@ impl> Backend { changeset.deleted.push(key); } } - // remove duplicate - let mut map_offstate: HashMap<_, _> = operation.db_updates.1.into_iter().collect(); - let mut offstate_changeset: state_db::ChangeSet> = state_db::ChangeSet::default(); - for (key, option_val) in map_offstate.drain() { - if let Some(val) = option_val { - offstate_changeset.inserted.push((key, val.to_vec())); - } else { - offstate_changeset.deleted.push(key); - } - } + // remove duplicate TODO EMCH very bad + let map_offstate: HashMap<_, _> = operation.db_updates.1.into_iter().collect(); + // TODO EMCH !!! keep map_offstate for OffstateChangeSet type??? + let offstate_changeset: state_db::OffstateChangeSet> = map_offstate.into_iter().collect(); let number_u64 = number.saturated_into::(); let commit = self.storage.state_db.insert_block( &hash, @@ -1392,6 +1386,7 @@ impl> Backend { } fn apply_state_commit(transaction: &mut DBTransaction, commit: state_db::CommitSet>) { + // TODO EMCH unimplemented commit of offstate for (key, val) in commit.data.inserted.into_iter() { transaction.put(columns::STATE, &key[..], &val); } @@ -1583,9 +1578,9 @@ impl client::backend::Backend for Backend whe Ok(Some(ref hdr)) => { let hash = hdr.hash(); if let Ok(()) = self.storage.state_db.pin(&hash) { - let range = self.storage.state_db.get_branch_range(&hash); - let root = H256::from_slice(hdr.state_root().as_ref()); let block_number = hdr.number().clone().saturated_into::(); + let range = self.storage.state_db.get_branch_range(&hash, block_number); + let root = H256::from_slice(hdr.state_root().as_ref()); let offstate = StorageDbAt { storage_db: self.storage.clone(), state: (range.unwrap_or_else(Default::default), block_number), diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index e60816d993b6d..d52b3c945f426 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -147,6 +147,7 @@ impl RangeSet { pub fn branch_ranges_from_cache( &self, mut branch_index: u64, + block: Option, ) -> BranchRanges { // TODO EMCH transform this method to an iterator!!! // (avoid some intermediatory Vec (eg when building Hashset) @@ -154,7 +155,7 @@ impl RangeSet { if branch_index < self.treshold { return BranchRanges(result); } - let mut previous_start = u64::max_value(); + let mut previous_start = block.map(|b| b + 1).unwrap_or(u64::max_value()); loop { if let Some(Some(StatesBranch{ state, parent_branch_index, .. })) = self.storage.get(&branch_index) { // TODO EMCH consider vecdeque ?? @@ -365,6 +366,7 @@ impl RangeSet { /// Revert some ranges, without any way to revert. /// Returning ranges for the parent index. + /// TODO EMCH can remove ?? pub fn revert(&mut self, branch_ix: u64) -> BranchRanges { let parent_branch_index = if branch_ix != 0 { self.drop_state(branch_ix) @@ -374,7 +376,7 @@ impl RangeSet { 0 }; - self.branch_ranges_from_cache(parent_branch_index) + self.branch_ranges_from_cache(parent_branch_index, None) } #[cfg(test)] diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index 93b2901d1a83b..37c5e8e79f2a5 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -137,6 +137,8 @@ pub struct ChangeSet { pub deleted: Vec, } +/// A set of offstate state values changes. +pub type OffstateChangeSet = Vec<(H, Option)>; /// A set of changes to the backing database. #[derive(Default, Debug, Clone)] @@ -146,7 +148,7 @@ pub struct CommitSet { /// Metadata changes. pub meta: ChangeSet>, /// Offstate data changes. - pub offstate: ChangeSet, + pub offstate: OffstateChangeSet, } /// Pruning constraints. If none are specified pruning is @@ -233,13 +235,12 @@ impl StateDbSync { number: u64, parent_hash: &BlockHash, mut changeset: ChangeSet, - mut offstate_changeset: ChangeSet, + mut offstate_changeset: OffstateChangeSet, ) -> Result, Error> { match self.mode { PruningMode::ArchiveAll => { changeset.deleted.clear(); - offstate_changeset.deleted.clear(); // write changes immediately Ok(CommitSet { data: changeset, @@ -262,7 +263,6 @@ impl StateDbSync { Ok(()) => { if self.mode == PruningMode::ArchiveCanonical { commit.data.deleted.clear(); - commit.offstate.deleted.clear(); } } Err(e) => return Err(e), @@ -326,8 +326,8 @@ impl StateDbSync { } /// TODO EMCH - pub fn get_branch_range(&self, hash: &BlockHash) -> Option { - self.non_canonical.get_branch_range(hash) + pub fn get_branch_range(&self, hash: &BlockHash, number: u64) -> Option { + self.non_canonical.get_branch_range(hash, number) } pub fn pin(&mut self, hash: &BlockHash) -> Result<(), PinError> { @@ -383,7 +383,7 @@ impl StateDbSync { db: &D, ) -> Result, Error> { if let Some(value) = self.non_canonical.get_offstate(key, &state.0) { - return Ok(Some(value.clone())); + return Ok(value.clone()); } db.get_offstate(key, &Some(state.1)).map_err(|e| Error::Db(e)) } @@ -393,10 +393,8 @@ impl StateDbSync { state: &(BranchRanges, u64), db: &D, ) -> Vec<(OffstateKey, DBValue)> { - // TODO get non_canonical optional values and then filter - // db pairs over it : TODO -> db.get_offstate would return - // an iterator. - unimplemented!() + unimplemented!("TODO some filtering ever non canonical"); + Default::default() } @@ -443,7 +441,7 @@ impl StateDb { number: u64, parent_hash: &BlockHash, changeset: ChangeSet, - offstate_changeset: ChangeSet, + offstate_changeset: OffstateChangeSet, ) -> Result, Error> { self.db.write().insert_block(hash, number, parent_hash, changeset, offstate_changeset) } @@ -454,8 +452,8 @@ impl StateDb { } /// TODO EMCH - pub fn get_branch_range(&self, hash: &BlockHash) -> Option { - self.db.read().get_branch_range(hash) + pub fn get_branch_range(&self, hash: &BlockHash, number: u64) -> Option { + self.db.read().get_branch_range(hash, number) } /// Prevents pruning of specified block and its descendants. diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index bd9f073ecacfc..b9f1af6939f64 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -22,7 +22,10 @@ use std::fmt; use std::collections::{HashMap, VecDeque, hash_map::Entry, HashSet}; -use super::{Error, DBValue, ChangeSet, CommitSet, MetaDb, Hash, to_meta_key, OffstateKey}; +use super::{ + Error, DBValue, ChangeSet, OffstateChangeSet, CommitSet, MetaDb, Hash, + to_meta_key, OffstateKey, +}; use codec::{Encode, Decode}; use log::trace; use crate::branch::{RangeSet, BranchRanges}; @@ -47,7 +50,7 @@ pub struct NonCanonicalOverlay { pending_insertions: Vec, values: HashMap, //ref counted branches: RangeSet, - offstate_values: HashMap>, + offstate_values: HashMap>>, /// second value is offstate pinned index: used in order to determine if the pinned /// thread should block garbage collection. pinned: HashMap, GcIndex)>, @@ -82,7 +85,7 @@ impl OffstatePendingGC { fn try_gc( &mut self, - offstate_values: &mut HashMap>, + offstate_values: &mut HashMap>>, branches: &RangeSet, pinned: &HashMap, ) { @@ -143,8 +146,7 @@ struct JournalRecord { #[derive(Encode, Decode)] struct OffstateJournalRecord { - inserted: Vec<(OffstateKey, DBValue)>, - deleted: Vec, + inserted: Vec<(OffstateKey, Option)>, } fn to_journal_key(block: BlockNumber, index: u64) -> Vec { @@ -163,7 +165,6 @@ struct BlockOverlay { inserted: Vec, deleted: Vec, offstate_inserted: Vec, - offstate_deleted: Vec, } fn insert_values(values: &mut HashMap, inserted: Vec<(Key, DBValue)>) { @@ -176,8 +177,8 @@ fn insert_values(values: &mut HashMap, inserted: fn insert_offstate_values( state: &BranchRanges, - values: &mut HashMap>, - inserted: Vec<(Key, DBValue)>, + values: &mut HashMap>>, + inserted: Vec<(Key, Option)>, ) { for (k, v) in inserted { let entry = values.entry(k).or_insert_with(|| Default::default()); @@ -226,7 +227,7 @@ fn discard_descendants( levels: &mut VecDeque>>, mut values: &mut HashMap, index: usize, - parents: &mut HashMap, + parents: &mut HashMap, pinned: &mut HashMap, u64)>, offstate_gc: &mut OffstatePendingGC, hash: &BlockHash, @@ -288,18 +289,18 @@ impl NonCanonicalOverlay { // got a single element. // fetch parent info let parent_branch_index = parents.get(&record.parent_hash).map(|(_, i)| *i).unwrap_or(0); - let parent_branch_range = Some(branches.branch_ranges_from_cache(parent_branch_index)); + let parent_branch_range = Some(branches.branch_ranges_from_cache(parent_branch_index, Some(block - 1))); let (branch_range, branch_index) = branches.import( block, parent_branch_index, parent_branch_range, ); - let (offstate_record_inserted, offstate_record_deleted) = if let Some(record) = db + let offstate_record_inserted = if let Some(record) = db .get_meta(&offstate_journal_key).map_err(|e| Error::Db(e))? { let record = OffstateJournalRecord::decode(&mut record.as_slice())?; - (Some(record.inserted), Some(record.deleted)) - } else { (None, None) }; + Some(record.inserted) + } else { None }; let inserted = record.inserted.iter().map(|(k, _)| k.clone()).collect(); let offstate_inserted = offstate_record_inserted.as_ref() .map(|inserted| inserted.iter().map(|(k, _)| k.clone()).collect()) @@ -311,7 +312,6 @@ impl NonCanonicalOverlay { inserted: inserted, deleted: record.deleted, offstate_inserted: offstate_inserted, - offstate_deleted: offstate_record_deleted.unwrap_or(Vec::new()), }; insert_values(&mut values, record.inserted); if let Some(inserted) = offstate_record_inserted { @@ -319,13 +319,12 @@ impl NonCanonicalOverlay { } trace!( target: "state-db", - "Uncanonicalized journal entry {}.{} ({} {} inserted, {} {} deleted)", + "Uncanonicalized journal entry {}.{} ({} {} inserted, {} deleted)", block, index, overlay.inserted.len(), overlay.offstate_inserted.len(), overlay.deleted.len(), - overlay.offstate_deleted.len(), ); level.push(overlay); parents.insert(record.hash, (record.parent_hash, branch_index)); @@ -365,7 +364,7 @@ impl NonCanonicalOverlay { number: BlockNumber, parent_hash: &BlockHash, changeset: ChangeSet, - offstate_changeset: ChangeSet, + offstate_changeset: OffstateChangeSet, ) -> Result, Error> { let mut commit = CommitSet::default(); let front_block_number = self.front_block_number(); @@ -405,7 +404,7 @@ impl NonCanonicalOverlay { let offstate_journal_key = to_offstate_journal_key(number, index); let inserted = changeset.inserted.iter().map(|(k, _)| k.clone()).collect(); - let offstate_inserted = offstate_changeset.inserted.iter().map(|(k, _)| k.clone()).collect(); + let offstate_inserted = offstate_changeset.iter().map(|(k, _)| k.clone()).collect(); let overlay = BlockOverlay { hash: hash.clone(), journal_key: journal_key.clone(), @@ -413,11 +412,10 @@ impl NonCanonicalOverlay { inserted: inserted, deleted: changeset.deleted.clone(), offstate_inserted: offstate_inserted, - offstate_deleted: offstate_changeset.deleted.clone(), }; level.push(overlay); let parent_branch_index = self.parents.get(&parent_hash).map(|(_, i)| *i).unwrap_or(0); - let parent_branch_range = Some(self.branches.branch_ranges_from_cache(parent_branch_index)); + let parent_branch_range = Some(self.branches.branch_ranges_from_cache(parent_branch_index, Some(number - 1))); let (branch_range, branch_index) = self.branches.import( number, parent_branch_index, @@ -433,20 +431,18 @@ impl NonCanonicalOverlay { }; commit.meta.inserted.push((journal_key, journal_record.encode())); let offstate_journal_record = OffstateJournalRecord { - inserted: offstate_changeset.inserted, - deleted: offstate_changeset.deleted, + inserted: offstate_changeset, }; commit.meta.inserted.push((offstate_journal_key, offstate_journal_record.encode())); trace!( target: "state-db", - "Inserted uncanonicalized changeset {}.{} ({} {} inserted, {} {} deleted)", + "Inserted uncanonicalized changeset {}.{} ({} {} inserted, {} deleted)", number, index, journal_record.inserted.len(), offstate_journal_record.inserted.len(), journal_record.deleted.len(), - offstate_journal_record.deleted.len(), ); insert_values(&mut self.values, journal_record.inserted); insert_offstate_values(&branch_range, &mut self.offstate_values, offstate_journal_record.inserted); @@ -534,13 +530,14 @@ impl NonCanonicalOverlay { commit.data.inserted.extend(overlay.inserted.iter() .map(|k| (k.clone(), self.values.get(k).expect("For each key in overlays there's a value in values").1.clone()))); commit.data.deleted.extend(overlay.deleted.clone()); + let block_number = self.front_block_number() + self.pending_canonicalizations.len() as u64; if !overlay.offstate_inserted.is_empty() { // canonicalization is not frequent enough that we pass range // in parameter for now - // TODO EMCH it seems bad ide to maintain this commit offstate + // TODO EMCH it seems bad idea to maintain this commit offstate // field as his: simply delta of state could be better?? - if let Some(range) = self.get_branch_range(hash) { - commit.offstate.inserted.extend(overlay.offstate_inserted.iter() + if let Some(range) = self.get_branch_range(hash, block_number) { + commit.offstate.extend(overlay.offstate_inserted.iter() .map(|k| (k.clone(), self.offstate_values.get(k) .expect("For each key in overlays there's a value in values") .get(&range) @@ -548,10 +545,9 @@ impl NonCanonicalOverlay { .clone()))); } } - commit.offstate.deleted.extend(overlay.offstate_deleted.clone()); commit.meta.deleted.append(&mut discarded_journals); - let canonicalized = (hash.clone(), self.front_block_number() + self.pending_canonicalizations.len() as u64); + let canonicalized = (hash.clone(), block_number); commit.meta.inserted.push((to_meta_key(LAST_CANONICAL, &()), canonicalized.encode())); trace!(target: "state-db", "Discarding {} records", commit.meta.deleted.len()); self.pending_canonicalizations.push(hash.clone()); @@ -626,8 +622,10 @@ impl NonCanonicalOverlay { } /// Get a value from the node overlay. This searches in every existing changeset. - pub fn get_offstate(&self, key: &[u8], state: &BranchRanges) -> Option<&DBValue> { + pub fn get_offstate(&self, key: &[u8], state: &BranchRanges) -> Option<&Option> { + println!("b: {:?}", state); if let Some(value) = self.offstate_values.get(key) { + println!("v: {:?}", value); return value.get(state); } None @@ -695,9 +693,9 @@ impl NonCanonicalOverlay { } /// TODO EMCH aka get state for hash to query offstate storage. - pub fn get_branch_range(&self, hash: &BlockHash) -> Option { + pub fn get_branch_range(&self, hash: &BlockHash, number: BlockNumber) -> Option { self.parents.get(hash).map(|(_, branch_index)| *branch_index).map(|branch_index| { - self.branches.branch_ranges_from_cache(branch_index) + self.branches.branch_ranges_from_cache(branch_index, Some(number)) }) } @@ -714,7 +712,7 @@ mod tests { use std::collections::BTreeMap; use primitives::H256; use super::{NonCanonicalOverlay, to_journal_key}; - use crate::{ChangeSet, CommitSet, OffstateKey}; + use crate::{ChangeSet, OffstateChangeSet, CommitSet, OffstateKey}; use crate::test::{make_db, make_changeset, make_offstate_changeset}; use crate::branch::BranchRanges; @@ -722,26 +720,25 @@ mod tests { overlay.get(&H256::from_low_u64_be(key)) == Some(H256::from_low_u64_be(key).as_bytes().to_vec()) } - fn contains_offstate(overlay: &NonCanonicalOverlay, key: u64, state: &H256) -> bool { - overlay.get_branch_range(state).and_then(|state| { + fn contains_offstate(overlay: &NonCanonicalOverlay, key: u64, state: &H256, block: u64) -> bool { + overlay.get_branch_range(state, block).and_then(|state| { overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state) - }) == Some(&H256::from_low_u64_be(key).as_bytes().to_vec()) + }) == Some(&Some(H256::from_low_u64_be(key).as_bytes().to_vec())) } fn contains_offstate2(overlay: &NonCanonicalOverlay, key: u64, state: &BranchRanges) -> bool { overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state) - == Some(&H256::from_low_u64_be(key).as_bytes().to_vec()) + == Some(&Some(H256::from_low_u64_be(key).as_bytes().to_vec())) } - fn contains_both(overlay: &NonCanonicalOverlay, key: u64, state: &H256) -> bool { - contains(overlay, key) && contains_offstate(overlay, key, state) + fn contains_both(overlay: &NonCanonicalOverlay, key: u64, state: &H256, block: u64) -> bool { + contains(overlay, key) && contains_offstate(overlay, key, state, block) } - fn contains_any(overlay: &NonCanonicalOverlay, key: u64, state: &H256) -> bool { - contains(overlay, key) || contains_offstate(overlay, key, state) + fn contains_any(overlay: &NonCanonicalOverlay, key: u64, state: &H256, block: u64) -> bool { + contains(overlay, key) || contains_offstate(overlay, key, state, block) } - #[test] fn created_from_empty_db() { let db = make_db(&[]); @@ -769,11 +766,11 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); overlay.insert::( &h1, 2, &H256::default(), - ChangeSet::default(), ChangeSet::default(), + ChangeSet::default(), OffstateChangeSet::default(), ).unwrap(); overlay.insert::( &h2, 1, &h1, - ChangeSet::default(), ChangeSet::default(), + ChangeSet::default(), OffstateChangeSet::default(), ).unwrap(); } @@ -786,11 +783,11 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); overlay.insert::( &h1, 1, &H256::default(), - ChangeSet::default(), ChangeSet::default(), + ChangeSet::default(), OffstateChangeSet::default(), ).unwrap(); overlay.insert::( &h2, 3, &h1, - ChangeSet::default(), ChangeSet::default(), + ChangeSet::default(), OffstateChangeSet::default(), ).unwrap(); } @@ -803,11 +800,11 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); overlay.insert::( &h1, 1, &H256::default(), - ChangeSet::default(), ChangeSet::default(), + ChangeSet::default(), OffstateChangeSet::default(), ).unwrap(); overlay.insert::( &h2, 2, &H256::default(), - ChangeSet::default(), ChangeSet::default(), + ChangeSet::default(), OffstateChangeSet::default(), ).unwrap(); } @@ -820,7 +817,7 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); overlay.insert::( &h1, 1, &H256::default(), - ChangeSet::default(), ChangeSet::default(), + ChangeSet::default(), OffstateChangeSet::default(), ).unwrap(); let mut commit = CommitSet::default(); overlay.canonicalize::(&h2, &mut commit).unwrap(); @@ -840,8 +837,7 @@ mod tests { ).unwrap(); assert_eq!(insertion.data.inserted.len(), 0); assert_eq!(insertion.data.deleted.len(), 0); - assert_eq!(insertion.offstate.inserted.len(), 0); - assert_eq!(insertion.offstate.deleted.len(), 0); + assert_eq!(insertion.offstate.len(), 0); // last cannonical, journal_record and offstate_journal_record assert_eq!(insertion.meta.inserted.len(), 3); assert_eq!(insertion.meta.deleted.len(), 0); @@ -850,8 +846,7 @@ mod tests { overlay.canonicalize::(&h1, &mut finalization).unwrap(); assert_eq!(finalization.data.inserted.len(), changeset.inserted.len()); assert_eq!(finalization.data.deleted.len(), changeset.deleted.len()); - assert_eq!(finalization.offstate.inserted.len(), offstate_changeset.inserted.len()); - assert_eq!(finalization.offstate.deleted.len(), offstate_changeset.deleted.len()); + assert_eq!(finalization.offstate.len(), offstate_changeset.len()); assert_eq!(finalization.meta.inserted.len(), 1); // normal and offstate discarded journall assert_eq!(finalization.meta.deleted.len(), 2); @@ -929,26 +924,28 @@ mod tests { &h1, 1, &H256::default(), changeset1, offstate_changeset1, ).unwrap()); - assert!(contains_both(&overlay, 5, &h1)); + assert!(contains_both(&overlay, 5, &h1, 1)); db.commit(&overlay.insert::( &h2, 2, &h1, changeset2, offstate_changeset2, ).unwrap()); - assert!(contains_both(&overlay, 7, &h2)); - assert!(contains_both(&overlay, 5, &h2)); + assert!(contains_both(&overlay, 7, &h2, 2)); + assert!(!contains_offstate(&overlay, 5, &h2, 2)); + assert!(contains_offstate(&overlay, 5, &h1, 1)); + assert!(contains_both(&overlay, 5, &h1, 1)); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 2); let mut commit = CommitSet::default(); overlay.canonicalize::(&h1, &mut commit).unwrap(); db.commit(&commit); - assert!(contains_both(&overlay, 5, &h2)); + assert!(contains(&overlay, 5)); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 2); overlay.apply_pending(); assert_eq!(overlay.levels.len(), 1); assert_eq!(overlay.parents.len(), 1); - assert!(!contains_any(&overlay, 5, &h1)); - assert!(contains_both(&overlay, 7, &h2)); + assert!(!contains_any(&overlay, 5, &h1, 1)); + assert!(contains_both(&overlay, 7, &h2, 2)); let mut commit = CommitSet::default(); overlay.canonicalize::(&h2, &mut commit).unwrap(); db.commit(&commit); @@ -970,13 +967,13 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); db.commit(&overlay.insert::(&h_1, 1, &H256::default(), c_1, o_c_1).unwrap()); db.commit(&overlay.insert::(&h_2, 1, &H256::default(), c_2, o_c_2).unwrap()); - assert!(contains_both(&overlay, 1, &h_2)); + assert!(contains_both(&overlay, 1, &h_2, 1)); let mut commit = CommitSet::default(); overlay.canonicalize::(&h_1, &mut commit).unwrap(); db.commit(&commit); - assert!(contains_both(&overlay, 1, &h_2)); + assert!(contains_both(&overlay, 1, &h_2, 1)); overlay.apply_pending(); - assert!(!contains_any(&overlay, 1, &h_2)); + assert!(!contains_any(&overlay, 1, &h_2, 1)); } #[test] @@ -1012,7 +1009,7 @@ mod tests { fn make_both_changeset(inserted: &[u64], deleted: &[u64]) -> ( H256, ChangeSet, - ChangeSet, + OffstateChangeSet, ) { ( H256::random(), @@ -1067,12 +1064,12 @@ mod tests { db.commit(&overlay.insert::(&h_1_2_3, 3, &h_1_2, c_1_2_3, o_c_1_2_3).unwrap()); db.commit(&overlay.insert::(&h_2_1_1, 3, &h_2_1, c_2_1_1, o_c_2_1_1).unwrap()); - assert!(contains_both(&overlay, 2, &h_2_1_1)); - assert!(contains_both(&overlay, 11, &h_1_1_1)); - assert!(contains_both(&overlay, 21, &h_2_1_1)); - assert!(contains_both(&overlay, 111, &h_1_1_1)); - assert!(contains_both(&overlay, 122, &h_1_2_2)); - assert!(contains_both(&overlay, 211, &h_2_1_1)); + assert!(contains_both(&overlay, 2, &h_2_1_1, 3)); + assert!(contains_both(&overlay, 11, &h_1_1_1, 3)); + assert!(contains_both(&overlay, 21, &h_2_1_1, 3)); + assert!(contains_both(&overlay, 111, &h_1_1_1, 3)); + assert!(contains_both(&overlay, 122, &h_1_2_2, 3)); + assert!(contains_both(&overlay, 211, &h_2_1_1, 3)); assert_eq!(overlay.levels.len(), 3); assert_eq!(overlay.parents.len(), 11); assert_eq!(overlay.last_canonicalized, Some((H256::default(), 0))); @@ -1094,12 +1091,12 @@ mod tests { overlay.apply_pending(); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 6); - assert!(!contains_any(&overlay, 1, &h_1)); - assert!(!contains_any(&overlay, 2, &h_2)); - assert!(!contains_any(&overlay, 21, &h_2_1)); - assert!(!contains_any(&overlay, 22, &h_2_2)); - assert!(!contains_any(&overlay, 211, &h_2_1_1)); - assert!(contains_both(&overlay, 111, &h_1_1_1)); + assert!(!contains_any(&overlay, 1, &h_1, 1)); + assert!(!contains_any(&overlay, 2, &h_2, 1)); + assert!(!contains_any(&overlay, 21, &h_2_1, 2)); + assert!(!contains_any(&overlay, 22, &h_2_2, 2)); + assert!(!contains_any(&overlay, 211, &h_2_1_1, 3)); + assert!(contains_both(&overlay, 111, &h_1_1_1, 3)); // check that journals are deleted assert!(db.get_meta(&to_journal_key(1, 0)).unwrap().is_none()); assert!(db.get_meta(&to_journal_key(1, 1)).unwrap().is_none()); @@ -1114,11 +1111,11 @@ mod tests { overlay.apply_pending(); assert_eq!(overlay.levels.len(), 1); assert_eq!(overlay.parents.len(), 3); - assert!(!contains_any(&overlay, 11, &h_1_1)); - assert!(!contains_any(&overlay, 111, &h_1_1_1)); - assert!(contains_both(&overlay, 121, &h_1_2_1)); - assert!(contains_both(&overlay, 122, &h_1_2_2)); - assert!(contains_both(&overlay, 123, &h_1_2_3)); + assert!(!contains_any(&overlay, 11, &h_1_1, 2)); + assert!(!contains_any(&overlay, 111, &h_1_1_1, 3)); + assert!(contains_both(&overlay, 121, &h_1_2_1, 3)); + assert!(contains_both(&overlay, 122, &h_1_2_2, 3)); + assert!(contains_both(&overlay, 123, &h_1_2_3, 3)); assert!(overlay.have_block(&h_1_2_1)); assert!(!overlay.have_block(&h_1_2)); assert!(!overlay.have_block(&h_1_1)); @@ -1156,12 +1153,12 @@ mod tests { &h2, 2, &h1, changeset2, ochangeset2, ).unwrap()); - assert!(contains_both(&overlay, 7, &h2)); + assert!(contains_both(&overlay, 7, &h2, 2)); db.commit(&overlay.revert_one().unwrap()); assert_eq!(overlay.parents.len(), 1); - assert!(contains_both(&overlay, 5, &h1)); + assert!(contains_both(&overlay, 5, &h1, 1)); assert!(!contains(&overlay, 7)); - assert!(!contains_any(&overlay, 7, &h1)); + assert!(!contains_any(&overlay, 7, &h1, 1)); db.commit(&overlay.revert_one().unwrap()); assert_eq!(overlay.levels.len(), 0); assert_eq!(overlay.parents.len(), 0); @@ -1194,13 +1191,15 @@ mod tests { &h2_2, 2, &h1, changeset3, ochangeset3, ).unwrap(); - assert!(contains_both(&overlay, 7, &h2_1)); - assert!(contains_both(&overlay, 5, &h2_1)); - assert!(contains_both(&overlay, 9, &h2_2)); + assert!(contains_offstate(&overlay, 5, &h1, 1)); + assert!(contains_both(&overlay, 7, &h2_1, 2)); + assert!(!contains_offstate(&overlay, 5, &h2_1, 2)); + assert!(contains(&overlay, 5)); + assert!(contains_both(&overlay, 9, &h2_2, 2)); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 3); overlay.revert_pending(); - assert!(!contains_any(&overlay, 5, &h1)); + assert!(!contains_any(&overlay, 5, &h1, 1)); assert_eq!(overlay.levels.len(), 0); assert_eq!(overlay.parents.len(), 0); } @@ -1220,7 +1219,7 @@ mod tests { db.commit(&overlay.insert::(&h_2, 1, &H256::default(), c_2, o_c_2).unwrap()); overlay.pin(&h_1); - let h1_context = overlay.get_branch_range(&h_1).unwrap(); + let h1_context = overlay.get_branch_range(&h_1, 1).unwrap(); let mut commit = CommitSet::default(); overlay.canonicalize::(&h_2, &mut commit).unwrap(); @@ -1230,7 +1229,7 @@ mod tests { // we cannot use contains_offstate because offstate pining is relying on // asumption that pinned context memoïzed its branch state. assert!(contains_offstate2(&overlay, 1, &h1_context)); - assert!(!contains_offstate(&overlay, 1, &h_1)); + assert!(!contains_offstate(&overlay, 1, &h_1, 1)); overlay.unpin(&h_1); assert!(!contains(&overlay, 1)); assert!(!contains_offstate2(&overlay, 1, &h1_context)); diff --git a/core/state-db/src/pruning.rs b/core/state-db/src/pruning.rs index cb5e939e524a7..da4dd1e1ae273 100644 --- a/core/state-db/src/pruning.rs +++ b/core/state-db/src/pruning.rs @@ -37,8 +37,6 @@ pub struct RefWindow { death_rows: VecDeque>, /// An index that maps each key from `death_rows` to block number. death_index: HashMap, - /// An index that maps each key from `death_rows` to block number. - offstate_death_index: HashMap, /// Block number that corresponts to the front of `death_rows` pending_number: u64, /// Number of call of `note_canonical` after @@ -55,7 +53,7 @@ struct DeathRow { journal_key: Vec, offstate_journal_key: Vec, deleted: HashSet, - offstate_deleted: HashSet, + offstate_modified: HashSet, } #[derive(Encode, Decode)] @@ -67,8 +65,7 @@ struct JournalRecord { #[derive(Encode, Decode)] struct OffstateJournalRecord { - inserted: Vec, - deleted: Vec, + modified: Vec, } fn to_journal_key(block: u64) -> Vec { @@ -91,7 +88,6 @@ impl RefWindow { let mut pruning = RefWindow { death_rows: Default::default(), death_index: Default::default(), - offstate_death_index: Default::default(), pending_number: pending_number, pending_canonicalizations: 0, pending_prunings: 0, @@ -104,20 +100,19 @@ impl RefWindow { match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? { Some(record) => { let record: JournalRecord = Decode::decode(&mut record.as_slice())?; - let (offstate_record_inserted, offstate_record_deleted) = if let Some(record) = db + let offstate_record_inserted = if let Some(record) = db .get_meta(&offstate_journal_key).map_err(|e| Error::Db(e))? { let record = OffstateJournalRecord::decode(&mut record.as_slice())?; - (record.inserted, record.deleted) - } else { (Vec::new(), Vec::new()) }; + record.modified + } else { Vec::new() }; trace!( target: "state-db", - "Pruning journal entry {} ({} {} inserted, {} {} deleted)", + "Pruning journal entry {} ({} {} inserted, {} deleted)", block, record.inserted.len(), offstate_record_inserted.len(), record.deleted.len(), - offstate_record_deleted.len(), ); pruning.import( &record.hash, @@ -126,7 +121,6 @@ impl RefWindow { record.inserted.into_iter(), record.deleted, offstate_record_inserted.into_iter(), - offstate_record_deleted, ); }, None => break, @@ -144,7 +138,6 @@ impl RefWindow { inserted: I, deleted: Vec, offstate_inserted: I2, - offstate_deleted: Vec, ) { // remove all re-inserted keys from death rows for k in inserted { @@ -152,25 +145,18 @@ impl RefWindow { self.death_rows[(block - self.pending_number) as usize].deleted.remove(&k); } } - for k in offstate_inserted { - if let Some(block) = self.offstate_death_index.remove(&k) { - self.death_rows[(block - self.pending_number) as usize].offstate_deleted.remove(&k); - } - } // add new keys let imported_block = self.pending_number + self.death_rows.len() as u64; for k in deleted.iter() { self.death_index.insert(k.clone(), imported_block); } - for k in offstate_deleted.iter() { - self.offstate_death_index.insert(k.clone(), imported_block); - } self.death_rows.push_back( DeathRow { hash: hash.clone(), deleted: deleted.into_iter().collect(), - offstate_deleted: offstate_deleted.into_iter().collect(), + // TODO EMCH is it possible to change type to directly set ?? + offstate_modified: offstate_inserted.into_iter().collect(), journal_key, offstate_journal_key, } @@ -203,7 +189,6 @@ impl RefWindow { trace!(target: "state-db", "Pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len()); let index = self.pending_number + self.pending_prunings as u64; commit.data.deleted.extend(pruned.deleted.iter().cloned()); - commit.offstate.deleted.extend(pruned.offstate_deleted.iter().cloned()); commit.meta.inserted.push((to_meta_key(LAST_PRUNED, &()), index.encode())); commit.meta.deleted.push(pruned.journal_key.clone()); self.pending_prunings += 1; @@ -216,17 +201,15 @@ impl RefWindow { pub fn note_canonical(&mut self, hash: &BlockHash, commit: &mut CommitSet) { trace!(target: "state-db", "Adding to pruning window: {:?} ({} inserted, {} deleted)", hash, commit.data.inserted.len(), commit.data.deleted.len()); let inserted = commit.data.inserted.iter().map(|(k, _)| k.clone()).collect(); - let offstate_inserted = commit.offstate.inserted.iter().map(|(k, _)| k.clone()).collect(); + let offstate_modified = commit.offstate.iter().map(|(k, _)| k.clone()).collect(); let deleted = ::std::mem::replace(&mut commit.data.deleted, Vec::new()); - let offstate_deleted = ::std::mem::replace(&mut commit.offstate.deleted, Vec::new()); let journal_record = JournalRecord { hash: hash.clone(), inserted, deleted, }; let offstate_journal_record = OffstateJournalRecord { - inserted: offstate_inserted, - deleted: offstate_deleted, + modified: offstate_modified, }; let block = self.pending_number + self.death_rows.len() as u64; let journal_key = to_journal_key(block); @@ -239,8 +222,7 @@ impl RefWindow { offstate_journal_key, journal_record.inserted.into_iter(), journal_record.deleted, - offstate_journal_record.inserted.into_iter(), - offstate_journal_record.deleted, + offstate_journal_record.modified.into_iter(), ); self.pending_canonicalizations += 1; } @@ -254,9 +236,6 @@ impl RefWindow { for k in pruned.deleted.iter() { self.death_index.remove(&k); } - for k in pruned.offstate_deleted.iter() { - self.offstate_death_index.remove(k); - } self.pending_number += 1; } self.pending_prunings = 0; @@ -288,7 +267,6 @@ mod tests { assert_eq!(pruning.pending_number, restored.pending_number); assert_eq!(pruning.death_rows, restored.death_rows); assert_eq!(pruning.death_index, restored.death_index); - assert_eq!(pruning.offstate_death_index, restored.offstate_death_index); } #[test] @@ -298,19 +276,6 @@ mod tests { assert_eq!(pruning.pending_number, 0); assert!(pruning.death_rows.is_empty()); assert!(pruning.death_index.is_empty()); - assert!(pruning.offstate_death_index.is_empty()); - } - - #[test] - fn prune_empty() { - let db = make_db(&[]); - let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = CommitSet::default(); - pruning.prune_one(&mut commit); - assert_eq!(pruning.pending_number, 0); - assert!(pruning.death_rows.is_empty()); - assert!(pruning.death_index.is_empty()); - assert!(pruning.offstate_death_index.is_empty()); assert!(pruning.pending_prunings == 0); assert!(pruning.pending_canonicalizations == 0); } @@ -323,17 +288,18 @@ mod tests { let mut commit = make_commit_both(&[4, 5], &[1, 3]); commit.initialize_offstate(&[4, 5], &[1, 3]); let h = H256::random(); + assert!(!commit.data.deleted.is_empty()); pruning.note_canonical(&h, &mut commit); db.commit(&commit); assert!(pruning.have_block(&h)); pruning.apply_pending(); assert!(pruning.have_block(&h)); assert!(commit.data.deleted.is_empty()); - assert!(commit.offstate.deleted.is_empty()); + //assert!(commit.offstate.is_empty()); assert_eq!(pruning.death_rows.len(), 1); assert_eq!(pruning.death_index.len(), 2); - assert_eq!(pruning.offstate_death_index.len(), 2); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); + println!("{:?}", db.offstate); assert!(db.offstate_eq(&[1, 2, 3, 4, 5])); check_journal(&pruning, &db); @@ -347,7 +313,6 @@ mod tests { assert!(db.offstate_eq(&[2, 4, 5])); assert!(pruning.death_rows.is_empty()); assert!(pruning.death_index.is_empty()); - assert!(pruning.offstate_death_index.is_empty()); assert_eq!(pruning.pending_number, 1); } diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 3b10e6540859b..87f18d4c6f76e 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -18,13 +18,21 @@ use std::collections::HashMap; use primitives::H256; -use crate::{DBValue, ChangeSet, CommitSet, MetaDb, NodeDb, OffstateDb, OffstateKey}; +use crate::{ + DBValue, ChangeSet, OffstateChangeSet, CommitSet, MetaDb, NodeDb, OffstateDb, + OffstateKey, +}; +use historied_data::tree::Serialized; +use historied_data::linear::DefaultVersion; + +type Ser<'a> = Serialized<'a, DefaultVersion>; #[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct TestDb { pub data: HashMap, pub meta: HashMap, DBValue>, pub offstate: HashMap, + pub last_block: u64, } impl MetaDb for TestDb { @@ -39,11 +47,22 @@ impl OffstateDb> for TestDb { type Error = (); fn get_offstate(&self, key: &[u8], state: &Option) -> Result, ()> { - Ok(self.offstate.get(key).cloned()) + let state = state.unwrap_or(self.last_block); + Ok(self.offstate.get(key) + .map(|s| Ser::from(s.as_slice().into())) + .and_then(|s| s.get(state) + .map(Into::into) + )) + } fn get_offstate_pairs(&self, state: &Option) -> Vec<(OffstateKey, DBValue)> { - self.offstate.iter().map(|(a, b)| (a.clone(), b.clone())).collect() + let state = state.unwrap_or(self.last_block); + self.offstate.iter().filter_map(|(a, s)| ( + Ser::from(s.as_slice().into()) + .get(state) + .map(|v| (a.clone(), v.to_vec())) + )).collect() } } @@ -62,9 +81,12 @@ impl TestDb { for k in commit.data.deleted.iter() { self.data.remove(k); } - self.offstate.extend(commit.offstate.inserted.iter().cloned()); - for k in commit.offstate.deleted.iter() { - self.offstate.remove(k); + for (k, o) in commit.offstate.iter() { + if let Some(v) = o { + self.offstate.insert(k.to_vec(), v.to_vec()); + } else { + self.offstate.remove(k); + } } self.meta.extend(commit.meta.inserted.iter().cloned()); for k in commit.meta.deleted.iter() { @@ -78,12 +100,12 @@ impl TestDb { pub fn offstate_eq(&self, values: &[u64]) -> bool { let data = make_offstate_changeset(values, &[]); - self.offstate == data.inserted.into_iter().collect() + self.offstate == data.into_iter().filter_map(|(k, v)| v.map(|v| (k,v))).collect() } pub fn initialize_offstate(&mut self, inserted: &[u64]) { let data = make_offstate_changeset(inserted, &[]); - self.offstate = data.inserted.into_iter().collect(); + self.offstate = data.into_iter().filter_map(|(k, v)| v.map(|v| (k,v))).collect(); } } @@ -99,24 +121,25 @@ pub fn make_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet { } } -pub fn make_offstate_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet { - ChangeSet { - inserted: inserted - .iter() - .map(|v| {( - H256::from_low_u64_be(*v).as_bytes().to_vec(), - H256::from_low_u64_be(*v).as_bytes().to_vec(), - )}) - .collect(), - deleted: deleted.iter().map(|v| H256::from_low_u64_be(*v).as_bytes().to_vec()).collect(), - } +pub fn make_offstate_changeset(inserted: &[u64], deleted: &[u64]) -> OffstateChangeSet { + inserted.iter() + .map(|v| ( + H256::from_low_u64_be(*v).as_bytes().to_vec(), + Some(H256::from_low_u64_be(*v).as_bytes().to_vec()), + )) + .chain( + deleted + .iter() + .map(|v| (H256::from_low_u64_be(*v).as_bytes().to_vec(), None)) + ) + .collect() } pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet { CommitSet { data: make_changeset(inserted, deleted), meta: ChangeSet::default(), - offstate: ChangeSet::default(), + offstate: OffstateChangeSet::default(), } } @@ -145,6 +168,7 @@ pub fn make_db(inserted: &[u64]) -> TestDb { .collect(), meta: Default::default(), offstate: Default::default(), + last_block: Default::default(), } } diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index e032b2c653fd9..99d9b11ac0bdf 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -234,6 +234,13 @@ impl<'a, F: SerializedConfig> Default for Serialized<'a, F> { } } +impl<'a, F> Into> for &'a[u8] { + fn into(self) -> Serialized<'a, F> { + Serialized(Cow::Borrowed(self), PhantomData) + } +} + + // Utility function for basis implementation. impl<'a, F: SerializedConfig> Serialized<'a, F> { diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index 45d27b3ad99a2..99133757d71b0 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -152,6 +152,20 @@ impl<'a> BranchStateTrait for &'a BranchStateRef { } +/// u64 is use a a state target so it is implemented as +/// a upper bound. +impl<'a> BranchStateTrait for u64 { + + fn get_node(&self, i: u64) -> bool { + &i <= self + } + + fn last_index(&self) -> u64 { + *self + } + +} + impl Default for BranchState { @@ -787,6 +801,12 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { /// needed. pub struct Serialized<'a, F>(SerializedInner<'a, F>); +impl<'a, F> Into> for &'a[u8] { + fn into(self) -> Serialized<'a, F> { + Serialized(self.into()) + } +} + impl<'a, F> Into> for SerializedInner<'a, F> { fn into(self) -> Serialized<'a, F> { Serialized(self) From 2497575f92b6aa30fc10a34e15a64b0820b6a17f Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 3 Oct 2019 20:28:20 +0200 Subject: [PATCH 22/68] Figure out pruning (see prune_one/prune_two tests). --- core/state-db/src/lib.rs | 17 +++- core/state-db/src/noncanonical.rs | 74 +++++++++++----- core/state-db/src/pruning.rs | 44 ++++++++-- core/state-db/src/test.rs | 57 ++++++++++--- core/utils/historied-data/src/lib.rs | 8 ++ core/utils/historied-data/src/linear.rs | 86 +++++++++++++++++-- core/utils/historied-data/src/tree.rs | 108 +++++++++++++++++++----- 7 files changed, 328 insertions(+), 66 deletions(-) diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index 37c5e8e79f2a5..eed50ddd99d4b 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -37,7 +37,7 @@ mod branch; use std::fmt; use parking_lot::RwLock; use codec::Codec; -use std::collections::{HashMap, hash_map::Entry}; +use std::collections::{HashSet, HashMap, hash_map::Entry}; use noncanonical::NonCanonicalOverlay; use pruning::RefWindow; use log::trace; @@ -138,9 +138,21 @@ pub struct ChangeSet { } /// A set of offstate state values changes. +/// TODO EMCH note that this could really benefit from batching +/// the change set (need to change from client to run over +/// the whole range for offstate: then we can get prepare +/// insertion of batch values for history in db such as : +/// pub type OffstateChangeSet = Vec<(H, Vec(u64, Option))>; +/// ), +/// but it just need to be build from client (no need to change +/// it here except to extract faster). pub type OffstateChangeSet = Vec<(H, Option)>; +/// Info for pruning offstate: a last prune index and keys to prune. +pub type OffstateChangeSetPrune = Option<(u64, HashSet)>; + /// A set of changes to the backing database. +/// It only contain a single block change set. #[derive(Default, Debug, Clone)] pub struct CommitSet { /// State node changes. @@ -149,6 +161,8 @@ pub struct CommitSet { pub meta: ChangeSet>, /// Offstate data changes. pub offstate: OffstateChangeSet, + /// Offstate data changes good for prunning. + pub offstate_prune: OffstateChangeSetPrune, } /// Pruning constraints. If none are specified pruning is @@ -246,6 +260,7 @@ impl StateDbSync { data: changeset, meta: Default::default(), offstate: offstate_changeset, + offstate_prune: None, }) }, PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => { diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index b9f1af6939f64..9868e269cefda 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -146,7 +146,8 @@ struct JournalRecord { #[derive(Encode, Decode)] struct OffstateJournalRecord { - inserted: Vec<(OffstateKey, Option)>, + inserted: Vec<(OffstateKey, DBValue)>, + deleted: Vec, } fn to_journal_key(block: BlockNumber, index: u64) -> Vec { @@ -165,6 +166,7 @@ struct BlockOverlay { inserted: Vec, deleted: Vec, offstate_inserted: Vec, + offstate_deleted: Vec, } fn insert_values(values: &mut HashMap, inserted: Vec<(Key, DBValue)>) { @@ -178,11 +180,16 @@ fn insert_values(values: &mut HashMap, inserted: fn insert_offstate_values( state: &BranchRanges, values: &mut HashMap>>, - inserted: Vec<(Key, Option)>, + inserted: Vec<(Key, DBValue)>, + deleted: Vec, ) { for (k, v) in inserted { let entry = values.entry(k).or_insert_with(|| Default::default()); - entry.set(state, v); + entry.set(state, Some(v)); + } + for k in deleted { + let entry = values.entry(k).or_insert_with(|| Default::default()); + entry.set(state, None); } } @@ -212,6 +219,7 @@ fn discard_values( fn discard_offstate_values( inserted: Vec, + deleted: Vec, into: &mut OffstatePendingGC, ) { let into = if into.pending_canonicalisation_query.is_some() { @@ -220,6 +228,7 @@ fn discard_offstate_values( &mut into.keys_pending_gc }; into.extend(inserted); + into.extend(deleted); } @@ -243,6 +252,7 @@ fn discard_descendants( discard_values(&mut values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); discard_offstate_values( overlay.offstate_inserted, + overlay.offstate_deleted, offstate_gc, ); None @@ -296,35 +306,43 @@ impl NonCanonicalOverlay { parent_branch_range, ); - let offstate_record_inserted = if let Some(record) = db + let (offstate_record_inserted, offstate_record_deleted) = if let Some(record) = db .get_meta(&offstate_journal_key).map_err(|e| Error::Db(e))? { let record = OffstateJournalRecord::decode(&mut record.as_slice())?; - Some(record.inserted) - } else { None }; + (Some(record.inserted), Some(record.deleted)) + } else { (None, None) }; let inserted = record.inserted.iter().map(|(k, _)| k.clone()).collect(); let offstate_inserted = offstate_record_inserted.as_ref() .map(|inserted| inserted.iter().map(|(k, _)| k.clone()).collect()) .unwrap_or(Vec::new()); + let offstate_deleted = offstate_record_deleted.clone().unwrap_or(Vec::new()); let overlay = BlockOverlay { hash: record.hash.clone(), journal_key, offstate_journal_key, inserted: inserted, deleted: record.deleted, - offstate_inserted: offstate_inserted, + offstate_inserted, + offstate_deleted, }; insert_values(&mut values, record.inserted); - if let Some(inserted) = offstate_record_inserted { - insert_offstate_values(&branch_range, &mut offstate_values, inserted); + if offstate_record_inserted.is_some() || offstate_record_deleted.is_some() { + insert_offstate_values( + &branch_range, + &mut offstate_values, + offstate_record_inserted.unwrap_or(Vec::new()), + offstate_record_deleted.unwrap_or(Vec::new()), + ); } trace!( target: "state-db", - "Uncanonicalized journal entry {}.{} ({} {} inserted, {} deleted)", + "Uncanonicalized journal entry {}.{} ({} {} inserted, {} {} deleted)", block, index, overlay.inserted.len(), overlay.offstate_inserted.len(), overlay.deleted.len(), + overlay.offstate_deleted.len(), ); level.push(overlay); parents.insert(record.hash, (record.parent_hash, branch_index)); @@ -404,14 +422,25 @@ impl NonCanonicalOverlay { let offstate_journal_key = to_offstate_journal_key(number, index); let inserted = changeset.inserted.iter().map(|(k, _)| k.clone()).collect(); - let offstate_inserted = offstate_changeset.iter().map(|(k, _)| k.clone()).collect(); + let mut offstate_inserted = Vec::new(); + let mut offstate_inserted_value = Vec::new(); + let mut offstate_deleted = Vec::new(); + for (k, v) in offstate_changeset.into_iter() { + if let Some(v) = v { + offstate_inserted.push(k.clone()); + offstate_inserted_value.push((k, v)); + } else { + offstate_deleted.push(k); + } + } let overlay = BlockOverlay { hash: hash.clone(), journal_key: journal_key.clone(), offstate_journal_key: offstate_journal_key.clone(), inserted: inserted, deleted: changeset.deleted.clone(), - offstate_inserted: offstate_inserted, + offstate_inserted, + offstate_deleted: offstate_deleted.clone(), }; level.push(overlay); let parent_branch_index = self.parents.get(&parent_hash).map(|(_, i)| *i).unwrap_or(0); @@ -431,7 +460,8 @@ impl NonCanonicalOverlay { }; commit.meta.inserted.push((journal_key, journal_record.encode())); let offstate_journal_record = OffstateJournalRecord { - inserted: offstate_changeset, + inserted: offstate_inserted_value, + deleted: offstate_deleted, }; commit.meta.inserted.push((offstate_journal_key, offstate_journal_record.encode())); @@ -445,7 +475,12 @@ impl NonCanonicalOverlay { journal_record.deleted.len(), ); insert_values(&mut self.values, journal_record.inserted); - insert_offstate_values(&branch_range, &mut self.offstate_values, offstate_journal_record.inserted); + insert_offstate_values( + &branch_range, + &mut self.offstate_values, + offstate_journal_record.inserted, + offstate_journal_record.deleted, + ); self.pending_insertions.push(hash.clone()); Ok(commit) } @@ -531,13 +566,13 @@ impl NonCanonicalOverlay { .map(|k| (k.clone(), self.values.get(k).expect("For each key in overlays there's a value in values").1.clone()))); commit.data.deleted.extend(overlay.deleted.clone()); let block_number = self.front_block_number() + self.pending_canonicalizations.len() as u64; - if !overlay.offstate_inserted.is_empty() { + if !overlay.offstate_inserted.is_empty() || !overlay.offstate_deleted.is_empty() { // canonicalization is not frequent enough that we pass range // in parameter for now - // TODO EMCH it seems bad idea to maintain this commit offstate - // field as his: simply delta of state could be better?? if let Some(range) = self.get_branch_range(hash, block_number) { - commit.offstate.extend(overlay.offstate_inserted.iter() + commit.offstate.extend( + overlay.offstate_inserted.iter() + .chain(overlay.offstate_deleted.iter()) .map(|k| (k.clone(), self.offstate_values.get(k) .expect("For each key in overlays there's a value in values") .get(&range) @@ -586,6 +621,7 @@ impl NonCanonicalOverlay { discard_values(&mut self.values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); discard_offstate_values( overlay.offstate_inserted, + overlay.offstate_deleted, &mut self.offstate_gc, ); } @@ -648,7 +684,7 @@ impl NonCanonicalOverlay { self.branches.revert(branch_index); } discard_values(&mut self.values, overlay.inserted, None); - discard_offstate_values(overlay.offstate_inserted, &mut self.offstate_gc); + discard_offstate_values(overlay.offstate_inserted, overlay.offstate_deleted, &mut self.offstate_gc); } commit }) diff --git a/core/state-db/src/pruning.rs b/core/state-db/src/pruning.rs index da4dd1e1ae273..71c8fc57323ae 100644 --- a/core/state-db/src/pruning.rs +++ b/core/state-db/src/pruning.rs @@ -53,6 +53,9 @@ struct DeathRow { journal_key: Vec, offstate_journal_key: Vec, deleted: HashSet, + // TODO EMCH for offstate there is no need to put + // in memory so we can make it lazy (load from + // pruning journal on actual prune). offstate_modified: HashSet, } @@ -189,8 +192,18 @@ impl RefWindow { trace!(target: "state-db", "Pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len()); let index = self.pending_number + self.pending_prunings as u64; commit.data.deleted.extend(pruned.deleted.iter().cloned()); + if let Some(offstate) = commit.offstate_prune.as_mut() { + offstate.0 = std::cmp::max(offstate.0, index); + offstate.1.extend(pruned.offstate_modified.iter().cloned()); + } else { + commit.offstate_prune = Some(( + index, + pruned.offstate_modified.iter().cloned().collect(), + )); + } commit.meta.inserted.push((to_meta_key(LAST_PRUNED, &()), index.encode())); commit.meta.deleted.push(pruned.journal_key.clone()); + commit.meta.deleted.push(pruned.offstate_journal_key.clone()); self.pending_prunings += 1; } else { warn!(target: "state-db", "Trying to prune when there's nothing to prune"); @@ -299,8 +312,8 @@ mod tests { assert_eq!(pruning.death_rows.len(), 1); assert_eq!(pruning.death_index.len(), 2); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); - println!("{:?}", db.offstate); - assert!(db.offstate_eq(&[1, 2, 3, 4, 5])); + assert!(db.offstate_eq_at(&[1, 2, 3], Some(0))); + assert!(db.offstate_eq(&[2, 4, 5])); check_journal(&pruning, &db); let mut commit = CommitSet::default(); @@ -310,6 +323,8 @@ mod tests { pruning.apply_pending(); assert!(!pruning.have_block(&h)); assert!(db.data_eq(&make_db(&[2, 4, 5]))); + // two remains since it is still valid next + assert!(db.offstate_eq_at(&[2], Some(0))); assert!(db.offstate_eq(&[2, 4, 5])); assert!(pruning.death_rows.is_empty()); assert!(pruning.death_index.is_empty()); @@ -321,7 +336,7 @@ mod tests { let mut db = make_db(&[1, 2, 3]); db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = make_commit_both(&[4], &[1]); + let mut commit = make_commit_both(&[3, 4], &[1]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); let mut commit = make_commit_both(&[5], &[2]); @@ -329,7 +344,9 @@ mod tests { db.commit(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); - assert!(db.offstate_eq(&[1, 2, 3, 4, 5])); + assert!(db.offstate_eq_at(&[1, 2, 3], Some(0))); + assert!(db.offstate_eq_at(&[2, 3, 4], Some(1))); + assert!(db.offstate_eq(&[3, 4, 5])); check_journal(&pruning, &db); @@ -338,12 +355,17 @@ mod tests { db.commit(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[2, 3, 4, 5]))); - assert!(db.offstate_eq(&[2, 3, 4, 5])); + // 3 exists at 0 and 1 so 0 removed + assert!(db.offstate_eq_at(&[2], Some(0))); + assert!(db.offstate_eq_at(&[2, 3, 4], Some(1))); + assert!(db.offstate_eq(&[3, 4, 5])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[3, 4, 5]))); + assert!(db.offstate_eq_at(&[], Some(0))); + assert!(db.offstate_eq_at(&[3, 4], Some(1))); assert!(db.offstate_eq(&[3, 4, 5])); assert_eq!(pruning.pending_number, 2); } @@ -356,21 +378,27 @@ mod tests { let mut commit = make_commit_both(&[4], &[1]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); - let mut commit = make_commit_both(&[5], &[2]); + let mut commit = make_commit_both(&[3, 5], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); - assert!(db.offstate_eq(&[1, 2, 3, 4, 5])); + assert!(db.offstate_eq_at(&[1, 2, 3], Some(0))); + assert!(db.offstate_eq_at(&[2, 3, 4], Some(1))); + assert!(db.offstate_eq(&[3, 4, 5])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[2, 3, 4, 5]))); - assert!(db.offstate_eq(&[2, 3, 4, 5])); + assert!(db.offstate_eq_at(&[2, 3], Some(0))); + assert!(db.offstate_eq_at(&[2, 3, 4], Some(1))); + assert!(db.offstate_eq(&[3, 4, 5])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[3, 4, 5]))); + assert!(db.offstate_eq_at(&[], Some(0))); + assert!(db.offstate_eq_at(&[4], Some(1))); assert!(db.offstate_eq(&[3, 4, 5])); assert_eq!(pruning.pending_number, 2); } diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 87f18d4c6f76e..2428a32aaa803 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -16,13 +16,14 @@ //! Test utils -use std::collections::HashMap; +use std::collections::{HashMap, BTreeMap}; use primitives::H256; use crate::{ DBValue, ChangeSet, OffstateChangeSet, CommitSet, MetaDb, NodeDb, OffstateDb, OffstateKey, }; use historied_data::tree::Serialized; +use historied_data::PruneResult; use historied_data::linear::DefaultVersion; type Ser<'a> = Serialized<'a, DefaultVersion>; @@ -49,18 +50,19 @@ impl OffstateDb> for TestDb { fn get_offstate(&self, key: &[u8], state: &Option) -> Result, ()> { let state = state.unwrap_or(self.last_block); Ok(self.offstate.get(key) - .map(|s| Ser::from(s.as_slice().into())) + .map(|s| Ser::from_slice(s.as_slice())) .and_then(|s| s.get(state) + .unwrap_or(None) // flatten .map(Into::into) )) - } fn get_offstate_pairs(&self, state: &Option) -> Vec<(OffstateKey, DBValue)> { let state = state.unwrap_or(self.last_block); self.offstate.iter().filter_map(|(a, s)| ( - Ser::from(s.as_slice().into()) + Ser::from_slice(s.as_slice()) .get(state) + .unwrap_or(None) // flatten .map(|v| (a.clone(), v.to_vec())) )).collect() } @@ -77,15 +79,33 @@ impl NodeDb for TestDb { impl TestDb { pub fn commit(&mut self, commit: &CommitSet) { + self.last_block += 1; self.data.extend(commit.data.inserted.iter().cloned()); for k in commit.data.deleted.iter() { self.data.remove(k); } for (k, o) in commit.offstate.iter() { - if let Some(v) = o { - self.offstate.insert(k.to_vec(), v.to_vec()); - } else { - self.offstate.remove(k); + let encoded = self.offstate.entry(k.clone()) + .or_insert_with(|| Ser::default().into_vec()); + let mut ser = Ser::from_mut(&mut (*encoded)); + ser.push(self.last_block, o.as_ref().map(|v| v.as_slice())); + } + if let Some((block_prune, offstate_prune_key)) = commit.offstate_prune.as_ref() { + for k in offstate_prune_key.iter() { + match self.offstate.get_mut(k).map(|v| { + println!("t{}", *block_prune); + println!("t{:?}", k); + println!("ser{:?}", v); + let mut ser = Ser::from_mut(v); + let r = ser.prune(*block_prune); + println!("ser{:?}", ser.into_vec()); + r + }) { + Some(PruneResult::Cleared) => { let _ = self.offstate.remove(k); }, + Some(PruneResult::Changed) // changed applyied on mutable buffer without copy. + | Some(PruneResult::Unchanged) + | None => (), + } } } self.meta.extend(commit.meta.inserted.iter().cloned()); @@ -98,14 +118,27 @@ impl TestDb { self.data == other.data } - pub fn offstate_eq(&self, values: &[u64]) -> bool { + pub fn offstate_eq_at(&self, values: &[u64], block: Option) -> bool { let data = make_offstate_changeset(values, &[]); - self.offstate == data.into_iter().filter_map(|(k, v)| v.map(|v| (k,v))).collect() + let self_offstate: BTreeMap<_, _> = self.get_offstate_pairs(&block).into_iter().collect(); + println!("---{:?}", self_offstate); + self_offstate == data.into_iter().filter_map(|(k, v)| v.map(|v| (k,v))).collect() + } + + pub fn offstate_eq(&self, values: &[u64]) -> bool { + self.offstate_eq_at(values, None) } pub fn initialize_offstate(&mut self, inserted: &[u64]) { let data = make_offstate_changeset(inserted, &[]); - self.offstate = data.into_iter().filter_map(|(k, v)| v.map(|v| (k,v))).collect(); + self.offstate = data.into_iter() + .filter_map(|(k, v)| v.map(|v| (k,v))) + .map(|(k, v)| { + let mut ser = Ser::default(); + ser.push(self.last_block, Some(v.as_slice())); + (k, ser.into_vec()) + }) + .collect(); } } @@ -140,6 +173,7 @@ pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet { data: make_changeset(inserted, deleted), meta: ChangeSet::default(), offstate: OffstateChangeSet::default(), + offstate_prune: None, } } @@ -148,6 +182,7 @@ pub fn make_commit_both(inserted: &[u64], deleted: &[u64]) -> CommitSet { data: make_changeset(inserted, deleted), meta: ChangeSet::default(), offstate: make_offstate_changeset(inserted, deleted), + offstate_prune: None, } } diff --git a/core/utils/historied-data/src/lib.rs b/core/utils/historied-data/src/lib.rs index 0169829122d24..70b634845afee 100644 --- a/core/utils/historied-data/src/lib.rs +++ b/core/utils/historied-data/src/lib.rs @@ -80,3 +80,11 @@ fn as_i>(i: usize) -> I { Err(_) => panic!("historied value index underflow"), } } + +/// Prunning result to be able to proceed +/// with further update if the value needs it. +pub enum PruneResult { + Unchanged, + Changed, + Cleared, +} diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 99d9b11ac0bdf..1f4b417fe5fb3 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -47,8 +47,57 @@ const ALLOCATED_HISTORY: usize = 2; /// Can be written as is in underlying /// storage. /// Could be use for direct access memory to. -pub struct Serialized<'a, F>(Cow<'a, [u8]>, PhantomData); +pub struct Serialized<'a, F>(SerializedBuff<'a>, PhantomData); +#[derive(Debug)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +enum SerializedBuff<'a> { + Cow(Cow<'a, [u8]>), + Mut(&'a mut Vec), +} + +impl<'a> SerializedBuff<'a> { + pub fn to_mut(&mut self) -> &mut Vec { + match self { + SerializedBuff::Cow(c) => c.to_mut(), + SerializedBuff::Mut(m) => m, + } + } + pub fn into_owned(self) -> Vec { + match self { + SerializedBuff::Cow(c) => c.into_owned(), + SerializedBuff::Mut(m) => m.clone(), + } + } +} + +impl<'a> rstd::ops::Deref for SerializedBuff<'a> { + type Target = [u8]; + fn deref(&self) -> &Self::Target { + match self { + SerializedBuff::Cow(c) => c.deref(), + SerializedBuff::Mut(m) => m.deref(), + } + } +} + +impl<'a> rstd::ops::DerefMut for SerializedBuff<'a> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.to_mut()[..] + } +} + +impl<'a> Clone for SerializedBuff<'a> { + fn clone(&self) -> Self { + match self { + SerializedBuff::Cow(c) => SerializedBuff::Cow(c.clone()), + SerializedBuff::Mut(m) => { + let m: Vec = (*m).clone(); + SerializedBuff::Cow(Cow::Owned(m)) + } + } + } +} /// Serialized specific behavior. pub trait SerializedConfig { /// encoded empty slice @@ -90,7 +139,11 @@ const SIZE_BYTE_LEN: usize = 8; impl<'a, F: SerializedConfig> Serialized<'a, F> { pub fn into_owned(self) -> Serialized<'static, F> { - Serialized(Cow::from(self.0.into_owned()), PhantomData) + Serialized(SerializedBuff::Cow(Cow::from(self.0.into_owned())), PhantomData) + } + + pub fn into_vec(self) -> Vec { + self.0.into_owned() } pub(crate) fn len(&self) -> usize { @@ -98,7 +151,7 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { self.read_le_usize(len - SIZE_BYTE_LEN) as usize } - fn clear(&mut self) { + pub(crate) fn clear(&mut self) { self.write_le_usize(F::version_len(), 0); self.0.to_mut().truncate(F::version_len() + SIZE_BYTE_LEN); } @@ -148,6 +201,11 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { } pub(crate) fn push(&mut self, val: HistoriedValue<&[u8], u64>) { + self.push_extra(val, &[]) + } + + /// variant of push where part of the value is in a second slice. + pub(crate) fn push_extra(&mut self, val: HistoriedValue<&[u8], u64>, extra: &[u8]) { let len = self.len(); let start_ix = self.index_start(); let end_ix = self.0.len(); @@ -158,6 +216,7 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { self.0.to_mut().truncate(start_ix + SIZE_BYTE_LEN); self.write_le_u64(start_ix, val.index); self.0.to_mut().extend_from_slice(val.value); + self.0.to_mut().extend_from_slice(extra); self.0.to_mut().append(&mut new_ix); if len > 0 { self.write_le_usize(self.0.len() - SIZE_BYTE_LEN, start_ix); @@ -173,8 +232,11 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { } fn remove_range(&mut self, index: usize, end: usize) { + if end == 0 { + return; + } let len = self.len(); - if len == 1 && index == 0 { + if len <= end - index && index == 0 { self.clear(); return; } @@ -230,13 +292,25 @@ const DEFAULT_VERSION_EMPTY_SERIALIZED: [u8; SIZE_BYTE_LEN + 1] = { impl<'a, F: SerializedConfig> Default for Serialized<'a, F> { fn default() -> Self { - Serialized(Cow::Borrowed(F::empty()), PhantomData) + Serialized(SerializedBuff::Cow(Cow::Borrowed(F::empty())), PhantomData) } } impl<'a, F> Into> for &'a[u8] { fn into(self) -> Serialized<'a, F> { - Serialized(Cow::Borrowed(self), PhantomData) + Serialized(SerializedBuff::Cow(Cow::Borrowed(self)), PhantomData) + } +} + +impl Into> for Vec { + fn into(self) -> Serialized<'static, F> { + Serialized(SerializedBuff::Cow(Cow::Owned(self)), PhantomData) + } +} + +impl<'a, F> Into> for &'a mut Vec { + fn into(self) -> Serialized<'a, F> { + Serialized(SerializedBuff::Mut(self), PhantomData) } } diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index 99133757d71b0..ad45f99fbb8a7 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -737,7 +737,16 @@ impl History { } impl<'a, F: SerializedConfig> Serialized<'a, F> { - pub fn get (&self, state: S) -> Option<&[u8]> + + pub fn into_owned(self) -> Serialized<'static, F> { + Serialized(self.0.into_owned()) + } + + pub fn into_vec(self) -> Vec { + self.0.into_vec() + } + + pub fn get (&self, state: S) -> Option> where S: BranchStateTrait, I: Copy + Eq + TryFrom + TryInto, @@ -750,7 +759,12 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { index -= 1; let HistoriedValue { value, index: state_index } = self.0.get_state(index); if state.get_node(as_i(state_index as usize)) { - return Some(value) + // Note this extra byte is note optimal, should be part of index encoding + if value.len() > 0 { + return Some(Some(&value[..value.len() - 1])); + } else { + return Some(None); + } } } None @@ -758,7 +772,7 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { /// This append the value, and can only be use in an /// orderly fashion. - pub fn push(&mut self, state: S, value: &[u8]) + pub fn push(&mut self, state: S, value: Option<&[u8]>) where S: BranchStateTrait, I: Copy + Eq + TryFrom + TryInto, @@ -766,66 +780,118 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { let target_state_index = as_usize(state.last_index()) as u64; let index = self.0.len(); if index > 0 { - let last = self.0.get_state(index); + let last = self.0.get_state(index - 1); debug_assert!(target_state_index >= last.index); if target_state_index == last.index { self.0.pop(); } } - self.0.push(HistoriedValue {value, index: target_state_index}); + match value { + Some(value) => + self.0.push_extra(HistoriedValue {value, index: target_state_index}, &[0][..]), + None => + self.0.push(HistoriedValue {value: &[], index: target_state_index}), + } } - /// keep an single history before the state. - pub fn prune(&mut self, index: I) + /// keep a single with value history before the state. + pub fn prune(&mut self, index: I) -> crate::PruneResult where I: Copy + Eq + TryFrom + TryInto, { let from = as_usize(index) as u64; let len = self.0.len(); - for index in 0..len { + let mut last_index_with_value = None; + let mut index = 0; + println!("from {}", from); + while index < len { let history = self.0.get_state(index); - if history.index >= from { - // delete all index up to index - self.0.truncate_start(index); - break; + if history.index == from + 1 { + // new first content + if history.value.len() == 0 { + // delete so first content is after (if any) + last_index_with_value = None; + } else { + println!("start val: {}", index); + // start value over a value drop until here + last_index_with_value = Some(index + 1); + break; + } + } else if history.index > from { + if history.value.len() == 0 + && last_index_with_value.is_none() { + // delete on delete, continue + } else { + break; + } + } + if history.value.len() > 0 { + last_index_with_value = Some(index); + } else { + last_index_with_value = None; } + index += 1; } + + if let Some(last_index_with_value) = last_index_with_value { + if last_index_with_value > 1 { + self.0.truncate_start(last_index_with_value - 1); + return crate::PruneResult::Changed; + } + } else { + self.0.clear(); + return crate::PruneResult::Cleared; + } + + crate::PruneResult::Unchanged } } - #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test"), derive(PartialEq))] /// Serialized implementation when transaction support is not /// needed. pub struct Serialized<'a, F>(SerializedInner<'a, F>); -impl<'a, F> Into> for &'a[u8] { +impl<'a, F> Serialized<'a, F> { + pub fn from_slice(s: &'a [u8]) -> Serialized<'a, F> { + Serialized(s.into()) + } + + pub fn from_vec(s: Vec) -> Serialized<'static, F> { + Serialized(s.into()) + } + + pub fn from_mut(s: &'a mut Vec) -> Serialized<'a, F> { + Serialized(s.into()) + } +} + +impl<'a, F> Into> for &'a [u8] { fn into(self) -> Serialized<'a, F> { Serialized(self.into()) } } -impl<'a, F> Into> for SerializedInner<'a, F> { +impl<'a, F> Into> for &'a mut Vec { fn into(self) -> Serialized<'a, F> { - Serialized(self) + Serialized(self.into()) } } -impl<'a, F> Into> for Serialized<'a, F> { - fn into(self) -> SerializedInner<'a, F> { - self.0 +impl Into> for Vec { + fn into(self) -> Serialized<'static, F> { + Serialized(self.into()) } } impl<'a, F: SerializedConfig> Default for Serialized<'a, F> { fn default() -> Self { - SerializedInner::<'a, F>::default().into() + Serialized(SerializedInner::<'a, F>::default()) } } - #[cfg(test)] mod test { use super::*; From e86fdd2b0593ed5f6f0d5c2744b618079959d5bf Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 3 Oct 2019 22:35:11 +0200 Subject: [PATCH 23/68] end fixing pruning test, remainings root lib tests to adjust --- core/state-db/src/pruning.rs | 31 +++++++++++++++---------- core/utils/historied-data/src/linear.rs | 10 ++++---- core/utils/historied-data/src/tree.rs | 19 +++++++++++---- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/core/state-db/src/pruning.rs b/core/state-db/src/pruning.rs index 71c8fc57323ae..a6fd06489654a 100644 --- a/core/state-db/src/pruning.rs +++ b/core/state-db/src/pruning.rs @@ -273,7 +273,7 @@ mod tests { use super::RefWindow; use primitives::H256; use crate::CommitSet; - use crate::test::{make_db, make_commit_both, TestDb}; + use crate::test::{make_db, make_commit_both, TestDb, make_commit}; fn check_journal(pruning: &RefWindow, db: &TestDb) { let restored: RefWindow = RefWindow::new(db).unwrap(); @@ -419,7 +419,10 @@ mod tests { pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); - assert!(db.offstate_eq(&[1, 2, 3])); + assert!(db.offstate_eq_at(&[1, 2, 3], Some(0))); + assert!(db.offstate_eq_at(&[1, 3], Some(1))); + assert!(db.offstate_eq_at(&[1, 2, 3], Some(2))); + assert!(db.offstate_eq(&[1, 3])); pruning.apply_pending(); check_journal(&pruning, &db); @@ -428,15 +431,24 @@ mod tests { pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); - assert!(db.offstate_eq(&[1, 2, 3])); + assert!(db.offstate_eq_at(&[1, 3], Some(0))); + assert!(db.offstate_eq_at(&[1, 3], Some(1))); + assert!(db.offstate_eq_at(&[1, 2, 3], Some(2))); + assert!(db.offstate_eq(&[1, 3])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); - assert!(db.offstate_eq(&[1, 2, 3])); + assert!(db.offstate_eq_at(&[1, 3], Some(0))); + assert!(db.offstate_eq_at(&[1, 3], Some(1))); + assert!(db.offstate_eq_at(&[1, 2, 3], Some(2))); + assert!(db.offstate_eq(&[1, 3])); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 3]))); + assert!(db.offstate_eq_at(&[1, 3], Some(0))); + assert!(db.offstate_eq_at(&[1, 3], Some(1))); + assert!(db.offstate_eq_at(&[1, 3], Some(2))); assert!(db.offstate_eq(&[1, 3])); pruning.apply_pending(); assert_eq!(pruning.pending_number, 3); @@ -445,34 +457,29 @@ mod tests { #[test] fn reinserted_survivew_pending() { let mut db = make_db(&[1, 2, 3]); - db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = make_commit_both(&[], &[2]); + let mut commit = make_commit(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); - let mut commit = make_commit_both(&[2], &[]); + let mut commit = make_commit(&[2], &[]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); - let mut commit = make_commit_both(&[], &[2]); + let mut commit = make_commit(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); - assert!(db.offstate_eq(&[1, 2, 3])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); - assert!(db.offstate_eq(&[1, 2, 3])); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); - assert!(db.offstate_eq(&[1, 2, 3])); pruning.prune_one(&mut commit); db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 3]))); - assert!(db.offstate_eq(&[1, 3])); pruning.apply_pending(); assert_eq!(pruning.pending_number, 3); } diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 1f4b417fe5fb3..0e310fdb41d9f 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -175,7 +175,7 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { } // index stay in truncated content - pub(crate) fn truncate_start(&mut self, index: usize) { + pub(crate) fn truncate_until(&mut self, index: usize) { self.remove_range(0, index); } @@ -254,9 +254,9 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { let _ = self.0.to_mut().remove(elt_start); } let start_ix = start_ix - delete_size; - for i in 1..len - index - 1 { + for i in end..end + (end - index) - 1 { let old_value = self.read_le_usize(start_ix + i * SIZE_BYTE_LEN); - self.write_le_usize(start_ix + (i - 1) * SIZE_BYTE_LEN, old_value - delete_size); + self.write_le_usize(start_ix + (i - (end - index)) * SIZE_BYTE_LEN, old_value - delete_size); } let len = len - (end - index); let end_index = start_ix + len * SIZE_BYTE_LEN; @@ -417,12 +417,12 @@ mod test { assert_eq!(ser.get_state(0), (v2, 2).into()); ser.push((v1, 1).into()); ser.push((v3, 3).into()); - ser.truncate_start(1); + ser.truncate_until(1); assert_eq!(ser.len(), 2); assert_eq!(ser.get_state(0), (v1, 1).into()); assert_eq!(ser.get_state(1), (v3, 3).into()); ser.push((v2, 2).into()); - ser.truncate_start(2); + ser.truncate_until(2); assert_eq!(ser.len(), 1); assert_eq!(ser.get_state(0), (v2, 2).into()); diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index ad45f99fbb8a7..248468f1d2bea 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -787,9 +787,9 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { } } match value { - Some(value) => + Some(value) => self.0.push_extra(HistoriedValue {value, index: target_state_index}, &[0][..]), - None => + None => self.0.push(HistoriedValue {value: &[], index: target_state_index}), } } @@ -809,19 +809,27 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { if history.index == from + 1 { // new first content if history.value.len() == 0 { + println!("yoop"); // delete so first content is after (if any) last_index_with_value = None; } else { println!("start val: {}", index); // start value over a value drop until here - last_index_with_value = Some(index + 1); + last_index_with_value = Some(index); break; } } else if history.index > from { + println!("IN {}", history.index); if history.value.len() == 0 && last_index_with_value.is_none() { + println!("doo"); // delete on delete, continue } else { + println!("daa"); + if last_index_with_value.is_none() { + // first value, use this index + last_index_with_value = Some(index); + } break; } } @@ -834,8 +842,9 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { } if let Some(last_index_with_value) = last_index_with_value { - if last_index_with_value > 1 { - self.0.truncate_start(last_index_with_value - 1); + if last_index_with_value > 0 { + println!("IN {}", last_index_with_value); + self.0.truncate_until(last_index_with_value); return crate::PruneResult::Changed; } } else { From e48cf4335ecbe3fd79411f34f7b5df556c9e6c19 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 4 Oct 2019 13:54:55 +0200 Subject: [PATCH 24/68] fix gc call block index. --- core/state-db/src/branch.rs | 9 ++++--- core/state-db/src/lib.rs | 22 ++++++++++++++-- core/state-db/src/noncanonical.rs | 24 ++++++++++++++---- core/state-db/src/test.rs | 8 +++++- core/utils/historied-data/src/tree.rs | 36 ++++++++++++++++++++------- 5 files changed, 79 insertions(+), 20 deletions(-) diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index d52b3c945f426..96a0b55046a2e 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -299,7 +299,10 @@ impl RangeSet { full: bool, ) { - if branch_index <= self.treshold { + if branch_index < self.treshold { + return; + } + if branch_index == self.treshold && linear_index.is_none() { return; } // range set update @@ -307,7 +310,7 @@ impl RangeSet { // we do not finalize current branch cause it // can contains other blocks self.treshold = branch_index; - if branch_index == 0 || !full { + if (branch_index == 0 && linear_index.is_none()) || !full { // remove cached value under treshold only let new_storage = self.storage.split_off(&(self.treshold)); self.storage = new_storage; @@ -321,7 +324,7 @@ impl RangeSet { } else { let new_storage = self.storage.split_off(&(self.treshold)); self.storage = new_storage; - self.finalize_full(branch_index, None); + self.finalize_full(branch_index, linear_index); }; } diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index eed50ddd99d4b..a284458b2f606 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -146,6 +146,9 @@ pub struct ChangeSet { /// ), /// but it just need to be build from client (no need to change /// it here except to extract faster). +/// +/// This assumes that we only commit block per block (otherwhise +/// we need to inclued block number value here). pub type OffstateChangeSet = Vec<(H, Option)>; /// Info for pruning offstate: a last prune index and keys to prune. @@ -259,6 +262,12 @@ impl StateDbSync { Ok(CommitSet { data: changeset, meta: Default::default(), + // TODO EMCH no current support for archive all, + // this would require managing branch index at + // client level (was implemented in another branch: + // client-db-branch-ix) + // and use and ordered tuple (branchix, blocknumber) + // index to run. offstate: offstate_changeset, offstate_prune: None, }) @@ -278,6 +287,9 @@ impl StateDbSync { Ok(()) => { if self.mode == PruningMode::ArchiveCanonical { commit.data.deleted.clear(); + // TODO EMCH use pruning mode in statedb to avoid feeding that + // in the first place. + commit.offstate_prune = None; } } Err(e) => return Err(e), @@ -618,7 +630,9 @@ mod tests { fn full_archive_keeps_everything() { let (db, sdb) = make_test_db(PruningMode::ArchiveAll); assert!(db.data_eq(&make_db(&[1, 21, 22, 3, 4, 91, 921, 922, 93, 94]))); - assert!(db.offstate_eq(&[1, 21, 22, 3, 4, 81, 821, 822, 83, 84])); + + // TODO EMCH implement full for test db and test for offstate + assert!(!sdb.is_pruned(&H256::from_low_u64_be(0), 0)); } @@ -626,7 +640,11 @@ mod tests { fn canonical_archive_keeps_canonical() { let (db, _) = make_test_db(PruningMode::ArchiveCanonical); assert!(db.data_eq(&make_db(&[1, 21, 3, 91, 921, 922, 93, 94]))); - assert!(db.offstate_eq(&[1, 21, 3, 81, 821, 822, 83, 84])); + assert!(db.offstate_eq_at(&[81, 821, 822, 83, 84], Some(0))); + assert!(db.offstate_eq_at(&[1, 821, 822, 83, 84], Some(1))); + assert!(db.offstate_eq_at(&[21, 822, 83, 84], Some(2))); + assert!(db.offstate_eq_at(&[3, 21, 822, 84], Some(3))); + assert!(db.offstate_eq(&[3, 21, 822, 84])); } #[test] diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 9868e269cefda..bdebc9adb6001 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -95,9 +95,12 @@ impl OffstatePendingGC { // TODO EMCH double get is rather inefficient, see if it is possible // to reuse the hash of the hashset into the hashmap for key in self.keys_pending_gc.drain() { - offstate_values.get_mut(&key).map(|historied_value| { - historied_value.gc(branches.reverse_iter_ranges()); - }); + match offstate_values.get_mut(&key).map(|historied_value| { + historied_value.gc(branches.reverse_iter_ranges()) + }) { + Some(historied_data::PruneResult::Cleared) => { let _ = offstate_values.remove(&key); }, + _ => (), + } } self.pending_canonicalisation_query = None; @@ -572,12 +575,23 @@ impl NonCanonicalOverlay { if let Some(range) = self.get_branch_range(hash, block_number) { commit.offstate.extend( overlay.offstate_inserted.iter() - .chain(overlay.offstate_deleted.iter()) .map(|k| (k.clone(), self.offstate_values.get(k) .expect("For each key in overlays there's a value in values") .get(&range) .expect("For each key in overlays there's a historied entry in values") .clone()))); + // some deletion can be pruned if in first block so handle is + // a bit less restrictive + commit.offstate.extend( + overlay.offstate_deleted.iter() + .filter_map(|k| self.offstate_values.get(k) + .map(|v| (k.clone(), v.get(&range) + .expect("For each key in overlays there's a historied entry \ + in values, and pruned empty value are cleared") + .clone()) + ) + ) + ); } } @@ -633,7 +647,7 @@ impl NonCanonicalOverlay { if let Some(branch_index_cannonicalize) = last_index { // this needs to be call after parents update // TODO EMCH may be needed in 'canonicalize', and restore or commit here - self.branches.update_finalize_treshold(branch_index_cannonicalize, Some(block_number), true); + self.branches.update_finalize_treshold(branch_index_cannonicalize, Some(block_number + 1), true); // gc is at the right place self.offstate_gc.set_pending_gc(branch_index_cannonicalize); // try to run the garbage collection (can run later if there is diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 2428a32aaa803..8a30a014bda9c 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -33,6 +33,10 @@ pub struct TestDb { pub data: HashMap, pub meta: HashMap, DBValue>, pub offstate: HashMap, + // big heuristic to increment this for canonicalize transaction only + // by checking if there is value in offstate change set. + // If empty change set needed, this need to be change manually. + // TODO EMCH consider changing commit fun to use last block number explicitely. pub last_block: u64, } @@ -79,7 +83,9 @@ impl NodeDb for TestDb { impl TestDb { pub fn commit(&mut self, commit: &CommitSet) { - self.last_block += 1; + if commit.offstate.len() > 0 { + self.last_block += 1; + } self.data.extend(commit.data.inserted.iter().cloned()); for k in commit.data.deleted.iter() { self.data.remove(k); diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index 248468f1d2bea..66978f41d0c46 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -27,6 +27,7 @@ use crate::linear::{ SerializedConfig, }; use crate::HistoriedValue; +use crate::PruneResult; use crate::{as_usize, as_i}; use rstd::rc::Rc; use rstd::vec::Vec; @@ -639,12 +640,13 @@ impl History { /// Gc an historied value other its possible values. /// Iterator need to be reversed ordered by branch index. - pub fn gc(&mut self, mut states: IT) + pub fn gc(&mut self, mut states: IT) -> PruneResult where IT: Iterator, S: BranchStateTrait, I: Copy + Eq + TryFrom + TryInto, { + let mut changed = false; // state is likely bigger than history. let mut current_state = states.next(); for branch_index in (0..self.0.len()).rev() { @@ -663,29 +665,45 @@ impl History { if let Ok(node_index) = node_index.try_into() { if !state.0.get_node(node_index) { if history_index == len - 1 { - let _ = self.0[branch_index].history.pop(); + changed = self.0[branch_index] + .history.pop().is_some() || changed; } else { - let _ = self.0[branch_index].history.remove(history_index); + self.0[branch_index] + .history.remove(history_index); + changed = true; } } } } if self.0[branch_index].history.len() == 0 { - let _ = self.0.remove(branch_index); + self.0.remove(branch_index); + changed = true; } break; } else { - let _ = self.0.remove(branch_index); + self.0.remove(branch_index); + changed = true; break; } } self.0.remove(branch_index); + changed = true; break; } else { break; } } } + if changed { + if self.0.len() == 0 { + PruneResult::Cleared + } else { + PruneResult::Changed + } + + } else { + PruneResult::Unchanged + } } /* @@ -795,7 +813,7 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { } /// keep a single with value history before the state. - pub fn prune(&mut self, index: I) -> crate::PruneResult + pub fn prune(&mut self, index: I) -> PruneResult where I: Copy + Eq + TryFrom + TryInto, { @@ -845,14 +863,14 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { if last_index_with_value > 0 { println!("IN {}", last_index_with_value); self.0.truncate_until(last_index_with_value); - return crate::PruneResult::Changed; + return PruneResult::Changed; } } else { self.0.clear(); - return crate::PruneResult::Cleared; + return PruneResult::Cleared; } - crate::PruneResult::Unchanged + PruneResult::Unchanged } } From 0e3c65e6c3a069a8234edc765c6938601f743670 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 4 Oct 2019 17:59:40 +0200 Subject: [PATCH 25/68] state_db tests passing. --- core/state-db/src/branch.rs | 45 +++++++++++++------------ core/state-db/src/lib.rs | 18 ++++++++-- core/state-db/src/noncanonical.rs | 10 +++--- core/state-db/src/test.rs | 9 ++--- core/utils/historied-data/src/linear.rs | 14 +++++--- core/utils/historied-data/src/tree.rs | 11 ++---- 6 files changed, 58 insertions(+), 49 deletions(-) diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index 96a0b55046a2e..86eee7f81a7a0 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -160,6 +160,10 @@ impl RangeSet { if let Some(Some(StatesBranch{ state, parent_branch_index, .. })) = self.storage.get(&branch_index) { // TODO EMCH consider vecdeque ?? let state = if state.end > previous_start { + if state.start >= previous_start { + // empty branch stop here + break; + } BranchStateRef { start: state.start, end: previous_start, @@ -295,22 +299,19 @@ impl RangeSet { pub fn update_finalize_treshold( &mut self, branch_index: u64, - linear_index: Option, + linear_index: u64, full: bool, ) { if branch_index < self.treshold { return; } - if branch_index == self.treshold && linear_index.is_none() { - return; - } // range set update let old_treshold = self.treshold; // we do not finalize current branch cause it // can contains other blocks self.treshold = branch_index; - if (branch_index == 0 && linear_index.is_none()) || !full { + if !full { // remove cached value under treshold only let new_storage = self.storage.split_off(&(self.treshold)); self.storage = new_storage; @@ -330,17 +331,19 @@ impl RangeSet { /// Apply a post finalize without considering treshold. /// TODO EMCH rename as it is also use to prepare a gc - /// without moving the treshould first. + /// without moving the treshold first. + /// pub fn finalize_full( &mut self, branch_index: u64, - linear_index: Option, + linear_index: u64, ) { if let Some(Some(state)) = self.storage.remove(&branch_index) { let mut child_anchor: LinearStates = state.clone(); - child_anchor.state.start = linear_index.unwrap_or(child_anchor.state.start); + child_anchor.state.start = linear_index; let old_storage = std::mem::replace(&mut self.storage, BTreeMap::new()); - self.storage.insert(branch_index, Some(child_anchor)); + // insert anchor + self.storage.insert(branch_index, Some(child_anchor.clone())); for (index, state) in old_storage.into_iter().filter_map(|(k, v)| v.map(|v| (k, v))) { // ordered property of branch index allows to skip in depth branch search if let Some(Some(parent_state)) = self.storage.get(&state.parent_branch_index) { @@ -355,18 +358,17 @@ impl RangeSet { } } } + // remove anchor block + child_anchor.state.start += 1; + if child_anchor.state.start < child_anchor.state.end { + self.storage.insert(branch_index, Some(child_anchor.clone())); + } else { + self.storage.remove(&branch_index); + } } } - #[cfg(test)] - pub fn test_finalize_full( - &mut self, - branch_index: u64, - ) { - self.finalize_full(branch_index, None); - } - /// Revert some ranges, without any way to revert. /// Returning ranges for the parent index. /// TODO EMCH can remove ?? @@ -488,7 +490,7 @@ mod test { // 0 // |> 2: [10,2] // |> 3: [11,2] - fn build_finalize_set() -> (RangeSet, u64) { + fn build_finalize_set() -> (RangeSet, (u64, u64)) { let mut set = RangeSet::default(); set.import(10, 0, None); let (b1, anchor_1) = set.import(10, 0, None); @@ -499,7 +501,7 @@ mod test { set.import(12, finalize, Some(bf.clone())); set.import(12, finalize, Some(bf)); - (set, finalize) + (set, (finalize, 11)) } #[test] @@ -511,7 +513,7 @@ mod test { | { let (mut ranges, finalize) = build_finalize_set(); - let _ = ranges.update_finalize_treshold(finalize, None, full); + let _ = ranges.update_finalize_treshold(finalize.0, finalize.1, full); assert!(ranges.range_treshold() == 3); @@ -529,7 +531,8 @@ mod test { &[(1, 1), (2, 2)][..], ); with_full_finalize(true, - &[(3, 2), (5, 1)][..], + // (3, 1) because finalized block is removed + &[(3, 1), (5, 1)][..], &[(1, 1), (2, 2), (4, 2)][..], ); } diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index a284458b2f606..5f72d96e8d5a3 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -654,7 +654,11 @@ mod tests { max_mem: None, })); assert!(db.data_eq(&make_db(&[21, 3, 922, 94]))); - assert!(db.offstate_eq(&[21, 3, 822, 84])); + assert!(db.offstate_eq_at(&[822, 84], Some(0))); + assert!(db.offstate_eq_at(&[822, 84], Some(1))); + assert!(db.offstate_eq_at(&[21, 822, 84], Some(2))); + assert!(db.offstate_eq_at(&[3, 21, 822, 84], Some(3))); + assert!(db.offstate_eq(&[3, 21, 822, 84])); } #[test] @@ -668,7 +672,11 @@ mod tests { assert!(sdb.is_pruned(&H256::from_low_u64_be(21), 2)); assert!(sdb.is_pruned(&H256::from_low_u64_be(22), 2)); assert!(db.data_eq(&make_db(&[21, 3, 922, 93, 94]))); - assert!(db.offstate_eq(&[21, 3, 822, 83, 84])); + assert!(db.offstate_eq_at(&[822, 83, 84], Some(0))); + assert!(db.offstate_eq_at(&[822, 83, 84], Some(1))); + assert!(db.offstate_eq_at(&[21, 822, 83, 84], Some(2))); + assert!(db.offstate_eq_at(&[3, 21, 822, 84], Some(3))); + assert!(db.offstate_eq(&[3, 21, 822, 84])); } #[test] @@ -682,6 +690,10 @@ mod tests { assert!(!sdb.is_pruned(&H256::from_low_u64_be(21), 2)); assert!(sdb.is_pruned(&H256::from_low_u64_be(22), 2)); assert!(db.data_eq(&make_db(&[1, 21, 3, 921, 922, 93, 94]))); - assert!(db.offstate_eq(&[1, 21, 3, 821, 822, 83, 84])); + assert!(db.offstate_eq_at(&[821, 822, 83, 84], Some(0))); + assert!(db.offstate_eq_at(&[1, 821, 822, 83, 84], Some(1))); + assert!(db.offstate_eq_at(&[21, 822, 83, 84], Some(2))); + assert!(db.offstate_eq_at(&[3, 21, 822, 84], Some(3))); + assert!(db.offstate_eq(&[3, 21, 822, 84])); } } diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index bdebc9adb6001..d1ea356a094e7 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -98,7 +98,9 @@ impl OffstatePendingGC { match offstate_values.get_mut(&key).map(|historied_value| { historied_value.gc(branches.reverse_iter_ranges()) }) { - Some(historied_data::PruneResult::Cleared) => { let _ = offstate_values.remove(&key); }, + Some(historied_data::PruneResult::Cleared) => { + let _ = offstate_values.remove(&key); + }, _ => (), } } @@ -647,7 +649,7 @@ impl NonCanonicalOverlay { if let Some(branch_index_cannonicalize) = last_index { // this needs to be call after parents update // TODO EMCH may be needed in 'canonicalize', and restore or commit here - self.branches.update_finalize_treshold(branch_index_cannonicalize, Some(block_number + 1), true); + self.branches.update_finalize_treshold(branch_index_cannonicalize, block_number, true); // gc is at the right place self.offstate_gc.set_pending_gc(branch_index_cannonicalize); // try to run the garbage collection (can run later if there is @@ -673,9 +675,7 @@ impl NonCanonicalOverlay { /// Get a value from the node overlay. This searches in every existing changeset. pub fn get_offstate(&self, key: &[u8], state: &BranchRanges) -> Option<&Option> { - println!("b: {:?}", state); if let Some(value) = self.offstate_values.get(key) { - println!("v: {:?}", value); return value.get(state); } None @@ -1147,6 +1147,7 @@ mod tests { assert!(!contains_any(&overlay, 22, &h_2_2, 2)); assert!(!contains_any(&overlay, 211, &h_2_1_1, 3)); assert!(contains_both(&overlay, 111, &h_1_1_1, 3)); + assert!(contains_both(&overlay, 12, &h_1_2, 2)); // check that journals are deleted assert!(db.get_meta(&to_journal_key(1, 0)).unwrap().is_none()); assert!(db.get_meta(&to_journal_key(1, 1)).unwrap().is_none()); @@ -1270,6 +1271,7 @@ mod tests { overlay.pin(&h_1); let h1_context = overlay.get_branch_range(&h_1, 1).unwrap(); + let h2_context = overlay.get_branch_range(&h_2, 1).unwrap(); let mut commit = CommitSet::default(); overlay.canonicalize::(&h_2, &mut commit).unwrap(); diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 8a30a014bda9c..71852a9aa4eff 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -99,13 +99,8 @@ impl TestDb { if let Some((block_prune, offstate_prune_key)) = commit.offstate_prune.as_ref() { for k in offstate_prune_key.iter() { match self.offstate.get_mut(k).map(|v| { - println!("t{}", *block_prune); - println!("t{:?}", k); - println!("ser{:?}", v); let mut ser = Ser::from_mut(v); - let r = ser.prune(*block_prune); - println!("ser{:?}", ser.into_vec()); - r + ser.prune(*block_prune) }) { Some(PruneResult::Cleared) => { let _ = self.offstate.remove(k); }, Some(PruneResult::Changed) // changed applyied on mutable buffer without copy. @@ -127,7 +122,7 @@ impl TestDb { pub fn offstate_eq_at(&self, values: &[u64], block: Option) -> bool { let data = make_offstate_changeset(values, &[]); let self_offstate: BTreeMap<_, _> = self.get_offstate_pairs(&block).into_iter().collect(); - println!("---{:?}", self_offstate); +println!("of_eq {:?}", self_offstate); self_offstate == data.into_iter().filter_map(|(k, v)| v.map(|v| (k,v))).collect() } diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 0e310fdb41d9f..7e965253c61fc 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -251,14 +251,18 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { }; let delete_size = elt_end - elt_start; for _ in elt_start..elt_end { - let _ = self.0.to_mut().remove(elt_start); + let _ = self.0.to_mut().remove(elt_start); // TODO EMCH slice copy instead of that horror } let start_ix = start_ix - delete_size; - for i in end..end + (end - index) - 1 { - let old_value = self.read_le_usize(start_ix + i * SIZE_BYTE_LEN); - self.write_le_usize(start_ix + (i - (end - index)) * SIZE_BYTE_LEN, old_value - delete_size); - } + let len = len - (end - index); + for i in index..end { + let pos = i + (end - index); + if pos < len { + let old_value = self.read_le_usize(start_ix + pos * SIZE_BYTE_LEN); + self.write_le_usize(start_ix + i * SIZE_BYTE_LEN, old_value - delete_size); + } + } let end_index = start_ix + len * SIZE_BYTE_LEN; self.write_le_usize(end_index - SIZE_BYTE_LEN, len); self.0.to_mut().truncate(end_index); diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index 66978f41d0c46..8261298db12e4 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -640,6 +640,7 @@ impl History { /// Gc an historied value other its possible values. /// Iterator need to be reversed ordered by branch index. + /// TODO EMCH this needs rewrite and proper testing!! pub fn gc(&mut self, mut states: IT) -> PruneResult where IT: Iterator, @@ -686,11 +687,10 @@ impl History { break; } } + } else { self.0.remove(branch_index); changed = true; break; - } else { - break; } } } @@ -821,29 +821,23 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { let len = self.0.len(); let mut last_index_with_value = None; let mut index = 0; - println!("from {}", from); while index < len { let history = self.0.get_state(index); if history.index == from + 1 { // new first content if history.value.len() == 0 { - println!("yoop"); // delete so first content is after (if any) last_index_with_value = None; } else { - println!("start val: {}", index); // start value over a value drop until here last_index_with_value = Some(index); break; } } else if history.index > from { - println!("IN {}", history.index); if history.value.len() == 0 && last_index_with_value.is_none() { - println!("doo"); // delete on delete, continue } else { - println!("daa"); if last_index_with_value.is_none() { // first value, use this index last_index_with_value = Some(index); @@ -861,7 +855,6 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { if let Some(last_index_with_value) = last_index_with_value { if last_index_with_value > 0 { - println!("IN {}", last_index_with_value); self.0.truncate_until(last_index_with_value); return PruneResult::Changed; } From 13ea0886585091cc57aea31aaf681d52be6c02ca Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 4 Oct 2019 18:11:22 +0200 Subject: [PATCH 26/68] fix underflow --- core/state-db/src/noncanonical.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index d1ea356a094e7..0c870d2ffe48e 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -449,7 +449,11 @@ impl NonCanonicalOverlay { }; level.push(overlay); let parent_branch_index = self.parents.get(&parent_hash).map(|(_, i)| *i).unwrap_or(0); - let parent_branch_range = Some(self.branches.branch_ranges_from_cache(parent_branch_index, Some(number - 1))); + let parent_branch_range = if number == 0 { + None + } else { + Some(self.branches.branch_ranges_from_cache(parent_branch_index, Some(number - 1))) + }; let (branch_range, branch_index) = self.branches.import( number, parent_branch_index, From 92c699cd5d2262b09413efd2d02598e43c00f7a0 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 4 Oct 2019 19:23:35 +0200 Subject: [PATCH 27/68] before switching to absolute indexing (option to u64). --- Cargo.lock | 1 + core/client/db/Cargo.toml | 1 + core/client/db/src/lib.rs | 22 +++++++++++++++++++--- core/state-db/src/lib.rs | 16 +++++++++++++--- core/state-db/src/noncanonical.rs | 8 ++++++++ 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d84402a9755e..f504fdc4d9c36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4706,6 +4706,7 @@ version = "2.0.0" dependencies = [ "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "historied-data 2.0.0", "kvdb 0.1.0 (git+https://github.com/paritytech/parity-common?rev=b0317f649ab2c665b7987b8475878fc4d2e1f81d)", "kvdb-memorydb 0.1.0 (git+https://github.com/paritytech/parity-common?rev=b0317f649ab2c665b7987b8475878fc4d2e1f81d)", "kvdb-rocksdb 0.1.4 (git+https://github.com/paritytech/parity-common?rev=b0317f649ab2c665b7987b8475878fc4d2e1f81d)", diff --git a/core/client/db/Cargo.toml b/core/client/db/Cargo.toml index 89a1dcc0cbc8a..ebe4d4a64bfe0 100644 --- a/core/client/db/Cargo.toml +++ b/core/client/db/Cargo.toml @@ -22,6 +22,7 @@ executor = { package = "substrate-executor", path = "../../executor" } state_db = { package = "substrate-state-db", path = "../../state-db" } trie = { package = "substrate-trie", path = "../../trie" } consensus_common = { package = "substrate-consensus-common", path = "../../consensus/common" } +historied-data = { path = "../../utils/historied-data" } [dev-dependencies] substrate-keyring = { path = "../../keyring" } diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 2c1f9cea7eabc..315a3e6a9f53c 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -71,6 +71,11 @@ use state_db::BranchRanges; use crate::storage_cache::{CachingState, SharedCache, new_shared_cache}; use log::{trace, debug, warn}; pub use state_db::PruningMode; +use historied_data::tree::Serialized; +use historied_data::PruneResult; +use historied_data::linear::DefaultVersion; + +type Ser<'a> = Serialized<'a, DefaultVersion>; #[cfg(feature = "test-helpers")] use client::in_mem::Backend as InMemoryBackend; @@ -610,12 +615,23 @@ impl state_db::OffstateDb> for StorageDb { type Error = io::Error; fn get_offstate(&self, key: &[u8], state: &Option) -> Result>, Self::Error> { - self.db.get(columns::OFFSTATE, key); - unimplemented!("TODO map with history fetch at requested block") + let state = state.unwrap_or(self.last_block); + Ok(self.db.get(columns::OFFSTATE, key)? + .map(|s| Ser::from_slice(&s[..])) + .and_then(|s| s.get(state) + .unwrap_or(None) // flatten + .map(Into::into) + )) } fn get_offstate_pairs(&self, state: &Option) -> Vec<(Vec, Vec)> { - unimplemented!("TODO need to be an iterator") + let state = state.unwrap_or(self.last_block); + self.db.iter(columns::OFFSTATE).filter_map(|(k, v)| + Ser::from_slice(&v[..]).get(state) + .unwrap_or(None) // flatten + .map(Into::into) + .map(|v| (k.to_vec(), v)) + ).collect() } } diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index 5f72d96e8d5a3..b0f6f6d4d73ba 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -420,11 +420,21 @@ impl StateDbSync { state: &(BranchRanges, u64), db: &D, ) -> Vec<(OffstateKey, DBValue)> { - unimplemented!("TODO some filtering ever non canonical"); - Default::default() + let mut result = Vec::new(); + let mut filter = HashSet::new(); + for (k, o) in self.non_canonical.offstate_iter(&state.0) { + if let Some(v) = o.as_ref() { + result.push((k.clone(), v.clone())); + } + filter.insert(k.clone()); + } + result.extend( + db.get_offstate_pairs(&Some(state.1)).into_iter() + .filter(|kv| !filter.contains(&kv.0)) + ); + result } - pub fn apply_pending(&mut self) { self.non_canonical.apply_pending(); if let Some(pruning) = &mut self.pruning { diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 0c870d2ffe48e..d4cd2d1279854 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -758,6 +758,14 @@ impl NonCanonicalOverlay { self.pinned.remove(hash); self.offstate_gc.try_gc(&mut self.offstate_values, &self.branches, &self.pinned); } + + /// Iterator over values at a given state. Deletion are included in the result as a None value. + pub fn offstate_iter<'a>(&'a self, state: &'a BranchRanges) -> impl Iterator)> { + let state = state.clone(); + self.offstate_values.iter().filter_map(move |(k, v)| { + v.get(&state).map(|v| (k, v)) + }) + } } #[cfg(test)] From bf79879e1710e309ba7763ba35259036ba0d57b3 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 4 Oct 2019 20:41:09 +0200 Subject: [PATCH 28/68] complete first test in client db, new todo. --- core/client/db/src/lib.rs | 88 ++++++++++++++++++++++----- core/client/db/src/utils.rs | 2 +- core/state-db/src/lib.rs | 12 ++-- core/state-db/src/test.rs | 13 ++-- core/utils/historied-data/src/tree.rs | 2 - 5 files changed, 87 insertions(+), 30 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 315a3e6a9f53c..bc34f2975effd 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -610,24 +610,23 @@ impl state_db::NodeDb for StorageDb { } } -impl state_db::OffstateDb> for StorageDb { +impl state_db::OffstateDb for StorageDb { type Error = io::Error; - fn get_offstate(&self, key: &[u8], state: &Option) -> Result>, Self::Error> { - let state = state.unwrap_or(self.last_block); + fn get_offstate(&self, key: &[u8], state: &u64) -> Result>, Self::Error> { Ok(self.db.get(columns::OFFSTATE, key)? + .as_ref() .map(|s| Ser::from_slice(&s[..])) - .and_then(|s| s.get(state) + .and_then(|s| s.get(*state) .unwrap_or(None) // flatten .map(Into::into) )) } - fn get_offstate_pairs(&self, state: &Option) -> Vec<(Vec, Vec)> { - let state = state.unwrap_or(self.last_block); + fn get_offstate_pairs(&self, state: &u64) -> Vec<(Vec, Vec)> { self.db.iter(columns::OFFSTATE).filter_map(|(k, v)| - Ser::from_slice(&v[..]).get(state) + Ser::from_slice(&v[..]).get(*state) .unwrap_or(None) // flatten .map(Into::into) .map(|v| (k.to_vec(), v)) @@ -1162,7 +1161,11 @@ impl> Backend { trace!(target: "db", "Canonicalize block #{} ({:?})", new_canonical, hash); let commit = self.storage.state_db.canonicalize_block(&hash) .map_err(|e: state_db::Error| client::error::Error::from(format!("State database error: {:?}", e)))?; - apply_state_commit(transaction, commit); + apply_state_commit(transaction, commit, &self.storage.db, number_u64).map_err(|err| + client::error::Error::Backend( + format!("Error building commit transaction : {}", err) + ) + )?; }; Ok(()) @@ -1251,8 +1254,15 @@ impl> Backend { offstate_changeset, ).map_err(|e: state_db::Error| client::error::Error::from(format!("State database error: {:?}", e)))?; - apply_state_commit(&mut transaction, commit); - + // TODO EMCH wrong block number: there is no offstate change on block insert + // TODO split apply_state_commit and do an assertion in apply_state_commit + // that there is no offstate stuff anymore + apply_state_commit(&mut transaction, commit, &self.storage.db, number_u64).map_err(|err| + client::error::Error::Backend( + format!("Error building commit transaction : {}", err) + ) + )?; + // Check if need to finalize. Genesis is always finalized instantly. let finalized = number_u64 == 0 || pending_block.leaf_state.is_final(); @@ -1377,13 +1387,19 @@ impl> Backend { if self.storage.state_db.best_canonical().map(|c| f_num.saturated_into::() > c).unwrap_or(true) { let parent_hash = f_header.parent_hash().clone(); + let number = f_header.number().clone(); + let number_u64 = number.saturated_into::(); let lookup_key = utils::number_and_hash_to_lookup_key(f_num, f_hash.clone())?; transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &lookup_key); let commit = self.storage.state_db.canonicalize_block(&f_hash) .map_err(|e: state_db::Error| client::error::Error::from(format!("State database error: {:?}", e)))?; - apply_state_commit(transaction, commit); + apply_state_commit(transaction, commit, &self.storage.db, number_u64).map_err(|err| + client::error::Error::Backend( + format!("Error building commit transaction : {}", err) + ) + )?; let changes_trie_config = self.changes_trie_config(parent_hash)?; if let Some(changes_trie_config) = changes_trie_config { @@ -1401,8 +1417,44 @@ impl> Backend { } } -fn apply_state_commit(transaction: &mut DBTransaction, commit: state_db::CommitSet>) { - // TODO EMCH unimplemented commit of offstate +fn apply_state_commit( + transaction: &mut DBTransaction, + commit: state_db::CommitSet>, + db: &Arc, + last_block: u64, +) -> Result<(), io::Error> { + + for (key, o) in commit.offstate.iter() { + let mut ser = if let Some(stored) = db.get(columns::OFFSTATE, key)? { + Ser::from_vec(stored.to_vec()) + } else { + if let Some(value) = o { + Ser::default() + } else { + break; + } + }; + ser.push(last_block, o.as_ref().map(|v| v.as_slice())); + transaction.put(columns::OFFSTATE, &key[..], &ser.into_vec()) + } + if let Some((block_prune, offstate_prune_key)) = commit.offstate_prune.as_ref() { + for key in offstate_prune_key.iter() { + // TODO EMCH should we prune less often (wait on pruning journals for instance) + // or make it out of this context just stored the block prune and do asynch + let mut ser = if let Some(stored) = db.get(columns::OFFSTATE, key)? { + Ser::from_vec(stored.to_vec()) + } else { + break; + }; + + match ser.prune(*block_prune) { + PruneResult::Cleared => transaction.delete(columns::OFFSTATE, &key), + PruneResult::Changed => transaction.put(columns::OFFSTATE, &key[..], &ser.into_vec()), + PruneResult::Unchanged => (), + } + } + } + for (key, val) in commit.data.inserted.into_iter() { transaction.put(columns::STATE, &key[..], &val); } @@ -1415,6 +1467,7 @@ fn apply_state_commit(transaction: &mut DBTransaction, commit: state_db::CommitS for key in commit.meta.deleted.into_iter() { transaction.delete(columns::STATE_META, &key[..]); } + Ok(()) } impl client::backend::AuxStore for Backend where Block: BlockT { @@ -1538,7 +1591,14 @@ impl client::backend::Backend for Backend whe let mut transaction = DBTransaction::new(); match self.storage.state_db.revert_one() { Some(commit) => { - apply_state_commit(&mut transaction, commit); + // TODO EMCH: here we have a list of key and we need a new method similar to + // gc but for dropping n last state -> samething do not do one block per one + // block. + apply_state_commit(&mut transaction, commit, &self.storage.db, c).map_err(|err| + client::error::Error::Backend( + format!("Error building commit transaction : {}", err) + ) + )?; let removed = self.blockchain.header(BlockId::Number(best))?.ok_or_else( || client::error::Error::UnknownBlock( format!("Error reverting to {}. Block hash not found.", best)))?; diff --git a/core/client/db/src/utils.rs b/core/client/db/src/utils.rs index 70f0ff20588bc..2ad2d82590250 100644 --- a/core/client/db/src/utils.rs +++ b/core/client/db/src/utils.rs @@ -39,7 +39,7 @@ use crate::DatabaseSettings; /// Number of columns in the db. Must be the same for both full && light dbs. /// Otherwise RocksDb will fail to open database && check its type. -pub const NUM_COLUMNS: u32 = 10; +pub const NUM_COLUMNS: u32 = 11; /// Meta column. The set of keys in the column is shared by full && light storages. pub const COLUMN_META: Option = Some(0); diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index b0f6f6d4d73ba..ae437327ed988 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -403,7 +403,7 @@ impl StateDbSync { db.get(key.as_ref()).map_err(|e| Error::Db(e)) } - pub fn get_offstate>>( + pub fn get_offstate>( &self, key: &[u8], state: &(BranchRanges, u64), @@ -412,10 +412,10 @@ impl StateDbSync { if let Some(value) = self.non_canonical.get_offstate(key, &state.0) { return Ok(value.clone()); } - db.get_offstate(key, &Some(state.1)).map_err(|e| Error::Db(e)) + db.get_offstate(key, &state.1).map_err(|e| Error::Db(e)) } - pub fn get_offstate_pairs>>( + pub fn get_offstate_pairs>( &self, state: &(BranchRanges, u64), db: &D, @@ -429,7 +429,7 @@ impl StateDbSync { filter.insert(k.clone()); } result.extend( - db.get_offstate_pairs(&Some(state.1)).into_iter() + db.get_offstate_pairs(&state.1).into_iter() .filter(|kv| !filter.contains(&kv.0)) ); result @@ -511,7 +511,7 @@ impl StateDb { } /// Get a value from non-canonical/pruning overlay or the backing DB. - pub fn get_offstate>>( + pub fn get_offstate>( &self, key: &[u8], state: &(BranchRanges, u64), @@ -521,7 +521,7 @@ impl StateDb { } /// Get pairs values offstate. - pub fn get_offstate_pairs>>( + pub fn get_offstate_pairs>( &self, state: &(BranchRanges, u64), db: &D, diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 71852a9aa4eff..faf0baa89e8b5 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -48,24 +48,22 @@ impl MetaDb for TestDb { } } -impl OffstateDb> for TestDb { +impl OffstateDb for TestDb { type Error = (); - fn get_offstate(&self, key: &[u8], state: &Option) -> Result, ()> { - let state = state.unwrap_or(self.last_block); + fn get_offstate(&self, key: &[u8], state: &u64) -> Result, ()> { Ok(self.offstate.get(key) .map(|s| Ser::from_slice(s.as_slice())) - .and_then(|s| s.get(state) + .and_then(|s| s.get(*state) .unwrap_or(None) // flatten .map(Into::into) )) } - fn get_offstate_pairs(&self, state: &Option) -> Vec<(OffstateKey, DBValue)> { - let state = state.unwrap_or(self.last_block); + fn get_offstate_pairs(&self, state: &u64) -> Vec<(OffstateKey, DBValue)> { self.offstate.iter().filter_map(|(a, s)| ( Ser::from_slice(s.as_slice()) - .get(state) + .get(*state) .unwrap_or(None) // flatten .map(|v| (a.clone(), v.to_vec())) )).collect() @@ -120,6 +118,7 @@ impl TestDb { } pub fn offstate_eq_at(&self, values: &[u64], block: Option) -> bool { + let block = block.unwrap_or(self.last_block); let data = make_offstate_changeset(values, &[]); let self_offstate: BTreeMap<_, _> = self.get_offstate_pairs(&block).into_iter().collect(); println!("of_eq {:?}", self_offstate); diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index 8261298db12e4..1c789b3ce5e01 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -167,8 +167,6 @@ impl<'a> BranchStateTrait for u64 { } - - impl Default for BranchState { // initialize with one element fn default() -> Self { From 4dc2c8d7f9e3beb10e9aac80a089b4c39913212b Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 11:17:53 +0200 Subject: [PATCH 29/68] serialized with default --- core/state-db/src/branch.rs | 4 +- core/utils/historied-data/src/tree.rs | 108 ++++++++++++++------------ 2 files changed, 59 insertions(+), 53 deletions(-) diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index 86eee7f81a7a0..7e7fbb0218caf 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -22,7 +22,7 @@ use std::cmp::Reverse; use crate::Error; use std::hash::Hash as StdHash; use std::convert::TryInto; -use historied_data::tree::{TreeStateTrait, BranchStateTrait, StatesBranchRef, BranchStateRef}; +use historied_data::tree::{BranchesStateTrait, BranchStateTrait, StatesBranchRef, BranchStateRef}; #[derive(Clone, Default, Debug)] /// Reference to state that is enough for query updates, but not @@ -35,7 +35,7 @@ use historied_data::tree::{TreeStateTrait, BranchStateTrait, StatesBranchRef, Br /// copied representation in relation to an actual use case. pub struct BranchRanges(Vec); -impl<'a> TreeStateTrait for &'a BranchRanges { +impl<'a> BranchesStateTrait for &'a BranchRanges { type Branch = &'a BranchStateRef; type Iter = BranchRangesIter<'a>; diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index 1c789b3ce5e01..5a93dce18005f 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -35,9 +35,9 @@ use rstd::collections::btree_map::BTreeMap; use rstd::convert::{TryFrom, TryInto}; /// Trait defining a state for querying or modifying a tree. -/// This is therefore the representation of a branch. -/// TODO EMCH rename to BranchesStateTrait? -pub trait TreeStateTrait { +/// This is a collection of branches index, corresponding +/// to a tree path. +pub trait BranchesStateTrait { type Branch: BranchStateTrait; type Iter: Iterator; @@ -60,7 +60,7 @@ pub trait BranchStateTrait { fn last_index(&self) -> I; } -impl<'a> TreeStateTrait for &'a StatesRef { +impl<'a> BranchesStateTrait for &'a StatesRef { type Branch = (&'a BranchStateRef, Option); type Iter = StatesRefIter<'a>; @@ -480,14 +480,26 @@ impl TestStates { /// TODO EMCH consider removing second field (not use out of test) #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test"), derive(PartialEq))] -pub struct History(Vec>, Option); +pub struct History(Vec>); + +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct HistoryWithDefault(History, Option); + impl Default for History { fn default() -> Self { - History(Vec::new(), None) + History(Vec::new()) + } +} + +impl Default for HistoryWithDefault { + fn default() -> Self { + HistoryWithDefault(History::default(), None) } } + #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test"), derive(PartialEq))] pub struct HistoryBranch { @@ -500,7 +512,7 @@ impl History { /// Set or update value for a given state. pub fn set(&mut self, state: S, value: V) where - S: TreeStateTrait, + S: BranchesStateTrait, I: Copy + Eq + TryFrom + TryInto, BI: Copy + Eq + TryFrom + TryInto, { @@ -583,7 +595,7 @@ impl History { /// When possible please use `get_mut` as it can free some memory. pub fn get (&self, state: S) -> Option<&V> where - S: TreeStateTrait, + S: BranchesStateTrait, I: Copy + Eq + TryFrom + TryInto, BI: Copy + Eq + TryFrom + TryInto, { @@ -592,7 +604,7 @@ impl History { // along a branch (no state containing unordered branch_index // and no history containing unorderd branch_index). if index == 0 { - return self.1.as_ref(); + return None; } // TODO EMCH switch loops ? probably. @@ -613,7 +625,7 @@ impl History { break; } } - self.1.as_ref() + None } fn branch_get(&self, index: usize, state: &S) -> Option<&V> @@ -703,55 +715,49 @@ impl History { PruneResult::Unchanged } } - -/* - /// Access to last valid value (non dropped state in history). - /// When possible please use `get_mut` as it can free some memory. - pub fn get_mut(&mut self, state: &StatesRef) -> Option<&mut V> { - let mut index = self.0.len(); - let mut index_state = state.history.len() - 1; - // note that we expect branch index to be linearily set - // along a branch (no state containing unordered branch_index - // and no history containing unorderd branch_index). - if index == 0 || index_state == 0 { - return self.1.as_mut(); - } - while index > 0 && index_state > 0 { - index -= 1; - let branch_index = self.0[index].branch_index; +} - while index_state > 0 && state.history[index_state].branch_index > branch_index { - index_state -= 1; - } - if state.history[index_state].branch_index == branch_index { - if let Some(result) = self.branch_get_unchecked_mut(branch_index, &state.history[index_state]) { - return Some(result) - } - } - } - self.1.as_mut() +impl HistoryWithDefault { + + /// See `set` for History. + pub fn set(&mut self, state: S, value: V) + where + S: BranchesStateTrait, + I: Copy + Eq + TryFrom + TryInto, + BI: Copy + Eq + TryFrom + TryInto, + { + self.0.set(state, value) } - fn branch_get_unchecked_mut(&mut self, branch_index: u64, state: &StatesBranchRef) -> Option<&mut V> { - let history = &mut self.0[branch_index as usize]; - let mut index = history.history.len(); - if index == 0 { - return None; - } - while index > 0 { - index -= 1; - if let Some(&mut v) = history.history.get_mut(index).as_mut() { - if v.index < state.state.end { - return Some(&mut v.value); - } - } + /// See `get` for History. + pub fn get (&self, state: S) -> Option<&V> + where + S: BranchesStateTrait, + I: Copy + Eq + TryFrom + TryInto, + BI: Copy + Eq + TryFrom + TryInto, + { + let result = self.0.get(state); + if result.is_none() { + self.1.as_ref() + } else { + result } - None } -*/ + + /// See `gc` for History. + pub fn gc(&mut self, states: IT) -> PruneResult + where + IT: Iterator, + S: BranchStateTrait, + I: Copy + Eq + TryFrom + TryInto, + { + self.0.gc(states) + } + } + impl<'a, F: SerializedConfig> Serialized<'a, F> { pub fn into_owned(self) -> Serialized<'static, F> { From 03080c4bd78c3f86512fa4de378c34be677b5a37 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 11:18:35 +0200 Subject: [PATCH 30/68] serialized with default is useless --- core/utils/historied-data/src/tree.rs | 52 --------------------------- 1 file changed, 52 deletions(-) diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index 5a93dce18005f..d85873b340a8c 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -482,24 +482,12 @@ impl TestStates { #[cfg_attr(any(test, feature = "test"), derive(PartialEq))] pub struct History(Vec>); -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] -pub struct HistoryWithDefault(History, Option); - - impl Default for History { fn default() -> Self { History(Vec::new()) } } -impl Default for HistoryWithDefault { - fn default() -> Self { - HistoryWithDefault(History::default(), None) - } -} - - #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test"), derive(PartialEq))] pub struct HistoryBranch { @@ -718,46 +706,6 @@ impl History { } -impl HistoryWithDefault { - - /// See `set` for History. - pub fn set(&mut self, state: S, value: V) - where - S: BranchesStateTrait, - I: Copy + Eq + TryFrom + TryInto, - BI: Copy + Eq + TryFrom + TryInto, - { - self.0.set(state, value) - } - - /// See `get` for History. - pub fn get (&self, state: S) -> Option<&V> - where - S: BranchesStateTrait, - I: Copy + Eq + TryFrom + TryInto, - BI: Copy + Eq + TryFrom + TryInto, - { - let result = self.0.get(state); - if result.is_none() { - self.1.as_ref() - } else { - result - } - } - - /// See `gc` for History. - pub fn gc(&mut self, states: IT) -> PruneResult - where - IT: Iterator, - S: BranchStateTrait, - I: Copy + Eq + TryFrom + TryInto, - { - self.0.gc(states) - } - -} - - impl<'a, F: SerializedConfig> Serialized<'a, F> { pub fn into_owned(self) -> Serialized<'static, F> { From 8eb5e07a040ab0e7eaa532bf345b79c8be14c29e Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 12:02:39 +0200 Subject: [PATCH 31/68] few warning fix --- core/client/db/src/lib.rs | 1 - core/state-db/src/branch.rs | 70 +++++++------------------ core/state-db/src/lib.rs | 2 +- core/state-db/src/noncanonical.rs | 15 ------ core/utils/historied-data/src/linear.rs | 2 +- core/utils/historied-data/src/tree.rs | 22 ++++---- 6 files changed, 31 insertions(+), 81 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index bc05afc0d6a51..3171061957f1c 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -62,7 +62,6 @@ use executor::RuntimeInfo; use state_machine::{ DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, ChangesTrieBuildCache, backend::Backend as StateBackend, - TODO, TODO2, }; use crate::utils::{Meta, db_err, meta_keys, read_db, read_meta}; use client::leaves::{LeafSet, FinalizationDisplaced}; diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index 7e7fbb0218caf..f2d8ad0167d10 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -17,12 +17,9 @@ //! Helper for managing the set of indexed available branches. -use std::collections::{BTreeMap, BTreeSet, HashMap, btree_map::Entry}; -use std::cmp::Reverse; +use std::collections::{BTreeMap}; use crate::Error; -use std::hash::Hash as StdHash; -use std::convert::TryInto; -use historied_data::tree::{BranchesStateTrait, BranchStateTrait, StatesBranchRef, BranchStateRef}; +use historied_data::tree::{BranchesStateTrait, BranchStatesRef, BranchStateRef}; #[derive(Clone, Default, Debug)] /// Reference to state that is enough for query updates, but not @@ -33,7 +30,7 @@ use historied_data::tree::{BranchesStateTrait, BranchStateTrait, StatesBranchRef /// Note that an alternative could be a pointer to a full state /// branch for a given branch index, here we use an in memory /// copied representation in relation to an actual use case. -pub struct BranchRanges(Vec); +pub struct BranchRanges(Vec); impl<'a> BranchesStateTrait for &'a BranchRanges { type Branch = &'a BranchStateRef; @@ -87,17 +84,17 @@ impl<'a> Iterator for BranchRangesIter<'a> { /// numbers. /// /// New branches index are using `last_index`. -/// `treshold` is a branch index under which branches are undefined. /// /// Also acts as a cache, storage can store /// unknown db value as `None`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RangeSet { // TODO EMCH using a option value makes not sense when all in memory - storage: BTreeMap>, + storage: BTreeMap>, // TODO EMCH remove this? last_index: u64, - // TODO EMCH remove this? + /// treshold for possible node value, correspond + /// roughly to last cannonical block branch index. treshold: u64, } @@ -123,20 +120,10 @@ impl RangeSet { #[cfg(test)] /// test access to strage. - pub fn inner_storage(&self) -> &BTreeMap> { + pub fn inner_storage(&self) -> &BTreeMap> { &self.storage } - - /// Construct a new range set - pub fn new(treshold: u64, last_index: u64) -> Self { - RangeSet { - storage: BTreeMap::new(), - last_index, - treshold, - } - } - /// Iterator over all its range sets. pub fn reverse_iter_ranges(&self) -> impl Iterator { self.storage.iter().rev().filter_map(|(k, v)| v.as_ref().map(|v| (&v.state, *k))) @@ -157,7 +144,7 @@ impl RangeSet { } let mut previous_start = block.map(|b| b + 1).unwrap_or(u64::max_value()); loop { - if let Some(Some(StatesBranch{ state, parent_branch_index, .. })) = self.storage.get(&branch_index) { + if let Some(Some(BranchStates{ state, parent_branch_index, .. })) = self.storage.get(&branch_index) { // TODO EMCH consider vecdeque ?? let state = if state.end > previous_start { if state.start >= previous_start { @@ -172,7 +159,7 @@ impl RangeSet { previous_start = state.start; - result.insert(0, StatesBranchRef { + result.insert(0, BranchStatesRef { state, branch_index, }); @@ -252,7 +239,7 @@ impl RangeSet { self.last_index += 1; - let state = StatesBranch::new(number, branch_index); + let state = BranchStates::new(number, branch_index); self.storage.insert(self.last_index, Some(state)); Ok(self.last_index) } else { @@ -262,9 +249,9 @@ impl RangeSet { // TODO EMCH this access can be optimize at multiple places (by returning ref // instead of an anchor_id). - pub fn state_ref(&self, branch_index: u64) -> Option { + pub fn state_ref(&self, branch_index: u64) -> Option { self.storage.get(&branch_index).and_then(|v| v.as_ref().map(|v| v.state_ref())) - .map(|state| StatesBranchRef { + .map(|state| BranchStatesRef { branch_index, state, }) @@ -306,8 +293,6 @@ impl RangeSet { if branch_index < self.treshold { return; } - // range set update - let old_treshold = self.treshold; // we do not finalize current branch cause it // can contains other blocks self.treshold = branch_index; @@ -339,7 +324,7 @@ impl RangeSet { linear_index: u64, ) { if let Some(Some(state)) = self.storage.remove(&branch_index) { - let mut child_anchor: LinearStates = state.clone(); + let mut child_anchor: BranchStates = state.clone(); child_anchor.state.start = linear_index; let old_storage = std::mem::replace(&mut self.storage, BTreeMap::new()); // insert anchor @@ -395,29 +380,27 @@ impl RangeSet { } +/// Stored states for a branch, it contains branch reference information, +/// structural information (index of parent branch) and fork tree building +/// information (is branch appendable). #[derive(Debug, Clone, PartialEq, Eq)] -pub struct StatesBranch { +pub struct BranchStates { state: BranchStateRef, can_append: bool, parent_branch_index: u64, } - -// TODO EMCH temporary structs until merge with historied-data branch. - -type LinearStates = StatesBranch; - -impl Default for LinearStates { +impl Default for BranchStates { // initialize with one element fn default() -> Self { Self::new(0, 0) } } -impl LinearStates { +impl BranchStates { pub fn new(offset: u64, parent_branch_index: u64) -> Self { let offset = offset as u64; - LinearStates { + BranchStates { state: BranchStateRef { start: offset, end: offset + 1, @@ -454,24 +437,11 @@ impl LinearStates { } } - /// Return true if state exists. - pub fn get_state(&self, index: u64) -> bool { - index < self.state.end && index >= self.state.start - } - /// Return true if you can add this index. pub fn can_add(&self, index: u64) -> bool { index == self.state.end } - pub fn latest_ix(&self) -> Option { - if self.state.end - self.state.start > 0 { - Some(self.state.end - 1) - } else { - None - } - } - } // TODO EMCH end from historied - data diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index ae437327ed988..bbfc1564d0fc1 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -252,7 +252,7 @@ impl StateDbSync { number: u64, parent_hash: &BlockHash, mut changeset: ChangeSet, - mut offstate_changeset: OffstateChangeSet, + offstate_changeset: OffstateChangeSet, ) -> Result, Error> { match self.mode { diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index d4cd2d1279854..2ea9b5e1ec164 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -124,23 +124,8 @@ impl OffstatePendingGC { min } - // TODO is it of any use?? - fn revert(&mut self) { - self.pending_canonicalisation_query = None; - // TODO EMCH here reverting on a double set_pending_gc - // will fail: need to stack those offstatepending_gc?? - self.keys_pending_gc.clear(); - self.next_keys_pending_gc.clear(); - } - } -struct OffstatePinnedThread { - /// index of branch which was pinned - branch_index: u64, - - -} #[derive(Encode, Decode)] struct JournalRecord { hash: BlockHash, diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 7e965253c61fc..e6bfda01d94de 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -251,7 +251,7 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { }; let delete_size = elt_end - elt_start; for _ in elt_start..elt_end { - let _ = self.0.to_mut().remove(elt_start); // TODO EMCH slice copy instead of that horror + let _ = self.0.to_mut().remove(elt_start); } let start_ix = start_ix - delete_size; diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index d85873b340a8c..ab83dacd89bb8 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -255,8 +255,8 @@ pub struct TestStates { } impl StatesBranch { - pub fn branch_ref(&self) -> StatesBranchRef { - StatesBranchRef { + pub fn branch_ref(&self) -> BranchStatesRef { + BranchStatesRef { branch_index: self.branch_index, state: self.state.state_ref(), } @@ -288,7 +288,7 @@ pub struct StatesBranch { } #[derive(Debug, Clone, PartialEq, Eq)] -pub struct StatesBranchRef { +pub struct BranchStatesRef { pub branch_index: u64, pub state: BranchStateRef, } @@ -302,7 +302,7 @@ pub struct StatesBranchRef { /// tree state with a defined branch index implementing an iterator. pub struct StatesRef { /// Oredered by branch index linear branch states. - history: Rc>, + history: Rc>, /// Index is included, acts as length of history. upper_branch_index: Option, /// Index is included, acts as a branch ref end value. @@ -477,7 +477,6 @@ impl TestStates { /// First field is the actual history against which we run /// the state. /// Second field is an optional value for the no match case. -/// TODO EMCH consider removing second field (not use out of test) #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test"), derive(PartialEq))] pub struct History(Vec>); @@ -777,10 +776,7 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { let history = self.0.get_state(index); if history.index == from + 1 { // new first content - if history.value.len() == 0 { - // delete so first content is after (if any) - last_index_with_value = None; - } else { + if history.value.len() != 0 { // start value over a value drop until here last_index_with_value = Some(index); break; @@ -907,11 +903,11 @@ mod test { fn test_state_refs() { let states = test_states(); let ref_3 = vec![ - StatesBranchRef { + BranchStatesRef { branch_index: 1, state: BranchStateRef { start: 0, end: 2 }, }, - StatesBranchRef { + BranchStatesRef { branch_index: 3, state: BranchStateRef { start: 0, end: 1 }, }, @@ -922,11 +918,11 @@ mod test { assert_eq!(states.create_branch(1, 1, Some(0)), Some(6)); let ref_6 = vec![ - StatesBranchRef { + BranchStatesRef { branch_index: 1, state: BranchStateRef { start: 0, end: 1 }, }, - StatesBranchRef { + BranchStatesRef { branch_index: 6, state: BranchStateRef { start: 0, end: 1 }, }, From c89274d54f5b3da9ebd43be6f19b1d9032043771 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 12:21:52 +0200 Subject: [PATCH 32/68] simplify state machine offstate traits, TODO2 will be simple hashmap --- core/client/db/src/lib.rs | 20 +----- core/state-machine/src/backend.rs | 2 +- core/state-machine/src/lib.rs | 2 +- core/state-machine/src/offstate_backend.rs | 81 ---------------------- core/state-machine/src/proving_backend.rs | 2 +- core/state-machine/src/trie_backend.rs | 2 +- 6 files changed, 6 insertions(+), 103 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 3171061957f1c..7bdb3543f86df 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -61,7 +61,7 @@ use sr_primitives::traits::{ use executor::RuntimeInfo; use state_machine::{ DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, ChangesTrieBuildCache, - backend::Backend as StateBackend, + backend::Backend as StateBackend, TODO2, }; use crate::utils::{Meta, db_err, meta_keys, read_db, read_meta}; use client::leaves::{LeafSet, FinalizationDisplaced}; @@ -660,14 +660,6 @@ impl state_db::OffstateDb for StorageDb { impl state_machine::OffstateStorage<(BranchRanges, u64)> for StorageDbAt { - fn state(&self) -> &(BranchRanges, u64) { - &self.state - } - - fn change_state(&mut self, state: (BranchRanges, u64)) -> (BranchRanges, u64) { - std::mem::replace(&mut self.state, state) - } - fn get(&self, key: &[u8]) -> Result>, String> { self.storage_db.state_db.get_offstate(key, &self.state, self.storage_db.deref()) .map_err(|e| format!("Database backend error: {:?}", e)) @@ -680,14 +672,6 @@ impl state_machine::OffstateStorage<(BranchRanges, u64)> for Stor impl state_machine::OffstateStorage for TODO2At { - fn state(&self) -> &S { - &self.state - } - - fn change_state(&mut self, state: S) -> S { - std::mem::replace(&mut self.state, state) - } - fn get(&self, key: &[u8]) -> Result>, String> { self.storage_db.get(key) .map_err(|e| format!("Database backend error: {:?}", e)) @@ -1459,7 +1443,7 @@ fn apply_state_commit( let mut ser = if let Some(stored) = db.get(columns::OFFSTATE, key)? { Ser::from_vec(stored.to_vec()) } else { - if let Some(value) = o { + if o.is_some() { Ser::default() } else { break; diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index a6bd1cdd0e63f..efc2ed9bb7604 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -21,7 +21,7 @@ use log::warn; use hash_db::Hasher; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::TrieBackendStorage; -use crate::offstate_backend::{OffstateBackendStorage, TODO, TODO2}; +use crate::offstate_backend::{OffstateBackendStorage, TODO2}; use trie::{ TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::{TrieDBMut, Layout}, diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index b77327394b1c1..f898fd9317275 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -30,7 +30,7 @@ use primitives::{ traits::{BareCryptoStorePtr, CodeExecutor}, hexdisplay::HexDisplay, }; -pub use crate::offstate_backend::{OffstateStorage, OffstateBackendStorage, TODO, TODO2}; +pub use crate::offstate_backend::{OffstateStorage, OffstateBackendStorage, TODO2}; pub mod backend; mod changes_trie; diff --git a/core/state-machine/src/offstate_backend.rs b/core/state-machine/src/offstate_backend.rs index f75ff77466fd4..6a548ab66777a 100644 --- a/core/state-machine/src/offstate_backend.rs +++ b/core/state-machine/src/offstate_backend.rs @@ -26,22 +26,12 @@ pub trait OffstateStorage: Send + Sync { /// Retrieve a value from storage under given key. fn get(&self, key: &[u8]) -> Result>, String>; - /// Access inner state to query. - fn state(&self) -> &State; - - /// Change inner state to query, return old state - fn change_state(&mut self, state: State) -> State; - /// Return in memory all values for this backend, mainly for /// tests. fn pairs(&self) -> Vec<(Vec, Vec)>; } pub trait OffstateBackendStorage: Send + Sync { -/* /// state type for querying data - /// (similar to hash for a trie_backend). - trait ChanState;*/ - /// Retrieve a value from storage under given key. fn get(&self, key: &[u8]) -> Result>, String>; @@ -51,33 +41,6 @@ pub trait OffstateBackendStorage: Send + Sync { } -/*// TODO EMCH is it use?? -pub trait OffstateStorage { - - /// Persist a value in storage under given key. - fn set(&mut self, key: &[u8], value: &[u8]); - - /// Retrieve a value from storage under given key. - fn get(&self, key: &[u8]) -> Option>; - - /// Return in memory all values for this backend, mainly for - /// tests. - fn pairs(&self) -> Vec<(Vec, Vec)>; - -}*/ - -impl OffstateBackendStorage for TODO { - - fn get(&self, key: &[u8]) -> Result>, String> { - unimplemented!() - } - - fn pairs(&self) -> Vec<(Vec, Vec)> { - unimplemented!() - } - -} - impl OffstateBackendStorage for TODO2 { fn get(&self, key: &[u8]) -> Result>, String> { @@ -100,39 +63,9 @@ impl OffstateBackendStorage for Arc> { } } - - -impl OffstateStorage for TODO { - - fn state(&self) -> &N { - &self.0 - } - - fn change_state(&mut self, state: N) -> N { - std::mem::replace(&mut self.0, state) - } - - fn get(&self, key: &[u8]) -> Result>, String> { - ::get(&self, key) - } - - fn pairs(&self) -> Vec<(Vec, Vec)> { - unimplemented!() - } - -} - // TODO EMCH remove?? impl OffstateStorage<()> for TODO2 { - fn state(&self) -> &() { - &() - } - - fn change_state(&mut self, _state: ()) -> () { - () - } - fn get(&self, key: &[u8]) -> Result>, String> { ::get(&self, key) } @@ -144,21 +77,7 @@ impl OffstateStorage<()> for TODO2 { } -/// TODO EMCH implement, no branch ranges. -pub struct TODO(N); - -/// TODO EMCH variant for proof check or no /// need to keep multiple block state. /// TODO rename to something like SingleState pub struct TODO2; -impl TODO { - /// Build for a given block number. - /// TODO EMCH may or may not need a branch index to - pub fn new(block_number: N) -> Self { - TODO(block_number) - } -} - -impl TODO { -} diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index 8be71e21263ad..62e464c37fdf5 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -28,7 +28,7 @@ pub use trie::trie_types::{Layout, TrieError}; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend}; -use crate::offstate_backend::{OffstateBackendStorage, TODO, TODO2}; +use crate::offstate_backend::{OffstateBackendStorage, TODO2}; /// Patricia trie-based backend essence which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index 866812597bf11..23a633512918a 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -26,7 +26,7 @@ use crate::offstate_backend::OffstateBackendStorage; use primitives::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. -/// TODO EMCH with offstaet in backend this should be renamed eg StateBackend. +/// TODO EMCH with offstate in backend this should be renamed eg StateBackend. pub struct TrieBackend, H: Hasher, O: OffstateBackendStorage> { essence: TrieBackendEssence, offstate_storage: O, From 249770b95c99c650e3c9fb2e9a9760a46c1415fb Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 12:44:27 +0200 Subject: [PATCH 33/68] further simplify --- core/client/db/src/lib.rs | 32 +++---------------- core/client/db/src/storage_cache.rs | 4 +-- core/client/src/call_executor.rs | 4 +-- core/client/src/light/backend.rs | 4 +-- core/client/src/light/call_executor.rs | 4 +-- core/state-machine/src/backend.rs | 12 ++++---- core/state-machine/src/lib.rs | 8 ++--- core/state-machine/src/offstate_backend.rs | 36 +++------------------- core/state-machine/src/proving_backend.rs | 12 ++++---- core/state-machine/src/trie_backend.rs | 14 ++++----- 10 files changed, 41 insertions(+), 89 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 7bdb3543f86df..6420aeaa10853 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -93,7 +93,7 @@ const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); pub type DbState = state_machine::TrieBackend< Arc>, Blake2Hasher, - Arc>, + Arc, >; /// A reference tracking state. @@ -134,7 +134,7 @@ impl StateBackend for RefTrackingState { type Error = >::Error; type Transaction = >::Transaction; type TrieBackendStorage = >::TrieBackendStorage; - type OffstateBackendStorage = >::OffstateBackendStorage; + type OffstateBackend = >::OffstateBackend; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { self.state.storage(key) @@ -218,7 +218,7 @@ impl StateBackend for RefTrackingState { } fn as_trie_backend(&mut self) -> Option< - &state_machine::TrieBackend + &state_machine::TrieBackend > { self.state.as_trie_backend() } @@ -610,13 +610,6 @@ struct StorageDbAt { pub state: State, } -// Semantic is one block statedb with block in todo2, -// state is here for compatibility only. -struct TODO2At { - pub storage_db: TODO2, - pub state: State, -} - impl state_machine::Storage for StorageDb { fn get(&self, key: &H256, prefix: Prefix) -> Result, String> { let key = prefixed_key::(key, prefix); @@ -658,7 +651,7 @@ impl state_db::OffstateDb for StorageDb { } } -impl state_machine::OffstateStorage<(BranchRanges, u64)> for StorageDbAt { +impl state_machine::OffstateBackend for StorageDbAt { fn get(&self, key: &[u8]) -> Result>, String> { self.storage_db.state_db.get_offstate(key, &self.state, self.storage_db.deref()) @@ -670,18 +663,6 @@ impl state_machine::OffstateStorage<(BranchRanges, u64)> for Stor } } -impl state_machine::OffstateStorage for TODO2At { - - fn get(&self, key: &[u8]) -> Result>, String> { - self.storage_db.get(key) - .map_err(|e| format!("Database backend error: {:?}", e)) - } - fn pairs(&self) -> Vec<(Vec, Vec)> { - self.storage_db.pairs() - } -} - - struct DbGenesisStorage(pub H256); impl DbGenesisStorage { @@ -1658,10 +1639,7 @@ impl client::backend::Backend for Backend whe let genesis_storage = DbGenesisStorage::new(); let root = genesis_storage.0.clone(); // TODO EMCH see genesis impl: that is empty storage - let genesis_offstate = TODO2At { - storage_db: TODO2, - state: Default::default(), - }; + let genesis_offstate = TODO2; let db_state = DbState::new(Arc::new(genesis_storage), root, Arc::new(genesis_offstate)); let state = RefTrackingState::new(db_state, self.storage.clone(), None); return Ok(CachingState::new(state, self.shared_cache.clone(), None)); diff --git a/core/client/db/src/storage_cache.rs b/core/client/db/src/storage_cache.rs index baed84db041f0..5e492d28c2c9c 100644 --- a/core/client/db/src/storage_cache.rs +++ b/core/client/db/src/storage_cache.rs @@ -470,7 +470,7 @@ impl, B: BlockT> StateBackend for CachingState< type Error = S::Error; type Transaction = S::Transaction; type TrieBackendStorage = S::TrieBackendStorage; - type OffstateBackendStorage = S::OffstateBackendStorage; + type OffstateBackend = S::OffstateBackend; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { let local_cache = self.cache.local_cache.upgradable_read(); @@ -603,7 +603,7 @@ impl, B: BlockT> StateBackend for CachingState< } fn as_trie_backend(&mut self) -> Option< - &TrieBackend + &TrieBackend > { self.state.as_trie_backend() } diff --git a/core/client/src/call_executor.rs b/core/client/src/call_executor.rs index 4891a30f22589..2d481f784624f 100644 --- a/core/client/src/call_executor.rs +++ b/core/client/src/call_executor.rs @@ -145,7 +145,7 @@ where /// No changes are made. fn prove_at_trie_state< S: state_machine::TrieBackendStorage, - O: state_machine::OffstateBackendStorage, + O: state_machine::OffstateBackend, >( &self, trie_state: &state_machine::TrieBackend, @@ -383,7 +383,7 @@ where fn prove_at_trie_state< S: state_machine::TrieBackendStorage, - O: state_machine::OffstateBackendStorage, + O: state_machine::OffstateBackend, >( &self, trie_state: &state_machine::TrieBackend, diff --git a/core/client/src/light/backend.rs b/core/client/src/light/backend.rs index 7f15d19328936..a24c61f6a6170 100644 --- a/core/client/src/light/backend.rs +++ b/core/client/src/light/backend.rs @@ -344,7 +344,7 @@ impl StateBackend for GenesisOrUnavailableState type Error = ClientError; type Transaction = (); type TrieBackendStorage = MemoryDB; - type OffstateBackendStorage = state_machine::offstate_backend::TODO2; + type OffstateBackend = state_machine::offstate_backend::TODO2; fn storage(&self, key: &[u8]) -> ClientResult>> { match *self { @@ -464,7 +464,7 @@ impl StateBackend for GenesisOrUnavailableState } fn as_trie_backend(&mut self) -> Option< - &TrieBackend + &TrieBackend > { match self { GenesisOrUnavailableState::Genesis(ref mut state) => state.as_trie_backend(), diff --git a/core/client/src/light/call_executor.rs b/core/client/src/light/call_executor.rs index f456fd7fc3be6..f557c22e19db6 100644 --- a/core/client/src/light/call_executor.rs +++ b/core/client/src/light/call_executor.rs @@ -179,7 +179,7 @@ impl CallExecutor for fn prove_at_trie_state< S: state_machine::TrieBackendStorage, - O: state_machine::OffstateBackendStorage, + O: state_machine::OffstateBackend, >( &self, _state: &state_machine::TrieBackend, @@ -389,7 +389,7 @@ mod tests { fn prove_at_trie_state< S: state_machine::TrieBackendStorage, - O: state_machine::OffstateBackendStorage, + O: state_machine::OffstateBackend, >( &self, _trie_state: &state_machine::TrieBackend, diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index efc2ed9bb7604..14948034136ef 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -21,7 +21,7 @@ use log::warn; use hash_db::Hasher; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::TrieBackendStorage; -use crate::offstate_backend::{OffstateBackendStorage, TODO2}; +use crate::offstate_backend::{OffstateBackend, TODO2}; use trie::{ TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::{TrieDBMut, Layout}, @@ -43,7 +43,7 @@ pub trait Backend: std::fmt::Debug { /// Type of trie backend storage. TODO EMCH move to OffstateBackend /// after implementated. - type OffstateBackendStorage: OffstateBackendStorage; + type OffstateBackend: OffstateBackend; /// Get keyed storage or None if there is nothing associated. fn storage(&self, key: &[u8]) -> Result>, Self::Error>; @@ -138,7 +138,7 @@ pub trait Backend: std::fmt::Debug { /// Try convert into trie backend. fn as_trie_backend(&mut self) -> Option< - &TrieBackend + &TrieBackend > { None } @@ -185,7 +185,7 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { type Error = T::Error; type Transaction = T::Transaction; type TrieBackendStorage = T::TrieBackendStorage; - type OffstateBackendStorage = T::OffstateBackendStorage; + type OffstateBackend = T::OffstateBackend; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { (*self).storage(key) @@ -466,7 +466,7 @@ impl Backend for InMemory { type Error = Void; type Transaction = InMemoryTransaction; type TrieBackendStorage = MemoryDB; - type OffstateBackendStorage = TODO2; + type OffstateBackend = TODO2; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone))) @@ -591,7 +591,7 @@ impl Backend for InMemory { } fn as_trie_backend(&mut self)-> Option< - &TrieBackend + &TrieBackend > { let mut mdb = MemoryDB::default(); let mut root = None; diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index f898fd9317275..58cea55fceeb0 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -30,7 +30,7 @@ use primitives::{ traits::{BareCryptoStorePtr, CodeExecutor}, hexdisplay::HexDisplay, }; -pub use crate::offstate_backend::{OffstateStorage, OffstateBackendStorage, TODO2}; +pub use crate::offstate_backend::{OffstateBackend, TODO2}; pub mod backend; mod changes_trie; @@ -480,7 +480,7 @@ pub fn prove_execution_on_trie_backend( ) -> Result<(Vec, Vec>), Box> where S: trie_backend_essence::TrieBackendStorage, - O: OffstateBackendStorage, + O: OffstateBackend, H: Hasher, Exec: CodeExecutor, H::Out: Ord + 'static, @@ -589,7 +589,7 @@ where S: trie_backend_essence::TrieBackendStorage, H: Hasher, H::Out: Ord, - O: OffstateBackendStorage, + O: OffstateBackend, I: IntoIterator, I::Item: AsRef<[u8]>, { @@ -612,7 +612,7 @@ where S: trie_backend_essence::TrieBackendStorage, H: Hasher, H::Out: Ord, - O: OffstateBackendStorage, + O: OffstateBackend, I: IntoIterator, I::Item: AsRef<[u8]>, { diff --git a/core/state-machine/src/offstate_backend.rs b/core/state-machine/src/offstate_backend.rs index 6a548ab66777a..2a207af7b5385 100644 --- a/core/state-machine/src/offstate_backend.rs +++ b/core/state-machine/src/offstate_backend.rs @@ -16,22 +16,10 @@ //! Backend for storing data without a state. -/// TODO EMCH merge offstate storage and offstate backend storage ?? - use std::sync::Arc; use std::ops::Deref; -pub trait OffstateStorage: Send + Sync { - - /// Retrieve a value from storage under given key. - fn get(&self, key: &[u8]) -> Result>, String>; - - /// Return in memory all values for this backend, mainly for - /// tests. - fn pairs(&self) -> Vec<(Vec, Vec)>; -} - -pub trait OffstateBackendStorage: Send + Sync { +pub trait OffstateBackend: Send + Sync { /// Retrieve a value from storage under given key. fn get(&self, key: &[u8]) -> Result>, String>; @@ -41,7 +29,7 @@ pub trait OffstateBackendStorage: Send + Sync { } -impl OffstateBackendStorage for TODO2 { +impl OffstateBackend for TODO2 { fn get(&self, key: &[u8]) -> Result>, String> { unimplemented!() @@ -53,29 +41,15 @@ impl OffstateBackendStorage for TODO2 { } -// This implementation is used by normal storage trie clients. -impl OffstateBackendStorage for Arc> { +impl OffstateBackend for Arc { fn get(&self, key: &[u8]) -> Result>, String> { - OffstateStorage::get(self.deref(), key) + OffstateBackend::get(self.deref(), key) } fn pairs(&self) -> Vec<(Vec, Vec)> { - OffstateStorage::pairs(self.deref()) + OffstateBackend::pairs(self.deref()) } } -// TODO EMCH remove?? -impl OffstateStorage<()> for TODO2 { - - fn get(&self, key: &[u8]) -> Result>, String> { - ::get(&self, key) - } - - fn pairs(&self) -> Vec<(Vec, Vec)> { - unimplemented!() - } - -} - /// need to keep multiple block state. /// TODO rename to something like SingleState diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index 62e464c37fdf5..8f6ebbb67c88d 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -28,7 +28,7 @@ pub use trie::trie_types::{Layout, TrieError}; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend}; -use crate::offstate_backend::{OffstateBackendStorage, TODO2}; +use crate::offstate_backend::{OffstateBackend, TODO2}; /// Patricia trie-based backend essence which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. @@ -107,7 +107,7 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> pub struct ProvingBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher, - O: 'a + OffstateBackendStorage, + O: 'a + OffstateBackend, > { backend: &'a TrieBackend, proof_recorder: Rc>>, @@ -116,7 +116,7 @@ pub struct ProvingBackend<'a, impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher, - O: 'a + OffstateBackendStorage, + O: 'a + OffstateBackend, > ProvingBackend<'a, S, H, O> { /// Create new proving backend. pub fn new(backend: &'a TrieBackend) -> Self { @@ -151,7 +151,7 @@ impl<'a, impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher, - O: 'a + OffstateBackendStorage, + O: 'a + OffstateBackend, > std::fmt::Debug for ProvingBackend<'a, S, H, O> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "ProvingBackend") @@ -163,12 +163,12 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> S: 'a + TrieBackendStorage, H: 'a + Hasher, H::Out: Ord, - O: 'a + OffstateBackendStorage, + O: 'a + OffstateBackend, { type Error = String; type Transaction = (S::Overlay, Vec<(Vec, Option>)>); type TrieBackendStorage = PrefixedMemoryDB; - type OffstateBackendStorage = O; + type OffstateBackend = O; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { ProvingBackendEssence { diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index 23a633512918a..76ed3de78c2a9 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -22,17 +22,17 @@ use trie::{Trie, delta_trie_root, default_child_trie_root, child_delta_trie_root use trie::trie_types::{TrieDB, TrieError, Layout}; use crate::trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral}; use crate::Backend; -use crate::offstate_backend::OffstateBackendStorage; +use crate::offstate_backend::OffstateBackend; use primitives::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. /// TODO EMCH with offstate in backend this should be renamed eg StateBackend. -pub struct TrieBackend, H: Hasher, O: OffstateBackendStorage> { +pub struct TrieBackend, H: Hasher, O: OffstateBackend> { essence: TrieBackendEssence, offstate_storage: O, } -impl, O: OffstateBackendStorage, H: Hasher> TrieBackend { +impl, O: OffstateBackend, H: Hasher> TrieBackend { /// Create new trie-based backend. pub fn new(storage: S, root: H::Out, offstate_storage: O) -> Self { TrieBackend { @@ -73,7 +73,7 @@ impl, O: OffstateBackendStorage, H: Hasher> TrieBackend impl< S: TrieBackendStorage, H: Hasher, - O: OffstateBackendStorage, + O: OffstateBackend, > std::fmt::Debug for TrieBackend { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "TrieBackend") @@ -83,7 +83,7 @@ impl< impl< S: TrieBackendStorage, H: Hasher, - O: OffstateBackendStorage, + O: OffstateBackend, > Backend for TrieBackend where H::Out: Ord, { @@ -91,7 +91,7 @@ impl< type Transaction = (S::Overlay, Vec<(Vec, Option>)>); type TrieBackendStorage = S; // TODO EMCH this does not make sens : split as a OffstateBackend from trait. - type OffstateBackendStorage = O; + type OffstateBackend = O; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { self.essence.storage(key) @@ -274,7 +274,7 @@ impl< } fn as_trie_backend(&mut self) -> Option< - &TrieBackend + &TrieBackend > { Some(self) } From 69fef43f8905aaf0ab7f15842e7fb54b9449e8d0 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 15:27:30 +0200 Subject: [PATCH 34/68] Remove stub state machine implementation. --- core/client/db/src/lib.rs | 4 +- core/client/src/cht.rs | 4 +- core/client/src/light/backend.rs | 2 +- core/client/src/light/fetcher.rs | 5 +- core/state-machine/src/backend.rs | 59 ++++++++++++++++------ core/state-machine/src/lib.rs | 8 +-- core/state-machine/src/offstate_backend.rs | 26 +++++----- core/state-machine/src/proving_backend.rs | 11 ++-- core/state-machine/src/trie_backend.rs | 14 ++--- 9 files changed, 80 insertions(+), 53 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 6420aeaa10853..90e5a9bb9f62f 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -61,7 +61,7 @@ use sr_primitives::traits::{ use executor::RuntimeInfo; use state_machine::{ DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, ChangesTrieBuildCache, - backend::Backend as StateBackend, TODO2, + backend::Backend as StateBackend, InMemoryOffstateBackend, }; use crate::utils::{Meta, db_err, meta_keys, read_db, read_meta}; use client::leaves::{LeafSet, FinalizationDisplaced}; @@ -1639,7 +1639,7 @@ impl client::backend::Backend for Backend whe let genesis_storage = DbGenesisStorage::new(); let root = genesis_storage.0.clone(); // TODO EMCH see genesis impl: that is empty storage - let genesis_offstate = TODO2; + let genesis_offstate = InMemoryOffstateBackend::default(); let db_state = DbState::new(Arc::new(genesis_storage), root, Arc::new(genesis_offstate)); let state = RefTrackingState::new(db_state, self.storage.clone(), None); return Ok(CachingState::new(state, self.shared_cache.clone(), None)); diff --git a/core/client/src/cht.rs b/core/client/src/cht.rs index b4015dd33ec7e..47bfbad1a9b24 100644 --- a/core/client/src/cht.rs +++ b/core/client/src/cht.rs @@ -33,7 +33,7 @@ use state_machine::backend::InMemory as InMemoryState; use state_machine::backend::InMemoryTransaction; use state_machine::{MemoryDB, TrieBackend, Backend as StateBackend, prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend}; -use state_machine::offstate_backend::TODO2; +use state_machine::offstate_backend::InMemory as OffstateBackend; use crate::error::{Error as ClientError, Result as ClientResult}; @@ -148,7 +148,7 @@ pub fn check_proof_on_proving_backend( local_root: Header::Hash, local_number: Header::Number, remote_hash: Header::Hash, - proving_backend: &TrieBackend, Hasher, TODO2>, + proving_backend: &TrieBackend, Hasher, OffstateBackend>, ) -> ClientResult<()> where Header: HeaderT, diff --git a/core/client/src/light/backend.rs b/core/client/src/light/backend.rs index a24c61f6a6170..a847cf4f927be 100644 --- a/core/client/src/light/backend.rs +++ b/core/client/src/light/backend.rs @@ -344,7 +344,7 @@ impl StateBackend for GenesisOrUnavailableState type Error = ClientError; type Transaction = (); type TrieBackendStorage = MemoryDB; - type OffstateBackend = state_machine::offstate_backend::TODO2; + type OffstateBackend = state_machine::InMemoryOffstateBackend; fn storage(&self, key: &[u8]) -> ClientResult>> { match *self { diff --git a/core/client/src/light/fetcher.rs b/core/client/src/light/fetcher.rs index b59f342df6db1..d96b7e87b95f6 100644 --- a/core/client/src/light/fetcher.rs +++ b/core/client/src/light/fetcher.rs @@ -350,8 +350,9 @@ impl> LightDataChecker { } // using empty offstate as light do not use offstate information - // (things being fetch proved). - let offstate = state_machine::TODO2; + // (things being fetch proved and proof currently do not rely on + // offstate). + let offstate = state_machine::InMemoryOffstateBackend::default(); // check proof for single changes trie root let proving_backend = TrieBackend::new(storage, cht_root, offstate); diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index 14948034136ef..f7348bf984583 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -21,7 +21,7 @@ use log::warn; use hash_db::Hasher; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::TrieBackendStorage; -use crate::offstate_backend::{OffstateBackend, TODO2}; +use crate::offstate_backend::{OffstateBackend, InMemory as InMemoryOffstateBackend}; use trie::{ TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::{TrieDBMut, Layout}, @@ -41,8 +41,7 @@ pub trait Backend: std::fmt::Debug { /// Type of trie backend storage. type TrieBackendStorage: TrieBackendStorage; - /// Type of trie backend storage. TODO EMCH move to OffstateBackend - /// after implementated. + /// Type of trie backend storage. type OffstateBackend: OffstateBackend; /// Get keyed storage or None if there is nothing associated. @@ -271,7 +270,7 @@ impl Consolidate for Vec { } impl Consolidate for (U, V) { - fn consolidate(&mut self, mut other: Self) { + fn consolidate(&mut self, other: Self) { self.0.consolidate(other.0); self.1.consolidate(other.1); } @@ -310,8 +309,8 @@ impl error::Error for Void { /// tests. pub struct InMemory { inner: HashMap>, HashMap, Vec>>, - trie: Option, H, TODO2>>, - offstate: HashMap, Vec>, + offstate: Option, + trie: Option, H, InMemoryOffstateBackend>>, _hasher: PhantomData, } @@ -326,7 +325,7 @@ impl Default for InMemory { InMemory { inner: Default::default(), trie: None, - offstate: Default::default(), + offstate: Some(Default::default()), _hasher: PhantomData, } } @@ -346,15 +345,42 @@ impl Clone for InMemory { impl PartialEq for InMemory { fn eq(&self, other: &Self) -> bool { self.inner.eq(&other.inner) - && self.offstate.eq(&other.offstate) + && self.offstate().eq(other.offstate()) } } impl InMemory { + fn offstate(&self) -> &InMemoryOffstateBackend { + if let Some(offstate) = self.offstate.as_ref() { + offstate + } else { + &self.trie.as_ref().unwrap().offstate_storage + } + } + + fn offstate_mut(&mut self) -> &mut InMemoryOffstateBackend { + if let Some(offstate) = self.offstate.as_mut() { + offstate + } else { + &mut self.trie.as_mut().unwrap().offstate_storage + } + } + + fn extract_offstate(&mut self) -> InMemoryOffstateBackend { + if let Some(offstate) = self.offstate.take() { + offstate + } else { + std::mem::replace( + &mut self.trie.as_mut().unwrap().offstate_storage, + Default::default(), + ) + } + } + /// Copy the state, with applied updates pub fn update(&self, changes: >::Transaction) -> Self { let mut inner: HashMap<_, _> = self.inner.clone(); - let mut offstate: HashMap<_, _> = self.offstate.clone(); + let mut offstate: HashMap<_, _> = self.offstate().clone(); for (storage_key, key, val) in changes.storage { match val { Some(v) => { inner.entry(storage_key).or_default().insert(key, v); }, @@ -368,6 +394,7 @@ impl InMemory { } } + let offstate = Some(offstate); InMemory { inner, offstate, trie: None, _hasher: PhantomData } } } @@ -377,7 +404,7 @@ impl From>, HashMap, Vec>>> for In InMemory { inner: inner, trie: None, - offstate: Default::default(), + offstate: Some(Default::default()), _hasher: PhantomData, } } @@ -397,7 +424,7 @@ impl From<( InMemory { inner: inner, trie: None, - offstate: Default::default(), + offstate: Some(Default::default()), _hasher: PhantomData, } } @@ -410,7 +437,7 @@ impl From, Vec>> for InMemory { InMemory { inner: expanded, trie: None, - offstate: Default::default(), + offstate: Some(Default::default()), _hasher: PhantomData, } } @@ -437,7 +464,7 @@ impl From for InMemory { } } let mut result: InMemory = expanded.into(); - result.offstate.extend(inner.offstate.into_iter() + result.offstate_mut().extend(inner.offstate.into_iter().into_iter() .filter_map(|(k, v)| v.map(|v| (k, v)))); result } @@ -466,7 +493,7 @@ impl Backend for InMemory { type Error = Void; type Transaction = InMemoryTransaction; type TrieBackendStorage = MemoryDB; - type OffstateBackend = TODO2; + type OffstateBackend = InMemoryOffstateBackend; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone))) @@ -573,7 +600,7 @@ impl Backend for InMemory { } fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { - self.offstate.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + self.offstate().iter().map(|(k, v)| (k.clone(), v.clone())).collect() } fn keys(&self, prefix: &[u8]) -> Vec> { @@ -619,7 +646,7 @@ impl Backend for InMemory { // Since we are running on a memorydb (not a prefixed memory db), content // is not collision free, so an empty offtrie state can be use (no need // for keyspace). - self.trie = Some(TrieBackend::new(mdb, root, TODO2)); + self.trie = Some(TrieBackend::new(mdb, root, self.extract_offstate())); self.trie.as_ref() } } diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 58cea55fceeb0..e5dd5ad746e2f 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -30,7 +30,7 @@ use primitives::{ traits::{BareCryptoStorePtr, CodeExecutor}, hexdisplay::HexDisplay, }; -pub use crate::offstate_backend::{OffstateBackend, TODO2}; +pub use crate::offstate_backend::{OffstateBackend, InMemory as InMemoryOffstateBackend}; pub mod backend; mod changes_trie; @@ -520,7 +520,7 @@ where /// Check execution proof on proving backend, generated by `prove_execution` call. pub fn execution_proof_check_on_trie_backend( - trie_backend: &TrieBackend, H, TODO2>, + trie_backend: &TrieBackend, H, InMemoryOffstateBackend>, overlay: &mut OverlayedChanges, exec: &Exec, method: &str, @@ -674,7 +674,7 @@ where /// Check storage read proof on pre-created proving backend. pub fn read_proof_check_on_proving_backend( - proving_backend: &TrieBackend, H, TODO2>, + proving_backend: &TrieBackend, H, InMemoryOffstateBackend>, key: &[u8], ) -> Result>, Box> where @@ -686,7 +686,7 @@ where /// Check child storage read proof on pre-created proving backend. pub fn read_child_proof_check_on_proving_backend( - proving_backend: &TrieBackend, H, TODO2>, + proving_backend: &TrieBackend, H, InMemoryOffstateBackend>, storage_key: &[u8], key: &[u8], ) -> Result>, Box> diff --git a/core/state-machine/src/offstate_backend.rs b/core/state-machine/src/offstate_backend.rs index 2a207af7b5385..cb376de9526ce 100644 --- a/core/state-machine/src/offstate_backend.rs +++ b/core/state-machine/src/offstate_backend.rs @@ -17,41 +17,41 @@ //! Backend for storing data without a state. use std::sync::Arc; +use std::collections::HashMap; use std::ops::Deref; +/// This covers offstate values access. +/// It target a single history state (state machine +/// only run for a single history state). pub trait OffstateBackend: Send + Sync { /// Retrieve a value from storage under given key. fn get(&self, key: &[u8]) -> Result>, String>; - /// Return in memory all values for this backend, mainly for - /// tests. + /// Return all values (in memory) for this backend, mainly for + /// tests. This method should only be use for testing or + /// for small offstate. fn pairs(&self) -> Vec<(Vec, Vec)>; - } -impl OffstateBackend for TODO2 { +/// need to keep multiple block state. +pub type InMemory = HashMap, Vec>; +impl OffstateBackend for InMemory { fn get(&self, key: &[u8]) -> Result>, String> { - unimplemented!() + Ok(self.get(key).map(Clone::clone)) } fn pairs(&self) -> Vec<(Vec, Vec)> { - unimplemented!() + self.iter().map(|(k, v)| (k.clone(), v.clone())).collect() } - } impl OffstateBackend for Arc { fn get(&self, key: &[u8]) -> Result>, String> { OffstateBackend::get(self.deref(), key) } + fn pairs(&self) -> Vec<(Vec, Vec)> { OffstateBackend::pairs(self.deref()) } } - - -/// need to keep multiple block state. -/// TODO rename to something like SingleState -pub struct TODO2; - diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index 8f6ebbb67c88d..c01de053a0f51 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -28,7 +28,7 @@ pub use trie::trie_types::{Layout, TrieError}; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend}; -use crate::offstate_backend::{OffstateBackend, TODO2}; +use crate::offstate_backend::{OffstateBackend, InMemory as InMemoryOffstateBackend}; /// Patricia trie-based backend essence which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. @@ -253,14 +253,17 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> pub fn create_proof_check_backend( root: H::Out, proof: Vec> -) -> Result, H, TODO2>, Box> +) -> Result, H, InMemoryOffstateBackend>, Box> where H: Hasher, { let db = create_proof_check_backend_storage(proof); + // run on empty offstate (current proof does not require + // offstate). + let offstate = InMemoryOffstateBackend::default(); if db.contains(&root, EMPTY_PREFIX) { - Ok(TrieBackend::new(db, root, TODO2)) + Ok(TrieBackend::new(db, root, offstate)) } else { Err(Box::new(ExecutionError::InvalidProof)) } @@ -289,7 +292,7 @@ mod tests { // TODO this need an actual in momery with possibly content // as the test uses a prefixed memory db. - type OffstateBackend = TODO2; + type OffstateBackend = InMemoryOffstateBackend; fn test_proving<'a>( trie_backend: &'a TrieBackend, Blake2Hasher, OffstateBackend>, diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index 76ed3de78c2a9..9922d0d7a0039 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -29,7 +29,7 @@ use primitives::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; /// TODO EMCH with offstate in backend this should be renamed eg StateBackend. pub struct TrieBackend, H: Hasher, O: OffstateBackend> { essence: TrieBackendEssence, - offstate_storage: O, + pub offstate_storage: O, } impl, O: OffstateBackend, H: Hasher> TrieBackend { @@ -90,7 +90,6 @@ impl< type Error = String; type Transaction = (S::Overlay, Vec<(Vec, Option>)>); type TrieBackendStorage = S; - // TODO EMCH this does not make sens : split as a OffstateBackend from trait. type OffstateBackend = O; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { @@ -99,7 +98,6 @@ impl< fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - // TODO EMCH PROTO: remove before pr. TODO test it when implemented!! // let keyspace = self.child_keyspace(storage_key); // Then change essence functions to use keyspace as input. @@ -288,9 +286,7 @@ pub mod tests { use trie::{TrieMut, PrefixedMemoryDB, trie_types::TrieDBMut}; use super::*; - // TODO this need an actual in momery with possibly content - // as the test uses a prefixed memory db. - type OffstateBackend = crate::offstate_backend::TODO2; + type OffstateBackend = crate::offstate_backend::InMemory; fn test_db() -> (PrefixedMemoryDB, H256, OffstateBackend) { let mut root = H256::default(); @@ -315,7 +311,7 @@ pub mod tests { } } // empty history. - let offstate = crate::offstate_backend::TODO2; + let offstate = crate::offstate_backend::InMemory::default(); // TODO EMCH add a block in offstate or use an actual implementation of // offstate that do not use history (a test implementation most likely) // TODO EMCH feed offstate with keyspace for roots. @@ -345,9 +341,9 @@ pub mod tests { #[test] fn pairs_are_empty_on_empty_storage() { assert!(TrieBackend::, Blake2Hasher, OffstateBackend>::new( - PrefixedMemoryDB::default(), Default::default(), - crate::offstate_backend::TODO2, + Default::default(), + Default::default(), ).pairs().is_empty()); } From a547560088752b72c8dd5e636ed0d20b8f059f15 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 15:41:16 +0200 Subject: [PATCH 35/68] init offstate for tests, but tests will only be interesting with child trie implementation --- core/state-machine/src/trie_backend.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index 9922d0d7a0039..ceeeb7fbf61b1 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -311,10 +311,9 @@ pub mod tests { } } // empty history. - let offstate = crate::offstate_backend::InMemory::default(); - // TODO EMCH add a block in offstate or use an actual implementation of - // offstate that do not use history (a test implementation most likely) - // TODO EMCH feed offstate with keyspace for roots. + let mut offstate = crate::offstate_backend::InMemory::default(); + offstate.insert(b"offstate1", b"off_value1"); + offstate.insert(b"offstate2", b"off_value2"); (mdb, root, offstate) } From 397d51049249106d3a9104a9028d9756b25fb641 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 16:29:46 +0200 Subject: [PATCH 36/68] rem todos --- core/client/db/src/lib.rs | 2 -- core/state-machine/src/proving_backend.rs | 2 -- core/utils/historied-data/src/tree.rs | 5 ----- 3 files changed, 9 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 90e5a9bb9f62f..921f7c3068664 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -1835,7 +1835,6 @@ mod tests { ).0.into(); let hash = header.hash(); - // TODO EMCH implement for offstate in another pr (need also build genesis). op.reset_storage(storage.iter().cloned().collect(), Default::default()).unwrap(); op.set_block_data( header.clone(), @@ -1894,7 +1893,6 @@ mod tests { assert_eq!(state.storage(&[1, 3, 5]).unwrap(), None); assert_eq!(state.storage(&[1, 2, 3]).unwrap(), Some(vec![9, 9, 9])); assert_eq!(state.storage(&[5, 5, 5]).unwrap(), Some(vec![4, 5, 6])); - // TODO EMCH have a offstate get method assert_eq!(state.offstate_pairs(), vec![(vec![5, 5, 5], vec![4, 5, 6])]); let state = db.state_at(BlockId::Number(0)).unwrap(); assert_eq!(state.offstate_pairs(), vec![]); diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index c01de053a0f51..5eca9e3bcef5f 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -290,8 +290,6 @@ mod tests { use super::*; use primitives::{Blake2Hasher, child_storage_key::ChildStorageKey}; - // TODO this need an actual in momery with possibly content - // as the test uses a prefixed memory db. type OffstateBackend = InMemoryOffstateBackend; fn test_proving<'a>( diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index ab83dacd89bb8..cf69ef49330c6 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -503,9 +503,6 @@ impl History { I: Copy + Eq + TryFrom + TryInto, BI: Copy + Eq + TryFrom + TryInto, { - // TODO EMCH it does not seem stricly needed to pass - // a full state, double index looks enough. - // but this api can help using consistent state. if let Some((state_branch, state_index)) = state.iter().next() { if let Ok(state_index_usize) = state_index.try_into() { let state_index_u64 = state_index_usize as u64; @@ -637,7 +634,6 @@ impl History { /// Gc an historied value other its possible values. /// Iterator need to be reversed ordered by branch index. - /// TODO EMCH this needs rewrite and proper testing!! pub fn gc(&mut self, mut states: IT) -> PruneResult where IT: Iterator, @@ -1022,7 +1018,6 @@ mod test { assert_eq!(item1.get(&states.state_ref(a.0)), None); } } - } } From c133a0bf3847ac5b3b63272c3cc042cf788da222 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 17:04:35 +0200 Subject: [PATCH 37/68] compact casts --- Cargo.lock | 1 + core/state-machine/src/trie_backend.rs | 4 +- core/utils/historied-data/Cargo.toml | 2 + core/utils/historied-data/src/lib.rs | 14 +- core/utils/historied-data/src/tree.rs | 198 ++++++++++++------------- 5 files changed, 101 insertions(+), 118 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a34dc9cd1a639..dfa5d00bf9901 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1213,6 +1213,7 @@ dependencies = [ name = "historied-data" version = "2.0.0" dependencies = [ + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", ] diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index ceeeb7fbf61b1..e21b631c4cc81 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -312,8 +312,8 @@ pub mod tests { } // empty history. let mut offstate = crate::offstate_backend::InMemory::default(); - offstate.insert(b"offstate1", b"off_value1"); - offstate.insert(b"offstate2", b"off_value2"); + offstate.insert(b"offstate1".to_vec(), b"off_value1".to_vec()); + offstate.insert(b"offstate2".to_vec(), b"off_value2".to_vec()); (mdb, root, offstate) } diff --git a/core/utils/historied-data/Cargo.toml b/core/utils/historied-data/Cargo.toml index 79ab10dd8658b..0a1012a7a2e9b 100644 --- a/core/utils/historied-data/Cargo.toml +++ b/core/utils/historied-data/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" [dependencies] rstd = { package = "sr-std", path = "../../sr-std", default-features = false } smallvec = { version = "0.6", optional = true } +num-traits = { version = "0.2.8", default-features = false } [dev-dependencies] @@ -15,6 +16,7 @@ smallvec = { version = "0.6", optional = true } default = ["std"] std = [ "rstd/std", + "num-traits/std", "smallvec", ] test = [] diff --git a/core/utils/historied-data/src/lib.rs b/core/utils/historied-data/src/lib.rs index 70b634845afee..6b431d8639b0b 100644 --- a/core/utils/historied-data/src/lib.rs +++ b/core/utils/historied-data/src/lib.rs @@ -20,7 +20,7 @@ #![cfg_attr(not(feature = "std"), no_std)] -use rstd::convert::{TryFrom, TryInto}; +use rstd::convert::TryInto; pub mod tree; pub mod linear; @@ -66,18 +66,10 @@ impl HistoriedValue { } // utility function for panicking cast (similar to a `as` cast for number). -fn as_usize>(i: I) -> usize { +fn as_u>(i: I) -> U { match i.try_into() { Ok(index) => index, - Err(_) => panic!("historied value index overflow"), - } -} - -// utility function for panicking cast (similar to a `as` cast for number). -fn as_i>(i: usize) -> I { - match i.try_into() { - Ok(index) => index, - Err(_) => panic!("historied value index underflow"), + Err(_) => ::max_value(), } } diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index cf69ef49330c6..dfa19e05a7394 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -28,11 +28,12 @@ use crate::linear::{ }; use crate::HistoriedValue; use crate::PruneResult; -use crate::{as_usize, as_i}; +use crate::as_u; use rstd::rc::Rc; use rstd::vec::Vec; use rstd::collections::btree_map::BTreeMap; use rstd::convert::{TryFrom, TryInto}; +use num_traits::Bounded; /// Trait defining a state for querying or modifying a tree. /// This is a collection of branches index, corresponding @@ -500,46 +501,42 @@ impl History { pub fn set(&mut self, state: S, value: V) where S: BranchesStateTrait, - I: Copy + Eq + TryFrom + TryInto, - BI: Copy + Eq + TryFrom + TryInto, + I: Copy + Eq + TryFrom + TryInto, + BI: Copy + Eq + TryFrom + TryInto, { if let Some((state_branch, state_index)) = state.iter().next() { - if let Ok(state_index_usize) = state_index.try_into() { - let state_index_u64 = state_index_usize as u64; - let mut i = self.0.len(); - let (branch_position, new_branch) = loop { - if i == 0 { - break (0, true); - } - let branch_index = self.0[i - 1].branch_index; - if branch_index == state_index_u64 { - break (i - 1, false); + let state_index_u64 = as_u(state_index); + let mut i = self.0.len(); + let (branch_position, new_branch) = loop { + if i == 0 { + break (0, true); + } + let branch_index = self.0[i - 1].branch_index; + if branch_index == state_index_u64 { + break (i - 1, false); } else if branch_index < state_index_u64 { - break (i, true); - } - i -= 1; + break (i, true); + } + i -= 1; + }; + if new_branch { + let index = as_u(state_branch.last_index()); + let mut history = BranchBackend::::default(); + history.push(HistoriedValue { + value, + index, + }); + let h_value = HistoryBranch { + branch_index: state_index_u64, + history, }; - if new_branch { - if let Ok(index_usize) = state_branch.last_index().try_into() { - let index = index_usize as u64; - let mut history = BranchBackend::::default(); - history.push(HistoriedValue { - value, - index, - }); - let h_value = HistoryBranch { - branch_index: state_index_u64, - history, - }; - if branch_position == self.0.len() { - self.0.push(h_value); - } else { - self.0.insert(branch_position, h_value); - } - } + if branch_position == self.0.len() { + self.0.push(h_value); } else { - self.node_set(branch_position, &state_branch, value) + self.0.insert(branch_position, h_value); } + } else { + self.node_set(branch_position, &state_branch, value) } } } @@ -547,31 +544,29 @@ impl History { fn node_set(&mut self, branch_index: usize, state: &S, value: V) where S: BranchStateTrait, - I: Copy + Eq + TryFrom + TryInto, + I: Copy + Eq + TryFrom + TryInto, { - if let Ok(node_index_usize) = state.last_index().try_into() { - let node_index_u64 = node_index_usize as u64; - let history = &mut self.0[branch_index]; - let mut index = history.history.len(); - debug_assert!(index > 0); - loop { - if index == 0 || history.history[index - 1].index < node_index_u64 { - let h_value = HistoriedValue { - value, - index: node_index_u64 - }; - if index == history.history.len() { - history.history.push(h_value); - } else { - history.history.insert(index, h_value); - } - break; - } else if history.history[index - 1].index == node_index_u64 { - history.history[index - 1].value = value; - break; + let node_index_u64 = as_u(state.last_index()); + let history = &mut self.0[branch_index]; + let mut index = history.history.len(); + debug_assert!(index > 0); + loop { + if index == 0 || history.history[index - 1].index < node_index_u64 { + let h_value = HistoriedValue { + value, + index: node_index_u64 + }; + if index == history.history.len() { + history.history.push(h_value); + } else { + history.history.insert(index, h_value); } - index -= 1; + break; + } else if history.history[index - 1].index == node_index_u64 { + history.history[index - 1].value = value; + break; } + index -= 1; } } @@ -580,8 +575,8 @@ impl History { pub fn get (&self, state: S) -> Option<&V> where S: BranchesStateTrait, - I: Copy + Eq + TryFrom + TryInto, - BI: Copy + Eq + TryFrom + TryInto, + I: Copy + Eq + TryFrom + TryInto, + BI: Copy + Eq + TryFrom + TryInto + Bounded, { let mut index = self.0.len(); // note that we expect branch index to be linearily set @@ -595,13 +590,11 @@ impl History { for (state_branch, state_index) in state.iter() { while index > 0 { index -= 1; - if let Ok(branch_index) = self.0[index].branch_index.try_into() { - if let Ok(state_index) = state_index.try_into() { - if state_index == branch_index { - if let Some(result) = self.branch_get(index, &state_branch) { - return Some(result) - } - } + let branch_index = as_u(self.0[index].branch_index); + let state_index = as_u(state_index); + if state_index == branch_index { + if let Some(result) = self.branch_get(index, &state_branch) { + return Some(result) } } } @@ -615,17 +608,16 @@ impl History { fn branch_get(&self, index: usize, state: &S) -> Option<&V> where S: BranchStateTrait, - I: Copy + Eq + TryFrom + TryInto, + I: Copy + Eq + TryFrom + TryInto + Bounded, { let history = &self.0[index]; let mut index = history.history.len(); while index > 0 { index -= 1; if let Some(&v) = history.history.get(index).as_ref() { - if let Ok(i) = (v.index as usize).try_into() { - if state.get_node(i) { - return Some(&v.value); - } + let i = as_u(v.index); + if state.get_node(i) { + return Some(&v.value); } } } @@ -638,7 +630,7 @@ impl History { where IT: Iterator, S: BranchStateTrait, - I: Copy + Eq + TryFrom + TryInto, + I: Copy + Eq + TryFrom + TryInto + Bounded, { let mut changed = false; // state is likely bigger than history. @@ -647,38 +639,33 @@ impl History { let history_branch = self.0[branch_index].branch_index; loop { if let Some(state) = current_state.as_ref() { - if let Ok(state_index_usize) = state.1.try_into() { - let state_index_u64 = state_index_usize as u64; - if history_branch < state_index_u64 { - current_state = states.next(); - } else if history_branch == state_index_u64 { - let len = self.0[branch_index].history.len(); - for history_index in (0..len).rev() { - - let node_index = self.0[branch_index].history[history_index].index as usize; - if let Ok(node_index) = node_index.try_into() { - if !state.0.get_node(node_index) { - if history_index == len - 1 { - changed = self.0[branch_index] - .history.pop().is_some() || changed; - } else { - self.0[branch_index] - .history.remove(history_index); - changed = true; - } - } + let state_index_u64 = as_u(state.1); + if history_branch < state_index_u64 { + current_state = states.next(); + } else if history_branch == state_index_u64 { + let len = self.0[branch_index].history.len(); + for history_index in (0..len).rev() { + let node_index = as_u(self.0[branch_index].history[history_index].index); + if !state.0.get_node(node_index) { + if history_index == len - 1 { + changed = self.0[branch_index] + .history.pop().is_some() || changed; + } else { + self.0[branch_index] + .history.remove(history_index); + changed = true; } } - if self.0[branch_index].history.len() == 0 { - self.0.remove(branch_index); - changed = true; - } - break; - } else { + } + if self.0[branch_index].history.len() == 0 { self.0.remove(branch_index); changed = true; - break; } + break; + } else { + self.0.remove(branch_index); + changed = true; + break; } } else { self.0.remove(branch_index); @@ -714,7 +701,7 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { pub fn get (&self, state: S) -> Option> where S: BranchStateTrait, - I: Copy + Eq + TryFrom + TryInto, + I: Copy + Eq + TryFrom + TryInto + Bounded, { let mut index = self.0.len(); if index == 0 { @@ -723,7 +710,8 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { while index > 0 { index -= 1; let HistoriedValue { value, index: state_index } = self.0.get_state(index); - if state.get_node(as_i(state_index as usize)) { + let state_index = as_u(state_index); + if state.get_node(state_index) { // Note this extra byte is note optimal, should be part of index encoding if value.len() > 0 { return Some(Some(&value[..value.len() - 1])); @@ -740,9 +728,9 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { pub fn push(&mut self, state: S, value: Option<&[u8]>) where S: BranchStateTrait, - I: Copy + Eq + TryFrom + TryInto, + I: Copy + Eq + TryFrom + TryInto, { - let target_state_index = as_usize(state.last_index()) as u64; + let target_state_index = as_u(state.last_index()); let index = self.0.len(); if index > 0 { let last = self.0.get_state(index - 1); @@ -762,9 +750,9 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { /// keep a single with value history before the state. pub fn prune(&mut self, index: I) -> PruneResult where - I: Copy + Eq + TryFrom + TryInto, + I: Copy + Eq + TryFrom + TryInto, { - let from = as_usize(index) as u64; + let from = as_u(index); let len = self.0.len(); let mut last_index_with_value = None; let mut index = 0; From f197d7d2e3608e00628538d2827284a87fcd8aba Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 17:05:13 +0200 Subject: [PATCH 38/68] compact casts --- core/utils/historied-data/src/tree.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index dfa19e05a7394..3d448dff04f91 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -276,7 +276,7 @@ impl Default for TestStates { #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test"), derive(PartialEq, Eq))] -pub struct StatesBranch { +struct StatesBranch { // this is the key (need to growth unless full gc (can still have // content pointing to it even if it seems safe to reuse a previously // use ix). @@ -294,7 +294,6 @@ pub struct BranchStatesRef { pub state: BranchStateRef, } - #[derive(Clone)] /// Reference to state to use for query updates. /// It is a single brannch path with branches ordered by branch_index. From 9846613ce623b83a129ba4c366ac46bbc81fac71 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 18:02:19 +0200 Subject: [PATCH 39/68] quick prune test --- core/utils/historied-data/src/lib.rs | 1 + core/utils/historied-data/src/linear.rs | 6 ++ core/utils/historied-data/src/tree.rs | 73 ++++++++++++++++++++++++- 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/core/utils/historied-data/src/lib.rs b/core/utils/historied-data/src/lib.rs index 6b431d8639b0b..d63216f011b3f 100644 --- a/core/utils/historied-data/src/lib.rs +++ b/core/utils/historied-data/src/lib.rs @@ -73,6 +73,7 @@ fn as_u>(i: I) -> U { } } +#[cfg_attr(any(test, feature = "test"), derive(PartialEq, Debug))] /// Prunning result to be able to proceed /// with further update if the value needs it. pub enum PruneResult { diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index e6bfda01d94de..5ef55bba9943f 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -98,6 +98,7 @@ impl<'a> Clone for SerializedBuff<'a> { } } } + /// Serialized specific behavior. pub trait SerializedConfig { /// encoded empty slice @@ -105,9 +106,14 @@ pub trait SerializedConfig { /// size at start for encoding version. fn version_len() -> usize; } + +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] /// Serialize without versioning. pub struct NoVersion; +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] /// Serialize with default verison pub struct DefaultVersion; diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index 3d448dff04f91..c917b8152a30e 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -453,7 +453,7 @@ impl TestStates { } /// this function can go into deep recursion with full scan, it indicates - /// that the in memory model use here should only be use for small data or + /// that the tree model use here should only be use for small data or /// tests. pub fn apply_drop_state(&mut self, branch_index: u64, node_index: u64) { let mut to_delete = Vec::new(); @@ -1007,4 +1007,75 @@ mod test { } } + #[test] + fn test_prune() { + let mut item: Serialized = Default::default(); + // setting value respecting branch build order + for i in 1..6 { + item.push(i, Some(&[i as u8])); + } + + for a in 1..6 { + assert_eq!(item.get(a), Some(Some(&[a as u8][..]))); + } + item.prune(1); + assert_eq!(item.get(1), None); + for a in 2..6 { + assert_eq!(item.get(a), Some(Some(&[a as u8][..]))); + } + + item.prune(4); + for a in 1..5 { + assert_eq!(item.get(a), None); + } + for a in 5..6 { + assert_eq!(item.get(a), Some(Some(&[a as u8][..]))); + } + + item.prune(80); + for a in 1..4 { + assert_eq!(item.get(a), None); + } + // pruning preserve last valid value + for a in 5..11 { + assert_eq!(item.get(a), Some(Some(&[5 as u8][..]))); + } + + // prune skip unrelevant delete + let mut item: Serialized = Default::default(); + item.push(1, Some(&[1 as u8])); + item.push(2, None); + item.push(3, Some(&[3 as u8])); + assert_eq!(item.get(1), Some(Some(&[1][..]))); + assert_eq!(item.get(2), Some(None)); + assert_eq!(item.get(3), Some(Some(&[3][..]))); + assert_eq!(item.0.len(), 3); + item.prune(1); + assert_eq!(item.0.len(), 1); + assert_eq!(item.get(1), None); + assert_eq!(item.get(2), None); + assert_eq!(item.get(3), Some(Some(&[3][..]))); + + // prune skip unrelevant delete + let mut item: Serialized = Default::default(); + item.push(1, Some(&[1 as u8])); + item.push(3, None); + item.push(4, Some(&[4 as u8])); + assert_eq!(item.get(1), Some(Some(&[1][..]))); + assert_eq!(item.get(2), Some(Some(&[1][..]))); + assert_eq!(item.get(3), Some(None)); + assert_eq!(item.get(4), Some(Some(&[4][..]))); + assert_eq!(item.0.len(), 3); + // 1 needed for state two + assert_eq!(PruneResult::Unchanged, item.prune(1)); + // 3 unneeded + item.prune(2); + assert_eq!(item.0.len(), 1); + assert_eq!(item.get(1), None); + assert_eq!(item.get(2), None); + assert_eq!(item.get(3), None); + assert_eq!(item.get(4), Some(Some(&[4][..]))); + + } + } From d51e98fbb296279d1b7aecf6d98977db8c4185ce Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 18:25:53 +0200 Subject: [PATCH 40/68] break some lines --- core/client/db/src/lib.rs | 37 ++++++++++++++------ core/state-db/src/branch.rs | 12 +++++-- core/state-db/src/noncanonical.rs | 48 +++++++++++++++++++++----- core/state-db/src/test.rs | 7 ++-- core/state-machine/src/backend.rs | 6 +++- core/state-machine/src/trie_backend.rs | 3 +- 6 files changed, 85 insertions(+), 28 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 921f7c3068664..d10b3529b7319 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -524,10 +524,7 @@ where Block: BlockT, // Currently cache isn't implemented on full nodes. } - fn update_db_storage( - &mut self, - update: (PrefixedMemoryDB, Vec<(Vec, Option>)>), - ) -> ClientResult<()> { + fn update_db_storage(&mut self, update: PrefixedMemoryDB) -> ClientResult<()> { self.db_updates = update; Ok(()) } @@ -964,7 +961,8 @@ impl> Backend { let justification = self.blockchain.justification(id).unwrap(); let body = self.blockchain.body(id).unwrap(); let state = self.state_at(id).unwrap(); - let mut storage: Vec<_> = state.pairs().into_iter().map(|(k, v)| (None, k, Some(v))).collect(); + let mut storage: Vec<_> = state.pairs().into_iter() + .map(|(k, v)| (None, k, Some(v))).collect(); for child_key in state.children_storage_keys() { storage.extend(state.child_pairs(child_key.as_slice()) .into_iter().map(|(k, v)| (Some(child_key.clone()), k, Some(v)))); @@ -981,7 +979,8 @@ impl> Backend { op.set_block_data(header, body, justification, new_block_state).unwrap(); op.update_db_storage(InMemoryTransaction { storage, - offstate: state.offstate_pairs().into_iter().map(|(k, v)| (k, Some(v))).collect(), + offstate: state.offstate_pairs().into_iter() + .map(|(k, v)| (k, Some(v))).collect(), }).unwrap(); inmem.commit_operation(op).unwrap(); } @@ -1240,7 +1239,8 @@ impl> Backend { // remove duplicate TODO EMCH very bad let map_offstate: HashMap<_, _> = operation.db_updates.1.into_iter().collect(); // TODO EMCH !!! keep map_offstate for OffstateChangeSet type??? - let offstate_changeset: state_db::OffstateChangeSet> = map_offstate.into_iter().collect(); + let offstate_changeset: state_db::OffstateChangeSet> = map_offstate.into_iter() + .collect(); let number_u64 = number.saturated_into::(); let commit = self.storage.state_db.insert_block( &hash, @@ -1293,9 +1293,20 @@ impl> Backend { displaced_leaf }; - let mut children = children::read_children(&*self.storage.db, columns::META, meta_keys::CHILDREN_PREFIX, parent_hash)?; + let mut children = children::read_children( + &*self.storage.db, + columns::META, + meta_keys::CHILDREN_PREFIX, + parent_hash, + )?; children.push(hash); - children::write_children(&mut transaction, columns::META, meta_keys::CHILDREN_PREFIX, parent_hash, children); + children::write_children( + &mut transaction, + columns::META, + meta_keys::CHILDREN_PREFIX, + parent_hash, + children, + ); meta_updates.push((hash, number, pending_block.leaf_state.is_best(), finalized)); @@ -1381,7 +1392,9 @@ impl> Backend { { let f_num = f_header.number().clone(); - if self.storage.state_db.best_canonical().map(|c| f_num.saturated_into::() > c).unwrap_or(true) { + if self.storage.state_db.best_canonical() + .map(|c| f_num.saturated_into::() > c) + .unwrap_or(true) { let parent_hash = f_header.parent_hash().clone(); let number = f_header.number().clone(); let number_u64 = number.saturated_into::(); @@ -1390,7 +1403,9 @@ impl> Backend { transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &lookup_key); let commit = self.storage.state_db.canonicalize_block(&f_hash) - .map_err(|e: state_db::Error| client::error::Error::from(format!("State database error: {:?}", e)))?; + .map_err(|e: state_db::Error| + client::error::Error::from(format!("State database error: {:?}", e)) + )?; apply_state_commit(transaction, commit, &self.storage.db, number_u64).map_err(|err| client::error::Error::Backend( format!("Error building commit transaction : {}", err) diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index f2d8ad0167d10..9edefa3aa6c42 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -144,7 +144,11 @@ impl RangeSet { } let mut previous_start = block.map(|b| b + 1).unwrap_or(u64::max_value()); loop { - if let Some(Some(BranchStates{ state, parent_branch_index, .. })) = self.storage.get(&branch_index) { + if let Some(Some(BranchStates{ + state, + parent_branch_index, + .. + })) = self.storage.get(&branch_index) { // TODO EMCH consider vecdeque ?? let state = if state.end > previous_start { if state.start >= previous_start { @@ -276,7 +280,10 @@ impl RangeSet { } else { let anchor_index = self.add_state(parent_branch_index, number) .expect("coherent branch index state"); // TODO EMCH fail in add_state - (BranchRanges(vec![self.state_ref(anchor_index).expect("added just above")]), anchor_index) + ( + BranchRanges(vec![self.state_ref(anchor_index).expect("added just above")]), + anchor_index, + ) } } @@ -508,4 +515,3 @@ mod test { } } - diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 2ea9b5e1ec164..6eaad11fd3e44 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -288,8 +288,11 @@ impl NonCanonicalOverlay { // TODO EMCh following two range fetch can be optimize when parent level // got a single element. // fetch parent info - let parent_branch_index = parents.get(&record.parent_hash).map(|(_, i)| *i).unwrap_or(0); - let parent_branch_range = Some(branches.branch_ranges_from_cache(parent_branch_index, Some(block - 1))); + let parent_branch_index = parents.get(&record.parent_hash) + .map(|(_, i)| *i).unwrap_or(0); + let parent_branch_range = Some( + branches.branch_ranges_from_cache(parent_branch_index, Some(block - 1)) + ); let (branch_range, branch_index) = branches.import( block, parent_branch_index, @@ -632,7 +635,8 @@ impl NonCanonicalOverlay { } } if let Some(hash) = last { - let block_number = self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1); + let block_number = self.last_canonicalized.as_ref() + .map(|(_, n)| n + count).unwrap_or(count - 1); self.last_canonicalized = Some((hash, block_number)); if let Some(branch_index_cannonicalize) = last_index { @@ -687,7 +691,11 @@ impl NonCanonicalOverlay { self.branches.revert(branch_index); } discard_values(&mut self.values, overlay.inserted, None); - discard_offstate_values(overlay.offstate_inserted, overlay.offstate_deleted, &mut self.offstate_gc); + discard_offstate_values( + overlay.offstate_inserted, + overlay.offstate_deleted, + &mut self.offstate_gc, + ); } commit }) @@ -745,7 +753,10 @@ impl NonCanonicalOverlay { } /// Iterator over values at a given state. Deletion are included in the result as a None value. - pub fn offstate_iter<'a>(&'a self, state: &'a BranchRanges) -> impl Iterator)> { + pub fn offstate_iter<'a>( + &'a self, + state: &'a BranchRanges, + ) -> impl Iterator)> { let state = state.clone(); self.offstate_values.iter().filter_map(move |(k, v)| { v.get(&state).map(|v| (k, v)) @@ -767,22 +778,41 @@ mod tests { overlay.get(&H256::from_low_u64_be(key)) == Some(H256::from_low_u64_be(key).as_bytes().to_vec()) } - fn contains_offstate(overlay: &NonCanonicalOverlay, key: u64, state: &H256, block: u64) -> bool { + fn contains_offstate( + overlay: &NonCanonicalOverlay, + key: u64, + state: &H256, + block: u64, + ) -> bool { overlay.get_branch_range(state, block).and_then(|state| { overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state) }) == Some(&Some(H256::from_low_u64_be(key).as_bytes().to_vec())) } - fn contains_offstate2(overlay: &NonCanonicalOverlay, key: u64, state: &BranchRanges) -> bool { + fn contains_offstate2( + overlay: &NonCanonicalOverlay, + key: u64, + state: &BranchRanges, + ) -> bool { overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state) == Some(&Some(H256::from_low_u64_be(key).as_bytes().to_vec())) } - fn contains_both(overlay: &NonCanonicalOverlay, key: u64, state: &H256, block: u64) -> bool { + fn contains_both( + overlay: &NonCanonicalOverlay, + key: u64, + state: &H256, + block: u64, + ) -> bool { contains(overlay, key) && contains_offstate(overlay, key, state, block) } - fn contains_any(overlay: &NonCanonicalOverlay, key: u64, state: &H256, block: u64) -> bool { + fn contains_any( + overlay: &NonCanonicalOverlay, + key: u64, + state: &H256, + block: u64, + ) -> bool { contains(overlay, key) || contains_offstate(overlay, key, state, block) } diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index faf0baa89e8b5..6abd1235777e4 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -121,7 +121,6 @@ impl TestDb { let block = block.unwrap_or(self.last_block); let data = make_offstate_changeset(values, &[]); let self_offstate: BTreeMap<_, _> = self.get_offstate_pairs(&block).into_iter().collect(); -println!("of_eq {:?}", self_offstate); self_offstate == data.into_iter().filter_map(|(k, v)| v.map(|v| (k,v))).collect() } @@ -154,7 +153,10 @@ pub fn make_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet { } } -pub fn make_offstate_changeset(inserted: &[u64], deleted: &[u64]) -> OffstateChangeSet { +pub fn make_offstate_changeset( + inserted: &[u64], + deleted: &[u64], +) -> OffstateChangeSet { inserted.iter() .map(|v| ( H256::from_low_u64_be(*v).as_bytes().to_vec(), @@ -206,4 +208,3 @@ pub fn make_db(inserted: &[u64]) -> TestDb { last_block: Default::default(), } } - diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index f7348bf984583..b851f56d28a04 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -570,7 +570,11 @@ impl Backend for InMemory { let is_default = root == default_child_trie_root::>(&storage_key); - (root, is_default, InMemoryTransaction { storage: full_transaction, offstate: Default::default() }) + ( + root, + is_default, + InMemoryTransaction { storage: full_transaction, offstate: Default::default() }, + ) } fn offstate_transaction(&self, delta: I) -> Self::Transaction diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index e21b631c4cc81..4dc40ead2a24e 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -317,7 +317,8 @@ pub mod tests { (mdb, root, offstate) } - pub(crate) fn test_trie() -> TrieBackend, Blake2Hasher, OffstateBackend> { + pub(crate) fn test_trie( + ) -> TrieBackend, Blake2Hasher, OffstateBackend> { let (mdb, root, offstate) = test_db(); TrieBackend::new(mdb, root, offstate) } From fff7cb50fff05b3630026540af3bcf11f40230c6 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Oct 2019 19:54:29 +0200 Subject: [PATCH 41/68] fix --- core/client/db/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index d10b3529b7319..accf4c4a0d963 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -524,7 +524,10 @@ where Block: BlockT, // Currently cache isn't implemented on full nodes. } - fn update_db_storage(&mut self, update: PrefixedMemoryDB) -> ClientResult<()> { + fn update_db_storage( + &mut self, + update: (PrefixedMemoryDB, Vec<(Vec, Option>)>), + ) -> ClientResult<()> { self.db_updates = update; Ok(()) } From d55211a94f284461f10036809ff3c14715b34fc4 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Oct 2019 12:12:35 +0200 Subject: [PATCH 42/68] split commit set to have offstate pruning only when needed. --- core/client/db/src/lib.rs | 50 +++++++----- core/state-db/src/branch.rs | 6 -- core/state-db/src/lib.rs | 50 ++++++++---- core/state-db/src/noncanonical.rs | 1 - core/state-db/src/pruning.rs | 110 ++++++++++++++------------ core/state-db/src/test.rs | 21 ++--- core/utils/historied-data/src/tree.rs | 11 +++ 7 files changed, 145 insertions(+), 104 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index accf4c4a0d963..b24f9187a5033 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -1155,7 +1155,7 @@ impl> Backend { trace!(target: "db", "Canonicalize block #{} ({:?})", new_canonical, hash); let commit = self.storage.state_db.canonicalize_block(&hash) .map_err(|e: state_db::Error| client::error::Error::from(format!("State database error: {:?}", e)))?; - apply_state_commit(transaction, commit, &self.storage.db, number_u64).map_err(|err| + apply_state_commit_canonical(transaction, commit, &self.storage.db, number_u64).map_err(|err| client::error::Error::Backend( format!("Error building commit transaction : {}", err) ) @@ -1409,11 +1409,10 @@ impl> Backend { .map_err(|e: state_db::Error| client::error::Error::from(format!("State database error: {:?}", e)) )?; - apply_state_commit(transaction, commit, &self.storage.db, number_u64).map_err(|err| - client::error::Error::Backend( + apply_state_commit_canonical(transaction, commit, &self.storage.db, number_u64) + .map_err(|err| client::error::Error::Backend( format!("Error building commit transaction : {}", err) - ) - )?; + ))?; let changes_trie_config = self.changes_trie_config(parent_hash)?; if let Some(changes_trie_config) = changes_trie_config { @@ -1451,7 +1450,31 @@ fn apply_state_commit( ser.push(last_block, o.as_ref().map(|v| v.as_slice())); transaction.put(columns::OFFSTATE, &key[..], &ser.into_vec()) } - if let Some((block_prune, offstate_prune_key)) = commit.offstate_prune.as_ref() { + for (key, val) in commit.data.inserted.into_iter() { + transaction.put(columns::STATE, &key[..], &val); + } + for key in commit.data.deleted.into_iter() { + transaction.delete(columns::STATE, &key[..]); + } + for (key, val) in commit.meta.inserted.into_iter() { + transaction.put(columns::STATE_META, &key[..], &val); + } + for key in commit.meta.deleted.into_iter() { + transaction.delete(columns::STATE_META, &key[..]); + } + Ok(()) +} + +fn apply_state_commit_canonical( + transaction: &mut DBTransaction, + commit: state_db::CommitSetCanonical>, + db: &Arc, + last_block: u64, +) -> Result<(), io::Error> { + apply_state_commit(transaction, commit.0, db, last_block)?; + + if let Some((block_prune, offstate_prune_key)) = commit.1 { + // no need to into_iter for key in offstate_prune_key.iter() { // TODO EMCH should we prune less often (wait on pruning journals for instance) // or make it out of this context just stored the block prune and do asynch @@ -1461,26 +1484,13 @@ fn apply_state_commit( break; }; - match ser.prune(*block_prune) { + match ser.prune(block_prune) { PruneResult::Cleared => transaction.delete(columns::OFFSTATE, &key), PruneResult::Changed => transaction.put(columns::OFFSTATE, &key[..], &ser.into_vec()), PruneResult::Unchanged => (), } } } - - for (key, val) in commit.data.inserted.into_iter() { - transaction.put(columns::STATE, &key[..], &val); - } - for key in commit.data.deleted.into_iter() { - transaction.delete(columns::STATE, &key[..]); - } - for (key, val) in commit.meta.inserted.into_iter() { - transaction.put(columns::STATE_META, &key[..], &val); - } - for key in commit.meta.deleted.into_iter() { - transaction.delete(columns::STATE_META, &key[..]); - } Ok(()) } diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index 9edefa3aa6c42..2b1c3da91ebf5 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -118,12 +118,6 @@ impl RangeSet { self.treshold } - #[cfg(test)] - /// test access to strage. - pub fn inner_storage(&self) -> &BTreeMap> { - &self.storage - } - /// Iterator over all its range sets. pub fn reverse_iter_ranges(&self) -> impl Iterator { self.storage.iter().rev().filter_map(|(k, v)| v.as_ref().map(|v| (&v.state, *k))) diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index bbfc1564d0fc1..ce48eaa07794b 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -152,7 +152,14 @@ pub struct ChangeSet { pub type OffstateChangeSet = Vec<(H, Option)>; /// Info for pruning offstate: a last prune index and keys to prune. -pub type OffstateChangeSetPrune = Option<(u64, HashSet)>; +/// Is set to none when not initialized. +pub type OffstateChangeSetPrune = Option<(u64, HashSet)>; + +/// Info for reverting offstate of a canonical backend. +pub type OffstateChangeSetRevert = u64; + +/// Commit set on canonical operation. +pub type CommitSetCanonical = (CommitSet, OffstateChangeSetPrune); /// A set of changes to the backing database. /// It only contain a single block change set. @@ -164,8 +171,6 @@ pub struct CommitSet { pub meta: ChangeSet>, /// Offstate data changes. pub offstate: OffstateChangeSet, - /// Offstate data changes good for prunning. - pub offstate_prune: OffstateChangeSetPrune, } /// Pruning constraints. If none are specified pruning is @@ -269,7 +274,6 @@ impl StateDbSync { // and use and ordered tuple (branchix, blocknumber) // index to run. offstate: offstate_changeset, - offstate_prune: None, }) }, PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => { @@ -278,24 +282,24 @@ impl StateDbSync { } } - pub fn canonicalize_block(&mut self, hash: &BlockHash) -> Result, Error> { - let mut commit = CommitSet::default(); + pub fn canonicalize_block( + &mut self, + hash: &BlockHash, + ) -> Result, Error> { + let mut commit = (CommitSet::default(), None); if self.mode == PruningMode::ArchiveAll { return Ok(commit) } - match self.non_canonical.canonicalize(&hash, &mut commit) { + match self.non_canonical.canonicalize(&hash, &mut commit.0) { Ok(()) => { if self.mode == PruningMode::ArchiveCanonical { - commit.data.deleted.clear(); - // TODO EMCH use pruning mode in statedb to avoid feeding that - // in the first place. - commit.offstate_prune = None; + commit.0.data.deleted.clear(); } } Err(e) => return Err(e), }; if let Some(ref mut pruning) = self.pruning { - pruning.note_canonical(&hash, &mut commit); + pruning.note_canonical(&hash, &mut commit.0); } self.prune(&mut commit); Ok(commit) @@ -318,7 +322,7 @@ impl StateDbSync { } } - fn prune(&mut self, commit: &mut CommitSet) { + fn prune(&mut self, commit: &mut CommitSetCanonical) { if let (&mut Some(ref mut pruning), &PruningMode::Constrained(ref constraints)) = (&mut self.pruning, &self.mode) { loop { if pruning.window_size() <= constraints.max_blocks.unwrap_or(0) as u64 { @@ -484,7 +488,10 @@ impl StateDb { } /// Finalize a previously inserted block. - pub fn canonicalize_block(&self, hash: &BlockHash) -> Result, Error> { + pub fn canonicalize_block( + &self, + hash: &BlockHash, + ) -> Result, Error> { self.db.write().canonicalize_block(hash) } @@ -614,7 +621,10 @@ mod tests { .unwrap(), ); state_db.apply_pending(); - db.commit(&state_db.canonicalize_block::(&H256::from_low_u64_be(1)).unwrap()); + db.commit_canonical( + &state_db.canonicalize_block::(&H256::from_low_u64_be(1)) + .unwrap() + ); state_db.apply_pending(); db.commit( &state_db @@ -628,9 +638,15 @@ mod tests { .unwrap(), ); state_db.apply_pending(); - db.commit(&state_db.canonicalize_block::(&H256::from_low_u64_be(21)).unwrap()); + db.commit_canonical( + &state_db.canonicalize_block::(&H256::from_low_u64_be(21)) + .unwrap() + ); state_db.apply_pending(); - db.commit(&state_db.canonicalize_block::(&H256::from_low_u64_be(3)).unwrap()); + db.commit_canonical( + &state_db.canonicalize_block::(&H256::from_low_u64_be(3)) + .unwrap() + ); state_db.apply_pending(); (db, state_db) diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 6eaad11fd3e44..4b57e38d08cf6 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -1298,7 +1298,6 @@ mod tests { overlay.pin(&h_1); let h1_context = overlay.get_branch_range(&h_1, 1).unwrap(); - let h2_context = overlay.get_branch_range(&h_2, 1).unwrap(); let mut commit = CommitSet::default(); overlay.canonicalize::(&h_2, &mut commit).unwrap(); diff --git a/core/state-db/src/pruning.rs b/core/state-db/src/pruning.rs index a6fd06489654a..1dd21416fe347 100644 --- a/core/state-db/src/pruning.rs +++ b/core/state-db/src/pruning.rs @@ -24,7 +24,8 @@ use std::collections::{HashMap, HashSet, VecDeque}; use codec::{Encode, Decode}; -use crate::{CommitSet, Error, MetaDb, to_meta_key, Hash, OffstateKey}; +use crate::{CommitSet, CommitSetCanonical, Error, MetaDb, to_meta_key, Hash, + OffstateKey}; use log::{trace, warn}; const LAST_PRUNED: &[u8] = b"last_pruned"; @@ -140,7 +141,7 @@ impl RefWindow { offstate_journal_key: Vec, inserted: I, deleted: Vec, - offstate_inserted: I2, + offstate_modified: I2, ) { // remove all re-inserted keys from death rows for k in inserted { @@ -154,12 +155,13 @@ impl RefWindow { for k in deleted.iter() { self.death_index.insert(k.clone(), imported_block); } + // TODO EMCH is it possible to change type to directly set ?? + let offstate_modified = offstate_modified.into_iter().collect(); self.death_rows.push_back( DeathRow { hash: hash.clone(), deleted: deleted.into_iter().collect(), - // TODO EMCH is it possible to change type to directly set ?? - offstate_modified: offstate_inserted.into_iter().collect(), + offstate_modified, journal_key, offstate_journal_key, } @@ -186,17 +188,23 @@ impl RefWindow { self.death_rows.iter().skip(self.pending_prunings).any(|r| r.hash == *hash) } - /// Prune next block. Expects at least one block in the window. Adds changes to `commit`. - pub fn prune_one(&mut self, commit: &mut CommitSet) { + /// Prune next block. Expects at least one block in the window. + /// Adds changes to `commit`. + /// `offstate_prune` to None indicates archive mode. + pub fn prune_one( + &mut self, + commit: &mut CommitSetCanonical, + ) { + let (commit, offstate_prune) = commit; if let Some(pruned) = self.death_rows.get(self.pending_prunings) { trace!(target: "state-db", "Pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len()); let index = self.pending_number + self.pending_prunings as u64; commit.data.deleted.extend(pruned.deleted.iter().cloned()); - if let Some(offstate) = commit.offstate_prune.as_mut() { + if let Some(offstate) = offstate_prune.as_mut() { offstate.0 = std::cmp::max(offstate.0, index); offstate.1.extend(pruned.offstate_modified.iter().cloned()); } else { - commit.offstate_prune = Some(( + *offstate_prune = Some(( index, pruned.offstate_modified.iter().cloned().collect(), )); @@ -272,7 +280,7 @@ impl RefWindow { mod tests { use super::RefWindow; use primitives::H256; - use crate::CommitSet; + use crate::CommitSetCanonical; use crate::test::{make_db, make_commit_both, TestDb, make_commit}; fn check_journal(pruning: &RefWindow, db: &TestDb) { @@ -298,16 +306,16 @@ mod tests { let mut db = make_db(&[1, 2, 3]); db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = make_commit_both(&[4, 5], &[1, 3]); - commit.initialize_offstate(&[4, 5], &[1, 3]); + let mut commit = (make_commit_both(&[4, 5], &[1, 3]), None); + commit.0.initialize_offstate(&[4, 5], &[1, 3]); let h = H256::random(); - assert!(!commit.data.deleted.is_empty()); - pruning.note_canonical(&h, &mut commit); - db.commit(&commit); + assert!(!commit.0.data.deleted.is_empty()); + pruning.note_canonical(&h, &mut commit.0); + db.commit_canonical(&commit); assert!(pruning.have_block(&h)); pruning.apply_pending(); assert!(pruning.have_block(&h)); - assert!(commit.data.deleted.is_empty()); + assert!(commit.0.data.deleted.is_empty()); //assert!(commit.offstate.is_empty()); assert_eq!(pruning.death_rows.len(), 1); assert_eq!(pruning.death_index.len(), 2); @@ -316,10 +324,10 @@ mod tests { assert!(db.offstate_eq(&[2, 4, 5])); check_journal(&pruning, &db); - let mut commit = CommitSet::default(); + let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); assert!(!pruning.have_block(&h)); - db.commit(&commit); + db.commit_canonical(&commit); pruning.apply_pending(); assert!(!pruning.have_block(&h)); assert!(db.data_eq(&make_db(&[2, 4, 5]))); @@ -350,18 +358,18 @@ mod tests { check_journal(&pruning, &db); - let mut commit = CommitSet::default(); + let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); - db.commit(&commit); + db.commit_canonical(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[2, 3, 4, 5]))); // 3 exists at 0 and 1 so 0 removed assert!(db.offstate_eq_at(&[2], Some(0))); assert!(db.offstate_eq_at(&[2, 3, 4], Some(1))); assert!(db.offstate_eq(&[3, 4, 5])); - let mut commit = CommitSet::default(); + let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); - db.commit(&commit); + db.commit_canonical(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[3, 4, 5]))); assert!(db.offstate_eq_at(&[], Some(0))); @@ -375,26 +383,26 @@ mod tests { let mut db = make_db(&[1, 2, 3]); db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = make_commit_both(&[4], &[1]); - pruning.note_canonical(&H256::random(), &mut commit); - db.commit(&commit); - let mut commit = make_commit_both(&[3, 5], &[2]); - pruning.note_canonical(&H256::random(), &mut commit); - db.commit(&commit); + let mut commit = (make_commit_both(&[4], &[1]), None); + pruning.note_canonical(&H256::random(), &mut commit.0); + db.commit_canonical(&commit); + let mut commit = (make_commit_both(&[3, 5], &[2]), None); + pruning.note_canonical(&H256::random(), &mut commit.0); + db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); assert!(db.offstate_eq_at(&[1, 2, 3], Some(0))); assert!(db.offstate_eq_at(&[2, 3, 4], Some(1))); assert!(db.offstate_eq(&[3, 4, 5])); - let mut commit = CommitSet::default(); + let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); - db.commit(&commit); + db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[2, 3, 4, 5]))); assert!(db.offstate_eq_at(&[2, 3], Some(0))); assert!(db.offstate_eq_at(&[2, 3, 4], Some(1))); assert!(db.offstate_eq(&[3, 4, 5])); - let mut commit = CommitSet::default(); + let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); - db.commit(&commit); + db.commit_canonical(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[3, 4, 5]))); assert!(db.offstate_eq_at(&[], Some(0))); @@ -408,16 +416,16 @@ mod tests { let mut db = make_db(&[1, 2, 3]); db.initialize_offstate(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = make_commit_both(&[], &[2]); - commit.initialize_offstate(&[], &[2]); - pruning.note_canonical(&H256::random(), &mut commit); - db.commit(&commit); - let mut commit = make_commit_both(&[2], &[]); - pruning.note_canonical(&H256::random(), &mut commit); - db.commit(&commit); - let mut commit = make_commit_both(&[], &[2]); - pruning.note_canonical(&H256::random(), &mut commit); - db.commit(&commit); + let mut commit = (make_commit_both(&[], &[2]), None); + commit.0.initialize_offstate(&[], &[2]); + pruning.note_canonical(&H256::random(), &mut commit.0); + db.commit_canonical(&commit); + let mut commit = (make_commit_both(&[2], &[]), None); + pruning.note_canonical(&H256::random(), &mut commit.0); + db.commit_canonical(&commit); + let mut commit = (make_commit_both(&[], &[2]), None); + pruning.note_canonical(&H256::random(), &mut commit.0); + db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); assert!(db.offstate_eq_at(&[1, 2, 3], Some(0))); assert!(db.offstate_eq_at(&[1, 3], Some(1))); @@ -427,24 +435,24 @@ mod tests { check_journal(&pruning, &db); - let mut commit = CommitSet::default(); + let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); - db.commit(&commit); + db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); assert!(db.offstate_eq_at(&[1, 3], Some(0))); assert!(db.offstate_eq_at(&[1, 3], Some(1))); assert!(db.offstate_eq_at(&[1, 2, 3], Some(2))); assert!(db.offstate_eq(&[1, 3])); - let mut commit = CommitSet::default(); + let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); - db.commit(&commit); + db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); assert!(db.offstate_eq_at(&[1, 3], Some(0))); assert!(db.offstate_eq_at(&[1, 3], Some(1))); assert!(db.offstate_eq_at(&[1, 2, 3], Some(2))); assert!(db.offstate_eq(&[1, 3])); pruning.prune_one(&mut commit); - db.commit(&commit); + db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 3]))); assert!(db.offstate_eq_at(&[1, 3], Some(0))); assert!(db.offstate_eq_at(&[1, 3], Some(1))); @@ -469,16 +477,16 @@ mod tests { db.commit(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); - let mut commit = CommitSet::default(); + let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); - db.commit(&commit); + db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); - let mut commit = CommitSet::default(); + let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); - db.commit(&commit); + db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); pruning.prune_one(&mut commit); - db.commit(&commit); + db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 3]))); pruning.apply_pending(); assert_eq!(pruning.pending_number, 3); diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 6abd1235777e4..142397912e8d5 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -19,8 +19,8 @@ use std::collections::{HashMap, BTreeMap}; use primitives::H256; use crate::{ - DBValue, ChangeSet, OffstateChangeSet, CommitSet, MetaDb, NodeDb, OffstateDb, - OffstateKey, + DBValue, ChangeSet, OffstateChangeSet, OffstateKey, MetaDb, NodeDb, OffstateDb, + CommitSet, CommitSetCanonical, }; use historied_data::tree::Serialized; use historied_data::PruneResult; @@ -94,7 +94,16 @@ impl TestDb { let mut ser = Ser::from_mut(&mut (*encoded)); ser.push(self.last_block, o.as_ref().map(|v| v.as_slice())); } - if let Some((block_prune, offstate_prune_key)) = commit.offstate_prune.as_ref() { + self.meta.extend(commit.meta.inserted.iter().cloned()); + for k in commit.meta.deleted.iter() { + self.meta.remove(k); + } + } + + pub fn commit_canonical(&mut self, commit: &CommitSetCanonical) { + self.commit(&commit.0); + + if let Some((block_prune, offstate_prune_key)) = commit.1.as_ref() { for k in offstate_prune_key.iter() { match self.offstate.get_mut(k).map(|v| { let mut ser = Ser::from_mut(v); @@ -107,10 +116,6 @@ impl TestDb { } } } - self.meta.extend(commit.meta.inserted.iter().cloned()); - for k in commit.meta.deleted.iter() { - self.meta.remove(k); - } } pub fn data_eq(&self, other: &TestDb) -> bool { @@ -175,7 +180,6 @@ pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet { data: make_changeset(inserted, deleted), meta: ChangeSet::default(), offstate: OffstateChangeSet::default(), - offstate_prune: None, } } @@ -184,7 +188,6 @@ pub fn make_commit_both(inserted: &[u64], deleted: &[u64]) -> CommitSet { data: make_changeset(inserted, deleted), meta: ChangeSet::default(), offstate: make_offstate_changeset(inserted, deleted), - offstate_prune: None, } } diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index c917b8152a30e..7cfec75fcd9d2 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -1076,6 +1076,17 @@ mod test { assert_eq!(item.get(3), None); assert_eq!(item.get(4), Some(Some(&[4][..]))); + // prune delete at block + let mut item: Serialized = Default::default(); + item.push(0, Some(&[0 as u8])); + item.push(1, None); + assert_eq!(item.get(0), Some(Some(&[0][..]))); + assert_eq!(item.get(1), Some(None)); + item.prune(0); + assert_eq!(item.get(0), None); + assert_eq!(item.get(1), None); + assert_eq!(item.0.len(), 0); + } } From 27ca685171248f9cff0896095e36089344b00d3e Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Oct 2019 14:48:45 +0200 Subject: [PATCH 43/68] Fix logic of pruning in client (prior it did not take account of last block modification and worst could erase changes). --- core/client/db/src/lib.rs | 74 ++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index b24f9187a5033..75a47d70ccb14 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -1436,20 +1436,63 @@ fn apply_state_commit( db: &Arc, last_block: u64, ) -> Result<(), io::Error> { + apply_state_commit_inner(transaction, commit, db, last_block, None) +} + +fn apply_state_commit_inner( + transaction: &mut DBTransaction, + commit: state_db::CommitSet>, + db: &Arc, + last_block: u64, + mut offstate_prune_key: state_db::OffstateChangeSetPrune, +) -> Result<(), io::Error> { for (key, o) in commit.offstate.iter() { - let mut ser = if let Some(stored) = db.get(columns::OFFSTATE, key)? { - Ser::from_vec(stored.to_vec()) + let (mut ser, new) = if let Some(stored) = db.get(columns::OFFSTATE, key)? { + (Ser::from_vec(stored.to_vec()), false) } else { if o.is_some() { - Ser::default() + (Ser::default(), true) } else { break; } }; ser.push(last_block, o.as_ref().map(|v| v.as_slice())); - transaction.put(columns::OFFSTATE, &key[..], &ser.into_vec()) + if let Some((block_prune, offstate_prune_keys)) = offstate_prune_key.as_mut() { + if !new && offstate_prune_keys.remove(key) { + match ser.prune(*block_prune) { + PruneResult::Cleared => transaction.delete(columns::OFFSTATE, &key), + PruneResult::Changed + | PruneResult::Unchanged => transaction.put(columns::OFFSTATE, &key[..], &ser.into_vec()), + } + } else { + transaction.put(columns::OFFSTATE, &key[..], &ser.into_vec()) + } + } else { + transaction.put(columns::OFFSTATE, &key[..], &ser.into_vec()) + } + } + + if let Some((block_prune, offstate_prune_key)) = offstate_prune_key { + // no need to into_iter + for key in offstate_prune_key.iter() { + // TODO EMCH should we prune less often (wait on pruning journals for instance) + // or make it out of this context just stored the block prune and do asynch + let mut ser = if let Some(stored) = db.get(columns::OFFSTATE, key)? { + Ser::from_vec(stored.to_vec()) + } else { + break; + }; + + match ser.prune(block_prune) { + PruneResult::Cleared => transaction.delete(columns::OFFSTATE, &key), + PruneResult::Changed => transaction.put(columns::OFFSTATE, &key[..], &ser.into_vec()), + PruneResult::Unchanged => (), + } + } + } + for (key, val) in commit.data.inserted.into_iter() { transaction.put(columns::STATE, &key[..], &val); } @@ -1465,33 +1508,14 @@ fn apply_state_commit( Ok(()) } + fn apply_state_commit_canonical( transaction: &mut DBTransaction, commit: state_db::CommitSetCanonical>, db: &Arc, last_block: u64, ) -> Result<(), io::Error> { - apply_state_commit(transaction, commit.0, db, last_block)?; - - if let Some((block_prune, offstate_prune_key)) = commit.1 { - // no need to into_iter - for key in offstate_prune_key.iter() { - // TODO EMCH should we prune less often (wait on pruning journals for instance) - // or make it out of this context just stored the block prune and do asynch - let mut ser = if let Some(stored) = db.get(columns::OFFSTATE, key)? { - Ser::from_vec(stored.to_vec()) - } else { - break; - }; - - match ser.prune(block_prune) { - PruneResult::Cleared => transaction.delete(columns::OFFSTATE, &key), - PruneResult::Changed => transaction.put(columns::OFFSTATE, &key[..], &ser.into_vec()), - PruneResult::Unchanged => (), - } - } - } - Ok(()) + apply_state_commit_inner(transaction, commit.0, db, last_block, commit.1) } impl client::backend::AuxStore for Backend where Block: BlockT { From 64b4102bebcb0ea6296f9ea8776246a05a77ea06 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Oct 2019 15:29:39 +0200 Subject: [PATCH 44/68] No need for special revert unless we create archive mode with pruning (here we never revert canonical). --- core/client/db/src/lib.rs | 3 --- core/state-db/src/branch.rs | 10 +++------- core/state-db/src/lib.rs | 3 ++- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 75a47d70ccb14..6ea34d32293c3 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -1643,9 +1643,6 @@ impl client::backend::Backend for Backend whe let mut transaction = DBTransaction::new(); match self.storage.state_db.revert_one() { Some(commit) => { - // TODO EMCH: here we have a list of key and we need a new method similar to - // gc but for dropping n last state -> samething do not do one block per one - // block. apply_state_commit(&mut transaction, commit, &self.storage.db, c).map_err(|err| client::error::Error::Backend( format!("Error building commit transaction : {}", err) diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index 2b1c3da91ebf5..6201435f2bf87 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -174,7 +174,6 @@ impl RangeSet { BranchRanges(result) } - /// Return anchor index for this branch history: /// - same index as input if branch is not empty /// - parent index if branch is empty @@ -357,17 +356,14 @@ impl RangeSet { /// Revert some ranges, without any way to revert. /// Returning ranges for the parent index. - /// TODO EMCH can remove ?? - pub fn revert(&mut self, branch_ix: u64) -> BranchRanges { - let parent_branch_index = if branch_ix != 0 { + pub fn revert(&mut self, branch_ix: u64) -> u64 { + if branch_ix != 0 { self.drop_state(branch_ix) // silenced error .unwrap_or(0) } else { 0 - }; - - self.branch_ranges_from_cache(parent_branch_index, None) + } } #[cfg(test)] diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index ce48eaa07794b..f7522f01e0d69 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -356,7 +356,8 @@ impl StateDbSync { } } - /// TODO EMCH + /// For a a given block return its path in the block tree. + /// Note that using `number` is use to skip a query to block number for hash. pub fn get_branch_range(&self, hash: &BlockHash, number: u64) -> Option { self.non_canonical.get_branch_range(hash, number) } From 5072f85872af317a94a5667398bc543b5c5969f6 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Oct 2019 15:50:03 +0200 Subject: [PATCH 45/68] Do not keep deletion in branch storage. --- core/state-db/src/branch.rs | 52 +++++++++++++++---------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index 6201435f2bf87..fa5aa048b6be3 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -89,8 +89,7 @@ impl<'a> Iterator for BranchRangesIter<'a> { /// unknown db value as `None`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RangeSet { - // TODO EMCH using a option value makes not sense when all in memory - storage: BTreeMap>, + storage: BTreeMap, // TODO EMCH remove this? last_index: u64, /// treshold for possible node value, correspond @@ -120,7 +119,7 @@ impl RangeSet { /// Iterator over all its range sets. pub fn reverse_iter_ranges(&self) -> impl Iterator { - self.storage.iter().rev().filter_map(|(k, v)| v.as_ref().map(|v| (&v.state, *k))) + self.storage.iter().rev().map(|(k, v)| (&v.state, *k)) } // TODO EMCH can rw lock over the latest accessed range (lru) and @@ -138,11 +137,11 @@ impl RangeSet { } let mut previous_start = block.map(|b| b + 1).unwrap_or(u64::max_value()); loop { - if let Some(Some(BranchStates{ + if let Some(BranchStates{ state, parent_branch_index, .. - })) = self.storage.get(&branch_index) { + }) = self.storage.get(&branch_index) { // TODO EMCH consider vecdeque ?? let state = if state.end > previous_start { if state.start >= previous_start { @@ -180,10 +179,10 @@ impl RangeSet { pub fn drop_state( &mut self, branch_index: u64, - ) -> Result> { + ) -> u64 { let mut do_remove = None; match self.storage.get_mut(&branch_index) { - Some(Some(branch_state)) => { + Some(branch_state) => { if let Some(drop_index) = branch_state.drop_state() { if drop_index == 0 { do_remove = Some(branch_state.parent_branch_index); @@ -194,16 +193,14 @@ impl RangeSet { // deleted branch, do nothing } }, - Some(None) => (), // already dropped. - None => // TODO not sure keeping this error (we may want to clear storage) - return Err(Error::InvalidRange), + None => (), } if let Some(parent_index) = do_remove { self.storage.remove(&branch_index); - Ok(parent_index) + parent_index } else { - Ok(branch_index) + branch_index } } @@ -219,16 +216,14 @@ impl RangeSet { if branch_index == 0 || branch_index < self.treshold { create_new = true; } else { match self.storage.get_mut(&branch_index) { - Some(Some(branch_state)) => { + Some(branch_state) => { if branch_state.can_append && branch_state.can_add(number) { branch_state.add_state(); } else { create_new = true; } }, - Some(None) => - return Err(Error::InvalidRange), - None => // TODO not sure keeping this error (we may want to clear storage) + None => // TODO EMCH not sure keeping this error (we may want to clear storage) return Err(Error::InvalidRange), }} @@ -237,17 +232,15 @@ impl RangeSet { let state = BranchStates::new(number, branch_index); - self.storage.insert(self.last_index, Some(state)); + self.storage.insert(self.last_index, state); Ok(self.last_index) } else { Ok(branch_index) } } - // TODO EMCH this access can be optimize at multiple places (by returning ref - // instead of an anchor_id). pub fn state_ref(&self, branch_index: u64) -> Option { - self.storage.get(&branch_index).and_then(|v| v.as_ref().map(|v| v.state_ref())) + self.storage.get(&branch_index).map(|v| v.state_ref()) .map(|state| BranchStatesRef { branch_index, state, @@ -304,9 +297,8 @@ impl RangeSet { // at some point even if there is no branching). // Also if gc and treshold happen after this call, // ensure this branch can get remove. - // TODO EMCH this can also make sense for full self.storage.get_mut(&branch_index) - .map(|state| state.as_mut().map(|state| state.can_append = false)); + .map(|state| state.can_append = false); } else { let new_storage = self.storage.split_off(&(self.treshold)); self.storage = new_storage; @@ -323,15 +315,15 @@ impl RangeSet { branch_index: u64, linear_index: u64, ) { - if let Some(Some(state)) = self.storage.remove(&branch_index) { + if let Some(state) = self.storage.remove(&branch_index) { let mut child_anchor: BranchStates = state.clone(); child_anchor.state.start = linear_index; let old_storage = std::mem::replace(&mut self.storage, BTreeMap::new()); // insert anchor - self.storage.insert(branch_index, Some(child_anchor.clone())); - for (index, state) in old_storage.into_iter().filter_map(|(k, v)| v.map(|v| (k, v))) { + self.storage.insert(branch_index, child_anchor.clone()); + for (index, state) in old_storage.into_iter() { // ordered property of branch index allows to skip in depth branch search - if let Some(Some(parent_state)) = self.storage.get(&state.parent_branch_index) { + if let Some(parent_state) = self.storage.get(&state.parent_branch_index) { // first state is splitted // use property that start of a branch - 1 is origin of parent // this is obvious for parent is first branch (truncated), but also @@ -339,14 +331,14 @@ impl RangeSet { let linear_index_parent = state.state.start - 1; if linear_index_parent < parent_state.state.end && linear_index_parent >= parent_state.state.start { - self.storage.insert(index, Some(state)); + self.storage.insert(index, state); } } } // remove anchor block child_anchor.state.start += 1; if child_anchor.state.start < child_anchor.state.end { - self.storage.insert(branch_index, Some(child_anchor.clone())); + self.storage.insert(branch_index, child_anchor.clone()); } else { self.storage.remove(&branch_index); } @@ -359,8 +351,6 @@ impl RangeSet { pub fn revert(&mut self, branch_ix: u64) -> u64 { if branch_ix != 0 { self.drop_state(branch_ix) - // silenced error - .unwrap_or(0) } else { 0 } @@ -368,7 +358,7 @@ impl RangeSet { #[cfg(test)] pub fn contains_range(&self, branch_index: u64, size: u64) -> bool { - if let Some(Some(s)) = self.storage.get(&branch_index) { + if let Some(s) = self.storage.get(&branch_index) { (s.state_ref().end - s.state_ref().start) == size } else { false From 8f8ecc9f34fe9851d866a53e0600e34e0f7bf7ee Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Oct 2019 17:11:08 +0200 Subject: [PATCH 46/68] sed renaming of offstate to more generic and less incorect 'kv' (as key value). --- core/client/db/src/lib.rs | 76 ++-- core/client/db/src/storage_cache.rs | 12 +- core/client/src/call_executor.rs | 4 +- core/client/src/cht.rs | 6 +- core/client/src/client.rs | 4 +- core/client/src/light/backend.rs | 10 +- core/client/src/light/call_executor.rs | 4 +- core/client/src/light/fetcher.rs | 8 +- core/state-db/src/branch.rs | 12 +- core/state-db/src/lib.rs | 129 +++---- core/state-db/src/noncanonical.rs | 324 +++++++++--------- core/state-db/src/pruning.rs | 162 ++++----- core/state-db/src/test.rs | 64 ++-- core/state-machine/src/backend.rs | 106 +++--- core/state-machine/src/changes_trie/build.rs | 4 +- core/state-machine/src/ext.rs | 6 +- .../{offstate_backend.rs => kv_backend.rs} | 14 +- core/state-machine/src/lib.rs | 16 +- core/state-machine/src/overlayed_changes.rs | 48 +-- core/state-machine/src/proving_backend.rs | 40 +-- core/state-machine/src/testing.rs | 14 +- core/state-machine/src/trie_backend.rs | 62 ++-- core/utils/historied-data/src/tree.rs | 8 +- 23 files changed, 572 insertions(+), 561 deletions(-) rename core/state-machine/src/{offstate_backend.rs => kv_backend.rs} (85%) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 6ea34d32293c3..2c762a303711c 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -61,7 +61,7 @@ use sr_primitives::traits::{ use executor::RuntimeInfo; use state_machine::{ DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, ChangesTrieBuildCache, - backend::Backend as StateBackend, InMemoryOffstateBackend, + backend::Backend as StateBackend, InMemoryKvBackend, }; use crate::utils::{Meta, db_err, meta_keys, read_db, read_meta}; use client::leaves::{LeafSet, FinalizationDisplaced}; @@ -93,7 +93,7 @@ const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); pub type DbState = state_machine::TrieBackend< Arc>, Blake2Hasher, - Arc, + Arc, >; /// A reference tracking state. @@ -134,7 +134,7 @@ impl StateBackend for RefTrackingState { type Error = >::Error; type Transaction = >::Transaction; type TrieBackendStorage = >::TrieBackendStorage; - type OffstateBackend = >::OffstateBackend; + type KvBackend = >::KvBackend; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { self.state.storage(key) @@ -186,11 +186,11 @@ impl StateBackend for RefTrackingState { self.state.child_storage_root(storage_key, delta) } - fn offstate_transaction(&self, delta: I) -> Self::Transaction + fn kv_transaction(&self, delta: I) -> Self::Transaction where I: IntoIterator, Option>)> { - self.state.offstate_transaction(delta) + self.state.kv_transaction(delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -205,8 +205,8 @@ impl StateBackend for RefTrackingState { self.state.child_pairs(child_storage_key) } - fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { - self.state.offstate_pairs() + fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + self.state.kv_pairs() } fn keys(&self, prefix: &[u8]) -> Vec> { @@ -218,7 +218,7 @@ impl StateBackend for RefTrackingState { } fn as_trie_backend(&mut self) -> Option< - &state_machine::TrieBackend + &state_machine::TrieBackend > { self.state.as_trie_backend() } @@ -283,7 +283,7 @@ pub(crate) mod columns { pub const AUX: Option = Some(8); /// Offchain workers local storage pub const OFFCHAIN: Option = Some(9); - /// Offstate data + /// Kv data pub const OFFSTATE: Option = Some(10); } @@ -473,7 +473,7 @@ pub struct BlockImportOperation { db_updates: (PrefixedMemoryDB, Vec<(Vec, Option>)>), storage_updates: StorageCollection, child_storage_updates: ChildStorageCollection, - // TODO EMCH offstate update and offstate values in cache + // TODO EMCH kv update and kv values in cache changes_trie_updates: MemoryDB, changes_trie_cache_update: Option>>, pending_block: Option>, @@ -627,11 +627,11 @@ impl state_db::NodeDb for StorageDb { } } -impl state_db::OffstateDb for StorageDb { +impl state_db::KvDb for StorageDb { type Error = io::Error; - fn get_offstate(&self, key: &[u8], state: &u64) -> Result>, Self::Error> { + fn get_kv(&self, key: &[u8], state: &u64) -> Result>, Self::Error> { Ok(self.db.get(columns::OFFSTATE, key)? .as_ref() .map(|s| Ser::from_slice(&s[..])) @@ -641,8 +641,8 @@ impl state_db::OffstateDb for StorageDb { )) } - fn get_offstate_pairs(&self, state: &u64) -> Vec<(Vec, Vec)> { - self.db.iter(columns::OFFSTATE).filter_map(|(k, v)| + fn get_kv_pairs(&self, state: &u64) -> Vec<(Vec, Vec)> { + self.db.iter(columns::OFFSTATE).filter_map(|(k, v)| Ser::from_slice(&v[..]).get(*state) .unwrap_or(None) // flatten .map(Into::into) @@ -651,15 +651,15 @@ impl state_db::OffstateDb for StorageDb { } } -impl state_machine::OffstateBackend for StorageDbAt { +impl state_machine::KvBackend for StorageDbAt { fn get(&self, key: &[u8]) -> Result>, String> { - self.storage_db.state_db.get_offstate(key, &self.state, self.storage_db.deref()) + self.storage_db.state_db.get_kv(key, &self.state, self.storage_db.deref()) .map_err(|e| format!("Database backend error: {:?}", e)) } fn pairs(&self) -> Vec<(Vec, Vec)> { - self.storage_db.state_db.get_offstate_pairs(&self.state, self.storage_db.deref()) + self.storage_db.state_db.get_kv_pairs(&self.state, self.storage_db.deref()) } } @@ -982,7 +982,7 @@ impl> Backend { op.set_block_data(header, body, justification, new_block_state).unwrap(); op.update_db_storage(InMemoryTransaction { storage, - offstate: state.offstate_pairs().into_iter() + kv: state.kv_pairs().into_iter() .map(|(k, v)| (k, Some(v))).collect(), }).unwrap(); inmem.commit_operation(op).unwrap(); @@ -1240,9 +1240,9 @@ impl> Backend { } } // remove duplicate TODO EMCH very bad - let map_offstate: HashMap<_, _> = operation.db_updates.1.into_iter().collect(); - // TODO EMCH !!! keep map_offstate for OffstateChangeSet type??? - let offstate_changeset: state_db::OffstateChangeSet> = map_offstate.into_iter() + let map_kv: HashMap<_, _> = operation.db_updates.1.into_iter().collect(); + // TODO EMCH !!! keep map_kv for KvChangeSet type??? + let kv_changeset: state_db::KvChangeSet> = map_kv.into_iter() .collect(); let number_u64 = number.saturated_into::(); let commit = self.storage.state_db.insert_block( @@ -1250,18 +1250,18 @@ impl> Backend { number_u64, &pending_block.header.parent_hash(), changeset, - offstate_changeset, + kv_changeset, ).map_err(|e: state_db::Error| client::error::Error::from(format!("State database error: {:?}", e)))?; - // TODO EMCH wrong block number: there is no offstate change on block insert + // TODO EMCH wrong block number: there is no kv change on block insert // TODO split apply_state_commit and do an assertion in apply_state_commit - // that there is no offstate stuff anymore + // that there is no kv stuff anymore apply_state_commit(&mut transaction, commit, &self.storage.db, number_u64).map_err(|err| client::error::Error::Backend( format!("Error building commit transaction : {}", err) ) )?; - + // Check if need to finalize. Genesis is always finalized instantly. let finalized = number_u64 == 0 || pending_block.leaf_state.is_final(); @@ -1444,10 +1444,10 @@ fn apply_state_commit_inner( commit: state_db::CommitSet>, db: &Arc, last_block: u64, - mut offstate_prune_key: state_db::OffstateChangeSetPrune, + mut kv_prune_key: state_db::KvChangeSetPrune, ) -> Result<(), io::Error> { - for (key, o) in commit.offstate.iter() { + for (key, o) in commit.kv.iter() { let (mut ser, new) = if let Some(stored) = db.get(columns::OFFSTATE, key)? { (Ser::from_vec(stored.to_vec()), false) } else { @@ -1458,8 +1458,8 @@ fn apply_state_commit_inner( } }; ser.push(last_block, o.as_ref().map(|v| v.as_slice())); - if let Some((block_prune, offstate_prune_keys)) = offstate_prune_key.as_mut() { - if !new && offstate_prune_keys.remove(key) { + if let Some((block_prune, kv_prune_keys)) = kv_prune_key.as_mut() { + if !new && kv_prune_keys.remove(key) { match ser.prune(*block_prune) { PruneResult::Cleared => transaction.delete(columns::OFFSTATE, &key), PruneResult::Changed @@ -1473,9 +1473,9 @@ fn apply_state_commit_inner( } } - if let Some((block_prune, offstate_prune_key)) = offstate_prune_key { + if let Some((block_prune, kv_prune_key)) = kv_prune_key { // no need to into_iter - for key in offstate_prune_key.iter() { + for key in kv_prune_key.iter() { // TODO EMCH should we prune less often (wait on pruning journals for instance) // or make it out of this context just stored the block prune and do asynch let mut ser = if let Some(stored) = db.get(columns::OFFSTATE, key)? { @@ -1490,7 +1490,7 @@ fn apply_state_commit_inner( PruneResult::Unchanged => (), } } - + } for (key, val) in commit.data.inserted.into_iter() { @@ -1688,8 +1688,8 @@ impl client::backend::Backend for Backend whe let genesis_storage = DbGenesisStorage::new(); let root = genesis_storage.0.clone(); // TODO EMCH see genesis impl: that is empty storage - let genesis_offstate = InMemoryOffstateBackend::default(); - let db_state = DbState::new(Arc::new(genesis_storage), root, Arc::new(genesis_offstate)); + let genesis_kv = InMemoryKvBackend::default(); + let db_state = DbState::new(Arc::new(genesis_storage), root, Arc::new(genesis_kv)); let state = RefTrackingState::new(db_state, self.storage.clone(), None); return Ok(CachingState::new(state, self.shared_cache.clone(), None)); }, @@ -1703,11 +1703,11 @@ impl client::backend::Backend for Backend whe let block_number = hdr.number().clone().saturated_into::(); let range = self.storage.state_db.get_branch_range(&hash, block_number); let root = H256::from_slice(hdr.state_root().as_ref()); - let offstate = StorageDbAt { + let kv = StorageDbAt { storage_db: self.storage.clone(), state: (range.unwrap_or_else(Default::default), block_number), }; - let db_state = DbState::new(self.storage.clone(), root, Arc::new(offstate)); + let db_state = DbState::new(self.storage.clone(), root, Arc::new(kv)); let state = RefTrackingState::new(db_state, self.storage.clone(), Some(hash.clone())); Ok(CachingState::new(state, self.shared_cache.clone(), Some(hash))) } else { @@ -1942,9 +1942,9 @@ mod tests { assert_eq!(state.storage(&[1, 3, 5]).unwrap(), None); assert_eq!(state.storage(&[1, 2, 3]).unwrap(), Some(vec![9, 9, 9])); assert_eq!(state.storage(&[5, 5, 5]).unwrap(), Some(vec![4, 5, 6])); - assert_eq!(state.offstate_pairs(), vec![(vec![5, 5, 5], vec![4, 5, 6])]); + assert_eq!(state.kv_pairs(), vec![(vec![5, 5, 5], vec![4, 5, 6])]); let state = db.state_at(BlockId::Number(0)).unwrap(); - assert_eq!(state.offstate_pairs(), vec![]); + assert_eq!(state.kv_pairs(), vec![]); } } diff --git a/core/client/db/src/storage_cache.rs b/core/client/db/src/storage_cache.rs index 5e492d28c2c9c..09165810c2621 100644 --- a/core/client/db/src/storage_cache.rs +++ b/core/client/db/src/storage_cache.rs @@ -470,7 +470,7 @@ impl, B: BlockT> StateBackend for CachingState< type Error = S::Error; type Transaction = S::Transaction; type TrieBackendStorage = S::TrieBackendStorage; - type OffstateBackend = S::OffstateBackend; + type KvBackend = S::KvBackend; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { let local_cache = self.cache.local_cache.upgradable_read(); @@ -571,11 +571,11 @@ impl, B: BlockT> StateBackend for CachingState< self.state.child_storage_root(storage_key, delta) } - fn offstate_transaction(&self, delta: I) -> Self::Transaction + fn kv_transaction(&self, delta: I) -> Self::Transaction where I: IntoIterator, Option>)>, { - self.state.offstate_transaction(delta) + self.state.kv_transaction(delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -590,8 +590,8 @@ impl, B: BlockT> StateBackend for CachingState< self.state.child_pairs(storage_key) } - fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { - self.state.offstate_pairs() + fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + self.state.kv_pairs() } fn keys(&self, prefix: &[u8]) -> Vec> { @@ -603,7 +603,7 @@ impl, B: BlockT> StateBackend for CachingState< } fn as_trie_backend(&mut self) -> Option< - &TrieBackend + &TrieBackend > { self.state.as_trie_backend() } diff --git a/core/client/src/call_executor.rs b/core/client/src/call_executor.rs index 2d481f784624f..69884920304f4 100644 --- a/core/client/src/call_executor.rs +++ b/core/client/src/call_executor.rs @@ -145,7 +145,7 @@ where /// No changes are made. fn prove_at_trie_state< S: state_machine::TrieBackendStorage, - O: state_machine::OffstateBackend, + O: state_machine::KvBackend, >( &self, trie_state: &state_machine::TrieBackend, @@ -383,7 +383,7 @@ where fn prove_at_trie_state< S: state_machine::TrieBackendStorage, - O: state_machine::OffstateBackend, + O: state_machine::KvBackend, >( &self, trie_state: &state_machine::TrieBackend, diff --git a/core/client/src/cht.rs b/core/client/src/cht.rs index 47bfbad1a9b24..ae432ccbb1eb5 100644 --- a/core/client/src/cht.rs +++ b/core/client/src/cht.rs @@ -33,7 +33,7 @@ use state_machine::backend::InMemory as InMemoryState; use state_machine::backend::InMemoryTransaction; use state_machine::{MemoryDB, TrieBackend, Backend as StateBackend, prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend}; -use state_machine::offstate_backend::InMemory as OffstateBackend; +use state_machine::kv_backend::InMemory as KvBackend; use crate::error::{Error as ClientError, Result as ClientResult}; @@ -104,7 +104,7 @@ pub fn build_proof( .collect::>(); let mut storage = InMemoryState::::default().update(InMemoryTransaction { storage: transaction, - offstate: Default::default(), + kv: Default::default(), }); let trie_storage = storage.as_trie_backend() .expect("InMemoryState::as_trie_backend always returns Some; qed"); @@ -148,7 +148,7 @@ pub fn check_proof_on_proving_backend( local_root: Header::Hash, local_number: Header::Number, remote_hash: Header::Hash, - proving_backend: &TrieBackend, Hasher, OffstateBackend>, + proving_backend: &TrieBackend, Hasher, KvBackend>, ) -> ClientResult<()> where Header: HeaderT, diff --git a/core/client/src/client.rs b/core/client/src/client.rs index 017b52d4f1014..70ec1abfffcbd 100644 --- a/core/client/src/client.rs +++ b/core/client/src/client.rs @@ -1074,7 +1074,7 @@ impl Client where overlay.commit_prospective(); - let (top, children, offstate) = overlay.into_committed(); + let (top, children, kv) = overlay.into_committed(); let children = children.map(|(sk, it)| (sk, it.collect())).collect(); if import_headers.post().state_root() != &storage_update.1 { return Err(error::Error::InvalidStateRoot); @@ -1083,7 +1083,7 @@ impl Client where Ok(( Some(storage_update.0), Some(changes_update), - Some((top.collect(), children, offstate.collect())), + Some((top.collect(), children, kv.collect())), )) }, None => Ok((None, None, None)) diff --git a/core/client/src/light/backend.rs b/core/client/src/light/backend.rs index a847cf4f927be..12a4328920833 100644 --- a/core/client/src/light/backend.rs +++ b/core/client/src/light/backend.rs @@ -344,7 +344,7 @@ impl StateBackend for GenesisOrUnavailableState type Error = ClientError; type Transaction = (); type TrieBackendStorage = MemoryDB; - type OffstateBackend = state_machine::InMemoryOffstateBackend; + type KvBackend = state_machine::InMemoryKvBackend; fn storage(&self, key: &[u8]) -> ClientResult>> { match *self { @@ -421,7 +421,7 @@ impl StateBackend for GenesisOrUnavailableState } } - fn offstate_transaction(&self, _delta: I) -> Self::Transaction + fn kv_transaction(&self, _delta: I) -> Self::Transaction where I: IntoIterator, Option>)> { @@ -449,9 +449,9 @@ impl StateBackend for GenesisOrUnavailableState } } - fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { + fn kv_pairs(&self) -> Vec<(Vec, Vec)> { match *self { - GenesisOrUnavailableState::Genesis(ref state) => state.offstate_pairs(), + GenesisOrUnavailableState::Genesis(ref state) => state.kv_pairs(), GenesisOrUnavailableState::Unavailable => Vec::new(), } } @@ -464,7 +464,7 @@ impl StateBackend for GenesisOrUnavailableState } fn as_trie_backend(&mut self) -> Option< - &TrieBackend + &TrieBackend > { match self { GenesisOrUnavailableState::Genesis(ref mut state) => state.as_trie_backend(), diff --git a/core/client/src/light/call_executor.rs b/core/client/src/light/call_executor.rs index f557c22e19db6..a703ceb75b99d 100644 --- a/core/client/src/light/call_executor.rs +++ b/core/client/src/light/call_executor.rs @@ -179,7 +179,7 @@ impl CallExecutor for fn prove_at_trie_state< S: state_machine::TrieBackendStorage, - O: state_machine::OffstateBackend, + O: state_machine::KvBackend, >( &self, _state: &state_machine::TrieBackend, @@ -389,7 +389,7 @@ mod tests { fn prove_at_trie_state< S: state_machine::TrieBackendStorage, - O: state_machine::OffstateBackend, + O: state_machine::KvBackend, >( &self, _trie_state: &state_machine::TrieBackend, diff --git a/core/client/src/light/fetcher.rs b/core/client/src/light/fetcher.rs index d96b7e87b95f6..c0ee63c528591 100644 --- a/core/client/src/light/fetcher.rs +++ b/core/client/src/light/fetcher.rs @@ -349,13 +349,13 @@ impl> LightDataChecker { return Err(ClientError::InvalidCHTProof.into()); } - // using empty offstate as light do not use offstate information + // using empty kv as light do not use kv information // (things being fetch proved and proof currently do not rely on - // offstate). - let offstate = state_machine::InMemoryOffstateBackend::default(); + // kv). + let kv = state_machine::InMemoryKvBackend::default(); // check proof for single changes trie root - let proving_backend = TrieBackend::new(storage, cht_root, offstate); + let proving_backend = TrieBackend::new(storage, cht_root, kv); let remote_changes_trie_root = remote_roots[&block]; cht::check_proof_on_proving_backend::( local_cht_root, diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index fa5aa048b6be3..e3e60e544a3e1 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -122,15 +122,17 @@ impl RangeSet { self.storage.iter().rev().map(|(k, v)| (&v.state, *k)) } - // TODO EMCH can rw lock over the latest accessed range (lru) and - // return an clone of it (arc of the value so it will be fine). this is call by state_at that is call a lot!!! - pub fn branch_ranges_from_cache( + /// For a a given branch, return its path if the tree of ranges. + /// If block number is undefined, return the path up to the latest block + /// of the branch index. + /// + /// Note that this method is a bit costy and a caching could be use. + /// Similarily in some case using an iterator instead could make sense. + pub fn branch_ranges( &self, mut branch_index: u64, block: Option, ) -> BranchRanges { - // TODO EMCH transform this method to an iterator!!! - // (avoid some intermediatory Vec (eg when building Hashset) let mut result = Vec::new(); if branch_index < self.treshold { return BranchRanges(result); diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index f7522f01e0d69..fb4cae206239e 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -46,8 +46,8 @@ pub use branch::BranchRanges; /// Database value type. pub type DBValue = Vec; -/// Offstate storage key definition. -pub type OffstateKey = Vec; +/// Kv storage key definition. +pub type KvKey = Vec; /// Basic set of requirements for the Block hash and node key types. pub trait Hash: Send + Sync + Sized + Eq + PartialEq + Clone + Default + fmt::Debug + Codec + std::hash::Hash + 'static {} @@ -72,19 +72,19 @@ pub trait NodeDb { /// Backend database trait. Read-only. /// -/// State parameter indicate where to query offstate storage, +/// State parameter indicate where to query kv storage, /// this is a Block hash, as branch index are /// build on load, with a persistence for this /// data it would be more direct to use a /// tuple of block number and branchindex. -pub trait OffstateDb { +pub trait KvDb { type Error: fmt::Debug; /// Get state trie node. - fn get_offstate(&self, key: &[u8], state: &State) -> Result, Self::Error>; + fn get_kv(&self, key: &[u8], state: &State) -> Result, Self::Error>; /// Get all pairs of key at current state. - fn get_offstate_pairs(&self, state: &State) -> Vec<(OffstateKey, DBValue)>; + fn get_kv_pairs(&self, state: &State) -> Vec<(KvKey, DBValue)>; } /// Error type. @@ -137,29 +137,29 @@ pub struct ChangeSet { pub deleted: Vec, } -/// A set of offstate state values changes. +/// A set of kv state values changes. /// TODO EMCH note that this could really benefit from batching /// the change set (need to change from client to run over -/// the whole range for offstate: then we can get prepare +/// the whole range for kv: then we can get prepare /// insertion of batch values for history in db such as : -/// pub type OffstateChangeSet = Vec<(H, Vec(u64, Option))>; +/// pub type KvChangeSet = Vec<(H, Vec(u64, Option))>; /// ), /// but it just need to be build from client (no need to change /// it here except to extract faster). /// /// This assumes that we only commit block per block (otherwhise /// we need to inclued block number value here). -pub type OffstateChangeSet = Vec<(H, Option)>; +pub type KvChangeSet = Vec<(H, Option)>; -/// Info for pruning offstate: a last prune index and keys to prune. +/// Info for pruning kv: a last prune index and keys to prune. /// Is set to none when not initialized. -pub type OffstateChangeSetPrune = Option<(u64, HashSet)>; +pub type KvChangeSetPrune = Option<(u64, HashSet)>; -/// Info for reverting offstate of a canonical backend. -pub type OffstateChangeSetRevert = u64; +/// Info for reverting kv of a canonical backend. +pub type KvChangeSetRevert = u64; /// Commit set on canonical operation. -pub type CommitSetCanonical = (CommitSet, OffstateChangeSetPrune); +pub type CommitSetCanonical = (CommitSet, KvChangeSetPrune); /// A set of changes to the backing database. /// It only contain a single block change set. @@ -169,8 +169,8 @@ pub struct CommitSet { pub data: ChangeSet, /// Metadata changes. pub meta: ChangeSet>, - /// Offstate data changes. - pub offstate: OffstateChangeSet, + /// Kv data changes. + pub kv: KvChangeSet, } /// Pruning constraints. If none are specified pruning is @@ -257,7 +257,7 @@ impl StateDbSync { number: u64, parent_hash: &BlockHash, mut changeset: ChangeSet, - offstate_changeset: OffstateChangeSet, + kv_changeset: KvChangeSet, ) -> Result, Error> { match self.mode { @@ -273,11 +273,11 @@ impl StateDbSync { // client-db-branch-ix) // and use and ordered tuple (branchix, blocknumber) // index to run. - offstate: offstate_changeset, + kv: kv_changeset, }) }, PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => { - self.non_canonical.insert(hash, number, parent_hash, changeset, offstate_changeset) + self.non_canonical.insert(hash, number, parent_hash, changeset, kv_changeset) } } } @@ -408,33 +408,33 @@ impl StateDbSync { db.get(key.as_ref()).map_err(|e| Error::Db(e)) } - pub fn get_offstate>( + pub fn get_kv>( &self, key: &[u8], state: &(BranchRanges, u64), db: &D, ) -> Result, Error> { - if let Some(value) = self.non_canonical.get_offstate(key, &state.0) { + if let Some(value) = self.non_canonical.get_kv(key, &state.0) { return Ok(value.clone()); } - db.get_offstate(key, &state.1).map_err(|e| Error::Db(e)) + db.get_kv(key, &state.1).map_err(|e| Error::Db(e)) } - pub fn get_offstate_pairs>( + pub fn get_kv_pairs>( &self, state: &(BranchRanges, u64), db: &D, - ) -> Vec<(OffstateKey, DBValue)> { + ) -> Vec<(KvKey, DBValue)> { let mut result = Vec::new(); let mut filter = HashSet::new(); - for (k, o) in self.non_canonical.offstate_iter(&state.0) { + for (k, o) in self.non_canonical.kv_iter(&state.0) { if let Some(v) = o.as_ref() { result.push((k.clone(), v.clone())); } filter.insert(k.clone()); } result.extend( - db.get_offstate_pairs(&state.1).into_iter() + db.get_kv_pairs(&state.1).into_iter() .filter(|kv| !filter.contains(&kv.0)) ); result @@ -483,9 +483,9 @@ impl StateDb { number: u64, parent_hash: &BlockHash, changeset: ChangeSet, - offstate_changeset: OffstateChangeSet, + kv_changeset: KvChangeSet, ) -> Result, Error> { - self.db.write().insert_block(hash, number, parent_hash, changeset, offstate_changeset) + self.db.write().insert_block(hash, number, parent_hash, changeset, kv_changeset) } /// Finalize a previously inserted block. @@ -496,7 +496,8 @@ impl StateDb { self.db.write().canonicalize_block(hash) } - /// TODO EMCH + /// For a a given block return its path in the block tree. + /// Note that using `number` is use to skip a query to block number for hash. pub fn get_branch_range(&self, hash: &BlockHash, number: u64) -> Option { self.db.read().get_branch_range(hash, number) } @@ -519,22 +520,22 @@ impl StateDb { } /// Get a value from non-canonical/pruning overlay or the backing DB. - pub fn get_offstate>( + pub fn get_kv>( &self, key: &[u8], state: &(BranchRanges, u64), db: &D, ) -> Result, Error> { - self.db.read().get_offstate(key, state, db) + self.db.read().get_kv(key, state, db) } - /// Get pairs values offstate. - pub fn get_offstate_pairs>( + /// Get pairs values kv. + pub fn get_kv_pairs>( &self, state: &(BranchRanges, u64), db: &D, - ) -> Vec<(OffstateKey, DBValue)> { - self.db.read().get_offstate_pairs(state, db) + ) -> Vec<(KvKey, DBValue)> { + self.db.read().get_kv_pairs(state, db) } /// Revert all non-canonical blocks with the best block number. @@ -570,11 +571,11 @@ mod tests { use std::io; use primitives::H256; use crate::{StateDb, PruningMode, Constraints}; - use crate::test::{make_db, make_changeset, make_offstate_changeset, TestDb}; + use crate::test::{make_db, make_changeset, make_kv_changeset, TestDb}; fn make_test_db(settings: PruningMode) -> (TestDb, StateDb) { let mut db = make_db(&[91, 921, 922, 93, 94]); - db.initialize_offstate(&[81, 821, 822, 83, 84]); + db.initialize_kv(&[81, 821, 822, 83, 84]); let state_db = StateDb::new(settings, &db).unwrap(); db.commit( @@ -584,7 +585,7 @@ mod tests { 1, &H256::from_low_u64_be(0), make_changeset(&[1], &[91]), - make_offstate_changeset(&[1], &[81]), + make_kv_changeset(&[1], &[81]), ) .unwrap(), ); @@ -595,7 +596,7 @@ mod tests { 2, &H256::from_low_u64_be(1), make_changeset(&[21], &[921, 1]), - make_offstate_changeset(&[21], &[821, 1]), + make_kv_changeset(&[21], &[821, 1]), ) .unwrap(), ); @@ -606,7 +607,7 @@ mod tests { 2, &H256::from_low_u64_be(1), make_changeset(&[22], &[922]), - make_offstate_changeset(&[22], &[822]), + make_kv_changeset(&[22], &[822]), ) .unwrap(), ); @@ -617,7 +618,7 @@ mod tests { 3, &H256::from_low_u64_be(21), make_changeset(&[3], &[93]), - make_offstate_changeset(&[3], &[83]), + make_kv_changeset(&[3], &[83]), ) .unwrap(), ); @@ -634,7 +635,7 @@ mod tests { 4, &H256::from_low_u64_be(3), make_changeset(&[4], &[94]), - make_offstate_changeset(&[4], &[84]), + make_kv_changeset(&[4], &[84]), ) .unwrap(), ); @@ -658,7 +659,7 @@ mod tests { let (db, sdb) = make_test_db(PruningMode::ArchiveAll); assert!(db.data_eq(&make_db(&[1, 21, 22, 3, 4, 91, 921, 922, 93, 94]))); - // TODO EMCH implement full for test db and test for offstate + // TODO EMCH implement full for test db and test for kv assert!(!sdb.is_pruned(&H256::from_low_u64_be(0), 0)); } @@ -667,11 +668,11 @@ mod tests { fn canonical_archive_keeps_canonical() { let (db, _) = make_test_db(PruningMode::ArchiveCanonical); assert!(db.data_eq(&make_db(&[1, 21, 3, 91, 921, 922, 93, 94]))); - assert!(db.offstate_eq_at(&[81, 821, 822, 83, 84], Some(0))); - assert!(db.offstate_eq_at(&[1, 821, 822, 83, 84], Some(1))); - assert!(db.offstate_eq_at(&[21, 822, 83, 84], Some(2))); - assert!(db.offstate_eq_at(&[3, 21, 822, 84], Some(3))); - assert!(db.offstate_eq(&[3, 21, 822, 84])); + assert!(db.kv_eq_at(&[81, 821, 822, 83, 84], Some(0))); + assert!(db.kv_eq_at(&[1, 821, 822, 83, 84], Some(1))); + assert!(db.kv_eq_at(&[21, 822, 83, 84], Some(2))); + assert!(db.kv_eq_at(&[3, 21, 822, 84], Some(3))); + assert!(db.kv_eq(&[3, 21, 822, 84])); } #[test] @@ -681,11 +682,11 @@ mod tests { max_mem: None, })); assert!(db.data_eq(&make_db(&[21, 3, 922, 94]))); - assert!(db.offstate_eq_at(&[822, 84], Some(0))); - assert!(db.offstate_eq_at(&[822, 84], Some(1))); - assert!(db.offstate_eq_at(&[21, 822, 84], Some(2))); - assert!(db.offstate_eq_at(&[3, 21, 822, 84], Some(3))); - assert!(db.offstate_eq(&[3, 21, 822, 84])); + assert!(db.kv_eq_at(&[822, 84], Some(0))); + assert!(db.kv_eq_at(&[822, 84], Some(1))); + assert!(db.kv_eq_at(&[21, 822, 84], Some(2))); + assert!(db.kv_eq_at(&[3, 21, 822, 84], Some(3))); + assert!(db.kv_eq(&[3, 21, 822, 84])); } #[test] @@ -699,11 +700,11 @@ mod tests { assert!(sdb.is_pruned(&H256::from_low_u64_be(21), 2)); assert!(sdb.is_pruned(&H256::from_low_u64_be(22), 2)); assert!(db.data_eq(&make_db(&[21, 3, 922, 93, 94]))); - assert!(db.offstate_eq_at(&[822, 83, 84], Some(0))); - assert!(db.offstate_eq_at(&[822, 83, 84], Some(1))); - assert!(db.offstate_eq_at(&[21, 822, 83, 84], Some(2))); - assert!(db.offstate_eq_at(&[3, 21, 822, 84], Some(3))); - assert!(db.offstate_eq(&[3, 21, 822, 84])); + assert!(db.kv_eq_at(&[822, 83, 84], Some(0))); + assert!(db.kv_eq_at(&[822, 83, 84], Some(1))); + assert!(db.kv_eq_at(&[21, 822, 83, 84], Some(2))); + assert!(db.kv_eq_at(&[3, 21, 822, 84], Some(3))); + assert!(db.kv_eq(&[3, 21, 822, 84])); } #[test] @@ -717,10 +718,10 @@ mod tests { assert!(!sdb.is_pruned(&H256::from_low_u64_be(21), 2)); assert!(sdb.is_pruned(&H256::from_low_u64_be(22), 2)); assert!(db.data_eq(&make_db(&[1, 21, 3, 921, 922, 93, 94]))); - assert!(db.offstate_eq_at(&[821, 822, 83, 84], Some(0))); - assert!(db.offstate_eq_at(&[1, 821, 822, 83, 84], Some(1))); - assert!(db.offstate_eq_at(&[21, 822, 83, 84], Some(2))); - assert!(db.offstate_eq_at(&[3, 21, 822, 84], Some(3))); - assert!(db.offstate_eq(&[3, 21, 822, 84])); + assert!(db.kv_eq_at(&[821, 822, 83, 84], Some(0))); + assert!(db.kv_eq_at(&[1, 821, 822, 83, 84], Some(1))); + assert!(db.kv_eq_at(&[21, 822, 83, 84], Some(2))); + assert!(db.kv_eq_at(&[3, 21, 822, 84], Some(3))); + assert!(db.kv_eq(&[3, 21, 822, 84])); } } diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 4b57e38d08cf6..3fa18315d961f 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -23,8 +23,8 @@ use std::fmt; use std::collections::{HashMap, VecDeque, hash_map::Entry, HashSet}; use super::{ - Error, DBValue, ChangeSet, OffstateChangeSet, CommitSet, MetaDb, Hash, - to_meta_key, OffstateKey, + Error, DBValue, ChangeSet, KvChangeSet, CommitSet, MetaDb, Hash, + to_meta_key, KvKey, }; use codec::{Encode, Decode}; use log::trace; @@ -32,7 +32,7 @@ use crate::branch::{RangeSet, BranchRanges}; use historied_data::tree::History; const NON_CANONICAL_JOURNAL: &[u8] = b"noncanonical_journal"; -const NON_CANONICAL_OFFSTATE_JOURNAL: &[u8] = b"offstate_noncanonical_journal"; +const NON_CANONICAL_OFFSTATE_JOURNAL: &[u8] = b"kv_noncanonical_journal"; const LAST_CANONICAL: &[u8] = b"last_canonical"; type BranchIndex = u64; @@ -50,30 +50,30 @@ pub struct NonCanonicalOverlay { pending_insertions: Vec, values: HashMap, //ref counted branches: RangeSet, - offstate_values: HashMap>>, - /// second value is offstate pinned index: used in order to determine if the pinned + kv_values: HashMap>>, + /// second value is kv pinned index: used in order to determine if the pinned /// thread should block garbage collection. pinned: HashMap, GcIndex)>, - offstate_gc: OffstatePendingGC, + kv_gc: KvPendingGC, } #[derive(Default)] -/// offstate gc only happen when all pinned threads that where +/// kv gc only happen when all pinned threads that where /// running at the time of cannonicalisation are finished. -struct OffstatePendingGC { +struct KvPendingGC { /// Each gc call uses a new index. gc_last_index: GcIndex, /// Keep trace of last cannonicalization branch index height. /// All data in state are added after this value (branch is /// set as non modifiable on canonicalisation). - pending_canonicalisation_query: Option, + pending_canonicalisation_query: Option, /// keys to gc that got their journal removed. - keys_pending_gc: HashSet, + keys_pending_gc: HashSet, /// store keys while waiting for a gc. - next_keys_pending_gc: HashSet, + next_keys_pending_gc: HashSet, } -impl OffstatePendingGC { +impl KvPendingGC { fn set_pending_gc(&mut self, branch_index: u64) { self.gc_last_index += 1; if self.pending_canonicalisation_query.is_some() { @@ -85,7 +85,7 @@ impl OffstatePendingGC { fn try_gc( &mut self, - offstate_values: &mut HashMap>>, + kv_values: &mut HashMap>>, branches: &RangeSet, pinned: &HashMap, ) { @@ -95,11 +95,11 @@ impl OffstatePendingGC { // TODO EMCH double get is rather inefficient, see if it is possible // to reuse the hash of the hashset into the hashmap for key in self.keys_pending_gc.drain() { - match offstate_values.get_mut(&key).map(|historied_value| { + match kv_values.get_mut(&key).map(|historied_value| { historied_value.gc(branches.reverse_iter_ranges()) }) { Some(historied_data::PruneResult::Cleared) => { - let _ = offstate_values.remove(&key); + let _ = kv_values.remove(&key); }, _ => (), } @@ -135,16 +135,16 @@ struct JournalRecord { } #[derive(Encode, Decode)] -struct OffstateJournalRecord { - inserted: Vec<(OffstateKey, DBValue)>, - deleted: Vec, +struct KvJournalRecord { + inserted: Vec<(KvKey, DBValue)>, + deleted: Vec, } fn to_journal_key(block: BlockNumber, index: u64) -> Vec { to_meta_key(NON_CANONICAL_JOURNAL, &(block, index)) } -fn to_offstate_journal_key(block: BlockNumber, index: u64) -> Vec { +fn to_kv_journal_key(block: BlockNumber, index: u64) -> Vec { to_meta_key(NON_CANONICAL_OFFSTATE_JOURNAL, &(block, index)) } @@ -152,11 +152,11 @@ fn to_offstate_journal_key(block: BlockNumber, index: u64) -> Vec { struct BlockOverlay { hash: BlockHash, journal_key: Vec, - offstate_journal_key: Vec, + kv_journal_key: Vec, inserted: Vec, deleted: Vec, - offstate_inserted: Vec, - offstate_deleted: Vec, + kv_inserted: Vec, + kv_deleted: Vec, } fn insert_values(values: &mut HashMap, inserted: Vec<(Key, DBValue)>) { @@ -167,7 +167,7 @@ fn insert_values(values: &mut HashMap, inserted: } } -fn insert_offstate_values( +fn insert_kv_values( state: &BranchRanges, values: &mut HashMap>>, inserted: Vec<(Key, DBValue)>, @@ -207,10 +207,10 @@ fn discard_values( } } -fn discard_offstate_values( - inserted: Vec, - deleted: Vec, - into: &mut OffstatePendingGC, +fn discard_kv_values( + inserted: Vec, + deleted: Vec, + into: &mut KvPendingGC, ) { let into = if into.pending_canonicalisation_query.is_some() { &mut into.next_keys_pending_gc @@ -221,14 +221,13 @@ fn discard_offstate_values( into.extend(deleted); } - fn discard_descendants( levels: &mut VecDeque>>, mut values: &mut HashMap, index: usize, parents: &mut HashMap, pinned: &mut HashMap, u64)>, - offstate_gc: &mut OffstatePendingGC, + kv_gc: &mut KvPendingGC, hash: &BlockHash, ) { let mut discarded = Vec::new(); @@ -240,10 +239,10 @@ fn discard_descendants( discarded.push(overlay.hash); let mut pinned = pinned.get_mut(hash); discard_values(&mut values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); - discard_offstate_values( - overlay.offstate_inserted, - overlay.offstate_deleted, - offstate_gc, + discard_kv_values( + overlay.kv_inserted, + overlay.kv_deleted, + kv_gc, ); None } else { @@ -252,7 +251,7 @@ fn discard_descendants( }).collect(); } for hash in discarded { - discard_descendants(levels, values, index + 1, parents, pinned, offstate_gc, &hash); + discard_descendants(levels, values, index + 1, parents, pinned, kv_gc, &hash); } } @@ -268,7 +267,7 @@ impl NonCanonicalOverlay { let mut levels = VecDeque::new(); let mut parents = HashMap::new(); let mut values = HashMap::new(); - let mut offstate_values = HashMap::new(); + let mut kv_values = HashMap::new(); let mut branches = RangeSet::default(); if let Some((ref hash, mut block)) = last_canonicalized { // read the journal @@ -280,7 +279,7 @@ impl NonCanonicalOverlay { let mut level = Vec::new(); loop { let journal_key = to_journal_key(block, index); - let offstate_journal_key = to_offstate_journal_key(block, index); + let kv_journal_key = to_kv_journal_key(block, index); match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? { Some(record) => { let record: JournalRecord = Decode::decode(&mut record.as_slice())?; @@ -291,7 +290,7 @@ impl NonCanonicalOverlay { let parent_branch_index = parents.get(&record.parent_hash) .map(|(_, i)| *i).unwrap_or(0); let parent_branch_range = Some( - branches.branch_ranges_from_cache(parent_branch_index, Some(block - 1)) + branches.branch_ranges(parent_branch_index, Some(block - 1)) ); let (branch_range, branch_index) = branches.import( block, @@ -299,32 +298,32 @@ impl NonCanonicalOverlay { parent_branch_range, ); - let (offstate_record_inserted, offstate_record_deleted) = if let Some(record) = db - .get_meta(&offstate_journal_key).map_err(|e| Error::Db(e))? { - let record = OffstateJournalRecord::decode(&mut record.as_slice())?; + let (kv_record_inserted, kv_record_deleted) = if let Some(record) = db + .get_meta(&kv_journal_key).map_err(|e| Error::Db(e))? { + let record = KvJournalRecord::decode(&mut record.as_slice())?; (Some(record.inserted), Some(record.deleted)) } else { (None, None) }; let inserted = record.inserted.iter().map(|(k, _)| k.clone()).collect(); - let offstate_inserted = offstate_record_inserted.as_ref() + let kv_inserted = kv_record_inserted.as_ref() .map(|inserted| inserted.iter().map(|(k, _)| k.clone()).collect()) .unwrap_or(Vec::new()); - let offstate_deleted = offstate_record_deleted.clone().unwrap_or(Vec::new()); + let kv_deleted = kv_record_deleted.clone().unwrap_or(Vec::new()); let overlay = BlockOverlay { hash: record.hash.clone(), journal_key, - offstate_journal_key, + kv_journal_key, inserted: inserted, deleted: record.deleted, - offstate_inserted, - offstate_deleted, + kv_inserted, + kv_deleted, }; insert_values(&mut values, record.inserted); - if offstate_record_inserted.is_some() || offstate_record_deleted.is_some() { - insert_offstate_values( + if kv_record_inserted.is_some() || kv_record_deleted.is_some() { + insert_kv_values( &branch_range, - &mut offstate_values, - offstate_record_inserted.unwrap_or(Vec::new()), - offstate_record_deleted.unwrap_or(Vec::new()), + &mut kv_values, + kv_record_inserted.unwrap_or(Vec::new()), + kv_record_deleted.unwrap_or(Vec::new()), ); } trace!( @@ -333,9 +332,9 @@ impl NonCanonicalOverlay { block, index, overlay.inserted.len(), - overlay.offstate_inserted.len(), + overlay.kv_inserted.len(), overlay.deleted.len(), - overlay.offstate_deleted.len(), + overlay.kv_deleted.len(), ); level.push(overlay); parents.insert(record.hash, (record.parent_hash, branch_index)); @@ -361,8 +360,8 @@ impl NonCanonicalOverlay { pending_insertions: Default::default(), pinned: Default::default(), values, - offstate_values, - offstate_gc: Default::default(), + kv_values, + kv_gc: Default::default(), branches, }) } @@ -375,7 +374,7 @@ impl NonCanonicalOverlay { number: BlockNumber, parent_hash: &BlockHash, changeset: ChangeSet, - offstate_changeset: OffstateChangeSet, + kv_changeset: KvChangeSet, ) -> Result, Error> { let mut commit = CommitSet::default(); let front_block_number = self.front_block_number(); @@ -412,35 +411,35 @@ impl NonCanonicalOverlay { let index = level.len() as u64; let journal_key = to_journal_key(number, index); - let offstate_journal_key = to_offstate_journal_key(number, index); + let kv_journal_key = to_kv_journal_key(number, index); let inserted = changeset.inserted.iter().map(|(k, _)| k.clone()).collect(); - let mut offstate_inserted = Vec::new(); - let mut offstate_inserted_value = Vec::new(); - let mut offstate_deleted = Vec::new(); - for (k, v) in offstate_changeset.into_iter() { + let mut kv_inserted = Vec::new(); + let mut kv_inserted_value = Vec::new(); + let mut kv_deleted = Vec::new(); + for (k, v) in kv_changeset.into_iter() { if let Some(v) = v { - offstate_inserted.push(k.clone()); - offstate_inserted_value.push((k, v)); + kv_inserted.push(k.clone()); + kv_inserted_value.push((k, v)); } else { - offstate_deleted.push(k); + kv_deleted.push(k); } } let overlay = BlockOverlay { hash: hash.clone(), journal_key: journal_key.clone(), - offstate_journal_key: offstate_journal_key.clone(), + kv_journal_key: kv_journal_key.clone(), inserted: inserted, deleted: changeset.deleted.clone(), - offstate_inserted, - offstate_deleted: offstate_deleted.clone(), + kv_inserted, + kv_deleted: kv_deleted.clone(), }; level.push(overlay); let parent_branch_index = self.parents.get(&parent_hash).map(|(_, i)| *i).unwrap_or(0); let parent_branch_range = if number == 0 { None } else { - Some(self.branches.branch_ranges_from_cache(parent_branch_index, Some(number - 1))) + Some(self.branches.branch_ranges(parent_branch_index, Some(number - 1))) }; let (branch_range, branch_index) = self.branches.import( number, @@ -456,11 +455,11 @@ impl NonCanonicalOverlay { deleted: changeset.deleted, }; commit.meta.inserted.push((journal_key, journal_record.encode())); - let offstate_journal_record = OffstateJournalRecord { - inserted: offstate_inserted_value, - deleted: offstate_deleted, + let kv_journal_record = KvJournalRecord { + inserted: kv_inserted_value, + deleted: kv_deleted, }; - commit.meta.inserted.push((offstate_journal_key, offstate_journal_record.encode())); + commit.meta.inserted.push((kv_journal_key, kv_journal_record.encode())); trace!( target: "state-db", @@ -468,15 +467,15 @@ impl NonCanonicalOverlay { number, index, journal_record.inserted.len(), - offstate_journal_record.inserted.len(), + kv_journal_record.inserted.len(), journal_record.deleted.len(), ); insert_values(&mut self.values, journal_record.inserted); - insert_offstate_values( + insert_kv_values( &branch_range, - &mut self.offstate_values, - offstate_journal_record.inserted, - offstate_journal_record.deleted, + &mut self.kv_values, + kv_journal_record.inserted, + kv_journal_record.deleted, ); self.pending_insertions.push(hash.clone()); Ok(commit) @@ -495,7 +494,7 @@ impl NonCanonicalOverlay { .expect("there is a parent entry for each entry in levels; qed"); if parent == hash { discarded_journals.push(overlay.journal_key.clone()); - discarded_journals.push(overlay.offstate_journal_key.clone()); + discarded_journals.push(overlay.kv_journal_key.clone()); discarded_blocks.push(overlay.hash.clone()); self.discard_journals(level_index + 1, discarded_journals, discarded_blocks, &overlay.hash); } @@ -553,7 +552,7 @@ impl NonCanonicalOverlay { ); } discarded_journals.push(overlay.journal_key.clone()); - discarded_journals.push(overlay.offstate_journal_key.clone()); + discarded_journals.push(overlay.kv_journal_key.clone()); discarded_blocks.push(overlay.hash.clone()); } @@ -563,22 +562,22 @@ impl NonCanonicalOverlay { .map(|k| (k.clone(), self.values.get(k).expect("For each key in overlays there's a value in values").1.clone()))); commit.data.deleted.extend(overlay.deleted.clone()); let block_number = self.front_block_number() + self.pending_canonicalizations.len() as u64; - if !overlay.offstate_inserted.is_empty() || !overlay.offstate_deleted.is_empty() { + if !overlay.kv_inserted.is_empty() || !overlay.kv_deleted.is_empty() { // canonicalization is not frequent enough that we pass range // in parameter for now if let Some(range) = self.get_branch_range(hash, block_number) { - commit.offstate.extend( - overlay.offstate_inserted.iter() - .map(|k| (k.clone(), self.offstate_values.get(k) + commit.kv.extend( + overlay.kv_inserted.iter() + .map(|k| (k.clone(), self.kv_values.get(k) .expect("For each key in overlays there's a value in values") .get(&range) .expect("For each key in overlays there's a historied entry in values") .clone()))); // some deletion can be pruned if in first block so handle is // a bit less restrictive - commit.offstate.extend( - overlay.offstate_deleted.iter() - .filter_map(|k| self.offstate_values.get(k) + commit.kv.extend( + overlay.kv_deleted.iter() + .filter_map(|k| self.kv_values.get(k) .map(|v| (k.clone(), v.get(&range) .expect("For each key in overlays there's a historied entry \ in values, and pruned empty value are cleared") @@ -620,17 +619,17 @@ impl NonCanonicalOverlay { 0, &mut self.parents, &mut self.pinned, - &mut self.offstate_gc, + &mut self.kv_gc, &overlay.hash, ); } let mut pinned = self.pinned.get_mut(&overlay.hash); discard_values(&mut self.values, overlay.inserted, pinned.as_mut().map(|p| &mut p.0)); - discard_offstate_values( - overlay.offstate_inserted, - overlay.offstate_deleted, - &mut self.offstate_gc, + discard_kv_values( + overlay.kv_inserted, + overlay.kv_deleted, + &mut self.kv_gc, ); } } @@ -644,10 +643,10 @@ impl NonCanonicalOverlay { // TODO EMCH may be needed in 'canonicalize', and restore or commit here self.branches.update_finalize_treshold(branch_index_cannonicalize, block_number, true); // gc is at the right place - self.offstate_gc.set_pending_gc(branch_index_cannonicalize); + self.kv_gc.set_pending_gc(branch_index_cannonicalize); // try to run the garbage collection (can run later if there is // pinned process). - self.offstate_gc.try_gc(&mut self.offstate_values, &self.branches, &self.pinned); + self.kv_gc.try_gc(&mut self.kv_values, &self.branches, &self.pinned); } } @@ -667,8 +666,8 @@ impl NonCanonicalOverlay { } /// Get a value from the node overlay. This searches in every existing changeset. - pub fn get_offstate(&self, key: &[u8], state: &BranchRanges) -> Option<&Option> { - if let Some(value) = self.offstate_values.get(key) { + pub fn get_kv(&self, key: &[u8], state: &BranchRanges) -> Option<&Option> { + if let Some(value) = self.kv_values.get(key) { return value.get(state); } None @@ -686,15 +685,15 @@ impl NonCanonicalOverlay { let mut commit = CommitSet::default(); for overlay in level.into_iter() { commit.meta.deleted.push(overlay.journal_key); - commit.meta.deleted.push(overlay.offstate_journal_key); + commit.meta.deleted.push(overlay.kv_journal_key); if let Some((_, branch_index)) = self.parents.remove(&overlay.hash) { self.branches.revert(branch_index); } discard_values(&mut self.values, overlay.inserted, None); - discard_offstate_values( - overlay.offstate_inserted, - overlay.offstate_deleted, - &mut self.offstate_gc, + discard_kv_values( + overlay.kv_inserted, + overlay.kv_deleted, + &mut self.kv_gc, ); } commit @@ -713,8 +712,6 @@ impl NonCanonicalOverlay { let overlay = self.levels[level_index].pop().expect("Empty levels are not allowed in self.levels"); discard_values(&mut self.values, overlay.inserted, None); - // TODO EMCH we need discard state?? from update finalize treshold here!! see discard value - // usgage if self.levels[level_index].is_empty() { debug_assert_eq!(level_index, self.levels.len() - 1); self.levels.pop_back(); @@ -736,29 +733,30 @@ impl NonCanonicalOverlay { /// Pin state values in memory pub fn pin(&mut self, hash: &BlockHash) { - self.pinned.insert(hash.clone(), (Default::default(), self.offstate_gc.gc_last_index)); + self.pinned.insert(hash.clone(), (Default::default(), self.kv_gc.gc_last_index)); } - /// TODO EMCH aka get state for hash to query offstate storage. + /// For a a given block return its path in the block tree. + /// Note that using `number` is use to skip a query to block number for hash. pub fn get_branch_range(&self, hash: &BlockHash, number: BlockNumber) -> Option { self.parents.get(hash).map(|(_, branch_index)| *branch_index).map(|branch_index| { - self.branches.branch_ranges_from_cache(branch_index, Some(number)) + self.branches.branch_ranges(branch_index, Some(number)) }) } /// Discard pinned state pub fn unpin(&mut self, hash: &BlockHash) { self.pinned.remove(hash); - self.offstate_gc.try_gc(&mut self.offstate_values, &self.branches, &self.pinned); + self.kv_gc.try_gc(&mut self.kv_values, &self.branches, &self.pinned); } /// Iterator over values at a given state. Deletion are included in the result as a None value. - pub fn offstate_iter<'a>( + pub fn kv_iter<'a>( &'a self, state: &'a BranchRanges, - ) -> impl Iterator)> { + ) -> impl Iterator)> { let state = state.clone(); - self.offstate_values.iter().filter_map(move |(k, v)| { + self.kv_values.iter().filter_map(move |(k, v)| { v.get(&state).map(|v| (k, v)) }) } @@ -770,31 +768,31 @@ mod tests { use std::collections::BTreeMap; use primitives::H256; use super::{NonCanonicalOverlay, to_journal_key}; - use crate::{ChangeSet, OffstateChangeSet, CommitSet, OffstateKey}; - use crate::test::{make_db, make_changeset, make_offstate_changeset}; + use crate::{ChangeSet, KvChangeSet, CommitSet, KvKey}; + use crate::test::{make_db, make_changeset, make_kv_changeset}; use crate::branch::BranchRanges; fn contains(overlay: &NonCanonicalOverlay, key: u64) -> bool { overlay.get(&H256::from_low_u64_be(key)) == Some(H256::from_low_u64_be(key).as_bytes().to_vec()) } - fn contains_offstate( + fn contains_kv( overlay: &NonCanonicalOverlay, key: u64, state: &H256, block: u64, ) -> bool { overlay.get_branch_range(state, block).and_then(|state| { - overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state) + overlay.get_kv(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state) }) == Some(&Some(H256::from_low_u64_be(key).as_bytes().to_vec())) } - fn contains_offstate2( + fn contains_kv2( overlay: &NonCanonicalOverlay, key: u64, state: &BranchRanges, ) -> bool { - overlay.get_offstate(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state) + overlay.get_kv(&H256::from_low_u64_be(key).as_bytes().to_vec(), &state) == Some(&Some(H256::from_low_u64_be(key).as_bytes().to_vec())) } @@ -804,7 +802,7 @@ mod tests { state: &H256, block: u64, ) -> bool { - contains(overlay, key) && contains_offstate(overlay, key, state, block) + contains(overlay, key) && contains_kv(overlay, key, state, block) } fn contains_any( @@ -813,7 +811,7 @@ mod tests { state: &H256, block: u64, ) -> bool { - contains(overlay, key) || contains_offstate(overlay, key, state, block) + contains(overlay, key) || contains_kv(overlay, key, state, block) } #[test] @@ -843,11 +841,11 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); overlay.insert::( &h1, 2, &H256::default(), - ChangeSet::default(), OffstateChangeSet::default(), + ChangeSet::default(), KvChangeSet::default(), ).unwrap(); overlay.insert::( &h2, 1, &h1, - ChangeSet::default(), OffstateChangeSet::default(), + ChangeSet::default(), KvChangeSet::default(), ).unwrap(); } @@ -860,11 +858,11 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); overlay.insert::( &h1, 1, &H256::default(), - ChangeSet::default(), OffstateChangeSet::default(), + ChangeSet::default(), KvChangeSet::default(), ).unwrap(); overlay.insert::( &h2, 3, &h1, - ChangeSet::default(), OffstateChangeSet::default(), + ChangeSet::default(), KvChangeSet::default(), ).unwrap(); } @@ -877,11 +875,11 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); overlay.insert::( &h1, 1, &H256::default(), - ChangeSet::default(), OffstateChangeSet::default(), + ChangeSet::default(), KvChangeSet::default(), ).unwrap(); overlay.insert::( &h2, 2, &H256::default(), - ChangeSet::default(), OffstateChangeSet::default(), + ChangeSet::default(), KvChangeSet::default(), ).unwrap(); } @@ -894,7 +892,7 @@ mod tests { let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); overlay.insert::( &h1, 1, &H256::default(), - ChangeSet::default(), OffstateChangeSet::default(), + ChangeSet::default(), KvChangeSet::default(), ).unwrap(); let mut commit = CommitSet::default(); overlay.canonicalize::(&h2, &mut commit).unwrap(); @@ -904,18 +902,18 @@ mod tests { fn insert_canonicalize_one() { let h1 = H256::random(); let mut db = make_db(&[1, 2]); - db.initialize_offstate(&[1, 2]); + db.initialize_kv(&[1, 2]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); let changeset = make_changeset(&[3, 4], &[2]); - let offstate_changeset = make_offstate_changeset(&[3, 4], &[2]); + let kv_changeset = make_kv_changeset(&[3, 4], &[2]); let insertion = overlay.insert::( &h1, 1, &H256::default(), - changeset.clone(), offstate_changeset.clone(), + changeset.clone(), kv_changeset.clone(), ).unwrap(); assert_eq!(insertion.data.inserted.len(), 0); assert_eq!(insertion.data.deleted.len(), 0); - assert_eq!(insertion.offstate.len(), 0); - // last cannonical, journal_record and offstate_journal_record + assert_eq!(insertion.kv.len(), 0); + // last cannonical, journal_record and kv_journal_record assert_eq!(insertion.meta.inserted.len(), 3); assert_eq!(insertion.meta.deleted.len(), 0); db.commit(&insertion); @@ -923,13 +921,13 @@ mod tests { overlay.canonicalize::(&h1, &mut finalization).unwrap(); assert_eq!(finalization.data.inserted.len(), changeset.inserted.len()); assert_eq!(finalization.data.deleted.len(), changeset.deleted.len()); - assert_eq!(finalization.offstate.len(), offstate_changeset.len()); + assert_eq!(finalization.kv.len(), kv_changeset.len()); assert_eq!(finalization.meta.inserted.len(), 1); - // normal and offstate discarded journall + // normal and kv discarded journall assert_eq!(finalization.meta.deleted.len(), 2); db.commit(&finalization); assert!(db.data_eq(&make_db(&[1, 3, 4]))); - assert!(db.offstate_eq(&[1, 3, 4])); + assert!(db.kv_eq(&[1, 3, 4])); } #[test] @@ -937,17 +935,17 @@ mod tests { let h1 = H256::random(); let h2 = H256::random(); let mut db = make_db(&[1, 2]); - db.initialize_offstate(&[1, 2]); + db.initialize_kv(&[1, 2]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); db.commit(&overlay.insert::( &h1, 10, &H256::default(), make_changeset(&[3, 4], &[2]), - make_offstate_changeset(&[3, 4], &[2]), + make_kv_changeset(&[3, 4], &[2]), ).unwrap()); db.commit(&overlay.insert::( &h2, 11, &h1, make_changeset(&[5], &[3]), - make_offstate_changeset(&[5], &[3]), + make_kv_changeset(&[5], &[3]), ).unwrap()); assert_eq!(db.meta.len(), 5); @@ -962,17 +960,17 @@ mod tests { let h1 = H256::random(); let h2 = H256::random(); let mut db = make_db(&[1, 2]); - db.initialize_offstate(&[1, 2]); + db.initialize_kv(&[1, 2]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); db.commit(&overlay.insert::( &h1, 10, &H256::default(), make_changeset(&[3, 4], &[2]), - make_offstate_changeset(&[3, 4], &[2]), + make_kv_changeset(&[3, 4], &[2]), ).unwrap()); db.commit(&overlay.insert::( &h2,11, &h1, make_changeset(&[5], &[3]), - make_offstate_changeset(&[5], &[3]), + make_kv_changeset(&[5], &[3]), ).unwrap()); let mut commit = CommitSet::default(); overlay.canonicalize::(&h1, &mut commit).unwrap(); @@ -991,24 +989,24 @@ mod tests { let h1 = H256::random(); let h2 = H256::random(); let mut db = make_db(&[1, 2, 3, 4]); - db.initialize_offstate(&[1, 2, 3, 4]); + db.initialize_kv(&[1, 2, 3, 4]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); let changeset1 = make_changeset(&[5, 6], &[2]); let changeset2 = make_changeset(&[7, 8], &[5, 3]); - let offstate_changeset1 = make_offstate_changeset(&[5, 6], &[2]); - let offstate_changeset2 = make_offstate_changeset(&[7, 8], &[5, 3]); + let kv_changeset1 = make_kv_changeset(&[5, 6], &[2]); + let kv_changeset2 = make_kv_changeset(&[7, 8], &[5, 3]); db.commit(&overlay.insert::( &h1, 1, &H256::default(), - changeset1, offstate_changeset1, + changeset1, kv_changeset1, ).unwrap()); assert!(contains_both(&overlay, 5, &h1, 1)); db.commit(&overlay.insert::( &h2, 2, &h1, - changeset2, offstate_changeset2, + changeset2, kv_changeset2, ).unwrap()); assert!(contains_both(&overlay, 7, &h2, 2)); - assert!(!contains_offstate(&overlay, 5, &h2, 2)); - assert!(contains_offstate(&overlay, 5, &h1, 1)); + assert!(!contains_kv(&overlay, 5, &h2, 2)); + assert!(contains_kv(&overlay, 5, &h1, 1)); assert!(contains_both(&overlay, 5, &h1, 1)); assert_eq!(overlay.levels.len(), 2); assert_eq!(overlay.parents.len(), 2); @@ -1030,7 +1028,7 @@ mod tests { assert_eq!(overlay.levels.len(), 0); assert_eq!(overlay.parents.len(), 0); assert!(db.data_eq(&make_db(&[1, 4, 6, 7, 8]))); - assert!(db.offstate_eq(&[1, 4, 6, 7, 8])); + assert!(db.kv_eq(&[1, 4, 6, 7, 8])); } #[test] @@ -1038,8 +1036,8 @@ mod tests { let mut db = make_db(&[]); let (h_1, c_1) = (H256::random(), make_changeset(&[1], &[])); let (h_2, c_2) = (H256::random(), make_changeset(&[1], &[])); - let o_c_1 = make_offstate_changeset(&[1], &[]); - let o_c_2 = make_offstate_changeset(&[1], &[]); + let o_c_1 = make_kv_changeset(&[1], &[]); + let o_c_2 = make_kv_changeset(&[1], &[]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); db.commit(&overlay.insert::(&h_1, 1, &H256::default(), c_1, o_c_1).unwrap()); @@ -1061,7 +1059,7 @@ mod tests { let mut db = make_db(&[]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); let changeset = make_changeset(&[], &[]); - let ochangeset = make_offstate_changeset(&[], &[]); + let ochangeset = make_kv_changeset(&[], &[]); db.commit(&overlay.insert::( &h1, 1, &H256::default(), changeset.clone(), ochangeset.clone(), @@ -1086,12 +1084,12 @@ mod tests { fn make_both_changeset(inserted: &[u64], deleted: &[u64]) -> ( H256, ChangeSet, - OffstateChangeSet, + KvChangeSet, ) { ( H256::random(), make_changeset(inserted, deleted), - make_offstate_changeset(inserted, deleted), + make_kv_changeset(inserted, deleted), ) } @@ -1207,7 +1205,7 @@ mod tests { assert_eq!(overlay.levels.len(), 0); assert_eq!(overlay.parents.len(), 0); assert!(db.data_eq(&make_db(&[1, 12, 122]))); - assert!(db.offstate_eq(&[1, 12, 122])); + assert!(db.kv_eq(&[1, 12, 122])); assert_eq!(overlay.last_canonicalized, Some((h_1_2_2, 3))); } @@ -1216,13 +1214,13 @@ mod tests { let h1 = H256::random(); let h2 = H256::random(); let mut db = make_db(&[1, 2, 3, 4]); - db.initialize_offstate(&[1, 2, 3, 4]); + db.initialize_kv(&[1, 2, 3, 4]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); assert!(overlay.revert_one().is_none()); let changeset1 = make_changeset(&[5, 6], &[2]); - let ochangeset1 = make_offstate_changeset(&[5, 6], &[2]); + let ochangeset1 = make_kv_changeset(&[5, 6], &[2]); let changeset2 = make_changeset(&[7, 8], &[5, 3]); - let ochangeset2 = make_offstate_changeset(&[7, 8], &[5, 3]); + let ochangeset2 = make_kv_changeset(&[7, 8], &[5, 3]); db.commit(&overlay.insert::( &h1, 1, &H256::default(), changeset1, ochangeset1, @@ -1251,11 +1249,11 @@ mod tests { let db = make_db(&[]); let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); let changeset1 = make_changeset(&[5, 6], &[2]); - let ochangeset1 = make_offstate_changeset(&[5, 6], &[2]); + let ochangeset1 = make_kv_changeset(&[5, 6], &[2]); let changeset2 = make_changeset(&[7, 8], &[5, 3]); - let ochangeset2 = make_offstate_changeset(&[7, 8], &[5, 3]); + let ochangeset2 = make_kv_changeset(&[7, 8], &[5, 3]); let changeset3 = make_changeset(&[9], &[]); - let ochangeset3 = make_offstate_changeset(&[9], &[]); + let ochangeset3 = make_kv_changeset(&[9], &[]); overlay.insert::( &h1, 1, &H256::default(), changeset1, ochangeset1, @@ -1269,9 +1267,9 @@ mod tests { &h2_2, 2, &h1, changeset3, ochangeset3, ).unwrap(); - assert!(contains_offstate(&overlay, 5, &h1, 1)); + assert!(contains_kv(&overlay, 5, &h1, 1)); assert!(contains_both(&overlay, 7, &h2_1, 2)); - assert!(!contains_offstate(&overlay, 5, &h2_1, 2)); + assert!(!contains_kv(&overlay, 5, &h2_1, 2)); assert!(contains(&overlay, 5)); assert!(contains_both(&overlay, 9, &h2_2, 2)); assert_eq!(overlay.levels.len(), 2); @@ -1304,12 +1302,12 @@ mod tests { db.commit(&commit); overlay.apply_pending(); assert!(contains(&overlay, 1)); - // we cannot use contains_offstate because offstate pining is relying on + // we cannot use contains_kv because kv pining is relying on // asumption that pinned context memoïzed its branch state. - assert!(contains_offstate2(&overlay, 1, &h1_context)); - assert!(!contains_offstate(&overlay, 1, &h_1, 1)); + assert!(contains_kv2(&overlay, 1, &h1_context)); + assert!(!contains_kv(&overlay, 1, &h_1, 1)); overlay.unpin(&h_1); assert!(!contains(&overlay, 1)); - assert!(!contains_offstate2(&overlay, 1, &h1_context)); + assert!(!contains_kv2(&overlay, 1, &h1_context)); } } diff --git a/core/state-db/src/pruning.rs b/core/state-db/src/pruning.rs index 1dd21416fe347..66116124f9168 100644 --- a/core/state-db/src/pruning.rs +++ b/core/state-db/src/pruning.rs @@ -25,12 +25,12 @@ use std::collections::{HashMap, HashSet, VecDeque}; use codec::{Encode, Decode}; use crate::{CommitSet, CommitSetCanonical, Error, MetaDb, to_meta_key, Hash, - OffstateKey}; + KvKey}; use log::{trace, warn}; const LAST_PRUNED: &[u8] = b"last_pruned"; const PRUNING_JOURNAL: &[u8] = b"pruning_journal"; -const OFFSTATE_PRUNING_JOURNAL: &[u8] = b"offstate_pruning_journal"; +const OFFSTATE_PRUNING_JOURNAL: &[u8] = b"kv_pruning_journal"; /// See module documentation. pub struct RefWindow { @@ -52,12 +52,12 @@ pub struct RefWindow { struct DeathRow { hash: BlockHash, journal_key: Vec, - offstate_journal_key: Vec, + kv_journal_key: Vec, deleted: HashSet, - // TODO EMCH for offstate there is no need to put + // TODO EMCH for kv there is no need to put // in memory so we can make it lazy (load from // pruning journal on actual prune). - offstate_modified: HashSet, + kv_modified: HashSet, } #[derive(Encode, Decode)] @@ -68,15 +68,15 @@ struct JournalRecord { } #[derive(Encode, Decode)] -struct OffstateJournalRecord { - modified: Vec, +struct KvJournalRecord { + modified: Vec, } fn to_journal_key(block: u64) -> Vec { to_meta_key(PRUNING_JOURNAL, &block) } -fn to_offstate_journal_key(block: u64) -> Vec { +fn to_kv_journal_key(block: u64) -> Vec { to_meta_key(OFFSTATE_PRUNING_JOURNAL, &block) } @@ -100,13 +100,13 @@ impl RefWindow { trace!(target: "state-db", "Reading pruning journal. Pending #{}", pending_number); loop { let journal_key = to_journal_key(block); - let offstate_journal_key = to_offstate_journal_key(block); + let kv_journal_key = to_kv_journal_key(block); match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? { Some(record) => { let record: JournalRecord = Decode::decode(&mut record.as_slice())?; - let offstate_record_inserted = if let Some(record) = db - .get_meta(&offstate_journal_key).map_err(|e| Error::Db(e))? { - let record = OffstateJournalRecord::decode(&mut record.as_slice())?; + let kv_record_inserted = if let Some(record) = db + .get_meta(&kv_journal_key).map_err(|e| Error::Db(e))? { + let record = KvJournalRecord::decode(&mut record.as_slice())?; record.modified } else { Vec::new() }; @@ -115,16 +115,16 @@ impl RefWindow { "Pruning journal entry {} ({} {} inserted, {} deleted)", block, record.inserted.len(), - offstate_record_inserted.len(), + kv_record_inserted.len(), record.deleted.len(), ); pruning.import( &record.hash, journal_key, - offstate_journal_key, + kv_journal_key, record.inserted.into_iter(), record.deleted, - offstate_record_inserted.into_iter(), + kv_record_inserted.into_iter(), ); }, None => break, @@ -134,14 +134,14 @@ impl RefWindow { Ok(pruning) } - fn import, I2: IntoIterator>( + fn import, I2: IntoIterator>( &mut self, hash: &BlockHash, journal_key: Vec, - offstate_journal_key: Vec, + kv_journal_key: Vec, inserted: I, deleted: Vec, - offstate_modified: I2, + kv_modified: I2, ) { // remove all re-inserted keys from death rows for k in inserted { @@ -156,14 +156,14 @@ impl RefWindow { self.death_index.insert(k.clone(), imported_block); } // TODO EMCH is it possible to change type to directly set ?? - let offstate_modified = offstate_modified.into_iter().collect(); + let kv_modified = kv_modified.into_iter().collect(); self.death_rows.push_back( DeathRow { hash: hash.clone(), deleted: deleted.into_iter().collect(), - offstate_modified, + kv_modified, journal_key, - offstate_journal_key, + kv_journal_key, } ); } @@ -190,28 +190,28 @@ impl RefWindow { /// Prune next block. Expects at least one block in the window. /// Adds changes to `commit`. - /// `offstate_prune` to None indicates archive mode. + /// `kv_prune` to None indicates archive mode. pub fn prune_one( &mut self, commit: &mut CommitSetCanonical, ) { - let (commit, offstate_prune) = commit; + let (commit, kv_prune) = commit; if let Some(pruned) = self.death_rows.get(self.pending_prunings) { trace!(target: "state-db", "Pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len()); let index = self.pending_number + self.pending_prunings as u64; commit.data.deleted.extend(pruned.deleted.iter().cloned()); - if let Some(offstate) = offstate_prune.as_mut() { - offstate.0 = std::cmp::max(offstate.0, index); - offstate.1.extend(pruned.offstate_modified.iter().cloned()); + if let Some(kv) = kv_prune.as_mut() { + kv.0 = std::cmp::max(kv.0, index); + kv.1.extend(pruned.kv_modified.iter().cloned()); } else { - *offstate_prune = Some(( + *kv_prune = Some(( index, - pruned.offstate_modified.iter().cloned().collect(), + pruned.kv_modified.iter().cloned().collect(), )); } commit.meta.inserted.push((to_meta_key(LAST_PRUNED, &()), index.encode())); commit.meta.deleted.push(pruned.journal_key.clone()); - commit.meta.deleted.push(pruned.offstate_journal_key.clone()); + commit.meta.deleted.push(pruned.kv_journal_key.clone()); self.pending_prunings += 1; } else { warn!(target: "state-db", "Trying to prune when there's nothing to prune"); @@ -222,28 +222,28 @@ impl RefWindow { pub fn note_canonical(&mut self, hash: &BlockHash, commit: &mut CommitSet) { trace!(target: "state-db", "Adding to pruning window: {:?} ({} inserted, {} deleted)", hash, commit.data.inserted.len(), commit.data.deleted.len()); let inserted = commit.data.inserted.iter().map(|(k, _)| k.clone()).collect(); - let offstate_modified = commit.offstate.iter().map(|(k, _)| k.clone()).collect(); + let kv_modified = commit.kv.iter().map(|(k, _)| k.clone()).collect(); let deleted = ::std::mem::replace(&mut commit.data.deleted, Vec::new()); let journal_record = JournalRecord { hash: hash.clone(), inserted, deleted, }; - let offstate_journal_record = OffstateJournalRecord { - modified: offstate_modified, + let kv_journal_record = KvJournalRecord { + modified: kv_modified, }; let block = self.pending_number + self.death_rows.len() as u64; let journal_key = to_journal_key(block); - let offstate_journal_key = to_offstate_journal_key(block); + let kv_journal_key = to_kv_journal_key(block); commit.meta.inserted.push((journal_key.clone(), journal_record.encode())); - commit.meta.inserted.push((offstate_journal_key.clone(), offstate_journal_record.encode())); + commit.meta.inserted.push((kv_journal_key.clone(), kv_journal_record.encode())); self.import( &journal_record.hash, journal_key, - offstate_journal_key, + kv_journal_key, journal_record.inserted.into_iter(), journal_record.deleted, - offstate_journal_record.modified.into_iter(), + kv_journal_record.modified.into_iter(), ); self.pending_canonicalizations += 1; } @@ -304,10 +304,10 @@ mod tests { #[test] fn prune_one() { let mut db = make_db(&[1, 2, 3]); - db.initialize_offstate(&[1, 2, 3]); + db.initialize_kv(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); let mut commit = (make_commit_both(&[4, 5], &[1, 3]), None); - commit.0.initialize_offstate(&[4, 5], &[1, 3]); + commit.0.initialize_kv(&[4, 5], &[1, 3]); let h = H256::random(); assert!(!commit.0.data.deleted.is_empty()); pruning.note_canonical(&h, &mut commit.0); @@ -316,12 +316,12 @@ mod tests { pruning.apply_pending(); assert!(pruning.have_block(&h)); assert!(commit.0.data.deleted.is_empty()); - //assert!(commit.offstate.is_empty()); + //assert!(commit.kv.is_empty()); assert_eq!(pruning.death_rows.len(), 1); assert_eq!(pruning.death_index.len(), 2); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); - assert!(db.offstate_eq_at(&[1, 2, 3], Some(0))); - assert!(db.offstate_eq(&[2, 4, 5])); + assert!(db.kv_eq_at(&[1, 2, 3], Some(0))); + assert!(db.kv_eq(&[2, 4, 5])); check_journal(&pruning, &db); let mut commit = CommitSetCanonical::default(); @@ -332,8 +332,8 @@ mod tests { assert!(!pruning.have_block(&h)); assert!(db.data_eq(&make_db(&[2, 4, 5]))); // two remains since it is still valid next - assert!(db.offstate_eq_at(&[2], Some(0))); - assert!(db.offstate_eq(&[2, 4, 5])); + assert!(db.kv_eq_at(&[2], Some(0))); + assert!(db.kv_eq(&[2, 4, 5])); assert!(pruning.death_rows.is_empty()); assert!(pruning.death_index.is_empty()); assert_eq!(pruning.pending_number, 1); @@ -342,7 +342,7 @@ mod tests { #[test] fn prune_two() { let mut db = make_db(&[1, 2, 3]); - db.initialize_offstate(&[1, 2, 3]); + db.initialize_kv(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); let mut commit = make_commit_both(&[3, 4], &[1]); pruning.note_canonical(&H256::random(), &mut commit); @@ -352,9 +352,9 @@ mod tests { db.commit(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); - assert!(db.offstate_eq_at(&[1, 2, 3], Some(0))); - assert!(db.offstate_eq_at(&[2, 3, 4], Some(1))); - assert!(db.offstate_eq(&[3, 4, 5])); + assert!(db.kv_eq_at(&[1, 2, 3], Some(0))); + assert!(db.kv_eq_at(&[2, 3, 4], Some(1))); + assert!(db.kv_eq(&[3, 4, 5])); check_journal(&pruning, &db); @@ -364,24 +364,24 @@ mod tests { pruning.apply_pending(); assert!(db.data_eq(&make_db(&[2, 3, 4, 5]))); // 3 exists at 0 and 1 so 0 removed - assert!(db.offstate_eq_at(&[2], Some(0))); - assert!(db.offstate_eq_at(&[2, 3, 4], Some(1))); - assert!(db.offstate_eq(&[3, 4, 5])); + assert!(db.kv_eq_at(&[2], Some(0))); + assert!(db.kv_eq_at(&[2, 3, 4], Some(1))); + assert!(db.kv_eq(&[3, 4, 5])); let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); db.commit_canonical(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[3, 4, 5]))); - assert!(db.offstate_eq_at(&[], Some(0))); - assert!(db.offstate_eq_at(&[3, 4], Some(1))); - assert!(db.offstate_eq(&[3, 4, 5])); + assert!(db.kv_eq_at(&[], Some(0))); + assert!(db.kv_eq_at(&[3, 4], Some(1))); + assert!(db.kv_eq(&[3, 4, 5])); assert_eq!(pruning.pending_number, 2); } #[test] fn prune_two_pending() { let mut db = make_db(&[1, 2, 3]); - db.initialize_offstate(&[1, 2, 3]); + db.initialize_kv(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); let mut commit = (make_commit_both(&[4], &[1]), None); pruning.note_canonical(&H256::random(), &mut commit.0); @@ -390,34 +390,34 @@ mod tests { pruning.note_canonical(&H256::random(), &mut commit.0); db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); - assert!(db.offstate_eq_at(&[1, 2, 3], Some(0))); - assert!(db.offstate_eq_at(&[2, 3, 4], Some(1))); - assert!(db.offstate_eq(&[3, 4, 5])); + assert!(db.kv_eq_at(&[1, 2, 3], Some(0))); + assert!(db.kv_eq_at(&[2, 3, 4], Some(1))); + assert!(db.kv_eq(&[3, 4, 5])); let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[2, 3, 4, 5]))); - assert!(db.offstate_eq_at(&[2, 3], Some(0))); - assert!(db.offstate_eq_at(&[2, 3, 4], Some(1))); - assert!(db.offstate_eq(&[3, 4, 5])); + assert!(db.kv_eq_at(&[2, 3], Some(0))); + assert!(db.kv_eq_at(&[2, 3, 4], Some(1))); + assert!(db.kv_eq(&[3, 4, 5])); let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); db.commit_canonical(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[3, 4, 5]))); - assert!(db.offstate_eq_at(&[], Some(0))); - assert!(db.offstate_eq_at(&[4], Some(1))); - assert!(db.offstate_eq(&[3, 4, 5])); + assert!(db.kv_eq_at(&[], Some(0))); + assert!(db.kv_eq_at(&[4], Some(1))); + assert!(db.kv_eq(&[3, 4, 5])); assert_eq!(pruning.pending_number, 2); } #[test] fn reinserted_survives() { let mut db = make_db(&[1, 2, 3]); - db.initialize_offstate(&[1, 2, 3]); + db.initialize_kv(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); let mut commit = (make_commit_both(&[], &[2]), None); - commit.0.initialize_offstate(&[], &[2]); + commit.0.initialize_kv(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit.0); db.commit_canonical(&commit); let mut commit = (make_commit_both(&[2], &[]), None); @@ -427,10 +427,10 @@ mod tests { pruning.note_canonical(&H256::random(), &mut commit.0); db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); - assert!(db.offstate_eq_at(&[1, 2, 3], Some(0))); - assert!(db.offstate_eq_at(&[1, 3], Some(1))); - assert!(db.offstate_eq_at(&[1, 2, 3], Some(2))); - assert!(db.offstate_eq(&[1, 3])); + assert!(db.kv_eq_at(&[1, 2, 3], Some(0))); + assert!(db.kv_eq_at(&[1, 3], Some(1))); + assert!(db.kv_eq_at(&[1, 2, 3], Some(2))); + assert!(db.kv_eq(&[1, 3])); pruning.apply_pending(); check_journal(&pruning, &db); @@ -439,25 +439,25 @@ mod tests { pruning.prune_one(&mut commit); db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); - assert!(db.offstate_eq_at(&[1, 3], Some(0))); - assert!(db.offstate_eq_at(&[1, 3], Some(1))); - assert!(db.offstate_eq_at(&[1, 2, 3], Some(2))); - assert!(db.offstate_eq(&[1, 3])); + assert!(db.kv_eq_at(&[1, 3], Some(0))); + assert!(db.kv_eq_at(&[1, 3], Some(1))); + assert!(db.kv_eq_at(&[1, 2, 3], Some(2))); + assert!(db.kv_eq(&[1, 3])); let mut commit = CommitSetCanonical::default(); pruning.prune_one(&mut commit); db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); - assert!(db.offstate_eq_at(&[1, 3], Some(0))); - assert!(db.offstate_eq_at(&[1, 3], Some(1))); - assert!(db.offstate_eq_at(&[1, 2, 3], Some(2))); - assert!(db.offstate_eq(&[1, 3])); + assert!(db.kv_eq_at(&[1, 3], Some(0))); + assert!(db.kv_eq_at(&[1, 3], Some(1))); + assert!(db.kv_eq_at(&[1, 2, 3], Some(2))); + assert!(db.kv_eq(&[1, 3])); pruning.prune_one(&mut commit); db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 3]))); - assert!(db.offstate_eq_at(&[1, 3], Some(0))); - assert!(db.offstate_eq_at(&[1, 3], Some(1))); - assert!(db.offstate_eq_at(&[1, 3], Some(2))); - assert!(db.offstate_eq(&[1, 3])); + assert!(db.kv_eq_at(&[1, 3], Some(0))); + assert!(db.kv_eq_at(&[1, 3], Some(1))); + assert!(db.kv_eq_at(&[1, 3], Some(2))); + assert!(db.kv_eq(&[1, 3])); pruning.apply_pending(); assert_eq!(pruning.pending_number, 3); } diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 142397912e8d5..6424c28e5c3a6 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -19,7 +19,7 @@ use std::collections::{HashMap, BTreeMap}; use primitives::H256; use crate::{ - DBValue, ChangeSet, OffstateChangeSet, OffstateKey, MetaDb, NodeDb, OffstateDb, + DBValue, ChangeSet, KvChangeSet, KvKey, MetaDb, NodeDb, KvDb, CommitSet, CommitSetCanonical, }; use historied_data::tree::Serialized; @@ -32,9 +32,9 @@ type Ser<'a> = Serialized<'a, DefaultVersion>; pub struct TestDb { pub data: HashMap, pub meta: HashMap, DBValue>, - pub offstate: HashMap, + pub kv: HashMap, // big heuristic to increment this for canonicalize transaction only - // by checking if there is value in offstate change set. + // by checking if there is value in kv change set. // If empty change set needed, this need to be change manually. // TODO EMCH consider changing commit fun to use last block number explicitely. pub last_block: u64, @@ -48,11 +48,11 @@ impl MetaDb for TestDb { } } -impl OffstateDb for TestDb { +impl KvDb for TestDb { type Error = (); - fn get_offstate(&self, key: &[u8], state: &u64) -> Result, ()> { - Ok(self.offstate.get(key) + fn get_kv(&self, key: &[u8], state: &u64) -> Result, ()> { + Ok(self.kv.get(key) .map(|s| Ser::from_slice(s.as_slice())) .and_then(|s| s.get(*state) .unwrap_or(None) // flatten @@ -60,8 +60,8 @@ impl OffstateDb for TestDb { )) } - fn get_offstate_pairs(&self, state: &u64) -> Vec<(OffstateKey, DBValue)> { - self.offstate.iter().filter_map(|(a, s)| ( + fn get_kv_pairs(&self, state: &u64) -> Vec<(KvKey, DBValue)> { + self.kv.iter().filter_map(|(a, s)| ( Ser::from_slice(s.as_slice()) .get(*state) .unwrap_or(None) // flatten @@ -81,15 +81,15 @@ impl NodeDb for TestDb { impl TestDb { pub fn commit(&mut self, commit: &CommitSet) { - if commit.offstate.len() > 0 { + if commit.kv.len() > 0 { self.last_block += 1; } self.data.extend(commit.data.inserted.iter().cloned()); for k in commit.data.deleted.iter() { self.data.remove(k); } - for (k, o) in commit.offstate.iter() { - let encoded = self.offstate.entry(k.clone()) + for (k, o) in commit.kv.iter() { + let encoded = self.kv.entry(k.clone()) .or_insert_with(|| Ser::default().into_vec()); let mut ser = Ser::from_mut(&mut (*encoded)); ser.push(self.last_block, o.as_ref().map(|v| v.as_slice())); @@ -103,13 +103,13 @@ impl TestDb { pub fn commit_canonical(&mut self, commit: &CommitSetCanonical) { self.commit(&commit.0); - if let Some((block_prune, offstate_prune_key)) = commit.1.as_ref() { - for k in offstate_prune_key.iter() { - match self.offstate.get_mut(k).map(|v| { + if let Some((block_prune, kv_prune_key)) = commit.1.as_ref() { + for k in kv_prune_key.iter() { + match self.kv.get_mut(k).map(|v| { let mut ser = Ser::from_mut(v); ser.prune(*block_prune) }) { - Some(PruneResult::Cleared) => { let _ = self.offstate.remove(k); }, + Some(PruneResult::Cleared) => { let _ = self.kv.remove(k); }, Some(PruneResult::Changed) // changed applyied on mutable buffer without copy. | Some(PruneResult::Unchanged) | None => (), @@ -122,20 +122,20 @@ impl TestDb { self.data == other.data } - pub fn offstate_eq_at(&self, values: &[u64], block: Option) -> bool { + pub fn kv_eq_at(&self, values: &[u64], block: Option) -> bool { let block = block.unwrap_or(self.last_block); - let data = make_offstate_changeset(values, &[]); - let self_offstate: BTreeMap<_, _> = self.get_offstate_pairs(&block).into_iter().collect(); - self_offstate == data.into_iter().filter_map(|(k, v)| v.map(|v| (k,v))).collect() + let data = make_kv_changeset(values, &[]); + let self_kv: BTreeMap<_, _> = self.get_kv_pairs(&block).into_iter().collect(); + self_kv == data.into_iter().filter_map(|(k, v)| v.map(|v| (k,v))).collect() } - pub fn offstate_eq(&self, values: &[u64]) -> bool { - self.offstate_eq_at(values, None) + pub fn kv_eq(&self, values: &[u64]) -> bool { + self.kv_eq_at(values, None) } - pub fn initialize_offstate(&mut self, inserted: &[u64]) { - let data = make_offstate_changeset(inserted, &[]); - self.offstate = data.into_iter() + pub fn initialize_kv(&mut self, inserted: &[u64]) { + let data = make_kv_changeset(inserted, &[]); + self.kv = data.into_iter() .filter_map(|(k, v)| v.map(|v| (k,v))) .map(|(k, v)| { let mut ser = Ser::default(); @@ -158,10 +158,10 @@ pub fn make_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet { } } -pub fn make_offstate_changeset( +pub fn make_kv_changeset( inserted: &[u64], deleted: &[u64], -) -> OffstateChangeSet { +) -> KvChangeSet { inserted.iter() .map(|v| ( H256::from_low_u64_be(*v).as_bytes().to_vec(), @@ -179,7 +179,7 @@ pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet { CommitSet { data: make_changeset(inserted, deleted), meta: ChangeSet::default(), - offstate: OffstateChangeSet::default(), + kv: KvChangeSet::default(), } } @@ -187,14 +187,14 @@ pub fn make_commit_both(inserted: &[u64], deleted: &[u64]) -> CommitSet { CommitSet { data: make_changeset(inserted, deleted), meta: ChangeSet::default(), - offstate: make_offstate_changeset(inserted, deleted), + kv: make_kv_changeset(inserted, deleted), } } impl CommitSet { - pub fn initialize_offstate(&mut self, inserted: &[u64], deleted: &[u64]) { - let data = make_offstate_changeset(inserted, deleted); - self.offstate = data; + pub fn initialize_kv(&mut self, inserted: &[u64], deleted: &[u64]) { + let data = make_kv_changeset(inserted, deleted); + self.kv = data; } } @@ -207,7 +207,7 @@ pub fn make_db(inserted: &[u64]) -> TestDb { }) .collect(), meta: Default::default(), - offstate: Default::default(), + kv: Default::default(), last_block: Default::default(), } } diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index b851f56d28a04..83d315db2b17d 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -21,7 +21,7 @@ use log::warn; use hash_db::Hasher; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::TrieBackendStorage; -use crate::offstate_backend::{OffstateBackend, InMemory as InMemoryOffstateBackend}; +use crate::kv_backend::{KvBackend, InMemory as InMemoryKvBackend}; use trie::{ TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::{TrieDBMut, Layout}, @@ -42,7 +42,7 @@ pub trait Backend: std::fmt::Debug { type TrieBackendStorage: TrieBackendStorage; /// Type of trie backend storage. - type OffstateBackend: OffstateBackend; + type KvBackend: KvBackend; /// Get keyed storage or None if there is nothing associated. fn storage(&self, key: &[u8]) -> Result>, Self::Error>; @@ -104,8 +104,8 @@ pub trait Backend: std::fmt::Debug { I: IntoIterator, Option>)>, H::Out: Ord; - /// Produce transaction for a given offstate information deltas. - fn offstate_transaction(&self, delta: I) -> Self::Transaction + /// Produce transaction for a given kv information deltas. + fn kv_transaction(&self, delta: I) -> Self::Transaction where I: IntoIterator, Option>)>; @@ -118,8 +118,8 @@ pub trait Backend: std::fmt::Debug { /// Get all key/value pairs into a Vec for a child storage. fn child_pairs(&self, child_storage_key: &[u8]) -> Vec<(Vec, Vec)>; - /// Get all key/value pairs of offstate storage. - fn offstate_pairs(&self) -> Vec<(Vec, Vec)>; + /// Get all key/value pairs of kv storage. + fn kv_pairs(&self) -> Vec<(Vec, Vec)>; /// Get all keys with given prefix fn keys(&self, prefix: &[u8]) -> Vec> { @@ -137,7 +137,7 @@ pub trait Backend: std::fmt::Debug { /// Try convert into trie backend. fn as_trie_backend(&mut self) -> Option< - &TrieBackend + &TrieBackend > { None } @@ -149,7 +149,7 @@ pub trait Backend: std::fmt::Debug { &self, delta: I1, child_deltas: I2, - offstate_deltas: I3, + kv_deltas: I3, ) -> (H::Out, Self::Transaction) where I1: IntoIterator, Option>)>, @@ -158,7 +158,7 @@ pub trait Backend: std::fmt::Debug { I3: IntoIterator, Option>)>, ::Out: Ord, { - let mut txs: Self::Transaction = self.offstate_transaction(offstate_deltas); + let mut txs: Self::Transaction = self.kv_transaction(kv_deltas); let mut child_roots: Vec<_> = Default::default(); // child first @@ -184,7 +184,7 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { type Error = T::Error; type Transaction = T::Transaction; type TrieBackendStorage = T::TrieBackendStorage; - type OffstateBackend = T::OffstateBackend; + type KvBackend = T::KvBackend; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { (*self).storage(key) @@ -222,11 +222,11 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { (*self).child_storage_root(storage_key, delta) } - fn offstate_transaction(&self, delta: I) -> Self::Transaction + fn kv_transaction(&self, delta: I) -> Self::Transaction where I: IntoIterator, Option>)> { - (*self).offstate_transaction(delta) + (*self).kv_transaction(delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -241,8 +241,8 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { (*self).child_pairs(child_storage_key) } - fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { - (*self).offstate_pairs() + fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + (*self).kv_pairs() } @@ -280,7 +280,7 @@ impl Consolidate for (U, V) { impl Consolidate for InMemoryTransaction { fn consolidate(&mut self, other: Self) { self.storage.consolidate(other.storage); - self.offstate.consolidate(other.offstate); + self.kv.consolidate(other.kv); } } @@ -309,8 +309,8 @@ impl error::Error for Void { /// tests. pub struct InMemory { inner: HashMap>, HashMap, Vec>>, - offstate: Option, - trie: Option, H, InMemoryOffstateBackend>>, + kv: Option, + trie: Option, H, InMemoryKvBackend>>, _hasher: PhantomData, } @@ -325,7 +325,7 @@ impl Default for InMemory { InMemory { inner: Default::default(), trie: None, - offstate: Some(Default::default()), + kv: Some(Default::default()), _hasher: PhantomData, } } @@ -336,7 +336,7 @@ impl Clone for InMemory { InMemory { inner: self.inner.clone(), trie: None, - offstate: self.offstate.clone(), + kv: self.kv.clone(), _hasher: PhantomData, } } @@ -345,33 +345,33 @@ impl Clone for InMemory { impl PartialEq for InMemory { fn eq(&self, other: &Self) -> bool { self.inner.eq(&other.inner) - && self.offstate().eq(other.offstate()) + && self.kv().eq(other.kv()) } } impl InMemory { - fn offstate(&self) -> &InMemoryOffstateBackend { - if let Some(offstate) = self.offstate.as_ref() { - offstate + fn kv(&self) -> &InMemoryKvBackend { + if let Some(kv) = self.kv.as_ref() { + kv } else { - &self.trie.as_ref().unwrap().offstate_storage + self.trie.as_ref().unwrap().kv_backend() } } - fn offstate_mut(&mut self) -> &mut InMemoryOffstateBackend { - if let Some(offstate) = self.offstate.as_mut() { - offstate + fn kv_mut(&mut self) -> &mut InMemoryKvBackend { + if let Some(kv) = self.kv.as_mut() { + kv } else { - &mut self.trie.as_mut().unwrap().offstate_storage + self.trie.as_mut().unwrap().kv_backend_mut() } } - fn extract_offstate(&mut self) -> InMemoryOffstateBackend { - if let Some(offstate) = self.offstate.take() { - offstate + fn extract_kv(&mut self) -> InMemoryKvBackend { + if let Some(kv) = self.kv.take() { + kv } else { std::mem::replace( - &mut self.trie.as_mut().unwrap().offstate_storage, + self.trie.as_mut().unwrap().kv_backend_mut(), Default::default(), ) } @@ -380,22 +380,22 @@ impl InMemory { /// Copy the state, with applied updates pub fn update(&self, changes: >::Transaction) -> Self { let mut inner: HashMap<_, _> = self.inner.clone(); - let mut offstate: HashMap<_, _> = self.offstate().clone(); + let mut kv: HashMap<_, _> = self.kv().clone(); for (storage_key, key, val) in changes.storage { match val { Some(v) => { inner.entry(storage_key).or_default().insert(key, v); }, None => { inner.entry(storage_key).or_default().remove(&key); }, } } - for (key, val) in changes.offstate { + for (key, val) in changes.kv { match val { - Some(v) => { offstate.insert(key, v); }, - None => { offstate.remove(&key); }, + Some(v) => { kv.insert(key, v); }, + None => { kv.remove(&key); }, } } - let offstate = Some(offstate); - InMemory { inner, offstate, trie: None, _hasher: PhantomData } + let kv = Some(kv); + InMemory { inner, kv, trie: None, _hasher: PhantomData } } } @@ -404,7 +404,7 @@ impl From>, HashMap, Vec>>> for In InMemory { inner: inner, trie: None, - offstate: Some(Default::default()), + kv: Some(Default::default()), _hasher: PhantomData, } } @@ -424,7 +424,7 @@ impl From<( InMemory { inner: inner, trie: None, - offstate: Some(Default::default()), + kv: Some(Default::default()), _hasher: PhantomData, } } @@ -437,7 +437,7 @@ impl From, Vec>> for InMemory { InMemory { inner: expanded, trie: None, - offstate: Some(Default::default()), + kv: Some(Default::default()), _hasher: PhantomData, } } @@ -464,7 +464,7 @@ impl From for InMemory { } } let mut result: InMemory = expanded.into(); - result.offstate_mut().extend(inner.offstate.into_iter().into_iter() + result.kv_mut().extend(inner.kv.into_iter().into_iter() .filter_map(|(k, v)| v.map(|v| (k, v)))); result } @@ -486,14 +486,14 @@ pub struct InMemoryTransaction { /// State trie key values changes (both top and child trie). pub storage: Vec<(Option>, Vec, Option>)>, /// Changes to auxilliary data. - pub offstate: Vec<(Vec, Option>)>, + pub kv: Vec<(Vec, Option>)>, } impl Backend for InMemory { type Error = Void; type Transaction = InMemoryTransaction; type TrieBackendStorage = MemoryDB; - type OffstateBackend = InMemoryOffstateBackend; + type KvBackend = InMemoryKvBackend; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone))) @@ -543,7 +543,7 @@ impl Backend for InMemory { let full_transaction = transaction.into_iter().map(|(k, v)| (None, k, v)).collect(); - (root, InMemoryTransaction { storage: full_transaction, offstate: Default::default() }) + (root, InMemoryTransaction { storage: full_transaction, kv: Default::default() }) } fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) @@ -573,16 +573,16 @@ impl Backend for InMemory { ( root, is_default, - InMemoryTransaction { storage: full_transaction, offstate: Default::default() }, + InMemoryTransaction { storage: full_transaction, kv: Default::default() }, ) } - fn offstate_transaction(&self, delta: I) -> Self::Transaction + fn kv_transaction(&self, delta: I) -> Self::Transaction where I: IntoIterator, Option>)> { - let offstate = delta.into_iter().collect(); - InMemoryTransaction { storage: Default::default(), offstate} + let kv = delta.into_iter().collect(); + InMemoryTransaction { storage: Default::default(), kv} } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -603,8 +603,8 @@ impl Backend for InMemory { .collect() } - fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { - self.offstate().iter().map(|(k, v)| (k.clone(), v.clone())).collect() + fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + self.kv().iter().map(|(k, v)| (k.clone(), v.clone())).collect() } fn keys(&self, prefix: &[u8]) -> Vec> { @@ -622,7 +622,7 @@ impl Backend for InMemory { } fn as_trie_backend(&mut self)-> Option< - &TrieBackend + &TrieBackend > { let mut mdb = MemoryDB::default(); let mut root = None; @@ -650,7 +650,7 @@ impl Backend for InMemory { // Since we are running on a memorydb (not a prefixed memory db), content // is not collision free, so an empty offtrie state can be use (no need // for keyspace). - self.trie = Some(TrieBackend::new(mdb, root, self.extract_offstate())); + self.trie = Some(TrieBackend::new(mdb, root, self.extract_kv())); self.trie.as_ref() } } diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index a90bd1506f6b8..fca828959f441 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -428,7 +428,7 @@ mod test { }) ].into_iter().collect()), ].into_iter().collect(), - offstate: vec![].into_iter().collect(), + kv: vec![].into_iter().collect(), }, committed: OverlayedChangeSet { top: vec![ (EXTRINSIC_INDEX.to_vec(), OverlayedValue { @@ -452,7 +452,7 @@ mod test { }) ].into_iter().collect()), ].into_iter().collect(), - offstate: vec![].into_iter().collect(), + kv: vec![].into_iter().collect(), }, changes_trie_config: Some(config.clone()), }; diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index b80d1e2c19bf0..644060a675231 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -418,9 +418,9 @@ where H: Hasher, let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone())) .chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))); - let offstate_delta = self.overlay.committed.offstate.clone().into_iter() - .chain(self.overlay.prospective.offstate.clone().into_iter()); - let (root, transaction) = self.backend.full_storage_root(delta, child_delta_iter, offstate_delta); + let kv_delta = self.overlay.committed.kv.clone().into_iter() + .chain(self.overlay.prospective.kv.clone().into_iter()); + let (root, transaction) = self.backend.full_storage_root(delta, child_delta_iter, kv_delta); self.storage_transaction = Some((transaction, root)); trace!(target: "state-trace", "{:04x}: Root {}", self.id, diff --git a/core/state-machine/src/offstate_backend.rs b/core/state-machine/src/kv_backend.rs similarity index 85% rename from core/state-machine/src/offstate_backend.rs rename to core/state-machine/src/kv_backend.rs index cb376de9526ce..dae29f8dc0c9e 100644 --- a/core/state-machine/src/offstate_backend.rs +++ b/core/state-machine/src/kv_backend.rs @@ -20,23 +20,23 @@ use std::sync::Arc; use std::collections::HashMap; use std::ops::Deref; -/// This covers offstate values access. +/// This covers kv values access. /// It target a single history state (state machine /// only run for a single history state). -pub trait OffstateBackend: Send + Sync { +pub trait KvBackend: Send + Sync { /// Retrieve a value from storage under given key. fn get(&self, key: &[u8]) -> Result>, String>; /// Return all values (in memory) for this backend, mainly for /// tests. This method should only be use for testing or - /// for small offstate. + /// for small kv. fn pairs(&self) -> Vec<(Vec, Vec)>; } /// need to keep multiple block state. pub type InMemory = HashMap, Vec>; -impl OffstateBackend for InMemory { +impl KvBackend for InMemory { fn get(&self, key: &[u8]) -> Result>, String> { Ok(self.get(key).map(Clone::clone)) } @@ -46,12 +46,12 @@ impl OffstateBackend for InMemory { } } -impl OffstateBackend for Arc { +impl KvBackend for Arc { fn get(&self, key: &[u8]) -> Result>, String> { - OffstateBackend::get(self.deref(), key) + KvBackend::get(self.deref(), key) } fn pairs(&self) -> Vec<(Vec, Vec)> { - OffstateBackend::pairs(self.deref()) + KvBackend::pairs(self.deref()) } } diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index e5dd5ad746e2f..bfa5268e82ead 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -30,7 +30,7 @@ use primitives::{ traits::{BareCryptoStorePtr, CodeExecutor}, hexdisplay::HexDisplay, }; -pub use crate::offstate_backend::{OffstateBackend, InMemory as InMemoryOffstateBackend}; +pub use crate::kv_backend::{KvBackend, InMemory as InMemoryKvBackend}; pub mod backend; mod changes_trie; @@ -43,7 +43,7 @@ mod proving_backend; mod trie_backend; mod trie_backend_essence; // TODO EMCH make it private -pub mod offstate_backend; +pub mod kv_backend; use overlayed_changes::OverlayedChangeSet; pub use trie::{TrieMut, DBValue, MemoryDB}; @@ -480,7 +480,7 @@ pub fn prove_execution_on_trie_backend( ) -> Result<(Vec, Vec>), Box> where S: trie_backend_essence::TrieBackendStorage, - O: OffstateBackend, + O: KvBackend, H: Hasher, Exec: CodeExecutor, H::Out: Ord + 'static, @@ -520,7 +520,7 @@ where /// Check execution proof on proving backend, generated by `prove_execution` call. pub fn execution_proof_check_on_trie_backend( - trie_backend: &TrieBackend, H, InMemoryOffstateBackend>, + trie_backend: &TrieBackend, H, InMemoryKvBackend>, overlay: &mut OverlayedChanges, exec: &Exec, method: &str, @@ -589,7 +589,7 @@ where S: trie_backend_essence::TrieBackendStorage, H: Hasher, H::Out: Ord, - O: OffstateBackend, + O: KvBackend, I: IntoIterator, I::Item: AsRef<[u8]>, { @@ -612,7 +612,7 @@ where S: trie_backend_essence::TrieBackendStorage, H: Hasher, H::Out: Ord, - O: OffstateBackend, + O: KvBackend, I: IntoIterator, I::Item: AsRef<[u8]>, { @@ -674,7 +674,7 @@ where /// Check storage read proof on pre-created proving backend. pub fn read_proof_check_on_proving_backend( - proving_backend: &TrieBackend, H, InMemoryOffstateBackend>, + proving_backend: &TrieBackend, H, InMemoryKvBackend>, key: &[u8], ) -> Result>, Box> where @@ -686,7 +686,7 @@ where /// Check child storage read proof on pre-created proving backend. pub fn read_child_proof_check_on_proving_backend( - proving_backend: &TrieBackend, H, InMemoryOffstateBackend>, + proving_backend: &TrieBackend, H, InMemoryKvBackend>, storage_key: &[u8], key: &[u8], ) -> Result>, Box> diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index ae87c3678c7f1..267a4d014047c 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -57,8 +57,8 @@ pub struct OverlayedChangeSet { pub top: HashMap, OverlayedValue>, /// Child storage changes. pub children: HashMap, HashMap, OverlayedValue>>, - /// Offstate storage changes. - pub offstate: HashMap, Option>>, + /// Kv storage changes. + pub kv: HashMap, Option>>, } #[cfg(test)] @@ -67,7 +67,7 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { Self { top: iter.into_iter().collect(), children: Default::default(), - offstate: Default::default(), + kv: Default::default(), } } } @@ -75,14 +75,14 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { impl OverlayedChangeSet { /// Whether the change set is empty. pub fn is_empty(&self) -> bool { - self.top.is_empty() && self.children.is_empty() && self.offstate.is_empty() + self.top.is_empty() && self.children.is_empty() && self.kv.is_empty() } /// Clear the change set. pub fn clear(&mut self) { self.top.clear(); self.children.clear(); - self.offstate.clear(); + self.kv.clear(); } } @@ -139,9 +139,9 @@ impl OverlayedChanges { /// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. - pub fn offstate_storage(&self, key: &[u8]) -> Option> { - self.prospective.offstate.get(key) - .or_else(|| self.committed.offstate.get(key)) + pub fn kv_storage(&self, key: &[u8]) -> Option> { + self.prospective.kv.get(key) + .or_else(|| self.committed.kv.get(key)) .map(|x| x.as_ref().map(AsRef::as_ref)) } @@ -177,8 +177,8 @@ impl OverlayedChanges { /// Inserts the given key-value pair as an auxilliary data. /// /// `None` can be used to delete a value specified by the given key. - pub(crate) fn set_offstate_storage(&mut self, key: Vec, val: Option>) { - self.prospective.offstate.insert(key, val); + pub(crate) fn set_kv_storage(&mut self, key: Vec, val: Option>) { + self.prospective.kv.insert(key, val); } /// Clear child storage of given storage key. @@ -305,8 +305,8 @@ impl OverlayedChanges { .extend(prospective_extrinsics); } } - for (key, val) in self.prospective.offstate.drain() { - self.committed.offstate.insert(key, val); + for (key, val) in self.prospective.kv.drain() { + self.committed.kv.insert(key, val); } for (storage_key, mut map) in self.prospective.children.drain() { let map_dest = self.committed.children.entry(storage_key).or_default(); @@ -336,7 +336,7 @@ impl OverlayedChanges { (self.committed.top.into_iter().map(|(k, v)| (k, v.value)), self.committed.children.into_iter() .map(|(sk, v)| (sk, v.into_iter().map(|(k, v)| (k, v.value)))), - self.committed.offstate.into_iter()) + self.committed.kv.into_iter()) } /// Inserts storage entry responsible for current extrinsic index. @@ -395,39 +395,39 @@ mod tests { let mut overlayed = OverlayedChanges::default(); let key = vec![42, 69, 169, 142]; - let offstate_key = vec![43, 69, 169, 142]; + let kv_key = vec![43, 69, 169, 142]; assert!(overlayed.storage(&key).is_none()); overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); - overlayed.set_offstate_storage(offstate_key.clone(), Some(vec![1, 2, 2])); + overlayed.set_kv_storage(kv_key.clone(), Some(vec![1, 2, 2])); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); - assert_eq!(overlayed.offstate_storage(&offstate_key).unwrap(), Some(&[1, 2, 2][..])); + assert_eq!(overlayed.kv_storage(&kv_key).unwrap(), Some(&[1, 2, 2][..])); overlayed.commit_prospective(); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); - assert_eq!(overlayed.offstate_storage(&offstate_key).unwrap(), Some(&[1, 2, 2][..])); + assert_eq!(overlayed.kv_storage(&kv_key).unwrap(), Some(&[1, 2, 2][..])); overlayed.set_storage(key.clone(), Some(vec![])); - overlayed.set_offstate_storage(offstate_key.clone(), Some(vec![2])); + overlayed.set_kv_storage(kv_key.clone(), Some(vec![2])); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[][..])); - assert_eq!(overlayed.offstate_storage(&offstate_key).unwrap(), Some(&[2][..])); + assert_eq!(overlayed.kv_storage(&kv_key).unwrap(), Some(&[2][..])); overlayed.set_storage(key.clone(), None); - overlayed.set_offstate_storage(offstate_key.clone(), None); + overlayed.set_kv_storage(kv_key.clone(), None); assert!(overlayed.storage(&key).unwrap().is_none()); - assert!(overlayed.offstate_storage(&offstate_key).unwrap().is_none()); + assert!(overlayed.kv_storage(&kv_key).unwrap().is_none()); overlayed.discard_prospective(); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); - assert_eq!(overlayed.offstate_storage(&offstate_key).unwrap(), Some(&[1, 2, 2][..])); + assert_eq!(overlayed.kv_storage(&kv_key).unwrap(), Some(&[1, 2, 2][..])); overlayed.set_storage(key.clone(), None); - overlayed.set_offstate_storage(offstate_key.clone(), None); + overlayed.set_kv_storage(kv_key.clone(), None); overlayed.commit_prospective(); assert!(overlayed.storage(&key).unwrap().is_none()); - assert!(overlayed.offstate_storage(&offstate_key).unwrap().is_none()); + assert!(overlayed.kv_storage(&kv_key).unwrap().is_none()); } #[test] diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index 5eca9e3bcef5f..e615e9ec5c4a0 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -28,7 +28,7 @@ pub use trie::trie_types::{Layout, TrieError}; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend}; -use crate::offstate_backend::{OffstateBackend, InMemory as InMemoryOffstateBackend}; +use crate::kv_backend::{KvBackend, InMemory as InMemoryKvBackend}; /// Patricia trie-based backend essence which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. @@ -107,7 +107,7 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> pub struct ProvingBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher, - O: 'a + OffstateBackend, + O: 'a + KvBackend, > { backend: &'a TrieBackend, proof_recorder: Rc>>, @@ -116,7 +116,7 @@ pub struct ProvingBackend<'a, impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher, - O: 'a + OffstateBackend, + O: 'a + KvBackend, > ProvingBackend<'a, S, H, O> { /// Create new proving backend. pub fn new(backend: &'a TrieBackend) -> Self { @@ -151,7 +151,7 @@ impl<'a, impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher, - O: 'a + OffstateBackend, + O: 'a + KvBackend, > std::fmt::Debug for ProvingBackend<'a, S, H, O> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "ProvingBackend") @@ -163,12 +163,12 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> S: 'a + TrieBackendStorage, H: 'a + Hasher, H::Out: Ord, - O: 'a + OffstateBackend, + O: 'a + KvBackend, { type Error = String; type Transaction = (S::Overlay, Vec<(Vec, Option>)>); type TrieBackendStorage = PrefixedMemoryDB; - type OffstateBackend = O; + type KvBackend = O; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { ProvingBackendEssence { @@ -214,8 +214,8 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> self.backend.child_pairs(storage_key) } - fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { - self.backend.offstate_pairs() + fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + self.backend.kv_pairs() } fn keys(&self, prefix: &[u8]) -> Vec> { @@ -240,11 +240,11 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> self.backend.child_storage_root(storage_key, delta) } - fn offstate_transaction(&self, delta: I) -> Self::Transaction + fn kv_transaction(&self, delta: I) -> Self::Transaction where I: IntoIterator, Option>)> { - self.backend.offstate_transaction(delta) + self.backend.kv_transaction(delta) } } @@ -253,17 +253,17 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> pub fn create_proof_check_backend( root: H::Out, proof: Vec> -) -> Result, H, InMemoryOffstateBackend>, Box> +) -> Result, H, InMemoryKvBackend>, Box> where H: Hasher, { let db = create_proof_check_backend_storage(proof); - // run on empty offstate (current proof does not require - // offstate). - let offstate = InMemoryOffstateBackend::default(); + // run on empty kv (current proof does not require + // kv). + let kv = InMemoryKvBackend::default(); if db.contains(&root, EMPTY_PREFIX) { - Ok(TrieBackend::new(db, root, offstate)) + Ok(TrieBackend::new(db, root, kv)) } else { Err(Box::new(ExecutionError::InvalidProof)) } @@ -290,11 +290,11 @@ mod tests { use super::*; use primitives::{Blake2Hasher, child_storage_key::ChildStorageKey}; - type OffstateBackend = InMemoryOffstateBackend; + type KvBackend = InMemoryKvBackend; fn test_proving<'a>( - trie_backend: &'a TrieBackend, Blake2Hasher, OffstateBackend>, - ) -> ProvingBackend<'a, PrefixedMemoryDB, Blake2Hasher, OffstateBackend> { + trie_backend: &'a TrieBackend, Blake2Hasher, KvBackend>, + ) -> ProvingBackend<'a, PrefixedMemoryDB, Blake2Hasher, KvBackend> { ProvingBackend::new(trie_backend) } @@ -338,7 +338,7 @@ mod tests { let in_memory = InMemory::::default(); let mut in_memory = in_memory.update(InMemoryTransaction { storage: contents, - offstate: Default::default(), + kv: Default::default(), }); let in_memory_root = in_memory.storage_root(::std::iter::empty()).0; @@ -371,7 +371,7 @@ mod tests { let in_memory = InMemory::::default(); let mut in_memory = in_memory.update(InMemoryTransaction { storage: contents, - offstate: Default::default(), + kv: Default::default(), }); let in_memory_root = in_memory.full_storage_root::<_, Vec<_>, _, _>( ::std::iter::empty(), diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index 33a497e5c9280..abce0ffdf4589 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -84,15 +84,15 @@ impl TestExternalities { pub fn insert(&mut self, k: Vec, v: Vec) { self.backend = self.backend.update(InMemoryTransaction { storage: vec![(None, k, Some(v))], - offstate: Default::default(), + kv: Default::default(), }); } /// Insert key/value into ofstate information backend - pub fn insert_offstate(&mut self, k: Vec, v: Vec) { + pub fn insert_kv(&mut self, k: Vec, v: Vec) { self.backend = self.backend.update(InMemoryTransaction { storage: Default::default(), - offstate: vec![(k, Some(v))], + kv: vec![(k, Some(v))], }); } @@ -125,12 +125,12 @@ impl TestExternalities { .collect::>() }); - let offstate = self.overlay.committed.offstate.clone().into_iter() - .chain(self.overlay.prospective.offstate.clone().into_iter()); + let kv = self.overlay.committed.kv.clone().into_iter() + .chain(self.overlay.prospective.kv.clone().into_iter()); self.backend.update(InMemoryTransaction { storage: top.chain(children).collect(), - offstate: offstate.collect(), + kv: kv.collect(), }) } } @@ -263,7 +263,7 @@ impl Externalities for TestExternalities where let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone())) .chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))); - // transaction not used afterward, so not using offstate. + // transaction not used afterward, so not using kv. self.backend.full_storage_root(delta, child_delta_iter, None).0 } diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index 4dc40ead2a24e..90512c131104b 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -22,22 +22,22 @@ use trie::{Trie, delta_trie_root, default_child_trie_root, child_delta_trie_root use trie::trie_types::{TrieDB, TrieError, Layout}; use crate::trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral}; use crate::Backend; -use crate::offstate_backend::OffstateBackend; +use crate::kv_backend::KvBackend; use primitives::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. -/// TODO EMCH with offstate in backend this should be renamed eg StateBackend. -pub struct TrieBackend, H: Hasher, O: OffstateBackend> { +/// TODO EMCH with kv in backend this should be renamed eg StateBackend. +pub struct TrieBackend, H: Hasher, O: KvBackend> { essence: TrieBackendEssence, - pub offstate_storage: O, + kv_storage: O, } -impl, O: OffstateBackend, H: Hasher> TrieBackend { +impl, O: KvBackend, H: Hasher> TrieBackend { /// Create new trie-based backend. - pub fn new(storage: S, root: H::Out, offstate_storage: O) -> Self { + pub fn new(storage: S, root: H::Out, kv_storage: O) -> Self { TrieBackend { essence: TrieBackendEssence::new(storage, root), - offstate_storage, + kv_storage, } } @@ -51,6 +51,16 @@ impl, O: OffstateBackend, H: Hasher> TrieBackend &O { + &self.kv_storage + } + + /// Get key value storage backend mutable reference. + pub fn kv_backend_mut(&mut self) -> &mut O { + &mut self.kv_storage + } + /// Get trie root. pub fn root(&self) -> &H::Out { self.essence.root() @@ -63,9 +73,9 @@ impl, O: OffstateBackend, H: Hasher> TrieBackend Result>, String> { - const PREFIX_KEYSPACE: &'static[u8] = b"offstate_keyspace"; + const PREFIX_KEYSPACE: &'static[u8] = b"kv_keyspace"; // TODO EMCH do prefixing manually. - self.offstate_storage.get(key) + self.kv_storage.get(key) } } @@ -73,7 +83,7 @@ impl, O: OffstateBackend, H: Hasher> TrieBackend, H: Hasher, - O: OffstateBackend, + O: KvBackend, > std::fmt::Debug for TrieBackend { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "TrieBackend") @@ -83,14 +93,14 @@ impl< impl< S: TrieBackendStorage, H: Hasher, - O: OffstateBackend, + O: KvBackend, > Backend for TrieBackend where H::Out: Ord, { type Error = String; type Transaction = (S::Overlay, Vec<(Vec, Option>)>); type TrieBackendStorage = S; - type OffstateBackend = O; + type KvBackend = O; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { self.essence.storage(key) @@ -181,8 +191,8 @@ impl< } } - fn offstate_pairs(&self) -> Vec<(Vec, Vec)> { - self.offstate_storage.pairs() + fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + self.kv_storage.pairs() } fn keys(&self, prefix: &[u8]) -> Vec> { @@ -264,7 +274,7 @@ impl< (root, is_default, (write_overlay, Default::default())) } - fn offstate_transaction(&self, delta: I) -> Self::Transaction + fn kv_transaction(&self, delta: I) -> Self::Transaction where I: IntoIterator, Option>)> { @@ -272,7 +282,7 @@ impl< } fn as_trie_backend(&mut self) -> Option< - &TrieBackend + &TrieBackend > { Some(self) } @@ -286,9 +296,9 @@ pub mod tests { use trie::{TrieMut, PrefixedMemoryDB, trie_types::TrieDBMut}; use super::*; - type OffstateBackend = crate::offstate_backend::InMemory; + type KvBackend = crate::kv_backend::InMemory; - fn test_db() -> (PrefixedMemoryDB, H256, OffstateBackend) { + fn test_db() -> (PrefixedMemoryDB, H256, KvBackend) { let mut root = H256::default(); let mut mdb = PrefixedMemoryDB::::default(); { @@ -311,16 +321,16 @@ pub mod tests { } } // empty history. - let mut offstate = crate::offstate_backend::InMemory::default(); - offstate.insert(b"offstate1".to_vec(), b"off_value1".to_vec()); - offstate.insert(b"offstate2".to_vec(), b"off_value2".to_vec()); - (mdb, root, offstate) + let mut kv = crate::kv_backend::InMemory::default(); + kv.insert(b"kv1".to_vec(), b"kv_value1".to_vec()); + kv.insert(b"kv2".to_vec(), b"kv_value2".to_vec()); + (mdb, root, kv) } pub(crate) fn test_trie( - ) -> TrieBackend, Blake2Hasher, OffstateBackend> { - let (mdb, root, offstate) = test_db(); - TrieBackend::new(mdb, root, offstate) + ) -> TrieBackend, Blake2Hasher, KvBackend> { + let (mdb, root, kv) = test_db(); + TrieBackend::new(mdb, root, kv) } #[test] @@ -340,7 +350,7 @@ pub mod tests { #[test] fn pairs_are_empty_on_empty_storage() { - assert!(TrieBackend::, Blake2Hasher, OffstateBackend>::new( + assert!(TrieBackend::, Blake2Hasher, KvBackend>::new( Default::default(), Default::default(), Default::default(), diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index 7cfec75fcd9d2..35d282421b0e7 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -1023,7 +1023,7 @@ mod test { for a in 2..6 { assert_eq!(item.get(a), Some(Some(&[a as u8][..]))); } - + item.prune(4); for a in 1..5 { assert_eq!(item.get(a), None); @@ -1031,7 +1031,7 @@ mod test { for a in 5..6 { assert_eq!(item.get(a), Some(Some(&[a as u8][..]))); } - + item.prune(80); for a in 1..4 { assert_eq!(item.get(a), None); @@ -1055,7 +1055,7 @@ mod test { assert_eq!(item.get(1), None); assert_eq!(item.get(2), None); assert_eq!(item.get(3), Some(Some(&[3][..]))); - + // prune skip unrelevant delete let mut item: Serialized = Default::default(); item.push(1, Some(&[1 as u8])); @@ -1075,7 +1075,7 @@ mod test { assert_eq!(item.get(2), None); assert_eq!(item.get(3), None); assert_eq!(item.get(4), Some(Some(&[4][..]))); - + // prune delete at block let mut item: Serialized = Default::default(); item.push(0, Some(&[0 as u8])); From 35bc11e8cf72dc29b3eea94b638662ee5f8d34df Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Oct 2019 19:30:42 +0200 Subject: [PATCH 47/68] switch overlay transaction to the inner hashmap storage, only change to vec for compactness in state_db (previously we where removing duplicate with inner map, now we keep map up to). --- core/client/db/src/lib.rs | 30 ++++++++++++---------- core/client/db/src/storage_cache.rs | 2 +- core/client/src/light/backend.rs | 4 +-- core/state-machine/src/backend.rs | 31 ++++++++++++----------- core/state-machine/src/kv_backend.rs | 14 +++++----- core/state-machine/src/proving_backend.rs | 5 ++-- core/state-machine/src/testing.rs | 2 +- core/state-machine/src/trie_backend.rs | 13 ++++++---- 8 files changed, 55 insertions(+), 46 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 2c762a303711c..fcba31456cd16 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -205,7 +205,7 @@ impl StateBackend for RefTrackingState { self.state.child_pairs(child_storage_key) } - fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + fn kv_pairs(&self) -> HashMap, Option>> { self.state.kv_pairs() } @@ -470,10 +470,9 @@ impl HeaderMetadata for BlockchainDb { /// Database transaction pub struct BlockImportOperation { old_state: CachingState, Block>, - db_updates: (PrefixedMemoryDB, Vec<(Vec, Option>)>), + db_updates: (PrefixedMemoryDB, HashMap, Option>>), storage_updates: StorageCollection, child_storage_updates: ChildStorageCollection, - // TODO EMCH kv update and kv values in cache changes_trie_updates: MemoryDB, changes_trie_cache_update: Option>>, pending_block: Option>, @@ -526,7 +525,7 @@ where Block: BlockT, fn update_db_storage( &mut self, - update: (PrefixedMemoryDB, Vec<(Vec, Option>)>), + update: (PrefixedMemoryDB, HashMap, Option>>), ) -> ClientResult<()> { self.db_updates = update; Ok(()) @@ -658,8 +657,12 @@ impl state_machine::KvBackend for StorageDbAt Vec<(Vec, Vec)> { + fn pairs(&self) -> HashMap, Option>> { self.storage_db.state_db.get_kv_pairs(&self.state, self.storage_db.deref()) + // function pairs should not be call except for testing so the conversion + // should be fine here. Note that storage. + // TODO EMCH try delete this + .into_iter().map(|(k, v)| (k, Some(v))).collect() } } @@ -982,8 +985,7 @@ impl> Backend { op.set_block_data(header, body, justification, new_block_state).unwrap(); op.update_db_storage(InMemoryTransaction { storage, - kv: state.kv_pairs().into_iter() - .map(|(k, v)| (k, Some(v))).collect(), + kv: state.kv_pairs().clone(), }).unwrap(); inmem.commit_operation(op).unwrap(); } @@ -1239,11 +1241,9 @@ impl> Backend { changeset.deleted.push(key); } } - // remove duplicate TODO EMCH very bad - let map_kv: HashMap<_, _> = operation.db_updates.1.into_iter().collect(); - // TODO EMCH !!! keep map_kv for KvChangeSet type??? - let kv_changeset: state_db::KvChangeSet> = map_kv.into_iter() - .collect(); + let kv_changeset: state_db::KvChangeSet> = operation.db_updates.1 + // use as vec from hash map + .into_iter().collect(); let number_u64 = number.saturated_into::(); let commit = self.storage.state_db.insert_block( &hash, @@ -1942,9 +1942,11 @@ mod tests { assert_eq!(state.storage(&[1, 3, 5]).unwrap(), None); assert_eq!(state.storage(&[1, 2, 3]).unwrap(), Some(vec![9, 9, 9])); assert_eq!(state.storage(&[5, 5, 5]).unwrap(), Some(vec![4, 5, 6])); - assert_eq!(state.kv_pairs(), vec![(vec![5, 5, 5], vec![4, 5, 6])]); + assert_eq!(state.kv_pairs(), vec![ + (vec![5, 5, 5], Some(vec![4, 5, 6])) + ].into_iter().collect()); let state = db.state_at(BlockId::Number(0)).unwrap(); - assert_eq!(state.kv_pairs(), vec![]); + assert_eq!(state.kv_pairs(), vec![].into_iter().collect()); } } diff --git a/core/client/db/src/storage_cache.rs b/core/client/db/src/storage_cache.rs index 09165810c2621..9a0bc7df673f0 100644 --- a/core/client/db/src/storage_cache.rs +++ b/core/client/db/src/storage_cache.rs @@ -590,7 +590,7 @@ impl, B: BlockT> StateBackend for CachingState< self.state.child_pairs(storage_key) } - fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + fn kv_pairs(&self) -> HashMap, Option>> { self.state.kv_pairs() } diff --git a/core/client/src/light/backend.rs b/core/client/src/light/backend.rs index 12a4328920833..a161abaf6addf 100644 --- a/core/client/src/light/backend.rs +++ b/core/client/src/light/backend.rs @@ -449,10 +449,10 @@ impl StateBackend for GenesisOrUnavailableState } } - fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + fn kv_pairs(&self) -> HashMap, Option>> { match *self { GenesisOrUnavailableState::Genesis(ref state) => state.kv_pairs(), - GenesisOrUnavailableState::Unavailable => Vec::new(), + GenesisOrUnavailableState::Unavailable => Default::default(), } } diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index 83d315db2b17d..ce2c001217b6d 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -119,7 +119,7 @@ pub trait Backend: std::fmt::Debug { fn child_pairs(&self, child_storage_key: &[u8]) -> Vec<(Vec, Vec)>; /// Get all key/value pairs of kv storage. - fn kv_pairs(&self) -> Vec<(Vec, Vec)>; + fn kv_pairs(&self) -> HashMap, Option>>; /// Get all keys with given prefix fn keys(&self, prefix: &[u8]) -> Vec> { @@ -241,7 +241,7 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { (*self).child_pairs(child_storage_key) } - fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + fn kv_pairs(&self) -> HashMap, Option>> { (*self).kv_pairs() } @@ -269,6 +269,12 @@ impl Consolidate for Vec { } } +impl Consolidate for HashMap { + fn consolidate(&mut self, other: Self) { + self.extend(other); + } +} + impl Consolidate for (U, V) { fn consolidate(&mut self, other: Self) { self.0.consolidate(other.0); @@ -387,12 +393,7 @@ impl InMemory { None => { inner.entry(storage_key).or_default().remove(&key); }, } } - for (key, val) in changes.kv { - match val { - Some(v) => { kv.insert(key, v); }, - None => { kv.remove(&key); }, - } - } + kv.extend(changes.kv); let kv = Some(kv); InMemory { inner, kv, trie: None, _hasher: PhantomData } @@ -464,8 +465,7 @@ impl From for InMemory { } } let mut result: InMemory = expanded.into(); - result.kv_mut().extend(inner.kv.into_iter().into_iter() - .filter_map(|(k, v)| v.map(|v| (k, v)))); + *result.kv_mut() = inner.kv; result } } @@ -485,8 +485,8 @@ impl InMemory { pub struct InMemoryTransaction { /// State trie key values changes (both top and child trie). pub storage: Vec<(Option>, Vec, Option>)>, - /// Changes to auxilliary data. - pub kv: Vec<(Vec, Option>)>, + /// Changes to non trie key value datas. + pub kv: HashMap, Option>>, } impl Backend for InMemory { @@ -581,7 +581,8 @@ impl Backend for InMemory { where I: IntoIterator, Option>)> { - let kv = delta.into_iter().collect(); + let mut kv = self.kv().clone(); + kv.extend(delta.into_iter()); InMemoryTransaction { storage: Default::default(), kv} } @@ -603,8 +604,8 @@ impl Backend for InMemory { .collect() } - fn kv_pairs(&self) -> Vec<(Vec, Vec)> { - self.kv().iter().map(|(k, v)| (k.clone(), v.clone())).collect() + fn kv_pairs(&self) -> HashMap, Option>> { + self.kv().clone() } fn keys(&self, prefix: &[u8]) -> Vec> { diff --git a/core/state-machine/src/kv_backend.rs b/core/state-machine/src/kv_backend.rs index dae29f8dc0c9e..8f3cb30752128 100644 --- a/core/state-machine/src/kv_backend.rs +++ b/core/state-machine/src/kv_backend.rs @@ -30,19 +30,21 @@ pub trait KvBackend: Send + Sync { /// Return all values (in memory) for this backend, mainly for /// tests. This method should only be use for testing or /// for small kv. - fn pairs(&self) -> Vec<(Vec, Vec)>; + /// It returns a map to be aligned with internal in memory storage + /// types. + fn pairs(&self) -> HashMap, Option>>; } /// need to keep multiple block state. -pub type InMemory = HashMap, Vec>; +pub type InMemory = HashMap, Option>>; impl KvBackend for InMemory { fn get(&self, key: &[u8]) -> Result>, String> { - Ok(self.get(key).map(Clone::clone)) + Ok(self.get(key).map(Clone::clone).unwrap_or(None)) } - fn pairs(&self) -> Vec<(Vec, Vec)> { - self.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + fn pairs(&self) -> HashMap, Option>> { + self.clone() } } @@ -51,7 +53,7 @@ impl KvBackend for Arc { KvBackend::get(self.deref(), key) } - fn pairs(&self) -> Vec<(Vec, Vec)> { + fn pairs(&self) -> HashMap, Option>> { KvBackend::pairs(self.deref()) } } diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index e615e9ec5c4a0..f8b97fcccfc25 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -23,6 +23,7 @@ use trie::{ MemoryDB, PrefixedMemoryDB, default_child_trie_root, read_trie_value_with, read_child_trie_value_with, record_all_keys }; +use std::collections::HashMap; pub use trie::Recorder; pub use trie::trie_types::{Layout, TrieError}; use crate::trie_backend::TrieBackend; @@ -166,7 +167,7 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> O: 'a + KvBackend, { type Error = String; - type Transaction = (S::Overlay, Vec<(Vec, Option>)>); + type Transaction = (S::Overlay, HashMap, Option>>); type TrieBackendStorage = PrefixedMemoryDB; type KvBackend = O; @@ -214,7 +215,7 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> self.backend.child_pairs(storage_key) } - fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + fn kv_pairs(&self) -> HashMap, Option>> { self.backend.kv_pairs() } diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index abce0ffdf4589..b958c02e298bc 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -92,7 +92,7 @@ impl TestExternalities { pub fn insert_kv(&mut self, k: Vec, v: Vec) { self.backend = self.backend.update(InMemoryTransaction { storage: Default::default(), - kv: vec![(k, Some(v))], + kv: Some((k, Some(v))).into_iter().collect(), }); } diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index 90512c131104b..fb937b3bcd0dc 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -24,6 +24,7 @@ use crate::trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Epheme use crate::Backend; use crate::kv_backend::KvBackend; use primitives::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; +use std::collections::HashMap; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. /// TODO EMCH with kv in backend this should be renamed eg StateBackend. @@ -98,7 +99,7 @@ impl< H::Out: Ord, { type Error = String; - type Transaction = (S::Overlay, Vec<(Vec, Option>)>); + type Transaction = (S::Overlay, HashMap, Option>>); type TrieBackendStorage = S; type KvBackend = O; @@ -191,7 +192,7 @@ impl< } } - fn kv_pairs(&self) -> Vec<(Vec, Vec)> { + fn kv_pairs(&self) -> HashMap, Option>> { self.kv_storage.pairs() } @@ -278,7 +279,9 @@ impl< where I: IntoIterator, Option>)> { - (Default::default(), delta.into_iter().collect()) + let mut result = self.kv_storage.pairs(); + result.extend(delta.into_iter()); + (Default::default(), result) } fn as_trie_backend(&mut self) -> Option< @@ -322,8 +325,8 @@ pub mod tests { } // empty history. let mut kv = crate::kv_backend::InMemory::default(); - kv.insert(b"kv1".to_vec(), b"kv_value1".to_vec()); - kv.insert(b"kv2".to_vec(), b"kv_value2".to_vec()); + kv.insert(b"kv1".to_vec(), Some(b"kv_value1".to_vec())); + kv.insert(b"kv2".to_vec(), Some(b"kv_value2".to_vec())); (mdb, root, kv) } From d1112889980ccffe2f192acfba76daa005efc411 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Oct 2019 11:10:50 +0200 Subject: [PATCH 48/68] Remove commit kv from revert. --- core/client/db/src/lib.rs | 26 ++++++++++++++------------ core/state-machine/src/trie_backend.rs | 2 +- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index fcba31456cd16..881dcdb05817e 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -90,6 +90,7 @@ const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u32 = 32768; const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); /// DB-backed patricia trie state, transaction type is an overlay of changes to commit. +/// A simple key value backend is also accessible for direct key value storage. pub type DbState = state_machine::TrieBackend< Arc>, Blake2Hasher, @@ -659,9 +660,8 @@ impl state_machine::KvBackend for StorageDbAt HashMap, Option>> { self.storage_db.state_db.get_kv_pairs(&self.state, self.storage_db.deref()) - // function pairs should not be call except for testing so the conversion - // should be fine here. Note that storage. - // TODO EMCH try delete this + // Not that we do not store deletion. + // Function pairs should not really be call for this backend. .into_iter().map(|(k, v)| (k, Some(v))).collect() } } @@ -1253,9 +1253,6 @@ impl> Backend { kv_changeset, ).map_err(|e: state_db::Error| client::error::Error::from(format!("State database error: {:?}", e)))?; - // TODO EMCH wrong block number: there is no kv change on block insert - // TODO split apply_state_commit and do an assertion in apply_state_commit - // that there is no kv stuff anymore apply_state_commit(&mut transaction, commit, &self.storage.db, number_u64).map_err(|err| client::error::Error::Backend( format!("Error building commit transaction : {}", err) @@ -1492,6 +1489,14 @@ fn apply_state_commit_inner( } } + apply_state_commit_no_kv(transaction, commit, db); + Ok(()) +} + +fn apply_state_commit_no_kv( + transaction: &mut DBTransaction, + commit: state_db::CommitSet>, + db: &Arc) { for (key, val) in commit.data.inserted.into_iter() { transaction.put(columns::STATE, &key[..], &val); @@ -1505,7 +1510,7 @@ fn apply_state_commit_inner( for key in commit.meta.deleted.into_iter() { transaction.delete(columns::STATE_META, &key[..]); } - Ok(()) + } @@ -1643,11 +1648,8 @@ impl client::backend::Backend for Backend whe let mut transaction = DBTransaction::new(); match self.storage.state_db.revert_one() { Some(commit) => { - apply_state_commit(&mut transaction, commit, &self.storage.db, c).map_err(|err| - client::error::Error::Backend( - format!("Error building commit transaction : {}", err) - ) - )?; + debug_assert!(commit.kv.is_empty(), "revert do not change key value store"); + apply_state_commit_no_kv(&mut transaction, commit, &self.storage.db); let removed = self.blockchain.header(BlockId::Number(best))?.ok_or_else( || client::error::Error::UnknownBlock( format!("Error reverting to {}. Block hash not found.", best)))?; diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index fb937b3bcd0dc..c7b83d289ca74 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -27,7 +27,7 @@ use primitives::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; use std::collections::HashMap; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. -/// TODO EMCH with kv in backend this should be renamed eg StateBackend. +/// A simple key value backend is also accessible for direct key value storage. pub struct TrieBackend, H: Hasher, O: KvBackend> { essence: TrieBackendEssence, kv_storage: O, From dd9dddb0d260e3fe64429c3089c3127e2a0dbe12 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Oct 2019 11:32:28 +0200 Subject: [PATCH 49/68] test db better handling --- core/client/db/src/lib.rs | 7 +++---- core/state-db/src/pruning.rs | 20 ++++++++++---------- core/state-db/src/test.rs | 26 +++++++++++--------------- 3 files changed, 24 insertions(+), 29 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 881dcdb05817e..8e31d11071b6a 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -1489,14 +1489,14 @@ fn apply_state_commit_inner( } } - apply_state_commit_no_kv(transaction, commit, db); + apply_state_commit_no_kv(transaction, commit); Ok(()) } fn apply_state_commit_no_kv( transaction: &mut DBTransaction, commit: state_db::CommitSet>, - db: &Arc) { + ) { for (key, val) in commit.data.inserted.into_iter() { transaction.put(columns::STATE, &key[..], &val); @@ -1649,7 +1649,7 @@ impl client::backend::Backend for Backend whe match self.storage.state_db.revert_one() { Some(commit) => { debug_assert!(commit.kv.is_empty(), "revert do not change key value store"); - apply_state_commit_no_kv(&mut transaction, commit, &self.storage.db); + apply_state_commit_no_kv(&mut transaction, commit); let removed = self.blockchain.header(BlockId::Number(best))?.ok_or_else( || client::error::Error::UnknownBlock( format!("Error reverting to {}. Block hash not found.", best)))?; @@ -1689,7 +1689,6 @@ impl client::backend::Backend for Backend whe BlockId::Hash(h) if h == Default::default() => { let genesis_storage = DbGenesisStorage::new(); let root = genesis_storage.0.clone(); - // TODO EMCH see genesis impl: that is empty storage let genesis_kv = InMemoryKvBackend::default(); let db_state = DbState::new(Arc::new(genesis_storage), root, Arc::new(genesis_kv)); let state = RefTrackingState::new(db_state, self.storage.clone(), None); diff --git a/core/state-db/src/pruning.rs b/core/state-db/src/pruning.rs index 66116124f9168..c07dcf5b879b7 100644 --- a/core/state-db/src/pruning.rs +++ b/core/state-db/src/pruning.rs @@ -306,7 +306,7 @@ mod tests { let mut db = make_db(&[1, 2, 3]); db.initialize_kv(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = (make_commit_both(&[4, 5], &[1, 3]), None); + let mut commit = make_commit_both(&[4, 5], &[1, 3]); commit.0.initialize_kv(&[4, 5], &[1, 3]); let h = H256::random(); assert!(!commit.0.data.deleted.is_empty()); @@ -345,11 +345,11 @@ mod tests { db.initialize_kv(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); let mut commit = make_commit_both(&[3, 4], &[1]); - pruning.note_canonical(&H256::random(), &mut commit); - db.commit(&commit); + pruning.note_canonical(&H256::random(), &mut commit.0); + db.commit_canonical(&commit); let mut commit = make_commit_both(&[5], &[2]); - pruning.note_canonical(&H256::random(), &mut commit); - db.commit(&commit); + pruning.note_canonical(&H256::random(), &mut commit.0); + db.commit_canonical(&commit); pruning.apply_pending(); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); assert!(db.kv_eq_at(&[1, 2, 3], Some(0))); @@ -383,10 +383,10 @@ mod tests { let mut db = make_db(&[1, 2, 3]); db.initialize_kv(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = (make_commit_both(&[4], &[1]), None); + let mut commit = make_commit_both(&[4], &[1]); pruning.note_canonical(&H256::random(), &mut commit.0); db.commit_canonical(&commit); - let mut commit = (make_commit_both(&[3, 5], &[2]), None); + let mut commit = make_commit_both(&[3, 5], &[2]); pruning.note_canonical(&H256::random(), &mut commit.0); db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); @@ -416,14 +416,14 @@ mod tests { let mut db = make_db(&[1, 2, 3]); db.initialize_kv(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); - let mut commit = (make_commit_both(&[], &[2]), None); + let mut commit = make_commit_both(&[], &[2]); commit.0.initialize_kv(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit.0); db.commit_canonical(&commit); - let mut commit = (make_commit_both(&[2], &[]), None); + let mut commit = make_commit_both(&[2], &[]); pruning.note_canonical(&H256::random(), &mut commit.0); db.commit_canonical(&commit); - let mut commit = (make_commit_both(&[], &[2]), None); + let mut commit = make_commit_both(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit.0); db.commit_canonical(&commit); assert!(db.data_eq(&make_db(&[1, 2, 3]))); diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs index 6424c28e5c3a6..1ca80c57c5735 100644 --- a/core/state-db/src/test.rs +++ b/core/state-db/src/test.rs @@ -33,11 +33,8 @@ pub struct TestDb { pub data: HashMap, pub meta: HashMap, DBValue>, pub kv: HashMap, - // big heuristic to increment this for canonicalize transaction only - // by checking if there is value in kv change set. - // If empty change set needed, this need to be change manually. - // TODO EMCH consider changing commit fun to use last block number explicitely. - pub last_block: u64, + // Heuristic : increase on commit_canonical. + pub kv_last_block: u64, } impl MetaDb for TestDb { @@ -81,9 +78,6 @@ impl NodeDb for TestDb { impl TestDb { pub fn commit(&mut self, commit: &CommitSet) { - if commit.kv.len() > 0 { - self.last_block += 1; - } self.data.extend(commit.data.inserted.iter().cloned()); for k in commit.data.deleted.iter() { self.data.remove(k); @@ -92,7 +86,7 @@ impl TestDb { let encoded = self.kv.entry(k.clone()) .or_insert_with(|| Ser::default().into_vec()); let mut ser = Ser::from_mut(&mut (*encoded)); - ser.push(self.last_block, o.as_ref().map(|v| v.as_slice())); + ser.push(self.kv_last_block, o.as_ref().map(|v| v.as_slice())); } self.meta.extend(commit.meta.inserted.iter().cloned()); for k in commit.meta.deleted.iter() { @@ -101,6 +95,8 @@ impl TestDb { } pub fn commit_canonical(&mut self, commit: &CommitSetCanonical) { + + self.kv_last_block += 1; self.commit(&commit.0); if let Some((block_prune, kv_prune_key)) = commit.1.as_ref() { @@ -123,7 +119,7 @@ impl TestDb { } pub fn kv_eq_at(&self, values: &[u64], block: Option) -> bool { - let block = block.unwrap_or(self.last_block); + let block = block.unwrap_or(self.kv_last_block); let data = make_kv_changeset(values, &[]); let self_kv: BTreeMap<_, _> = self.get_kv_pairs(&block).into_iter().collect(); self_kv == data.into_iter().filter_map(|(k, v)| v.map(|v| (k,v))).collect() @@ -139,7 +135,7 @@ impl TestDb { .filter_map(|(k, v)| v.map(|v| (k,v))) .map(|(k, v)| { let mut ser = Ser::default(); - ser.push(self.last_block, Some(v.as_slice())); + ser.push(self.kv_last_block, Some(v.as_slice())); (k, ser.into_vec()) }) .collect(); @@ -183,12 +179,12 @@ pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet { } } -pub fn make_commit_both(inserted: &[u64], deleted: &[u64]) -> CommitSet { - CommitSet { +pub fn make_commit_both(inserted: &[u64], deleted: &[u64]) -> CommitSetCanonical { + (CommitSet { data: make_changeset(inserted, deleted), meta: ChangeSet::default(), kv: make_kv_changeset(inserted, deleted), - } + }, None) } impl CommitSet { @@ -208,6 +204,6 @@ pub fn make_db(inserted: &[u64]) -> TestDb { .collect(), meta: Default::default(), kv: Default::default(), - last_block: Default::default(), + kv_last_block: Default::default(), } } From df950a561dae2f144b1ac7e4b8ab448de25f3089 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Oct 2019 12:23:48 +0200 Subject: [PATCH 50/68] cleaning pass. --- core/state-db/src/branch.rs | 57 ++++++++++----------- core/state-db/src/lib.rs | 19 ++----- core/state-db/src/noncanonical.rs | 6 --- core/state-machine/src/backend.rs | 5 -- core/state-machine/src/overlayed_changes.rs | 2 +- core/state-machine/src/trie_backend.rs | 10 ++-- core/utils/historied-data/src/tree.rs | 19 +++---- 7 files changed, 47 insertions(+), 71 deletions(-) diff --git a/core/state-db/src/branch.rs b/core/state-db/src/branch.rs index e3e60e544a3e1..b32b830b36108 100644 --- a/core/state-db/src/branch.rs +++ b/core/state-db/src/branch.rs @@ -18,18 +18,21 @@ //! Helper for managing the set of indexed available branches. use std::collections::{BTreeMap}; -use crate::Error; use historied_data::tree::{BranchesStateTrait, BranchStatesRef, BranchStateRef}; #[derive(Clone, Default, Debug)] -/// Reference to state that is enough for query updates, but not -/// for gc. +/// State needed for query updates. +/// That is a subset of the full branch ranges set. +/// /// Values are ordered by branch_ix, /// and only a logic branch path should be present. /// -/// Note that an alternative could be a pointer to a full state -/// branch for a given branch index, here we use an in memory -/// copied representation in relation to an actual use case. +/// Note that an alternative could be a pointer to the full state +/// a branch index corresponding to the leaf for the fork. +/// Here we use an in memory copy of the path because it seems +/// to fit query at a given state with multiple operations +/// (block processing), that way we iterate on a vec rather than +/// hoping over linked branches. pub struct BranchRanges(Vec); impl<'a> BranchesStateTrait for &'a BranchRanges { @@ -90,7 +93,6 @@ impl<'a> Iterator for BranchRangesIter<'a> { #[derive(Debug, Clone, PartialEq, Eq)] pub struct RangeSet { storage: BTreeMap, - // TODO EMCH remove this? last_index: u64, /// treshold for possible node value, correspond /// roughly to last cannonical block branch index. @@ -213,34 +215,32 @@ impl RangeSet { &mut self, branch_index: u64, number: u64, - ) -> Result> { + ) -> u64 { let mut create_new = false; if branch_index == 0 || branch_index < self.treshold { create_new = true; - } else { match self.storage.get_mut(&branch_index) { - Some(branch_state) => { - if branch_state.can_append && branch_state.can_add(number) { - branch_state.add_state(); - } else { - create_new = true; - } - }, - None => // TODO EMCH not sure keeping this error (we may want to clear storage) - return Err(Error::InvalidRange), - }} + } else { + let branch_state = self.storage.get_mut(&branch_index) + .expect("Inconsistent state on new block"); + if branch_state.can_append && branch_state.can_add(number) { + branch_state.add_state(); + } else { + create_new = true; + } + } if create_new { self.last_index += 1; - let state = BranchStates::new(number, branch_index); self.storage.insert(self.last_index, state); - Ok(self.last_index) + self.last_index } else { - Ok(branch_index) + branch_index } } + /// Get the branch reference for a given branch index if it exists. pub fn state_ref(&self, branch_index: u64) -> Option { self.storage.get(&branch_index).map(|v| v.state_ref()) .map(|state| BranchStatesRef { @@ -249,7 +249,8 @@ impl RangeSet { }) } - /// update the range set on import. returns a displaced leaf if there was one. + /// Updates the range set on import (defined by its number in branch (block number). + /// Returns the corresponding branch ranges and the branch index for this block. pub fn import( &mut self, number: u64, @@ -258,16 +259,14 @@ impl RangeSet { ) -> (BranchRanges, u64) { if let Some(mut branch_ranges) = parent_branch_range { - let anchor_index = self.add_state(parent_branch_index, number) - .expect("coherent branch index state"); // TODO EMCH fail in add_state + let anchor_index = self.add_state(parent_branch_index, number); if anchor_index == parent_branch_index { branch_ranges.0.pop(); } branch_ranges.0.push(self.state_ref(anchor_index).expect("added just above")); (branch_ranges, anchor_index) } else { - let anchor_index = self.add_state(parent_branch_index, number) - .expect("coherent branch index state"); // TODO EMCH fail in add_state + let anchor_index = self.add_state(parent_branch_index, number); ( BranchRanges(vec![self.state_ref(anchor_index).expect("added just above")]), anchor_index, @@ -309,9 +308,6 @@ impl RangeSet { } /// Apply a post finalize without considering treshold. - /// TODO EMCH rename as it is also use to prepare a gc - /// without moving the treshold first. - /// pub fn finalize_full( &mut self, branch_index: u64, @@ -433,7 +429,6 @@ impl BranchStates { } -// TODO EMCH end from historied - data #[cfg(test)] mod test { use super::*; diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs index fb4cae206239e..c74b1027432d8 100644 --- a/core/state-db/src/lib.rs +++ b/core/state-db/src/lib.rs @@ -99,8 +99,6 @@ pub enum Error { InvalidBlockNumber, /// Trying to insert block with unknown parent. InvalidParent, - /// branch range access error - InvalidRange, } /// Pinning error type. @@ -123,7 +121,6 @@ impl fmt::Debug for Error { Error::InvalidBlock => write!(f, "Trying to canonicalize invalid block"), Error::InvalidBlockNumber => write!(f, "Trying to insert block with invalid number"), Error::InvalidParent => write!(f, "Trying to insert block with unknown parent"), - Error::InvalidRange => write!(f, "Trying to use invalid branch range"), } } } @@ -138,17 +135,10 @@ pub struct ChangeSet { } /// A set of kv state values changes. -/// TODO EMCH note that this could really benefit from batching -/// the change set (need to change from client to run over -/// the whole range for kv: then we can get prepare -/// insertion of batch values for history in db such as : -/// pub type KvChangeSet = Vec<(H, Vec(u64, Option))>; -/// ), -/// but it just need to be build from client (no need to change -/// it here except to extract faster). /// /// This assumes that we only commit block per block (otherwhise -/// we need to inclued block number value here). +/// we need to include block number value here and +/// implement a more efficient batching update). pub type KvChangeSet = Vec<(H, Option)>; /// Info for pruning kv: a last prune index and keys to prune. @@ -268,7 +258,7 @@ impl StateDbSync { data: changeset, meta: Default::default(), // TODO EMCH no current support for archive all, - // this would require managing branch index at + // it would require managing branch index at // client level (was implemented in another branch: // client-db-branch-ix) // and use and ordered tuple (branchix, blocknumber) @@ -659,7 +649,8 @@ mod tests { let (db, sdb) = make_test_db(PruningMode::ArchiveAll); assert!(db.data_eq(&make_db(&[1, 21, 22, 3, 4, 91, 921, 922, 93, 94]))); - // TODO EMCH implement full for test db and test for kv + // TODO EMCH implement archive all support for test db and complete this + // test for kv store. assert!(!sdb.is_pruned(&H256::from_low_u64_be(0), 0)); } diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 3fa18315d961f..73f95964eb135 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -77,7 +77,6 @@ impl KvPendingGC { fn set_pending_gc(&mut self, branch_index: u64) { self.gc_last_index += 1; if self.pending_canonicalisation_query.is_some() { - // TODO EMCH some better merge self.keys_pending_gc.extend(self.next_keys_pending_gc.drain()); } self.pending_canonicalisation_query = Some(branch_index); @@ -284,9 +283,6 @@ impl NonCanonicalOverlay { Some(record) => { let record: JournalRecord = Decode::decode(&mut record.as_slice())?; - // TODO EMCh following two range fetch can be optimize when parent level - // got a single element. - // fetch parent info let parent_branch_index = parents.get(&record.parent_hash) .map(|(_, i)| *i).unwrap_or(0); let parent_branch_range = Some( @@ -640,9 +636,7 @@ impl NonCanonicalOverlay { if let Some(branch_index_cannonicalize) = last_index { // this needs to be call after parents update - // TODO EMCH may be needed in 'canonicalize', and restore or commit here self.branches.update_finalize_treshold(branch_index_cannonicalize, block_number, true); - // gc is at the right place self.kv_gc.set_pending_gc(branch_index_cannonicalize); // try to run the garbage collection (can run later if there is // pinned process). diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index ce2c001217b6d..ea41ed44a79e0 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -470,8 +470,6 @@ impl From for InMemory { } } - - impl InMemory { /// child storage key iterator pub fn child_storage_keys(&self) -> impl Iterator { @@ -648,9 +646,6 @@ impl Backend for InMemory { Some(root) => root, None => insert_into_memory_db::(&mut mdb, ::std::iter::empty())?, }; - // Since we are running on a memorydb (not a prefixed memory db), content - // is not collision free, so an empty offtrie state can be use (no need - // for keyspace). self.trie = Some(TrieBackend::new(mdb, root, self.extract_kv())); self.trie.as_ref() } diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 267a4d014047c..8fe028d18d749 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -57,7 +57,7 @@ pub struct OverlayedChangeSet { pub top: HashMap, OverlayedValue>, /// Child storage changes. pub children: HashMap, HashMap, OverlayedValue>>, - /// Kv storage changes. + /// Non trie key value storage changes. pub kv: HashMap, Option>>, } diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index c7b83d289ca74..c8f6a0977b3bf 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -238,9 +238,9 @@ impl< } fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) - where - I: IntoIterator, Option>)>, - H::Out: Ord + where + I: IntoIterator, Option>)>, + H::Out: Ord { let default_root = default_child_trie_root::>(storage_key); @@ -276,8 +276,8 @@ impl< } fn kv_transaction(&self, delta: I) -> Self::Transaction - where - I: IntoIterator, Option>)> + where + I: IntoIterator, Option>)> { let mut result = self.kv_storage.pairs(); result.extend(delta.into_iter()); diff --git a/core/utils/historied-data/src/tree.rs b/core/utils/historied-data/src/tree.rs index 35d282421b0e7..892d71459b55b 100644 --- a/core/utils/historied-data/src/tree.rs +++ b/core/utils/historied-data/src/tree.rs @@ -42,12 +42,13 @@ pub trait BranchesStateTrait { type Branch: BranchStateTrait; type Iter: Iterator; + /// Get branch state for node at a given index. fn get_branch(self, index: I) -> Option; - /// Inclusive. + /// Get the last index for the state, inclusive. fn last_index(self) -> I; - /// Iterator. + /// Iterator over the branch states. fn iter(self) -> Self::Iter; } @@ -55,9 +56,10 @@ pub trait BranchesStateTrait { /// This is therefore the representation of a branch state. pub trait BranchStateTrait { + /// Get state for node at a given index. fn get_node(&self, i: I) -> S; - /// Inclusive. + /// Get the last index for the state, inclusive. fn last_index(&self) -> I; } @@ -679,7 +681,6 @@ impl History { } else { PruneResult::Changed } - } else { PruneResult::Unchanged } @@ -690,12 +691,12 @@ impl History { impl<'a, F: SerializedConfig> Serialized<'a, F> { pub fn into_owned(self) -> Serialized<'static, F> { - Serialized(self.0.into_owned()) - } + Serialized(self.0.into_owned()) + } pub fn into_vec(self) -> Vec { - self.0.into_vec() - } + self.0.into_vec() + } pub fn get (&self, state: S) -> Option> where @@ -766,7 +767,7 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> { } } else if history.index > from { if history.value.len() == 0 - && last_index_with_value.is_none() { + && last_index_with_value.is_none() { // delete on delete, continue } else { if last_index_with_value.is_none() { From 0ca65303dfe1b5748375154a6164f75d0d1610eb Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 9 Oct 2019 16:07:52 +0200 Subject: [PATCH 51/68] starting using keyspace, storing in the overlay next to the child trie values is a wrong direction (it would mean historying things at two levels), so keeping it in kv seems better. Still in transaction we want this keyspace from kv (kv transaction is not exploitable). --- core/primitives/src/child_trie.rs | 96 ++++++++++++++++++++ core/primitives/src/lib.rs | 1 + core/state-machine/src/backend.rs | 58 +++++++----- core/state-machine/src/changes_trie/build.rs | 6 +- core/state-machine/src/ext.rs | 8 +- core/state-machine/src/overlayed_changes.rs | 7 +- core/state-machine/src/testing.rs | 4 +- 7 files changed, 147 insertions(+), 33 deletions(-) create mode 100644 core/primitives/src/child_trie.rs diff --git a/core/primitives/src/child_trie.rs b/core/primitives/src/child_trie.rs new file mode 100644 index 0000000000000..ef840198d456e --- /dev/null +++ b/core/primitives/src/child_trie.rs @@ -0,0 +1,96 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Child trie related primitives + +use codec::{Encode, Decode, Compact}; +use rstd::prelude::*; +use rstd::ptr; +use crate::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; +#[cfg(feature = "std")] +pub use impl_serde::serialize as bytes; +use hash_db::Prefix; + +/// `KeySpace` type contains a unique child trie identifier. +/// It is used to isolate a child trie in its backing database. +/// This is done by prefixing its key with this value. +/// +/// A keyspace byte representation must not be the start of another +/// keyspace byte representation (otherwhise conflict may happen). +/// This guaranty is provided by the fact that keyspace is a scale +/// encoded representation. +/// +/// From a `TrieDB` perspective, accessing a child trie content +/// will requires both the child trie root, but also the `KeySpace`. +/// +/// The `KeySpace` of a child trie must be unique for the canonical chain. +/// This unicity is currently guaranted by build from a simple counter. +/// +/// If a child trie was to be moved between two chains, the underlying +/// key value would be all the keyvaluedb prefixed by this keyspace. +/// Moving those key will still need for every content to change keyspace +/// of the origin chain with a new keyspace of a destination chain. +/// Please notice that usage of keyspace on ALL data makes it possible, +/// +/// The same thing is true for a child trie deletion, there is no need to +/// remove all historic of state child trie keypair from a multiple TrieDB +/// perspective, but simply all keyvaluedb content with a key starting with +/// this keyspace. +pub type KeySpace = Vec; + + +/// Keyspace to use for the parent trie key. +pub const NO_CHILD_KEYSPACE: [u8;1] = [0]; + + +/// Produce a new keyspace from current state counter. +pub fn produce_keyspace(child_counter: u128) -> Vec { + codec::Encode::encode(&Compact(child_counter)) +} + +/// Decode keyspace to original counter value. +pub fn reverse_keyspace(keyspace: &[u8]) -> Result { + >::decode(&mut &keyspace[..]).map(|c| c.0) +} + +// see FIXME #2741 for removal of this allocation on every operation. +// Simplest would be to put an additional optional field in prefix. +/// Utility function used for merging `KeySpace` data and `prefix` data +/// before calling key value database primitives. +pub fn keyspace_as_prefix_alloc(ks: Option<&KeySpace>, prefix: Prefix) -> (Vec, Option) { + let ks = ks.map(|ks| ks.as_slice()).unwrap_or(&NO_CHILD_KEYSPACE[..]); + let mut result = rstd::vec![0; ks.len() + prefix.0.len()]; + result[..ks.len()].copy_from_slice(ks); + result[ks.len()..].copy_from_slice(prefix.0); + (result, prefix.1) +} + +#[test] +fn encode_empty_prefix() { + let empty = produce_keyspace(0); + + assert_eq!(&NO_CHILD_KEYSPACE[..], &empty[..]); +} + +#[test] +fn keyspace_codec() { + let sample: [u128; 3] = [0, 1, 1_000_000]; + for s in sample.iter() { + let keyspace = produce_keyspace(*s); + let dec_keyspace = reverse_keyspace(keyspace.as_slice()).unwrap(); + assert_eq!(*s, dec_keyspace); + } +} diff --git a/core/primitives/src/lib.rs b/core/primitives/src/lib.rs index 310f0133ef259..0dfbf1f894293 100644 --- a/core/primitives/src/lib.rs +++ b/core/primitives/src/lib.rs @@ -64,6 +64,7 @@ pub mod sandbox; pub mod storage; pub mod uint; mod changes_trie; +pub mod child_trie; pub mod traits; pub mod testing; diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index ea41ed44a79e0..7e0b8bd9fa5fa 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -26,6 +26,7 @@ use trie::{ TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::{TrieDBMut, Layout}, }; +use primitives::child_trie::{KeySpace, NO_CHILD_KEYSPACE}; /// A state backend is used to read state data and can have changes committed /// to it. @@ -314,7 +315,7 @@ impl error::Error for Void { /// In-memory backend. Fully recomputes tries on each commit but useful for /// tests. pub struct InMemory { - inner: HashMap>, HashMap, Vec>>, + inner: HashMap>, (KeySpace, HashMap, Vec>)>, kv: Option, trie: Option, H, InMemoryKvBackend>>, _hasher: PhantomData, @@ -387,10 +388,14 @@ impl InMemory { pub fn update(&self, changes: >::Transaction) -> Self { let mut inner: HashMap<_, _> = self.inner.clone(); let mut kv: HashMap<_, _> = self.kv().clone(); - for (storage_key, key, val) in changes.storage { + for (storage_key, keyspace, key, val) in changes.storage { + let mut entry = inner.entry(storage_key).or_insert_with(|| ( + keyspace.unwrap_or_else(|| NO_CHILD_KEYSPACE.to_vec()), + Default::default(), + )); match val { - Some(v) => { inner.entry(storage_key).or_default().insert(key, v); }, - None => { inner.entry(storage_key).or_default().remove(&key); }, + Some(v) => { entry.1.insert(key, v); }, + None => { entry.1.remove(&key); }, } } kv.extend(changes.kv); @@ -400,8 +405,8 @@ impl InMemory { } } -impl From>, HashMap, Vec>>> for InMemory { - fn from(inner: HashMap>, HashMap, Vec>>) -> Self { +impl From>, (KeySpace, HashMap, Vec>)>> for InMemory { + fn from(inner: HashMap>, (KeySpace, HashMap, Vec>)>) -> Self { InMemory { inner: inner, trie: None, @@ -413,15 +418,15 @@ impl From>, HashMap, Vec>>> for In impl From<( HashMap, Vec>, - HashMap, HashMap, Vec>>, + HashMap, (KeySpace, HashMap, Vec>)>, )> for InMemory { fn from(inners: ( HashMap, Vec>, - HashMap, HashMap, Vec>>, + HashMap, (KeySpace, HashMap, Vec>)>, )) -> Self { - let mut inner: HashMap>, HashMap, Vec>> + let mut inner: HashMap>, (KeySpace, HashMap, Vec>)> = inners.1.into_iter().map(|(k, v)| (Some(k), v)).collect(); - inner.insert(None, inners.0); + inner.insert(None, (NO_CHILD_KEYSPACE.to_vec(), inners.0)); InMemory { inner: inner, trie: None, @@ -434,7 +439,7 @@ impl From<( impl From, Vec>> for InMemory { fn from(inner: HashMap, Vec>) -> Self { let mut expanded = HashMap::new(); - expanded.insert(None, inner); + expanded.insert(None, (NO_CHILD_KEYSPACE.to_vec(), inner)); InMemory { inner: expanded, trie: None, @@ -444,12 +449,14 @@ impl From, Vec>> for InMemory { } } -impl From>, Vec, Option>)>> for InMemory { - fn from(inner: Vec<(Option>, Vec, Option>)>) -> Self { - let mut expanded: HashMap>, HashMap, Vec>> = HashMap::new(); - for (child_key, key, value) in inner { +impl From>, Option, Vec, Option>)>> for InMemory { + fn from(inner: Vec<(Option>, Option, Vec, Option>)>) -> Self { + let mut expanded: HashMap>, (KeySpace, HashMap, Vec>)> = HashMap::new(); + for (child_key, keyspace, key, value) in inner { if let Some(value) = value { - expanded.entry(child_key).or_default().insert(key, value); + expanded.entry(child_key).or_insert_with(|| + (keyspace.unwrap_or_else(|| NO_CHILD_KEYSPACE.to_vec()), Default::default()) + ).1.insert(key, value); } } expanded.into() @@ -458,10 +465,15 @@ impl From>, Vec, Option>)>> for InMem impl From for InMemory { fn from(inner: InMemoryTransaction) -> Self { - let mut expanded: HashMap>, HashMap, Vec>> = HashMap::new(); - for (child_key, key, value) in inner.storage { + let mut expanded: HashMap< + Option>, + (KeySpace, HashMap, Vec>), + > = HashMap::new(); + for (child_key, keyspace, key, value) in inner.storage { if let Some(value) = value { - expanded.entry(child_key).or_default().insert(key, value); + expanded.entry(child_key).or_insert_with(|| + (keyspace.unwrap_or_else(|| NO_CHILD_KEYSPACE.to_vec()), Default::default()) + ).1.insert(key, value); } } let mut result: InMemory = expanded.into(); @@ -482,7 +494,7 @@ impl InMemory { /// in memory storage. pub struct InMemoryTransaction { /// State trie key values changes (both top and child trie). - pub storage: Vec<(Option>, Vec, Option>)>, + pub storage: Vec<(Option>, Option, Vec, Option>)>, /// Changes to non trie key value datas. pub kv: HashMap, Option>>, } @@ -494,15 +506,15 @@ impl Backend for InMemory { type KvBackend = InMemoryKvBackend; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { - Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone))) + Ok(self.inner.get(&None).and_then(|map| map.1.get(key).map(Clone::clone))) } fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - Ok(self.inner.get(&Some(storage_key.to_vec())).and_then(|map| map.get(key).map(Clone::clone))) + Ok(self.inner.get(&Some(storage_key.to_vec())).and_then(|map| map.1.get(key).map(Clone::clone))) } fn exists_storage(&self, key: &[u8]) -> Result { - Ok(self.inner.get(&None).map(|map| map.get(key).is_some()).unwrap_or(false)) + Ok(self.inner.get(&None).map(|map| map.1.get(key).is_some()).unwrap_or(false)) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index fca828959f441..5a7c2f28ecf49 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -133,7 +133,11 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( Number: BlockNumber, { let (committed, prospective) = if let Some(sk) = storage_key.as_ref() { - (changes.committed.children.get(sk), changes.prospective.children.get(sk)) + // keyspace is ignored for change trie + ( + changes.committed.children.get(sk).as_ref().map(|v| &v.1), + changes.prospective.children.get(sk).as_ref().map(|v| &v.1), + ) } else { (Some(&changes.committed.top), Some(&changes.prospective.top)) }; diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 644060a675231..5e4713d6ffed6 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -408,10 +408,10 @@ where H: Hasher, let child_delta_iter = child_storage_keys.map(|storage_key| (storage_key.clone(), self.overlay.committed.children.get(storage_key) .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) + .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone()))) .chain(self.overlay.prospective.children.get(storage_key) .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))))); + .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone())))))); // compute and memoize @@ -448,10 +448,10 @@ where H: Hasher, let delta = self.overlay.committed.children.get(storage_key) .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) + .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone()))) .chain(self.overlay.prospective.children.get(storage_key) .into_iter() - .flat_map(|map| map.clone().into_iter().map(|(k, v)| (k.clone(), v.value.clone())))); + .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone())))); let root = self.backend.child_storage_root(storage_key, delta).0; diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 8fe028d18d749..d2d053d70dd2c 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -22,6 +22,7 @@ use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; +use primitives::child_trie::KeySpace; /// The overlayed changes to state to be queried on top of the backend. /// @@ -56,7 +57,7 @@ pub struct OverlayedChangeSet { /// Top level storage changes. pub top: HashMap, OverlayedValue>, /// Child storage changes. - pub children: HashMap, HashMap, OverlayedValue>>, + pub children: HashMap, (KeySpace, HashMap, OverlayedValue>)>, /// Non trie key value storage changes. pub kv: HashMap, Option>>, } @@ -329,13 +330,13 @@ impl OverlayedChanges { /// Will panic if there are any uncommitted prospective changes. pub fn into_committed(self) -> ( impl Iterator, Option>)>, - impl Iterator, impl Iterator, Option>)>)>, + impl Iterator, KeySpace, impl Iterator, Option>)>)>, impl Iterator, Option>)>, ){ assert!(self.prospective.is_empty()); (self.committed.top.into_iter().map(|(k, v)| (k, v.value)), self.committed.children.into_iter() - .map(|(sk, v)| (sk, v.into_iter().map(|(k, v)| (k, v.value)))), + .map(|(sk, v)| (sk, v.0, v.1.into_iter().map(|(k, v)| (k, v.value)))), self.committed.kv.into_iter()) } diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index b958c02e298bc..014d898c1ae3b 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -119,9 +119,9 @@ impl TestExternalities { let children = self.overlay.committed.children.clone().into_iter() .chain(self.overlay.prospective.children.clone().into_iter()) - .flat_map(|(keyspace, map)| { + .flat_map(|(storage_key, (keyspace, map))| { map.into_iter() - .map(|(k, v)| (Some(keyspace.clone()), k, v.value)) + .map(|(k, v)| (Some((storage_key.clone(), keyspace.clone())), k, v.value)) .collect::>() }); From 0dd36d0ec92a7d6ecdd604ad0f82172cd920a871 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Oct 2019 18:44:01 +0200 Subject: [PATCH 52/68] it is tricky, but clearly getting rid of keyspace in overlayed change will be good. --- core/primitives/src/child_trie.rs | 22 ++++++++ core/state-machine/src/backend.rs | 60 ++++++++++++++------- core/state-machine/src/overlayed_changes.rs | 8 ++- core/state-machine/src/proving_backend.rs | 5 ++ core/state-machine/src/testing.rs | 19 +++---- core/state-machine/src/trie_backend.rs | 14 +++-- 6 files changed, 90 insertions(+), 38 deletions(-) diff --git a/core/primitives/src/child_trie.rs b/core/primitives/src/child_trie.rs index ef840198d456e..065d5bb3a080d 100644 --- a/core/primitives/src/child_trie.rs +++ b/core/primitives/src/child_trie.rs @@ -55,6 +55,28 @@ pub type KeySpace = Vec; /// Keyspace to use for the parent trie key. pub const NO_CHILD_KEYSPACE: [u8;1] = [0]; +/// Prefix for keyspace in key value storage. +/// TODO EMCH consider using its own column +const KEYSPACE_PREFIX: &'static [u8;9] = b"ks_prefix"; + +/// Address of local keyspace counter. For allowing +/// archive node with prunning, this counter should be +/// global (shared between all block evaluation: so +/// a mutex protected value that shall be initialized from +/// aux and updated every time a block is written to db), but +/// with current storage that allows only canonical +/// counter in key value storage works. +/// TODO EMCH consider directly using a global counter. +pub const KEYSPACE_COUNTER: &'static [u8;10] = b"ks_counter"; + +/// Produce a new keyspace from current state counter. +/// TODO EMCH remove if using column +pub fn prefixed_keyspace_kv(key: &[u8]) -> Vec { + let mut resu = Vec::with_capacity(KEYSPACE_PREFIX.len() + key.len()); + resu.extend_from_slice(&KEYSPACE_PREFIX[..]); + resu.extend_from_slice(key); + resu +} /// Produce a new keyspace from current state counter. pub fn produce_keyspace(child_counter: u128) -> Vec { diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index 7e0b8bd9fa5fa..79759176577f3 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -26,7 +26,7 @@ use trie::{ TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::{TrieDBMut, Layout}, }; -use primitives::child_trie::{KeySpace, NO_CHILD_KEYSPACE}; +use primitives::child_trie::{KeySpace, NO_CHILD_KEYSPACE, prefixed_keyspace_kv}; /// A state backend is used to read state data and can have changes committed /// to it. @@ -61,6 +61,9 @@ pub trait Backend: std::fmt::Debug { self.child_storage(storage_key, key).map(|v| v.map(|v| H::hash(&v))) } + /// Get technical keyspace use for child storage key. + fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, Self::Error>; + /// true if a key exists in storage. fn exists_storage(&self, key: &[u8]) -> Result { Ok(self.storage(key)?.is_some()) @@ -195,6 +198,10 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { (*self).child_storage(storage_key, key) } + fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, Self::Error> { + (*self).get_child_keyspace(storage_key) + } + fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { (*self).for_keys_in_child_storage(storage_key, f) } @@ -513,26 +520,37 @@ impl Backend for InMemory { Ok(self.inner.get(&Some(storage_key.to_vec())).and_then(|map| map.1.get(key).map(Clone::clone))) } + fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, Self::Error> { + Ok(self.inner.get(&Some(storage_key.to_vec())) + .map(|(ks, _)| ks) + .map(Clone::clone) + .or_else(|| self.kv().get(&prefixed_keyspace_kv(storage_key)[..]) + .map(Clone::clone) + .unwrap_or(None) + ) + ) + } + fn exists_storage(&self, key: &[u8]) -> Result { Ok(self.inner.get(&None).map(|map| map.1.get(key).is_some()).unwrap_or(false)) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { - self.inner.get(&None).map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f)); + self.inner.get(&None).map(|map| map.1.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f)); } fn for_key_values_with_prefix(&self, prefix: &[u8], mut f: F) { - self.inner.get(&None).map(|map| map.iter().filter(|(key, _val)| key.starts_with(prefix)) + self.inner.get(&None).map(|map| map.1.iter().filter(|(key, _val)| key.starts_with(prefix)) .for_each(|(k, v)| f(k, v))); } fn for_keys_in_child_storage(&self, storage_key: &[u8], mut f: F) { - self.inner.get(&Some(storage_key.to_vec())).map(|map| map.keys().for_each(|k| f(&k))); + self.inner.get(&Some(storage_key.to_vec())).map(|map| map.1.keys().for_each(|k| f(&k))); } fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { self.inner.get(&Some(storage_key.to_vec())) - .map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f)); + .map(|map| map.1.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f)); } fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) @@ -542,7 +560,7 @@ impl Backend for InMemory { { let existing_pairs = self.inner.get(&None) .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); + .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); let transaction: Vec<_> = delta.into_iter().collect(); let root = Layout::::trie_root(existing_pairs.chain(transaction.iter().cloned()) @@ -551,7 +569,7 @@ impl Backend for InMemory { .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) ); - let full_transaction = transaction.into_iter().map(|(k, v)| (None, k, v)).collect(); + let full_transaction = transaction.into_iter().map(|(k, v)| (None, None, k, v)).collect(); (root, InMemoryTransaction { storage: full_transaction, kv: Default::default() }) } @@ -561,24 +579,28 @@ impl Backend for InMemory { I: IntoIterator, Option>)>, H::Out: Ord { - let storage_key = storage_key.to_vec(); + let storage_key = Some(storage_key.to_vec()); - let existing_pairs = self.inner.get(&Some(storage_key.clone())) + let existing_pairs = self.inner.get(&storage_key) .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); + .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); let transaction: Vec<_> = delta.into_iter().collect(); let root = child_trie_root::, _, _, _>( - &storage_key, + storage_key.as_ref().expect("Initialized to some"), existing_pairs.chain(transaction.iter().cloned()) .collect::>() .into_iter() .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) ); - let full_transaction = transaction.into_iter().map(|(k, v)| (Some(storage_key.clone()), k, v)).collect(); + let keyspace = self.inner.get(&storage_key).map(|t| t.0); + let full_transaction = transaction.into_iter() + .map(|(k, v)| (storage_key.clone(), keyspace.clone(), k, v)).collect(); - let is_default = root == default_child_trie_root::>(&storage_key); + let is_default = root == default_child_trie_root::>( + storage_key.as_ref().expect("Initialized to some") + ); ( root, @@ -599,7 +621,7 @@ impl Backend for InMemory { fn pairs(&self) -> Vec<(Vec, Vec)> { self.inner.get(&None) .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))) + .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone()))) .collect() } @@ -610,7 +632,7 @@ impl Backend for InMemory { fn child_pairs(&self, storage_key: &[u8]) -> Vec<(Vec, Vec)> { self.inner.get(&Some(storage_key.to_vec())) .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))) + .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone()))) .collect() } @@ -621,14 +643,14 @@ impl Backend for InMemory { fn keys(&self, prefix: &[u8]) -> Vec> { self.inner.get(&None) .into_iter() - .flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()) + .flat_map(|map| map.1.keys().filter(|k| k.starts_with(prefix)).cloned()) .collect() } fn child_keys(&self, storage_key: &[u8], prefix: &[u8]) -> Vec> { self.inner.get(&Some(storage_key.to_vec())) .into_iter() - .flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()) + .flat_map(|map| map.1.keys().filter(|k| k.starts_with(prefix)).cloned()) .collect() } @@ -641,7 +663,7 @@ impl Backend for InMemory { let mut root_map = None; for (storage_key, map) in &self.inner { if let Some(storage_key) = storage_key.as_ref() { - let ch = insert_into_memory_db::(&mut mdb, map.clone().into_iter())?; + let ch = insert_into_memory_db::(&mut mdb, map.1.clone().into_iter())?; new_child_roots.push((storage_key.clone(), ch.as_ref().into())); } else { root_map = Some(map); @@ -651,7 +673,7 @@ impl Backend for InMemory { if let Some(map) = root_map.take() { root = Some(insert_into_memory_db::( &mut mdb, - map.clone().into_iter().chain(new_child_roots.into_iter()) + map.1.clone().into_iter().chain(new_child_roots.into_iter()) )?); } let root = match root { diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index d2d053d70dd2c..46e37f3a5e628 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -57,6 +57,10 @@ pub struct OverlayedChangeSet { /// Top level storage changes. pub top: HashMap, OverlayedValue>, /// Child storage changes. + /// TODO EMCH keyspace storage is a bit redundant with 'kv' containing + /// keyspace change, but it still seems pretty convenient. + /// TODO EMCH if keeping it, maybe change to Option for lazy + /// keyspace creation of new children pub children: HashMap, (KeySpace, HashMap, OverlayedValue>)>, /// Non trie key value storage changes. pub kv: HashMap, Option>>, @@ -123,13 +127,13 @@ impl OverlayedChanges { /// value has been set. pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { if let Some(map) = self.prospective.children.get(storage_key) { - if let Some(val) = map.get(key) { + if let Some(val) = map.1.get(key) { return Some(val.value.as_ref().map(AsRef::as_ref)); } } if let Some(map) = self.committed.children.get(storage_key) { - if let Some(val) = map.get(key) { + if let Some(val) = map.1.get(key) { return Some(val.value.as_ref().map(AsRef::as_ref)); } } diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index f8b97fcccfc25..f65e2a0fdfb14 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -30,6 +30,7 @@ use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend}; use crate::kv_backend::{KvBackend, InMemory as InMemoryKvBackend}; +use primitives::child_trie::KeySpace; /// Patricia trie-based backend essence which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. @@ -187,6 +188,10 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> }.child_storage(storage_key, key) } + fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, Self::Error> { + self.backend.get_child_keyspace(storage_key) + } + fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { self.backend.for_keys_in_child_storage(storage_key, f) } diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index 014d898c1ae3b..140483ccad6d6 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -28,12 +28,13 @@ use crate::{ use primitives::{ storage::well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key}, traits::{BareCryptoStorePtr, Externalities}, offchain, child_storage_key::ChildStorageKey, + child_trie::{KeySpace, NO_CHILD_KEYSPACE}, }; use codec::Encode; const EXT_NOT_ALLOWED_TO_FAIL: &str = "Externalities not allowed to fail within runtime"; -type StorageTuple = (HashMap, Vec>, HashMap, HashMap, Vec>>); +type StorageTuple = (HashMap, Vec>, HashMap, (KeySpace, HashMap, Vec>)>); /// Simple HashMap-based Externalities impl. pub struct TestExternalities { @@ -68,7 +69,7 @@ impl TestExternalities { let backend: HashMap<_, _> = storage.1.into_iter() .map(|(keyspace, map)| (Some(keyspace), map)) - .chain(Some((None, storage.0)).into_iter()) + .chain(Some((None, (NO_CHILD_KEYSPACE.to_vec(), storage.0))).into_iter()) .collect(); TestExternalities { @@ -83,7 +84,7 @@ impl TestExternalities { /// Insert key/value into backend pub fn insert(&mut self, k: Vec, v: Vec) { self.backend = self.backend.update(InMemoryTransaction { - storage: vec![(None, k, Some(v))], + storage: vec![(None, None, k, Some(v))], kv: Default::default(), }); } @@ -115,13 +116,13 @@ impl TestExternalities { pub fn commit_all(&self) -> InMemory { let top = self.overlay.committed.top.clone().into_iter() .chain(self.overlay.prospective.top.clone().into_iter()) - .map(|(k, v)| (None, k, v.value)); + .map(|(k, v)| (None, None, k, v.value)); let children = self.overlay.committed.children.clone().into_iter() .chain(self.overlay.prospective.children.clone().into_iter()) .flat_map(|(storage_key, (keyspace, map))| { map.into_iter() - .map(|(k, v)| (Some((storage_key.clone(), keyspace.clone())), k, v.value)) + .map(|(k, v)| (Some(storage_key.clone()), Some(keyspace.clone()), k, v.value)) .collect::>() }); @@ -253,10 +254,10 @@ impl Externalities for TestExternalities where let child_delta_iter = child_storage_keys.map(|storage_key| (storage_key.clone(), self.overlay.committed.children.get(storage_key) .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) + .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone()))) .chain(self.overlay.prospective.children.get(storage_key) .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))))); + .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone())))))); // compute and memoize @@ -274,10 +275,10 @@ impl Externalities for TestExternalities where let (root, is_empty, _) = { let delta = self.overlay.committed.children.get(storage_key) .into_iter() - .flat_map(|map| map.clone().into_iter().map(|(k, v)| (k, v.value))) + .flat_map(|map| map.1.clone().into_iter().map(|(k, v)| (k, v.value))) .chain(self.overlay.prospective.children.get(storage_key) .into_iter() - .flat_map(|map| map.clone().into_iter().map(|(k, v)| (k, v.value)))); + .flat_map(|map| map.1.clone().into_iter().map(|(k, v)| (k, v.value)))); self.backend.child_storage_root(storage_key, delta) }; diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index c8f6a0977b3bf..2fbf7d3ccdde5 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -25,6 +25,7 @@ use crate::Backend; use crate::kv_backend::KvBackend; use primitives::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; use std::collections::HashMap; +use primitives::child_trie::{KeySpace, prefixed_keyspace_kv}; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. /// A simple key value backend is also accessible for direct key value storage. @@ -72,13 +73,6 @@ impl, O: KvBackend, H: Hasher> TrieBackend { self.essence.into_storage() } - // TODO EMCH PROTO: remove before pr. - pub fn child_keyspace(&self, key: &[u8]) -> Result>, String> { - const PREFIX_KEYSPACE: &'static[u8] = b"kv_keyspace"; - // TODO EMCH do prefixing manually. - self.kv_storage.get(key) - } - } impl< @@ -108,13 +102,17 @@ impl< } fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - +// TODO EMCHEMCH // let keyspace = self.child_keyspace(storage_key); // Then change essence functions to use keyspace as input. self.essence.child_storage(storage_key, key) } + fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, Self::Error> { + self.kv_storage.get(&prefixed_keyspace_kv(storage_key)[..]) + } + fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { self.essence.for_keys_with_prefix(prefix, f) } From 34cf1381646ecb80cab5e46064663286f4e489b1 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Oct 2019 19:41:04 +0200 Subject: [PATCH 53/68] wrong approach (the overlay db code for kv is not strictly needed), nothing should happen in overlay db regarding keyspace, do resolution in block_execution from client. Consider removing KeySpace from backend InMemory to (kv will get it). --- core/client/src/cht.rs | 2 +- core/client/src/client.rs | 2 ++ core/state-machine/src/backend.rs | 7 +++-- core/state-machine/src/changes_trie/build.rs | 6 +--- core/state-machine/src/ext.rs | 8 ++--- core/state-machine/src/overlayed_changes.rs | 33 ++++++++++++++------ core/state-machine/src/proving_backend.rs | 8 ++--- core/state-machine/src/testing.rs | 22 +++++++++---- 8 files changed, 56 insertions(+), 32 deletions(-) diff --git a/core/client/src/cht.rs b/core/client/src/cht.rs index ae432ccbb1eb5..44517b08c4aaf 100644 --- a/core/client/src/cht.rs +++ b/core/client/src/cht.rs @@ -100,7 +100,7 @@ pub fn build_proof( { let transaction = build_pairs::(cht_size, cht_num, hashes)? .into_iter() - .map(|(k, v)| (None, k, Some(v))) + .map(|(k, v)| (None, None, k, Some(v))) .collect::>(); let mut storage = InMemoryState::::default().update(InMemoryTransaction { storage: transaction, diff --git a/core/client/src/client.rs b/core/client/src/client.rs index 70ec1abfffcbd..9717992337fe0 100644 --- a/core/client/src/client.rs +++ b/core/client/src/client.rs @@ -1075,6 +1075,8 @@ impl Client where overlay.commit_prospective(); let (top, children, kv) = overlay.into_committed(); + // TODO EMCH can we resolve keyspace here: will need to get kv intact + // from into_committed and make it an iter in a next step let children = children.map(|(sk, it)| (sk, it.collect())).collect(); if import_headers.post().state_root() != &storage_update.1 { return Err(error::Error::InvalidStateRoot); diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index 79759176577f3..40cd3b8f04935 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -396,7 +396,10 @@ impl InMemory { let mut inner: HashMap<_, _> = self.inner.clone(); let mut kv: HashMap<_, _> = self.kv().clone(); for (storage_key, keyspace, key, val) in changes.storage { - let mut entry = inner.entry(storage_key).or_insert_with(|| ( + let entry = inner.entry(storage_key).or_insert_with(|| ( + // TODO EMCH here we need to assert storage key is none + // otherwhise it is undefined key leading to new keyspace + // creation. keyspace.unwrap_or_else(|| NO_CHILD_KEYSPACE.to_vec()), Default::default(), )); @@ -594,7 +597,7 @@ impl Backend for InMemory { .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) ); - let keyspace = self.inner.get(&storage_key).map(|t| t.0); + let keyspace = self.inner.get(&storage_key).map(|t| t.0.clone()); let full_transaction = transaction.into_iter() .map(|(k, v)| (storage_key.clone(), keyspace.clone(), k, v)).collect(); diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 5a7c2f28ecf49..fca828959f441 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -133,11 +133,7 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( Number: BlockNumber, { let (committed, prospective) = if let Some(sk) = storage_key.as_ref() { - // keyspace is ignored for change trie - ( - changes.committed.children.get(sk).as_ref().map(|v| &v.1), - changes.prospective.children.get(sk).as_ref().map(|v| &v.1), - ) + (changes.committed.children.get(sk), changes.prospective.children.get(sk)) } else { (Some(&changes.committed.top), Some(&changes.prospective.top)) }; diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 5e4713d6ffed6..19e9fce4cf25b 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -408,10 +408,10 @@ where H: Hasher, let child_delta_iter = child_storage_keys.map(|storage_key| (storage_key.clone(), self.overlay.committed.children.get(storage_key) .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone()))) + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) .chain(self.overlay.prospective.children.get(storage_key) .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone())))))); + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))))); // compute and memoize @@ -448,10 +448,10 @@ where H: Hasher, let delta = self.overlay.committed.children.get(storage_key) .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone()))) + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) .chain(self.overlay.prospective.children.get(storage_key) .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone())))); + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))); let root = self.backend.child_storage_root(storage_key, delta).0; diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 46e37f3a5e628..1e977cff85365 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -22,7 +22,7 @@ use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -use primitives::child_trie::KeySpace; +use primitives::child_trie::{KeySpace, prefixed_keyspace_kv}; /// The overlayed changes to state to be queried on top of the backend. /// @@ -57,11 +57,7 @@ pub struct OverlayedChangeSet { /// Top level storage changes. pub top: HashMap, OverlayedValue>, /// Child storage changes. - /// TODO EMCH keyspace storage is a bit redundant with 'kv' containing - /// keyspace change, but it still seems pretty convenient. - /// TODO EMCH if keeping it, maybe change to Option for lazy - /// keyspace creation of new children - pub children: HashMap, (KeySpace, HashMap, OverlayedValue>)>, + pub children: HashMap, HashMap, OverlayedValue>>, /// Non trie key value storage changes. pub kv: HashMap, Option>>, } @@ -127,13 +123,13 @@ impl OverlayedChanges { /// value has been set. pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { if let Some(map) = self.prospective.children.get(storage_key) { - if let Some(val) = map.1.get(key) { + if let Some(val) = map.get(key) { return Some(val.value.as_ref().map(AsRef::as_ref)); } } if let Some(map) = self.committed.children.get(storage_key) { - if let Some(val) = map.1.get(key) { + if let Some(val) = map.get(key) { return Some(val.value.as_ref().map(AsRef::as_ref)); } } @@ -328,19 +324,36 @@ impl OverlayedChanges { } } + /// Get current changed keyspace value return `Some(None)` if deleted. + pub fn get_child_keyspace(&self, storage_key: &[u8]) -> Option> { + self.prospective.kv.get(&prefixed_keyspace_kv(&storage_key)) + .or_else(|| self.committed.kv.get(&prefixed_keyspace_kv(&storage_key))) + .map(Clone::clone) + } + /// Consume `OverlayedChanges` and take committed set. /// + /// The child keyspace is only added if it was change in the + /// commit set. + /// /// Panics: /// Will panic if there are any uncommitted prospective changes. pub fn into_committed(self) -> ( impl Iterator, Option>)>, - impl Iterator, KeySpace, impl Iterator, Option>)>)>, + impl Iterator, impl Iterator, Option>)>)>, impl Iterator, Option>)>, ){ assert!(self.prospective.is_empty()); (self.committed.top.into_iter().map(|(k, v)| (k, v.value)), self.committed.children.into_iter() - .map(|(sk, v)| (sk, v.0, v.1.into_iter().map(|(k, v)| (k, v.value)))), + .map(|(sk, v)| { + // TODO EMCH bad design do not include keyspace ??? + // could we do a later resolution: commenting for now +/* let keyspace = self.committed.kv.get(&prefixed_keyspace_kv(&sk)) + .map(Clone::clone) + .unwrap_or(None);*/ + (sk, v.into_iter().map(|(k, v)| (k, v.value))) + }), self.committed.kv.into_iter()) } diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index f65e2a0fdfb14..4c7e777943c47 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -340,7 +340,7 @@ mod tests { #[test] fn proof_recorded_and_checked() { - let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))).collect::>(); + let contents = (0..64).map(|i| (None, None, vec![i], Some(vec![i]))).collect::>(); let in_memory = InMemory::::default(); let mut in_memory = in_memory.update(InMemoryTransaction { storage: contents, @@ -370,9 +370,9 @@ mod tests { let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub2").unwrap(); let own1 = subtrie1.into_owned(); let own2 = subtrie2.into_owned(); - let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))) - .chain((28..65).map(|i| (Some(own1.clone()), vec![i], Some(vec![i])))) - .chain((10..15).map(|i| (Some(own2.clone()), vec![i], Some(vec![i])))) + let contents = (0..64).map(|i| (None, None, vec![i], Some(vec![i]))) + .chain((28..65).map(|i| (Some(own1.clone()), None, vec![i], Some(vec![i])))) + .chain((10..15).map(|i| (Some(own2.clone()), None, vec![i], Some(vec![i])))) .collect::>(); let in_memory = InMemory::::default(); let mut in_memory = in_memory.update(InMemoryTransaction { diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index 140483ccad6d6..eb431d9c57619 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -112,6 +112,14 @@ impl TestExternalities { &mut self.changes_trie_storage } + fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, ()> { + if let Some(change) = self.overlay.get_child_keyspace(storage_key) { + return Ok(change); + } + + self.backend.get_child_keyspace(storage_key).map_err(|_|()) + } + /// Return a new backend with all pending value. pub fn commit_all(&self) -> InMemory { let top = self.overlay.committed.top.clone().into_iter() @@ -120,9 +128,11 @@ impl TestExternalities { let children = self.overlay.committed.children.clone().into_iter() .chain(self.overlay.prospective.children.clone().into_iter()) - .flat_map(|(storage_key, (keyspace, map))| { + .flat_map(|(storage_key, map)| { + let keyspace = self.get_child_keyspace(storage_key.as_slice()) + .expect("Test backend do not produce db error"); map.into_iter() - .map(|(k, v)| (Some(storage_key.clone()), Some(keyspace.clone()), k, v.value)) + .map(|(k, v)| (Some(storage_key.clone()), keyspace.clone(), k, v.value)) .collect::>() }); @@ -254,10 +264,10 @@ impl Externalities for TestExternalities where let child_delta_iter = child_storage_keys.map(|storage_key| (storage_key.clone(), self.overlay.committed.children.get(storage_key) .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone()))) + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) .chain(self.overlay.prospective.children.get(storage_key) .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.value.clone())))))); + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))))); // compute and memoize @@ -275,10 +285,10 @@ impl Externalities for TestExternalities where let (root, is_empty, _) = { let delta = self.overlay.committed.children.get(storage_key) .into_iter() - .flat_map(|map| map.1.clone().into_iter().map(|(k, v)| (k, v.value))) + .flat_map(|map| map.clone().into_iter().map(|(k, v)| (k, v.value))) .chain(self.overlay.prospective.children.get(storage_key) .into_iter() - .flat_map(|map| map.1.clone().into_iter().map(|(k, v)| (k, v.value)))); + .flat_map(|map| map.clone().into_iter().map(|(k, v)| (k, v.value)))); self.backend.child_storage_root(storage_key, delta) }; From a7322317e635dd7c760f4be3f4370ae2a66479bb Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Oct 2019 10:54:24 +0200 Subject: [PATCH 54/68] a todo --- core/state-machine/src/backend.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index 40cd3b8f04935..afcb9e8345ac8 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -322,6 +322,7 @@ impl error::Error for Void { /// In-memory backend. Fully recomputes tries on each commit but useful for /// tests. pub struct InMemory { + // TODO EMCH remove keyspace: it is into kv. inner: HashMap>, (KeySpace, HashMap, Vec>)>, kv: Option, trie: Option, H, InMemoryKvBackend>>, From ae180ed80c076c70f7110e50bc22db7d9d3ce7eb Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Oct 2019 10:58:53 +0200 Subject: [PATCH 55/68] Slight change to 'into_committed' proto, to allow resolving keyspace in the child trie case (need query of kv updates). --- core/client/src/client.rs | 3 ++- core/state-machine/src/overlayed_changes.rs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/client/src/client.rs b/core/client/src/client.rs index 70ec1abfffcbd..d4f6aa76d748a 100644 --- a/core/client/src/client.rs +++ b/core/client/src/client.rs @@ -1083,7 +1083,8 @@ impl Client where Ok(( Some(storage_update.0), Some(changes_update), - Some((top.collect(), children, kv.collect())), + // switching kv from map to vec for compactness + Some((top.collect(), children, kv.into_iter().collect())), )) }, None => Ok((None, None, None)) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 8fe028d18d749..ea3fd0138c762 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -330,13 +330,13 @@ impl OverlayedChanges { pub fn into_committed(self) -> ( impl Iterator, Option>)>, impl Iterator, impl Iterator, Option>)>)>, - impl Iterator, Option>)>, + HashMap, Option>>, ){ assert!(self.prospective.is_empty()); (self.committed.top.into_iter().map(|(k, v)| (k, v.value)), self.committed.children.into_iter() .map(|(sk, v)| (sk, v.into_iter().map(|(k, v)| (k, v.value)))), - self.committed.kv.into_iter()) + self.committed.kv) } /// Inserts storage entry responsible for current extrinsic index. From d53dbba034e2faa9400e1c08b3dcadf6bde7d29d Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Oct 2019 16:06:42 +0200 Subject: [PATCH 56/68] boilerplate changes. --- core/client/db/src/lib.rs | 5 + core/client/db/src/storage_cache.rs | 5 + core/client/src/cht.rs | 2 +- core/client/src/genesis.rs | 9 +- core/client/src/light/backend.rs | 12 +- core/executor/src/wasm_executor.rs | 4 +- core/state-machine/src/backend.rs | 133 ++++++++++------------ core/state-machine/src/proving_backend.rs | 8 +- core/state-machine/src/testing.rs | 39 +++++-- core/test-runtime/src/system.rs | 2 +- node/executor/src/lib.rs | 18 +-- srml/authority-discovery/src/lib.rs | 6 +- srml/contracts/src/tests.rs | 2 +- srml/democracy/src/lib.rs | 2 +- srml/executive/src/lib.rs | 2 +- srml/finality-tracker/src/lib.rs | 6 +- srml/session/src/historical.rs | 2 +- srml/session/src/lib.rs | 2 +- srml/system/src/lib.rs | 2 +- srml/timestamp/src/lib.rs | 6 +- 20 files changed, 149 insertions(+), 118 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 8e31d11071b6a..036567650119c 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -51,6 +51,7 @@ use trie::{MemoryDB, PrefixedMemoryDB, prefixed_key}; use parking_lot::{Mutex, RwLock}; use primitives::{H256, Blake2Hasher, ChangesTrieConfiguration, convert_hash, traits::CodeExecutor}; use primitives::storage::well_known_keys; +use primitives::child_trie::KeySpace; use sr_primitives::{ generic::{BlockId, DigestItem}, Justification, StorageOverlay, ChildrenStorageOverlay, BuildStorage, @@ -149,6 +150,10 @@ impl StateBackend for RefTrackingState { self.state.child_storage(storage_key, key) } + fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, Self::Error> { + self.state.get_child_keyspace(storage_key) + } + fn exists_storage(&self, key: &[u8]) -> Result { self.state.exists_storage(key) } diff --git a/core/client/db/src/storage_cache.rs b/core/client/db/src/storage_cache.rs index 9a0bc7df673f0..4032ad3b184d8 100644 --- a/core/client/db/src/storage_cache.rs +++ b/core/client/db/src/storage_cache.rs @@ -22,6 +22,7 @@ use parking_lot::{Mutex, RwLock, RwLockUpgradableReadGuard}; use linked_hash_map::{LinkedHashMap, Entry}; use hash_db::Hasher; use sr_primitives::traits::{Block as BlockT, Header}; +use primitives::child_trie::KeySpace; use state_machine::{backend::Backend as StateBackend, TrieBackend}; use log::trace; use super::{StorageCollection, ChildStorageCollection}; @@ -531,6 +532,10 @@ impl, B: BlockT> StateBackend for CachingState< Ok(value) } + fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, Self::Error> { + self.state.get_child_keyspace(storage_key) + } + fn exists_storage(&self, key: &[u8]) -> Result { Ok(self.storage(key)?.is_some()) } diff --git a/core/client/src/cht.rs b/core/client/src/cht.rs index 44517b08c4aaf..ae432ccbb1eb5 100644 --- a/core/client/src/cht.rs +++ b/core/client/src/cht.rs @@ -100,7 +100,7 @@ pub fn build_proof( { let transaction = build_pairs::(cht_size, cht_num, hashes)? .into_iter() - .map(|(k, v)| (None, None, k, Some(v))) + .map(|(k, v)| (None, k, Some(v))) .collect::>(); let mut storage = InMemoryState::::default().update(InMemoryTransaction { storage: transaction, diff --git a/core/client/src/genesis.rs b/core/client/src/genesis.rs index 0c0d49f8eaccc..bac65efc4dcdc 100644 --- a/core/client/src/genesis.rs +++ b/core/client/src/genesis.rs @@ -162,7 +162,8 @@ mod tests { ).genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); - let backend = InMemory::from(storage); + // TODO EMCH need to put kv in genesis + let backend = InMemory::from((storage.0, storage.1, Default::default())); let (b1data, _b1hash) = block1(genesis_hash, &backend); let mut overlay = OverlayedChanges::default(); @@ -192,7 +193,8 @@ mod tests { ).genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); - let backend = InMemory::from(storage); + // TODO EMCH need to put kv in genesis + let backend = InMemory::from((storage.0, storage.1, Default::default())); let (b1data, _b1hash) = block1(genesis_hash, &backend); let mut overlay = OverlayedChanges::default(); @@ -222,7 +224,8 @@ mod tests { ).genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); - let backend = InMemory::from(storage); + // TODO EMCH need to put kv in genesis + let backend = InMemory::from((storage.0, storage.1, Default::default())); let (b1data, _b1hash) = block1(genesis_hash, &backend); let mut overlay = OverlayedChanges::default(); diff --git a/core/client/src/light/backend.rs b/core/client/src/light/backend.rs index a161abaf6addf..aacada29787d5 100644 --- a/core/client/src/light/backend.rs +++ b/core/client/src/light/backend.rs @@ -21,6 +21,7 @@ use std::collections::HashMap; use std::sync::Arc; use parking_lot::{RwLock, Mutex}; +use primitives::child_trie::KeySpace; use sr_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay}; use state_machine::{Backend as StateBackend, TrieBackend, backend::InMemory as InMemoryState, ChangesTrieTransaction}; use sr_primitives::traits::{Block as BlockT, NumberFor, Zero, Header}; @@ -290,7 +291,8 @@ where storage.insert(Some(child_key), child_storage); } - let storage_update: InMemoryState = storage.into(); + // TODO EMCH need kv init here + let storage_update: InMemoryState = (storage, Default::default()).into(); let (storage_root, _) = storage_update.full_storage_root( ::std::iter::empty(), child_delta, @@ -362,6 +364,14 @@ impl StateBackend for GenesisOrUnavailableState } } + fn get_child_keyspace(&self, storage_key: &[u8]) -> ClientResult> { + match *self { + GenesisOrUnavailableState::Genesis(ref state) => + Ok(state.get_child_keyspace(storage_key).expect(IN_MEMORY_EXPECT_PROOF)), + GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient), + } + } + fn for_keys_with_prefix(&self, prefix: &[u8], action: A) { match *self { GenesisOrUnavailableState::Genesis(ref state) => state.for_keys_with_prefix(prefix, action), diff --git a/core/executor/src/wasm_executor.rs b/core/executor/src/wasm_executor.rs index 6ded0adad63cd..ba440e3ae1b3c 100644 --- a/core/executor/src/wasm_executor.rs +++ b/core/executor/src/wasm_executor.rs @@ -1667,7 +1667,7 @@ mod tests { b"input".to_vec() => b"Hello world".to_vec(), b"foo".to_vec() => b"bar".to_vec(), b"baz".to_vec() => b"bar".to_vec() - ], map![])); + ], map![], map![])); assert_eq!(ext, expected); } @@ -1690,7 +1690,7 @@ mod tests { b"aaa".to_vec() => b"1".to_vec(), b"aab".to_vec() => b"2".to_vec(), b"bbb".to_vec() => b"5".to_vec() - ], map![])); + ], map![], map![])); assert_eq!(expected, ext); } diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index afcb9e8345ac8..bd862a9a173ac 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -322,8 +322,7 @@ impl error::Error for Void { /// In-memory backend. Fully recomputes tries on each commit but useful for /// tests. pub struct InMemory { - // TODO EMCH remove keyspace: it is into kv. - inner: HashMap>, (KeySpace, HashMap, Vec>)>, + inner: HashMap>, HashMap, Vec>>, kv: Option, trie: Option, H, InMemoryKvBackend>>, _hasher: PhantomData, @@ -396,17 +395,10 @@ impl InMemory { pub fn update(&self, changes: >::Transaction) -> Self { let mut inner: HashMap<_, _> = self.inner.clone(); let mut kv: HashMap<_, _> = self.kv().clone(); - for (storage_key, keyspace, key, val) in changes.storage { - let entry = inner.entry(storage_key).or_insert_with(|| ( - // TODO EMCH here we need to assert storage key is none - // otherwhise it is undefined key leading to new keyspace - // creation. - keyspace.unwrap_or_else(|| NO_CHILD_KEYSPACE.to_vec()), - Default::default(), - )); + for (storage_key, key, val) in changes.storage { match val { - Some(v) => { entry.1.insert(key, v); }, - None => { entry.1.remove(&key); }, + Some(v) => { inner.entry(storage_key).or_default().insert(key, v); }, + None => { inner.entry(storage_key).or_default().remove(&key); }, } } kv.extend(changes.kv); @@ -416,32 +408,36 @@ impl InMemory { } } -impl From>, (KeySpace, HashMap, Vec>)>> for InMemory { - fn from(inner: HashMap>, (KeySpace, HashMap, Vec>)>) -> Self { +type TupleInit = ( + HashMap>, HashMap, Vec>>, + HashMap, Option>, +); + +impl From for InMemory { + fn from(inner: TupleInit) -> Self { InMemory { - inner: inner, + inner: inner.0, trie: None, - kv: Some(Default::default()), + kv: Some(inner.1), _hasher: PhantomData, } } } -impl From<( - HashMap, Vec>, - HashMap, (KeySpace, HashMap, Vec>)>, -)> for InMemory { - fn from(inners: ( - HashMap, Vec>, - HashMap, (KeySpace, HashMap, Vec>)>, - )) -> Self { - let mut inner: HashMap>, (KeySpace, HashMap, Vec>)> - = inners.1.into_iter().map(|(k, v)| (Some(k), v)).collect(); - inner.insert(None, (NO_CHILD_KEYSPACE.to_vec(), inners.0)); +type TupleInit2 = ( + HashMap, Vec>, + HashMap, HashMap, Vec>>, + HashMap, Option>, +); + +impl From for InMemory { + fn from(tuple: TupleInit2) -> Self { + let mut inner: HashMap<_, _> = tuple.1.into_iter().map(|(k, v)| (Some(k), v)).collect(); + inner.insert(None, tuple.0); InMemory { inner: inner, trie: None, - kv: Some(Default::default()), + kv: Some(tuple.2), _hasher: PhantomData, } } @@ -450,7 +446,7 @@ impl From<( impl From, Vec>> for InMemory { fn from(inner: HashMap, Vec>) -> Self { let mut expanded = HashMap::new(); - expanded.insert(None, (NO_CHILD_KEYSPACE.to_vec(), inner)); + expanded.insert(None, inner); InMemory { inner: expanded, trie: None, @@ -460,36 +456,33 @@ impl From, Vec>> for InMemory { } } -impl From>, Option, Vec, Option>)>> for InMemory { - fn from(inner: Vec<(Option>, Option, Vec, Option>)>) -> Self { - let mut expanded: HashMap>, (KeySpace, HashMap, Vec>)> = HashMap::new(); - for (child_key, keyspace, key, value) in inner { +type TupleTx = ( + Vec<(Option>, Vec, Option>)>, + HashMap, Option>, +); + + +impl From for InMemory { + fn from(inner: TupleTx) -> Self { + let mut expanded: HashMap>, HashMap, Vec>> = HashMap::new(); + for (child_key, key, value) in inner.0 { if let Some(value) = value { - expanded.entry(child_key).or_insert_with(|| - (keyspace.unwrap_or_else(|| NO_CHILD_KEYSPACE.to_vec()), Default::default()) - ).1.insert(key, value); + expanded.entry(child_key).or_default().insert(key, value); } } - expanded.into() + (expanded, inner.1).into() } } impl From for InMemory { fn from(inner: InMemoryTransaction) -> Self { - let mut expanded: HashMap< - Option>, - (KeySpace, HashMap, Vec>), - > = HashMap::new(); - for (child_key, keyspace, key, value) in inner.storage { + let mut expanded: HashMap>, HashMap, Vec>> = HashMap::new(); + for (child_key, key, value) in inner.storage { if let Some(value) = value { - expanded.entry(child_key).or_insert_with(|| - (keyspace.unwrap_or_else(|| NO_CHILD_KEYSPACE.to_vec()), Default::default()) - ).1.insert(key, value); + expanded.entry(child_key).or_default().insert(key, value); } } - let mut result: InMemory = expanded.into(); - *result.kv_mut() = inner.kv; - result + (expanded, inner.kv).into() } } @@ -505,7 +498,7 @@ impl InMemory { /// in memory storage. pub struct InMemoryTransaction { /// State trie key values changes (both top and child trie). - pub storage: Vec<(Option>, Option, Vec, Option>)>, + pub storage: Vec<(Option>, Vec, Option>)>, /// Changes to non trie key value datas. pub kv: HashMap, Option>>, } @@ -517,44 +510,41 @@ impl Backend for InMemory { type KvBackend = InMemoryKvBackend; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { - Ok(self.inner.get(&None).and_then(|map| map.1.get(key).map(Clone::clone))) + Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone))) } fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - Ok(self.inner.get(&Some(storage_key.to_vec())).and_then(|map| map.1.get(key).map(Clone::clone))) + Ok(self.inner.get(&Some(storage_key.to_vec())).and_then(|map| map.get(key).map(Clone::clone))) } fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, Self::Error> { - Ok(self.inner.get(&Some(storage_key.to_vec())) - .map(|(ks, _)| ks) - .map(Clone::clone) - .or_else(|| self.kv().get(&prefixed_keyspace_kv(storage_key)[..]) + Ok( + self.kv().get(&prefixed_keyspace_kv(storage_key)[..]) .map(Clone::clone) .unwrap_or(None) - ) ) } fn exists_storage(&self, key: &[u8]) -> Result { - Ok(self.inner.get(&None).map(|map| map.1.get(key).is_some()).unwrap_or(false)) + Ok(self.inner.get(&None).map(|map| map.get(key).is_some()).unwrap_or(false)) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { - self.inner.get(&None).map(|map| map.1.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f)); + self.inner.get(&None).map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f)); } fn for_key_values_with_prefix(&self, prefix: &[u8], mut f: F) { - self.inner.get(&None).map(|map| map.1.iter().filter(|(key, _val)| key.starts_with(prefix)) + self.inner.get(&None).map(|map| map.iter().filter(|(key, _val)| key.starts_with(prefix)) .for_each(|(k, v)| f(k, v))); } fn for_keys_in_child_storage(&self, storage_key: &[u8], mut f: F) { - self.inner.get(&Some(storage_key.to_vec())).map(|map| map.1.keys().for_each(|k| f(&k))); + self.inner.get(&Some(storage_key.to_vec())).map(|map| map.keys().for_each(|k| f(&k))); } fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { self.inner.get(&Some(storage_key.to_vec())) - .map(|map| map.1.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f)); + .map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f)); } fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) @@ -564,7 +554,7 @@ impl Backend for InMemory { { let existing_pairs = self.inner.get(&None) .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); let transaction: Vec<_> = delta.into_iter().collect(); let root = Layout::::trie_root(existing_pairs.chain(transaction.iter().cloned()) @@ -573,7 +563,7 @@ impl Backend for InMemory { .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) ); - let full_transaction = transaction.into_iter().map(|(k, v)| (None, None, k, v)).collect(); + let full_transaction = transaction.into_iter().map(|(k, v)| (None, k, v)).collect(); (root, InMemoryTransaction { storage: full_transaction, kv: Default::default() }) } @@ -587,7 +577,7 @@ impl Backend for InMemory { let existing_pairs = self.inner.get(&storage_key) .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); let transaction: Vec<_> = delta.into_iter().collect(); let root = child_trie_root::, _, _, _>( @@ -598,9 +588,8 @@ impl Backend for InMemory { .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) ); - let keyspace = self.inner.get(&storage_key).map(|t| t.0.clone()); let full_transaction = transaction.into_iter() - .map(|(k, v)| (storage_key.clone(), keyspace.clone(), k, v)).collect(); + .map(|(k, v)| (storage_key.clone(), k, v)).collect(); let is_default = root == default_child_trie_root::>( storage_key.as_ref().expect("Initialized to some") @@ -625,7 +614,7 @@ impl Backend for InMemory { fn pairs(&self) -> Vec<(Vec, Vec)> { self.inner.get(&None) .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone()))) + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))) .collect() } @@ -636,7 +625,7 @@ impl Backend for InMemory { fn child_pairs(&self, storage_key: &[u8]) -> Vec<(Vec, Vec)> { self.inner.get(&Some(storage_key.to_vec())) .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone()))) + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))) .collect() } @@ -647,14 +636,14 @@ impl Backend for InMemory { fn keys(&self, prefix: &[u8]) -> Vec> { self.inner.get(&None) .into_iter() - .flat_map(|map| map.1.keys().filter(|k| k.starts_with(prefix)).cloned()) + .flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()) .collect() } fn child_keys(&self, storage_key: &[u8], prefix: &[u8]) -> Vec> { self.inner.get(&Some(storage_key.to_vec())) .into_iter() - .flat_map(|map| map.1.keys().filter(|k| k.starts_with(prefix)).cloned()) + .flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()) .collect() } @@ -667,7 +656,7 @@ impl Backend for InMemory { let mut root_map = None; for (storage_key, map) in &self.inner { if let Some(storage_key) = storage_key.as_ref() { - let ch = insert_into_memory_db::(&mut mdb, map.1.clone().into_iter())?; + let ch = insert_into_memory_db::(&mut mdb, map.clone().into_iter())?; new_child_roots.push((storage_key.clone(), ch.as_ref().into())); } else { root_map = Some(map); @@ -677,7 +666,7 @@ impl Backend for InMemory { if let Some(map) = root_map.take() { root = Some(insert_into_memory_db::( &mut mdb, - map.1.clone().into_iter().chain(new_child_roots.into_iter()) + map.clone().into_iter().chain(new_child_roots.into_iter()) )?); } let root = match root { diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index 4c7e777943c47..f65e2a0fdfb14 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -340,7 +340,7 @@ mod tests { #[test] fn proof_recorded_and_checked() { - let contents = (0..64).map(|i| (None, None, vec![i], Some(vec![i]))).collect::>(); + let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))).collect::>(); let in_memory = InMemory::::default(); let mut in_memory = in_memory.update(InMemoryTransaction { storage: contents, @@ -370,9 +370,9 @@ mod tests { let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub2").unwrap(); let own1 = subtrie1.into_owned(); let own2 = subtrie2.into_owned(); - let contents = (0..64).map(|i| (None, None, vec![i], Some(vec![i]))) - .chain((28..65).map(|i| (Some(own1.clone()), None, vec![i], Some(vec![i])))) - .chain((10..15).map(|i| (Some(own2.clone()), None, vec![i], Some(vec![i])))) + let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))) + .chain((28..65).map(|i| (Some(own1.clone()), vec![i], Some(vec![i])))) + .chain((10..15).map(|i| (Some(own2.clone()), vec![i], Some(vec![i])))) .collect::>(); let in_memory = InMemory::::default(); let mut in_memory = in_memory.update(InMemoryTransaction { diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index eb431d9c57619..497c882e3782c 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -28,13 +28,17 @@ use crate::{ use primitives::{ storage::well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key}, traits::{BareCryptoStorePtr, Externalities}, offchain, child_storage_key::ChildStorageKey, - child_trie::{KeySpace, NO_CHILD_KEYSPACE}, + child_trie::{KeySpace, NO_CHILD_KEYSPACE, prefixed_keyspace_kv}, }; use codec::Encode; const EXT_NOT_ALLOWED_TO_FAIL: &str = "Externalities not allowed to fail within runtime"; -type StorageTuple = (HashMap, Vec>, HashMap, (KeySpace, HashMap, Vec>)>); +type StorageTuple = ( + HashMap, Vec>, + HashMap, HashMap, Vec>>, + HashMap, Option>, +); /// Simple HashMap-based Externalities impl. pub struct TestExternalities { @@ -46,6 +50,12 @@ pub struct TestExternalities { } impl TestExternalities { + // TOOD EMCH delete that when genesis support + /// Create a new instance of `TestExternalities` with storage. + pub fn new_todo(storage: StorageTuple2) -> Self { + Self::new_with_code(&[], (storage.0, storage.1, Default::default())) + } + /// Create a new instance of `TestExternalities` with storage. pub fn new(storage: StorageTuple) -> Self { Self::new_with_code(&[], storage) @@ -68,14 +78,14 @@ impl TestExternalities { storage.0.insert(CODE.to_vec(), code.to_vec()); let backend: HashMap<_, _> = storage.1.into_iter() - .map(|(keyspace, map)| (Some(keyspace), map)) - .chain(Some((None, (NO_CHILD_KEYSPACE.to_vec(), storage.0))).into_iter()) + .map(|(storage_key, map)| (Some(storage_key), map)) + .chain(Some((None, storage.0)).into_iter()) .collect(); TestExternalities { overlay, changes_trie_storage: ChangesTrieInMemoryStorage::new(), - backend: backend.into(), + backend: (backend, storage.2).into(), offchain: None, keystore: None, } @@ -84,7 +94,7 @@ impl TestExternalities { /// Insert key/value into backend pub fn insert(&mut self, k: Vec, v: Vec) { self.backend = self.backend.update(InMemoryTransaction { - storage: vec![(None, None, k, Some(v))], + storage: vec![(None, k, Some(v))], kv: Default::default(), }); } @@ -124,15 +134,13 @@ impl TestExternalities { pub fn commit_all(&self) -> InMemory { let top = self.overlay.committed.top.clone().into_iter() .chain(self.overlay.prospective.top.clone().into_iter()) - .map(|(k, v)| (None, None, k, v.value)); + .map(|(k, v)| (None, k, v.value)); let children = self.overlay.committed.children.clone().into_iter() .chain(self.overlay.prospective.children.clone().into_iter()) - .flat_map(|(storage_key, map)| { - let keyspace = self.get_child_keyspace(storage_key.as_slice()) - .expect("Test backend do not produce db error"); + .flat_map(|(keyspace, map)| { map.into_iter() - .map(|(k, v)| (Some(storage_key.clone()), keyspace.clone(), k, v.value)) + .map(|(k, v)| (Some(keyspace.clone()), k, v.value)) .collect::>() }); @@ -170,6 +178,15 @@ impl From for TestExternalit } } +// TODO EMCH this is only needed until genesis builds from kv to. +type StorageTuple2 = (HashMap, Vec>, HashMap, HashMap, Vec>>); + +impl From for TestExternalities { + fn from(storage: StorageTuple2) -> Self { + Self::new_todo(storage) + } +} + impl Externalities for TestExternalities where H: Hasher, N: ChangesTrieBlockNumber, diff --git a/core/test-runtime/src/system.rs b/core/test-runtime/src/system.rs index e61e72f3a0e85..716e3a42edf0e 100644 --- a/core/test-runtime/src/system.rs +++ b/core/test-runtime/src/system.rs @@ -337,7 +337,7 @@ mod tests { blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => { vec![111u8, 0, 0, 0, 0, 0, 0, 0] } - ], map![])) + ], map![], map![])) } fn block_import_works(block_executor: F) where F: Fn(Block, &mut TestExternalities) { diff --git a/node/executor/src/lib.rs b/node/executor/src/lib.rs index 864b51d6c75f0..758e018fe551d 100644 --- a/node/executor/src/lib.rs +++ b/node/executor/src/lib.rs @@ -140,7 +140,7 @@ mod tests { >::hashed_key_for(0) => { vec![0u8; 32] } - ], map![])); + ], map![], map![])); let r = executor().call::<_, NeverNativeValue, fn() -> _>( &mut t, @@ -176,7 +176,7 @@ mod tests { >::hashed_key_for(0) => { vec![0u8; 32] } - ], map![])); + ], map![], map![])); let r = executor().call::<_, NeverNativeValue, fn() -> _>( &mut t, @@ -208,7 +208,7 @@ mod tests { }, >::hashed_key().to_vec() => vec![0u8; 16], >::hashed_key_for(0) => vec![0u8; 32] - ], map![])); + ], map![], map![])); let r = executor().call::<_, NeverNativeValue, fn() -> _>( &mut t, @@ -244,7 +244,7 @@ mod tests { }, >::hashed_key().to_vec() => vec![0u8; 16], >::hashed_key_for(0) => vec![0u8; 32] - ], map![])); + ], map![], map![])); let r = executor().call::<_, NeverNativeValue, fn() -> _>( &mut t, @@ -270,9 +270,11 @@ mod tests { } fn new_test_ext(code: &[u8], support_changes_trie: bool) -> TestExternalities { + let storage = node_testing::genesis::config(support_changes_trie, Some(code)).build_storage().unwrap(); let mut ext = TestExternalities::new_with_code( code, - node_testing::genesis::config(support_changes_trie, Some(code)).build_storage().unwrap(), + // TODO EMCH get genesis config return kv store + (storage.0, storage.1, Default::default()), ); ext.changes_trie_storage().insert(0, GENESIS_HASH.into(), Default::default()); ext @@ -783,7 +785,7 @@ mod tests { }, >::hashed_key().to_vec() => vec![0u8; 16], >::hashed_key_for(0) => vec![0u8; 32] - ], map![])); + ], map![], map![])); let r = executor().call::<_, NeverNativeValue, fn() -> _>( &mut t, @@ -815,7 +817,7 @@ mod tests { }, >::hashed_key().to_vec() => vec![0u8; 16], >::hashed_key_for(0) => vec![0u8; 32] - ], map![])); + ], map![], map![])); let r = executor().call::<_, NeverNativeValue, fn() -> _>( &mut t, @@ -993,7 +995,7 @@ mod tests { }, >::hashed_key().to_vec() => vec![0u8; 16], >::hashed_key_for(0) => vec![0u8; 32] - ], map![])); + ], map![], map![])); let tip = 1_000_000; let xt = sign(CheckedExtrinsic { diff --git a/srml/authority-discovery/src/lib.rs b/srml/authority-discovery/src/lib.rs index 2ec9e988358cc..28d17a39d296f 100644 --- a/srml/authority-discovery/src/lib.rs +++ b/srml/authority-discovery/src/lib.rs @@ -260,7 +260,7 @@ mod tests { .unwrap(); // Create externalities. - let mut externalities = TestExternalities::new(t); + let mut externalities = TestExternalities::new_todo(t); externalities.set_keystore(key_store); with_externalities(&mut externalities, || { @@ -297,7 +297,7 @@ mod tests { .unwrap(); // Create externalities. - let mut externalities = TestExternalities::new(t); + let mut externalities = TestExternalities::new_todo(t); externalities.set_keystore(key_store); with_externalities(&mut externalities, || { @@ -334,7 +334,7 @@ mod tests { .unwrap(); // Create externalities. - let mut externalities = TestExternalities::new(t); + let mut externalities = TestExternalities::new_todo(t); externalities.set_keystore(key_store); with_externalities(&mut externalities, || { diff --git a/srml/contracts/src/tests.rs b/srml/contracts/src/tests.rs index f2ef8a275d66b..4ad998904c659 100644 --- a/srml/contracts/src/tests.rs +++ b/srml/contracts/src/tests.rs @@ -289,7 +289,7 @@ impl ExtBuilder { }, gas_price: self.gas_price, }.assimilate_storage(&mut t).unwrap(); - runtime_io::TestExternalities::new(t) + runtime_io::TestExternalities::new_todo(t) } } diff --git a/srml/democracy/src/lib.rs b/srml/democracy/src/lib.rs index b93f6f893aef9..440fab95b47cc 100644 --- a/srml/democracy/src/lib.rs +++ b/srml/democracy/src/lib.rs @@ -1091,7 +1091,7 @@ mod tests { vesting: vec![], }.assimilate_storage(&mut t).unwrap(); GenesisConfig::default().assimilate_storage(&mut t).unwrap(); - runtime_io::TestExternalities::new(t) + runtime_io::TestExternalities::new_todo(t) } type System = system::Module; diff --git a/srml/executive/src/lib.rs b/srml/executive/src/lib.rs index ad9cb7bf80742..298b5ba99b331 100644 --- a/srml/executive/src/lib.rs +++ b/srml/executive/src/lib.rs @@ -419,7 +419,7 @@ mod tests { }.assimilate_storage(&mut t).unwrap(); let xt = sr_primitives::testing::TestXt(sign_extra(1, 0, 0), Call::Balances(BalancesCall::transfer(2, 69))); let weight = xt.get_dispatch_info().weight as u64; - let mut t = runtime_io::TestExternalities::::new(t); + let mut t = runtime_io::TestExternalities::::new_todo(t); with_externalities(&mut t, || { Executive::initialize_block(&Header::new( 1, diff --git a/srml/finality-tracker/src/lib.rs b/srml/finality-tracker/src/lib.rs index 1bf9754b0a583..f079cec026e0c 100644 --- a/srml/finality-tracker/src/lib.rs +++ b/srml/finality-tracker/src/lib.rs @@ -321,7 +321,7 @@ mod tests { #[test] fn median_works() { let t = system::GenesisConfig::default().build_storage::().unwrap(); - with_externalities(&mut TestExternalities::new(t), || { + with_externalities(&mut TestExternalities::new_todo(t), || { FinalityTracker::update_hint(Some(500)); assert_eq!(FinalityTracker::median(), 250); assert!(NOTIFICATIONS.with(|n| n.borrow().is_empty())); @@ -331,7 +331,7 @@ mod tests { #[test] fn notifies_when_stalled() { let t = system::GenesisConfig::default().build_storage::().unwrap(); - with_externalities(&mut TestExternalities::new(t), || { + with_externalities(&mut TestExternalities::new_todo(t), || { let mut parent_hash = System::parent_hash(); for i in 2..106 { System::initialize(&i, &parent_hash, &Default::default(), &Default::default()); @@ -350,7 +350,7 @@ mod tests { #[test] fn recent_notifications_prevent_stalling() { let t = system::GenesisConfig::default().build_storage::().unwrap(); - with_externalities(&mut TestExternalities::new(t), || { + with_externalities(&mut TestExternalities::new_todo(t), || { let mut parent_hash = System::parent_hash(); for i in 2..106 { System::initialize(&i, &parent_hash, &Default::default(), &Default::default()); diff --git a/srml/session/src/historical.rs b/srml/session/src/historical.rs index 24821a0393260..b2b4787627fdf 100644 --- a/srml/session/src/historical.rs +++ b/srml/session/src/historical.rs @@ -330,7 +330,7 @@ mod tests { l.borrow().iter().cloned().map(|i| (i, UintAuthorityId(i).into())).collect() ), }.assimilate_storage(&mut t).unwrap(); - runtime_io::TestExternalities::new(t) + runtime_io::TestExternalities::new_todo(t) } #[test] diff --git a/srml/session/src/lib.rs b/srml/session/src/lib.rs index 208be4664aa6e..5b06e1c8b6df5 100644 --- a/srml/session/src/lib.rs +++ b/srml/session/src/lib.rs @@ -699,7 +699,7 @@ mod tests { l.borrow().iter().cloned().map(|i| (i, UintAuthorityId(i).into())).collect() ), }.assimilate_storage(&mut t).unwrap(); - runtime_io::TestExternalities::new(t) + runtime_io::TestExternalities::new_todo(t) } fn initialize_block(block: u64) { diff --git a/srml/system/src/lib.rs b/srml/system/src/lib.rs index 5a1115b90d71c..35f06acfff184 100644 --- a/srml/system/src/lib.rs +++ b/srml/system/src/lib.rs @@ -710,7 +710,7 @@ impl Module { >::hashed_key_for(T::BlockNumber::zero()) => [69u8; 32].encode(), >::hashed_key().to_vec() => T::BlockNumber::one().encode(), >::hashed_key().to_vec() => [69u8; 32].encode() - ], map![])) + ], map![], map![])) } /// Set the block number to something in particular. Can be used as an alternative to diff --git a/srml/timestamp/src/lib.rs b/srml/timestamp/src/lib.rs index 4af0de8bd1b18..53b53726386ba 100644 --- a/srml/timestamp/src/lib.rs +++ b/srml/timestamp/src/lib.rs @@ -369,7 +369,7 @@ mod tests { #[test] fn timestamp_works() { let t = system::GenesisConfig::default().build_storage::().unwrap(); - with_externalities(&mut TestExternalities::new(t), || { + with_externalities(&mut TestExternalities::new_todo(t), || { Timestamp::set_timestamp(42); assert_ok!(Timestamp::dispatch(Call::set(69), Origin::NONE)); assert_eq!(Timestamp::now(), 69); @@ -380,7 +380,7 @@ mod tests { #[should_panic(expected = "Timestamp must be updated only once in the block")] fn double_timestamp_should_fail() { let t = system::GenesisConfig::default().build_storage::().unwrap(); - with_externalities(&mut TestExternalities::new(t), || { + with_externalities(&mut TestExternalities::new_todo(t), || { Timestamp::set_timestamp(42); assert_ok!(Timestamp::dispatch(Call::set(69), Origin::NONE)); let _ = Timestamp::dispatch(Call::set(70), Origin::NONE); @@ -391,7 +391,7 @@ mod tests { #[should_panic(expected = "Timestamp must increment by at least between sequential blocks")] fn block_period_minimum_enforced() { let t = system::GenesisConfig::default().build_storage::().unwrap(); - with_externalities(&mut TestExternalities::new(t), || { + with_externalities(&mut TestExternalities::new_todo(t), || { Timestamp::set_timestamp(42); let _ = Timestamp::dispatch(Call::set(46), Origin::NONE); }); From 9a3d762d819ca8286653e0dacf08385fc2582469 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Oct 2019 16:35:33 +0200 Subject: [PATCH 57/68] Change trie crate from child_trie_soft_min --- core/trie/src/lib.rs | 150 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 126 insertions(+), 24 deletions(-) diff --git a/core/trie/src/lib.rs b/core/trie/src/lib.rs index eab05bab28af0..7bbc58f13db98 100644 --- a/core/trie/src/lib.rs +++ b/core/trie/src/lib.rs @@ -23,9 +23,11 @@ mod node_header; mod node_codec; mod trie_stream; +use primitives::child_trie::{KeySpace, keyspace_as_prefix_alloc}; use rstd::boxed::Box; use rstd::vec::Vec; -use hash_db::Hasher; +use rstd::marker::PhantomData; +use hash_db::{Hasher, Prefix}; /// Our `NodeCodec`-specific error. pub use error::Error; /// The Substrate format implementation of `TrieStream`. @@ -130,7 +132,8 @@ pub fn delta_trie_root( DB: hash_db::HashDB, { { - let mut trie = TrieDBMut::::from_existing(&mut *db, &mut root)?; + let mut db = KeySpacedDBMut::new(db, None); + let mut trie = TrieDBMut::::from_existing(&mut db, &mut root)?; for (key, change) in delta { match change { @@ -149,7 +152,9 @@ pub fn read_trie_value, key: &[u8] ) -> Result>, Box>> { - Ok(TrieDB::::new(&*db, root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) + let db = KeySpacedDB::new(db, None); + let db = TrieDB::::new(&db, root)?; + Ok(db.get(key).map(|x| x.map(|val| val.to_vec()))?) } /// Read a value from the trie with given Query. @@ -163,7 +168,9 @@ pub fn read_trie_value_with< key: &[u8], query: Q ) -> Result>, Box>> { - Ok(TrieDB::::new(&*db, root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) + let db = KeySpacedDB::new(db, None); + let db = TrieDB::::new(&db, root)?; + Ok(db.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) } /// Determine the default child trie root. @@ -187,6 +194,7 @@ pub fn child_trie_root(_storage_key: &[u8], input pub fn child_delta_trie_root( _storage_key: &[u8], db: &mut DB, + keyspace: &KeySpace, root_vec: Vec, delta: I ) -> Result, Box>> @@ -197,12 +205,11 @@ pub fn child_delta_trie_root( DB: hash_db::HashDB + hash_db::PlainDB, trie_db::DBValue>, { - let mut root = TrieHash::::default(); - // root is fetched from DB, not writable by runtime, so it's always valid. - root.as_mut().copy_from_slice(&root_vec); - + // keyspaced is needed (db can be init from this operation, this is not only root calculation) + let mut db = KeySpacedDBMut::new(&mut *db, Some(keyspace)); + let mut root = trie_root_as_hash::(root_vec); { - let mut trie = TrieDBMut::::from_existing(&mut *db, &mut root)?; + let mut trie = TrieDBMut::::from_existing(&mut db, &mut root)?; for (key, change) in delta { match change { @@ -211,14 +218,15 @@ pub fn child_delta_trie_root( }; } } - Ok(root.as_ref().to_vec()) + } /// Call `f` for all keys in a child trie. pub fn for_keys_in_child_trie( _storage_key: &[u8], db: &DB, + keyspace: &KeySpace, root_slice: &[u8], mut f: F ) -> Result<(), Box>> @@ -226,11 +234,9 @@ pub fn for_keys_in_child_trie( DB: hash_db::HashDBRef + hash_db::PlainDBRef, trie_db::DBValue>, { - let mut root = TrieHash::::default(); - // root is fetched from DB, not writable by runtime, so it's always valid. - root.as_mut().copy_from_slice(root_slice); - - let trie = TrieDB::::new(&*db, &root)?; + let root = trie_root_as_hash::(root_slice); + let db = KeySpacedDB::new(&*db, Some(keyspace)); + let trie = TrieDB::::new(&db, &root)?; let iter = trie.iter()?; for x in iter { @@ -249,7 +255,8 @@ pub fn record_all_keys( ) -> Result<(), Box>> where DB: hash_db::HashDBRef { - let trie = TrieDB::::new(&*db, root)?; + let db = KeySpacedDB::new(db, None); + let trie = TrieDB::::new(&db, root)?; let iter = trie.iter()?; for x in iter { @@ -268,6 +275,7 @@ pub fn record_all_keys( pub fn read_child_trie_value( _storage_key: &[u8], db: &DB, + keyspace: &KeySpace, root_slice: &[u8], key: &[u8] ) -> Result>, Box>> @@ -275,17 +283,17 @@ pub fn read_child_trie_value( DB: hash_db::HashDBRef + hash_db::PlainDBRef, trie_db::DBValue>, { - let mut root = TrieHash::::default(); - // root is fetched from DB, not writable by runtime, so it's always valid. - root.as_mut().copy_from_slice(root_slice); + let root = trie_root_as_hash::(root_slice); + let db = KeySpacedDB::new(&*db, Some(keyspace)); - Ok(TrieDB::::new(&*db, &root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) + Ok(TrieDB::::new(&db, &root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) } /// Read a value from the child trie with given query. pub fn read_child_trie_value_with, DB>( _storage_key: &[u8], db: &DB, + keyspace: &KeySpace, root_slice: &[u8], key: &[u8], query: Q @@ -294,13 +302,100 @@ pub fn read_child_trie_value_with + hash_db::PlainDBRef, trie_db::DBValue>, { - let mut root = TrieHash::::default(); - // root is fetched from DB, not writable by runtime, so it's always valid. - root.as_mut().copy_from_slice(root_slice); + let root = trie_root_as_hash::(root_slice); + let db = KeySpacedDB::new(&*db, Some(keyspace)); + Ok(TrieDB::::new(&db, &root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) +} + +/// `HashDB` implementation that append a encoded `KeySpace` (unique id in as bytes) with the +/// prefix of every key value. +pub struct KeySpacedDB<'a, DB, H>(&'a DB, Option<&'a KeySpace>, PhantomData); +/// `HashDBMut` implementation that append a encoded `KeySpace` (unique id in as bytes) with the +/// prefix of every key value. +/// +/// Mutable variant of `KeySpacedDB`, see [`KeySpacedDB`]. +pub struct KeySpacedDBMut<'a, DB, H>(&'a mut DB, Option<&'a KeySpace>, PhantomData); + + +impl<'a, DB, H> KeySpacedDB<'a, DB, H> where + H: Hasher, +{ + /// instantiate new keyspaced db + pub fn new(db: &'a DB, ks: Option<&'a KeySpace>) -> Self { + KeySpacedDB(db, ks, PhantomData) + } +} +impl<'a, DB, H> KeySpacedDBMut<'a, DB, H> where + H: Hasher, +{ + /// instantiate new keyspaced db + pub fn new(db: &'a mut DB, ks: Option<&'a KeySpace>) -> Self { + KeySpacedDBMut(db, ks, PhantomData) + } +} + +impl<'a, DB, H, T> hash_db::HashDBRef for KeySpacedDB<'a, DB, H> where + DB: hash_db::HashDBRef, + H: Hasher, + T: From<&'static [u8]>, +{ + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.get(key, (&derived_prefix.0, derived_prefix.1)) + } + + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.contains(key, (&derived_prefix.0, derived_prefix.1)) + } +} + +impl<'a, DB, H, T> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> where + DB: hash_db::HashDB, + H: Hasher, + T: Default + PartialEq + for<'b> From<&'b [u8]> + Clone + Send + Sync, +{ + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.get(key, (&derived_prefix.0, derived_prefix.1)) + } + + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.contains(key, (&derived_prefix.0, derived_prefix.1)) + } + + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.insert((&derived_prefix.0, derived_prefix.1), value) + } + + fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T) { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.emplace(key, (&derived_prefix.0, derived_prefix.1), value) + } + + fn remove(&mut self, key: &H::Out, prefix: Prefix) { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.remove(key, (&derived_prefix.0, derived_prefix.1)) + } +} + +impl<'a, DB, H, T> hash_db::AsHashDB for KeySpacedDBMut<'a, DB, H> where + DB: hash_db::HashDB, + H: Hasher, + T: Default + PartialEq + for<'b> From<&'b [u8]> + Clone + Send + Sync, +{ + fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { + &mut *self + } - Ok(TrieDB::::new(&*db, &root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) } + +// Utilities (not exported): + /// Constants used into trie simplification codec. mod trie_constants { pub const EMPTY_TRIE: u8 = 0; @@ -310,6 +405,13 @@ mod trie_constants { pub const BRANCH_WITH_MASK: u8 = 0b_11 << 6; } +fn trie_root_as_hash> (trie_root: R) -> TrieHash { + let mut root = >::default(); + // root is fetched from DB, not writable by runtime, so it's always valid. + root.as_mut().copy_from_slice(trie_root.as_ref()); + root +} + #[cfg(test)] mod tests { use super::*; From da23a0c6abddb22fb4c5ab14bfc4dc4d1c11c6a9 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Oct 2019 22:43:18 +0200 Subject: [PATCH 58/68] prepare change --- core/state-machine/src/backend.rs | 30 +++++-- core/state-machine/src/basic.rs | 6 +- core/state-machine/src/ext.rs | 15 +++- core/state-machine/src/proving_backend.rs | 23 +++-- core/state-machine/src/testing.rs | 7 +- core/state-machine/src/trie_backend.rs | 87 ++++++++++++++----- .../state-machine/src/trie_backend_essence.rs | 30 +++++-- core/test-runtime/src/lib.rs | 4 +- 8 files changed, 158 insertions(+), 44 deletions(-) diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index bd862a9a173ac..5ddb582bad179 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -103,7 +103,12 @@ pub trait Backend: std::fmt::Debug { /// Calculate the child storage root, with given delta over what is already stored in /// the backend, and produce a "transaction" that can be used to commit. The second argument /// is true if child storage root equals default storage root. - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + keyspace: &KeySpace, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord; @@ -165,10 +170,15 @@ pub trait Backend: std::fmt::Debug { let mut txs: Self::Transaction = self.kv_transaction(kv_deltas); let mut child_roots: Vec<_> = Default::default(); + // child first for (storage_key, child_delta) in child_deltas { + let keyspace = unimplemented!("get or new keyspace and put the \ + new keyspace to kv, ak tx consolidate kv transaction (first line with \ + those as kv_deltas)"); + let (child_root, empty, child_txs) = - self.child_storage_root(&storage_key[..], child_delta); + self.child_storage_root(&storage_key[..], &keyspace, child_delta); txs.consolidate(child_txs); if empty { child_roots.push((storage_key, None)); @@ -222,12 +232,17 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { (*self).storage_root(delta) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + keyspace: &KeySpace, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord, { - (*self).child_storage_root(storage_key, delta) + (*self).child_storage_root(storage_key, keyspace, delta) } fn kv_transaction(&self, delta: I) -> Self::Transaction @@ -568,7 +583,12 @@ impl Backend for InMemory { (root, InMemoryTransaction { storage: full_transaction, kv: Default::default() }) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + _keyspace: &KeySpace, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord diff --git a/core/state-machine/src/basic.rs b/core/state-machine/src/basic.rs index e45af45c9f80b..066506e5b34a0 100644 --- a/core/state-machine/src/basic.rs +++ b/core/state-machine/src/basic.rs @@ -25,6 +25,7 @@ use trie::trie_types::Layout; use primitives::{ storage::well_known_keys::is_child_storage_key, child_storage_key::ChildStorageKey, offchain, traits::Externalities, + child_trie::{KeySpace, NO_CHILD_KEYSPACE}, }; use log::warn; @@ -181,8 +182,9 @@ impl Externalities for BasicExternalities where H::Out: Ord { fn child_storage_root(&mut self, storage_key: ChildStorageKey) -> Vec { if let Some(child) = self.children.get(storage_key.as_ref()) { let delta = child.clone().into_iter().map(|(k, v)| (k, Some(v))); - - InMemory::::default().child_storage_root(storage_key.as_ref(), delta).0 + // no transaction produce, just send a dummy + let keyspace = NO_CHILD_KEYSPACE.to_vec(); + InMemory::::default().child_storage_root(storage_key.as_ref(), &keyspace, delta).0 } else { default_child_trie_root::>(storage_key.as_ref()) } diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 19e9fce4cf25b..ce3c7b9008769 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -29,6 +29,7 @@ use primitives::{ offchain, storage::well_known_keys::is_child_storage_key, traits::{BareCryptoStorePtr, Externalities}, child_storage_key::ChildStorageKey, hexdisplay::HexDisplay, + child_trie::{KeySpace, NO_CHILD_KEYSPACE}, }; use trie::{MemoryDB, default_child_trie_root}; use trie::trie_types::Layout; @@ -154,7 +155,6 @@ where } -#[cfg(test)] impl<'a, H, N, B, T, O> Ext<'a, H, N, B, T, O> where H: Hasher, @@ -163,6 +163,7 @@ where O: 'a + offchain::Externalities, N: crate::changes_trie::BlockNumber, { + #[cfg(test)] pub fn storage_pairs(&self) -> Vec<(Vec, Vec)> { use std::collections::HashMap; @@ -175,10 +176,15 @@ where .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) .collect() } + + fn get_child_keyspace(&self, storage_key: &[u8]) -> Option { + unimplemented!("query overlay then backend (overlay can have delete it)"); + } } impl<'a, B, T, H, N, O> Externalities for Ext<'a, H, N, B, T, O> -where H: Hasher, +where + H: Hasher, B: 'a + Backend, T: 'a + ChangesTrieStorage, H::Out: Ord + 'static, @@ -453,7 +459,10 @@ where H: Hasher, .into_iter() .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))); - let root = self.backend.child_storage_root(storage_key, delta).0; + let keyspace = self.get_child_keyspace(storage_key) + // no need to generate keyspace here (tx are dropped) + .unwrap_or_else(|| NO_CHILD_KEYSPACE.to_vec()); + let root = self.backend.child_storage_root(storage_key, &keyspace, delta).0; self.overlay.set_storage(storage_key.to_vec(), Some(root.to_vec())); diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index f65e2a0fdfb14..f07c45ace3c48 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -30,7 +30,7 @@ use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend}; use crate::kv_backend::{KvBackend, InMemory as InMemoryKvBackend}; -use primitives::child_trie::KeySpace; +use primitives::child_trie::{KeySpace, NO_CHILD_KEYSPACE}; /// Patricia trie-based backend essence which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. @@ -64,10 +64,11 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> pub fn child_storage( &mut self, storage_key: &[u8], + keyspace: &KeySpace, key: &[u8] ) -> Result>, String> { let root = self.storage(storage_key)? - .unwrap_or(default_child_trie_root::>(storage_key)); + .unwrap_or_else(|| default_child_trie_root::>(storage_key)); let mut read_overlay = S::Overlay::default(); let eph = Ephemeral::new( @@ -80,6 +81,7 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> read_child_trie_value_with::, _, _>( storage_key, &eph, + keyspace, &root, key, &mut *self.proof_recorder @@ -181,11 +183,17 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> } fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { + // default to using the main keyspace is not an issue + // here as we are in the case of non existing trie + // and will query default trie root to empty trie. + let keyspace = self.get_child_keyspace(storage_key)? + .unwrap_or_else(|| NO_CHILD_KEYSPACE.to_vec()); + ProvingBackendEssence { backend: self.backend.essence(), proof_recorder: &mut *self.proof_recorder.try_borrow_mut() .expect("only fails when already borrowed; child_storage() is non-reentrant; qed"), - }.child_storage(storage_key, key) + }.child_storage(storage_key, &keyspace, key) } fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, Self::Error> { @@ -238,12 +246,17 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> self.backend.storage_root(delta) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + keyspace: &KeySpace, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord { - self.backend.child_storage_root(storage_key, delta) + self.backend.child_storage_root(storage_key, keyspace, delta) } fn kv_transaction(&self, delta: I) -> Self::Transaction diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index 497c882e3782c..583b8b5e8bf39 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -307,7 +307,12 @@ impl Externalities for TestExternalities where .into_iter() .flat_map(|map| map.clone().into_iter().map(|(k, v)| (k, v.value)))); - self.backend.child_storage_root(storage_key, delta) + let keyspace = match self.get_child_keyspace(storage_key) { + Ok(Some(keyspace)) => keyspace, + // no transaction produced, can default when new trie + _ => NO_CHILD_KEYSPACE.to_vec(), + }; + self.backend.child_storage_root(storage_key, &keyspace, delta) }; if is_empty { self.overlay.set_storage(storage_key.into(), None); diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index 2fbf7d3ccdde5..bbd5aa99a30db 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -20,6 +20,7 @@ use log::{warn, debug}; use hash_db::Hasher; use trie::{Trie, delta_trie_root, default_child_trie_root, child_delta_trie_root}; use trie::trie_types::{TrieDB, TrieError, Layout}; +use trie::KeySpacedDB; use crate::trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral}; use crate::Backend; use crate::kv_backend::KvBackend; @@ -102,11 +103,13 @@ impl< } fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { -// TODO EMCHEMCH -// let keyspace = self.child_keyspace(storage_key); - // Then change essence functions to use keyspace as input. + let keyspace = if let Some(keyspace) = self.get_child_keyspace(storage_key)? { + keyspace + } else { + return Ok(None); + }; - self.essence.child_storage(storage_key, key) + self.essence.child_storage(storage_key, &keyspace, key) } fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, Self::Error> { @@ -122,7 +125,16 @@ impl< } fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { - self.essence.for_keys_in_child_storage(storage_key, f) + let keyspace = match self.get_child_keyspace(storage_key) { + Ok(Some(keyspace)) => keyspace, + Err(e) => { + warn!(target: "trie", "Failed to query keyspace: {}", e); + return; + }, + Ok(None) => return, + }; + + self.essence.for_keys_in_child_storage(storage_key, &keyspace, f) } fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { @@ -134,6 +146,7 @@ impl< let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay); let collect_all = || -> Result<_, Box>> { + let eph = KeySpacedDB::new(&eph, None); let trie = TrieDB::::new(&eph, self.essence.root())?; let mut v = Vec::new(); for x in trie.iter()? { @@ -170,12 +183,24 @@ impl< let mut read_overlay = S::Overlay::default(); let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay); + let keyspace = match self.get_child_keyspace(storage_key) { + Ok(ks) => ks, + Err(e) => { + debug!(target: "trie", "Error extracting child trie values: {}", e); + return Vec::new(); + } + }; + let collect_all = || -> Result<_, Box>> { - let trie = TrieDB::::new(&eph, &root)?; let mut v = Vec::new(); - for x in trie.iter()? { - let (key, value) = x?; - v.push((key.to_vec(), value.to_vec())); + // if no keyspace, the child trie just got created. + if let Some(keyspace) = keyspace { + let eph = KeySpacedDB::new(&eph, Some(&keyspace)); + let trie = TrieDB::::new(&eph, &root)?; + for x in trie.iter()? { + let (key, value) = x?; + v.push((key.to_vec(), value.to_vec())); + } } Ok(v) @@ -199,6 +224,7 @@ impl< let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay); let collect_all = || -> Result<_, Box>> { + let eph = KeySpacedDB::new(&eph, None); let trie = TrieDB::::new(&eph, self.essence.root())?; let mut v = Vec::new(); for x in trie.iter()? { @@ -235,7 +261,12 @@ impl< (root, (write_overlay, Default::default())) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + keyspace: &KeySpace, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord @@ -260,6 +291,7 @@ impl< match child_delta_trie_root::, _, _, _, _>( storage_key, &mut eph, + keyspace, root.clone(), delta ) { @@ -293,26 +325,31 @@ impl< pub mod tests { use std::collections::HashSet; use primitives::{Blake2Hasher, H256}; - use codec::Encode; - use trie::{TrieMut, PrefixedMemoryDB, trie_types::TrieDBMut}; + use primitives::child_trie::{produce_keyspace, KEYSPACE_COUNTER, reverse_keyspace}; + use trie::{TrieMut, PrefixedMemoryDB, KeySpacedDBMut, trie_types::TrieDBMut}; use super::*; type KvBackend = crate::kv_backend::InMemory; + const CHILD_KEY_1: &[u8; 27] = b":child_storage:default:sub1"; + fn test_db() -> (PrefixedMemoryDB, H256, KvBackend) { let mut root = H256::default(); let mut mdb = PrefixedMemoryDB::::default(); + + let keyspace1 = produce_keyspace(1); + let mut sub_root = H256::default(); { - let mut trie = TrieDBMut::new(&mut mdb, &mut root); + let mut mdb = KeySpacedDBMut::new(&mut mdb, Some(&keyspace1)); + let mut trie = TrieDBMut::new(&mut mdb, &mut sub_root); trie.insert(b"value3", &[142]).expect("insert failed"); trie.insert(b"value4", &[124]).expect("insert failed"); - }; - + } + { - let mut sub_root = Vec::new(); - root.encode_to(&mut sub_root); + let mut mdb = KeySpacedDBMut::new(&mut mdb, None); let mut trie = TrieDBMut::new(&mut mdb, &mut root); - trie.insert(b":child_storage:default:sub1", &sub_root).expect("insert failed"); + trie.insert(CHILD_KEY_1, &sub_root[..]).expect("insert failed"); trie.insert(b"key", b"value").expect("insert failed"); trie.insert(b"value1", &[42]).expect("insert failed"); trie.insert(b"value2", &[24]).expect("insert failed"); @@ -323,8 +360,8 @@ pub mod tests { } // empty history. let mut kv = crate::kv_backend::InMemory::default(); - kv.insert(b"kv1".to_vec(), Some(b"kv_value1".to_vec())); - kv.insert(b"kv2".to_vec(), Some(b"kv_value2".to_vec())); + kv.insert(KEYSPACE_COUNTER.to_vec(), Some(keyspace1.clone())); + kv.insert(prefixed_keyspace_kv(&CHILD_KEY_1[..]), Some(keyspace1)); (mdb, root, kv) } @@ -339,6 +376,15 @@ pub mod tests { assert_eq!(test_trie().storage(b"key").unwrap(), Some(b"value".to_vec())); } + #[test] + fn read_from_child_storage_returns_some() { + let test_trie = test_trie(); + assert_eq!( + test_trie.child_storage(CHILD_KEY_1, b"value3").unwrap(), + Some(vec![142u8]), + ); + } + #[test] fn read_from_storage_returns_none() { assert_eq!(test_trie().storage(b"non-existing-key").unwrap(), None); @@ -374,6 +420,7 @@ pub mod tests { fn storage_root_transaction_is_non_empty() { let (new_root, mut tx) = test_trie().storage_root(vec![(b"new-key".to_vec(), Some(b"new-value".to_vec()))]); assert!(!tx.0.drain().is_empty()); + assert!(tx.1.is_empty()); assert!(new_root != test_trie().storage_root(::std::iter::empty()).0); } diff --git a/core/state-machine/src/trie_backend_essence.rs b/core/state-machine/src/trie_backend_essence.rs index 5a5431963448c..625ca4864e332 100644 --- a/core/state-machine/src/trie_backend_essence.rs +++ b/core/state-machine/src/trie_backend_essence.rs @@ -23,9 +23,10 @@ use log::{debug, warn}; use hash_db::{self, Hasher, EMPTY_PREFIX, Prefix}; use trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue, default_child_trie_root, read_trie_value, read_child_trie_value, - for_keys_in_child_trie}; + for_keys_in_child_trie, KeySpacedDB}; use trie::trie_types::{TrieDB, TrieError, Layout}; use crate::backend::Consolidate; +use primitives::child_trie::KeySpace; /// Patricia trie-based storage trait. pub trait Storage: Send + Sync { @@ -77,9 +78,17 @@ impl, H: Hasher> TrieBackendEssence { } /// Get the value of child storage at given key. - pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, String> { - let root = self.storage(storage_key)? - .unwrap_or(default_child_trie_root::>(storage_key)); + pub fn child_storage( + &self, + storage_key: &[u8], + keyspace: &KeySpace, + key: &[u8], + ) -> Result>, String> { + let root = if let Some(root) = self.storage(storage_key)? { + root + } else { + return Ok(None) + }; let mut read_overlay = S::Overlay::default(); let eph = Ephemeral { @@ -89,11 +98,16 @@ impl, H: Hasher> TrieBackendEssence { let map_e = |e| format!("Trie lookup error: {}", e); - read_child_trie_value::, _>(storage_key, &eph, &root, key).map_err(map_e) + read_child_trie_value::, _>(storage_key, &eph, &keyspace, &root, key).map_err(map_e) } /// Retrieve all entries keys of child storage and call `f` for each of those keys. - pub fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { + pub fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + keyspace: &KeySpace, + f: F, + ) { let root = match self.storage(storage_key) { Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key)), Err(e) => { @@ -111,6 +125,7 @@ impl, H: Hasher> TrieBackendEssence { if let Err(e) = for_keys_in_child_trie::, _, Ephemeral>( storage_key, &eph, + keyspace, &root, f, ) { @@ -152,7 +167,8 @@ impl, H: Hasher> TrieBackendEssence { }; let mut iter = move || -> Result<(), Box>> { - let trie = TrieDB::::new(&eph, root)?; + let eph = KeySpacedDB::new(&eph, None); + let trie = TrieDB::::new(&eph, &self.root)?; let mut iter = trie.iter()?; iter.seek(prefix)?; diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index e75cb69149d8d..22b47ca04b487 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -36,7 +36,7 @@ use primitives::{ use app_crypto::{ed25519, sr25519, RuntimeAppPublic}; pub use app_crypto; use trie_db::{TrieMut, Trie}; -use substrate_trie::PrefixedMemoryDB; +use substrate_trie::{KeySpacedDBMut, KeySpacedDB, PrefixedMemoryDB}; use substrate_trie::trie_types::{TrieDB, TrieDBMut}; use substrate_client::{ @@ -431,6 +431,7 @@ fn code_using_trie() -> u64 { let mut root = rstd::default::Default::default(); let _ = { let v = &pairs; + let mut mdb = KeySpacedDBMut::new(&mut mdb, None); let mut t = TrieDBMut::::new(&mut mdb, &mut root); for i in 0..v.len() { let key: &[u8]= &v[i].0; @@ -442,6 +443,7 @@ fn code_using_trie() -> u64 { t }; + let mdb = KeySpacedDB::new(&mdb, None); if let Ok(trie) = TrieDB::::new(&mdb, &root) { if let Ok(iter) = trie.iter() { let mut iter_pairs = Vec::new(); From 3931280f7df898058053c9a4e4d681bcbc2278a2 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Oct 2019 23:05:25 +0200 Subject: [PATCH 59/68] fix proof test --- core/state-machine/src/trie_backend.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index bbd5aa99a30db..4c6a00943f8b6 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -26,7 +26,7 @@ use crate::Backend; use crate::kv_backend::KvBackend; use primitives::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; use std::collections::HashMap; -use primitives::child_trie::{KeySpace, prefixed_keyspace_kv}; +use primitives::child_trie::{KeySpace, prefixed_keyspace_kv, NO_CHILD_KEYSPACE}; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. /// A simple key value backend is also accessible for direct key value storage. @@ -106,7 +106,9 @@ impl< let keyspace = if let Some(keyspace) = self.get_child_keyspace(storage_key)? { keyspace } else { - return Ok(None); + // cannot early exit for case where we do not prefix db + // (proof): can be any dummy value here. + NO_CHILD_KEYSPACE.to_vec() }; self.essence.child_storage(storage_key, &keyspace, key) From f6268ab624fd54632216b18b392ef82baee3e4af Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Oct 2019 10:20:43 +0200 Subject: [PATCH 60/68] need a kv_storage function in backend. --- core/state-machine/src/backend.rs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index 5ddb582bad179..77a8c8a47e316 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -48,6 +48,9 @@ pub trait Backend: std::fmt::Debug { /// Get keyed storage or None if there is nothing associated. fn storage(&self, key: &[u8]) -> Result>, Self::Error>; + /// Access a value in the key value storage. + fn kv_storage(&self, key: &[u8]) -> Result>, Self::Error>; + /// Get keyed storage value hash or None if there is nothing associated. fn storage_hash(&self, key: &[u8]) -> Result, Self::Error> { self.storage(key).map(|v| v.map(|v| H::hash(&v))) @@ -62,7 +65,9 @@ pub trait Backend: std::fmt::Debug { } /// Get technical keyspace use for child storage key. - fn get_child_keyspace(&self, storage_key: &[u8]) -> Result, Self::Error>; + fn get_child_keyspace2(&self, storage_key: &[u8]) -> Result, Self::Error> { + self.kv_storage(&prefixed_keyspace_kv(&storage_key)) + } /// true if a key exists in storage. fn exists_storage(&self, key: &[u8]) -> Result { @@ -159,7 +164,7 @@ pub trait Backend: std::fmt::Debug { delta: I1, child_deltas: I2, kv_deltas: I3, - ) -> (H::Out, Self::Transaction) + ) -> Result<(H::Out, Self::Transaction), Self::Error> where I1: IntoIterator, Option>)>, I2i: IntoIterator, Option>)>, @@ -167,15 +172,21 @@ pub trait Backend: std::fmt::Debug { I3: IntoIterator, Option>)>, ::Out: Ord, { - let mut txs: Self::Transaction = self.kv_transaction(kv_deltas); + let mut txs: Self::Transaction = Default::default(); let mut child_roots: Vec<_> = Default::default(); + let mut counter_keyspace = None; + let mut new_keyspaces = Vec::new(); // child first for (storage_key, child_delta) in child_deltas { - let keyspace = unimplemented!("get or new keyspace and put the \ - new keyspace to kv, ak tx consolidate kv transaction (first line with \ - those as kv_deltas)"); + let keyspace = match self.get_child_keyspace(storage_key.as_slice())? { + Some(keyspace) => keyspace, + None => { + // new child trie keyspace needed (creation of ct). + self.new_child_keyspace(storage_key, &mut counter_keyspace, &mut new_keyspaces)? + }, + }; let (child_root, empty, child_txs) = self.child_storage_root(&storage_key[..], &keyspace, child_delta); @@ -190,7 +201,10 @@ pub trait Backend: std::fmt::Debug { delta.into_iter().chain(child_roots.into_iter()) ); txs.consolidate(parent_txs); - (root, txs) + txs.consolidate(self.kv_transaction( + kv_deltas.into_iter().chain(new_keyspaces.into_iter()) + )); + Ok((root, txs)) } } From 431f93d5689e202b35644abbe8ec09224950675a Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Oct 2019 10:49:55 +0200 Subject: [PATCH 61/68] add 'kv_store' to state machine backend --- core/client/db/src/lib.rs | 4 ++++ core/client/db/src/storage_cache.rs | 4 ++++ core/client/src/light/backend.rs | 8 ++++++++ core/state-machine/src/backend.rs | 15 +++++++++++++++ core/state-machine/src/proving_backend.rs | 4 ++++ core/state-machine/src/trie_backend.rs | 8 ++++---- 6 files changed, 39 insertions(+), 4 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 8e31d11071b6a..0f3db1fc805c5 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -149,6 +149,10 @@ impl StateBackend for RefTrackingState { self.state.child_storage(storage_key, key) } + fn kv_storage(&self, key: &[u8]) -> Result>, Self::Error> { + self.state.kv_storage(key) + } + fn exists_storage(&self, key: &[u8]) -> Result { self.state.exists_storage(key) } diff --git a/core/client/db/src/storage_cache.rs b/core/client/db/src/storage_cache.rs index 9a0bc7df673f0..fae9642149bc1 100644 --- a/core/client/db/src/storage_cache.rs +++ b/core/client/db/src/storage_cache.rs @@ -531,6 +531,10 @@ impl, B: BlockT> StateBackend for CachingState< Ok(value) } + fn kv_storage(&self, key: &[u8]) -> Result>, Self::Error> { + self.state.kv_storage(key) + } + fn exists_storage(&self, key: &[u8]) -> Result { Ok(self.storage(key)?.is_some()) } diff --git a/core/client/src/light/backend.rs b/core/client/src/light/backend.rs index a161abaf6addf..69b79801a89d6 100644 --- a/core/client/src/light/backend.rs +++ b/core/client/src/light/backend.rs @@ -362,6 +362,14 @@ impl StateBackend for GenesisOrUnavailableState } } + fn kv_storage(&self, key: &[u8]) -> ClientResult>> { + match *self { + GenesisOrUnavailableState::Genesis(ref state) => + Ok(state.kv_storage(key).expect(IN_MEMORY_EXPECT_PROOF)), + GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient), + } + } + fn for_keys_with_prefix(&self, prefix: &[u8], action: A) { match *self { GenesisOrUnavailableState::Genesis(ref state) => state.for_keys_with_prefix(prefix, action), diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index ea41ed44a79e0..de5a5078283c8 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -47,6 +47,9 @@ pub trait Backend: std::fmt::Debug { /// Get keyed storage or None if there is nothing associated. fn storage(&self, key: &[u8]) -> Result>, Self::Error>; + /// Access a value in the key value storage. + fn kv_storage(&self, key: &[u8]) -> Result>, Self::Error>; + /// Get keyed storage value hash or None if there is nothing associated. fn storage_hash(&self, key: &[u8]) -> Result, Self::Error> { self.storage(key).map(|v| v.map(|v| H::hash(&v))) @@ -194,6 +197,10 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { (*self).child_storage(storage_key, key) } + fn kv_storage(&self, key: &[u8]) -> Result>, Self::Error> { + (*self).kv_storage(key) + } + fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { (*self).for_keys_in_child_storage(storage_key, f) } @@ -501,6 +508,14 @@ impl Backend for InMemory { Ok(self.inner.get(&Some(storage_key.to_vec())).and_then(|map| map.get(key).map(Clone::clone))) } + fn kv_storage(&self, key: &[u8]) -> Result>, Self::Error> { + Ok( + self.kv().get(key) + .map(Clone::clone) + .unwrap_or(None) + ) + } + fn exists_storage(&self, key: &[u8]) -> Result { Ok(self.inner.get(&None).map(|map| map.get(key).is_some()).unwrap_or(false)) } diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index f8b97fcccfc25..b277f69181a03 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -187,6 +187,10 @@ impl<'a, S, H, O> Backend for ProvingBackend<'a, S, H, O> }.child_storage(storage_key, key) } + fn kv_storage(&self, key: &[u8]) -> Result>, Self::Error> { + self.backend.kv_storage(key) + } + fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { self.backend.for_keys_in_child_storage(storage_key, f) } diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs index c8f6a0977b3bf..013bdaf457a28 100644 --- a/core/state-machine/src/trie_backend.rs +++ b/core/state-machine/src/trie_backend.rs @@ -108,13 +108,13 @@ impl< } fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - -// let keyspace = self.child_keyspace(storage_key); - // Then change essence functions to use keyspace as input. - self.essence.child_storage(storage_key, key) } + fn kv_storage(&self, key: &[u8]) -> Result>, Self::Error> { + self.kv_storage.get(key) + } + fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { self.essence.for_keys_with_prefix(prefix, f) } From 2b4dad9e323e90eb3f0625db751883431004ef70 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Oct 2019 11:58:06 +0200 Subject: [PATCH 62/68] state machine missing functions. --- core/state-machine/src/backend.rs | 34 ++++++++++++++++++++++++++++--- core/state-machine/src/ext.rs | 10 +++++++-- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index 15012cd5deb15..a46f72415de6f 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -26,7 +26,10 @@ use trie::{ TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::{TrieDBMut, Layout}, }; -use primitives::child_trie::{KeySpace, NO_CHILD_KEYSPACE, prefixed_keyspace_kv}; +use primitives::child_trie::{ + KeySpace, NO_CHILD_KEYSPACE, prefixed_keyspace_kv, KEYSPACE_COUNTER, + reverse_keyspace, produce_keyspace, +}; /// A state backend is used to read state data and can have changes committed /// to it. @@ -183,11 +186,35 @@ pub trait Backend: std::fmt::Debug { let keyspace = match self.get_child_keyspace(storage_key.as_slice())? { Some(keyspace) => keyspace, None => { - // new child trie keyspace needed (creation of ct). - self.new_child_keyspace(storage_key, &mut counter_keyspace, &mut new_keyspaces)? + if counter_keyspace.is_none() { + let counter_keyspace_enc = self.kv_storage(KEYSPACE_COUNTER)? + .unwrap_or(produce_keyspace(0)); + let keyspace = reverse_keyspace(counter_keyspace_enc.as_slice()) + .expect("Keyspaces are never added manually so encoding is valid"); + counter_keyspace = Some(keyspace); + } + // increment counter + counter_keyspace.map(|c| { + c + 1 + }); + let enc_counter_keyspace = produce_keyspace( + *counter_keyspace.as_ref().expect("lazy init at start of this block") + ); + new_keyspaces.push(( + prefixed_keyspace_kv(storage_key.as_slice()), + Some(enc_counter_keyspace.clone()), + )); + enc_counter_keyspace }, }; + counter_keyspace.map(|c| { + new_keyspaces.push(( + KEYSPACE_COUNTER.to_vec(), + Some(produce_keyspace(c)), + )); + }); + let (child_root, empty, child_txs) = self.child_storage_root(&storage_key[..], &keyspace, child_delta); txs.consolidate(child_txs); @@ -206,6 +233,7 @@ pub trait Backend: std::fmt::Debug { )); Ok((root, txs)) } + } impl<'a, T: Backend, H: Hasher> Backend for &'a T { diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index c3d64dbe2912b..ead6c47b772ef 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -29,7 +29,7 @@ use primitives::{ offchain, storage::well_known_keys::is_child_storage_key, traits::{BareCryptoStorePtr, Externalities}, child_storage_key::ChildStorageKey, hexdisplay::HexDisplay, - child_trie::{KeySpace, NO_CHILD_KEYSPACE}, + child_trie::{KeySpace, NO_CHILD_KEYSPACE, prefixed_keyspace_kv}, }; use trie::{MemoryDB, default_child_trie_root}; use trie::trie_types::Layout; @@ -178,7 +178,13 @@ where } fn get_child_keyspace(&self, storage_key: &[u8]) -> Option { - unimplemented!("query overlay then backend (overlay can have delete it)"); + match self.overlay.kv_storage(prefixed_keyspace_kv(storage_key).as_slice()) { + Some(Some(keyspace)) => return Some(keyspace.to_vec()), + Some(None) => return None, + None => (), + } + self.backend.get_child_keyspace(storage_key) + .expect(EXT_NOT_ALLOWED_TO_FAIL) } } From fc16bae3c885e94497c17ea4ebe4c0c8b6e310bc Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Oct 2019 12:33:41 +0200 Subject: [PATCH 63/68] fix other modules compilation. --- core/client/db/src/lib.rs | 8 ++++---- core/client/db/src/storage_cache.rs | 9 +++++++-- core/client/src/in_mem.rs | 2 +- core/client/src/light/backend.rs | 6 +++--- core/test-runtime/src/lib.rs | 5 ++--- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index f18b59b809d1a..97cde37620e68 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -185,11 +185,11 @@ impl StateBackend for RefTrackingState { self.state.storage_root(delta) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root(&self, storage_key: &[u8], keyspace: &KeySpace, delta: I) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, { - self.state.child_storage_root(storage_key, delta) + self.state.child_storage_root(storage_key, keyspace, delta) } fn kv_transaction(&self, delta: I) -> Self::Transaction @@ -561,7 +561,7 @@ where Block: BlockT, top.into_iter().map(|(k, v)| (k, Some(v))), child_delta, None, - ); + )?; self.db_updates = transaction; Ok(root) @@ -1930,7 +1930,7 @@ mod tests { storage.iter().cloned(), child, storage.iter().cloned(), - ); + ).unwrap(); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); diff --git a/core/client/db/src/storage_cache.rs b/core/client/db/src/storage_cache.rs index 789db21f32471..32b712016896e 100644 --- a/core/client/db/src/storage_cache.rs +++ b/core/client/db/src/storage_cache.rs @@ -568,12 +568,17 @@ impl, B: BlockT> StateBackend for CachingState< self.state.storage_root(delta) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + keyspace: &KeySpace, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord { - self.state.child_storage_root(storage_key, delta) + self.state.child_storage_root(storage_key, keyspace, delta) } fn kv_transaction(&self, delta: I) -> Self::Transaction diff --git a/core/client/src/in_mem.rs b/core/client/src/in_mem.rs index acb7c4333753e..5210b6e095a79 100644 --- a/core/client/src/in_mem.rs +++ b/core/client/src/in_mem.rs @@ -513,7 +513,7 @@ where top.into_iter().map(|(k, v)| (k, Some(v))), child_delta, None, - ); + ).map_err(|e| error::Error::from(format!("full storage root: {}", e)))?; self.new_state = Some(InMemory::from(transaction)); Ok(root) diff --git a/core/client/src/light/backend.rs b/core/client/src/light/backend.rs index af47576bd91e4..ff464b8a5e008 100644 --- a/core/client/src/light/backend.rs +++ b/core/client/src/light/backend.rs @@ -297,7 +297,7 @@ where ::std::iter::empty(), child_delta, ::std::iter::empty(), - ); + ).map_err(|e| ClientError::from(format!("full storage root: {}", e)))?; self.storage_update = Some(storage_update); Ok(storage_root) @@ -418,13 +418,13 @@ impl StateBackend for GenesisOrUnavailableState } } - fn child_storage_root(&self, key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root(&self, key: &[u8], keyspace: &KeySpace, delta: I) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)> { match *self { GenesisOrUnavailableState::Genesis(ref state) => { - let (root, is_equal, _) = state.child_storage_root(key, delta); + let (root, is_equal, _) = state.child_storage_root(key, keyspace, delta); (root, is_equal, ()) }, GenesisOrUnavailableState::Unavailable => (H::Out::default().as_ref().to_vec(), true, ()), diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 22b47ca04b487..ff023f222cef4 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -429,7 +429,7 @@ fn code_using_trie() -> u64 { let mut mdb = PrefixedMemoryDB::default(); let mut root = rstd::default::Default::default(); - let _ = { + { let v = &pairs; let mut mdb = KeySpacedDBMut::new(&mut mdb, None); let mut t = TrieDBMut::::new(&mut mdb, &mut root); @@ -440,8 +440,7 @@ fn code_using_trie() -> u64 { return 101; } } - t - }; + } let mdb = KeySpacedDB::new(&mdb, None); if let Ok(trie) = TrieDB::::new(&mdb, &root) { From ec10b6d0ba5fa3f6370b77dcaf5ccdbe66813902 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Oct 2019 14:11:43 +0200 Subject: [PATCH 64/68] revert into_committed and TODO for caching that is major (child root are not being cached at this point). --- core/client/src/client.rs | 7 ++----- core/state-machine/src/overlayed_changes.rs | 11 +++++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/client/src/client.rs b/core/client/src/client.rs index 8bcf1956862ef..017b52d4f1014 100644 --- a/core/client/src/client.rs +++ b/core/client/src/client.rs @@ -1074,9 +1074,7 @@ impl Client where overlay.commit_prospective(); - let (top, children, kv) = overlay.into_committed(); - // TODO EMCH can we resolve keyspace here: will need to get kv intact - // from into_committed and make it an iter in a next step + let (top, children, offstate) = overlay.into_committed(); let children = children.map(|(sk, it)| (sk, it.collect())).collect(); if import_headers.post().state_root() != &storage_update.1 { return Err(error::Error::InvalidStateRoot); @@ -1085,8 +1083,7 @@ impl Client where Ok(( Some(storage_update.0), Some(changes_update), - // switching kv from map to vec for compactness - Some((top.collect(), children, kv.into_iter().collect())), + Some((top.collect(), children, offstate.collect())), )) }, None => Ok((None, None, None)) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 2dc692c565a83..1277a00bfc3f1 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -326,21 +326,24 @@ impl OverlayedChanges { /// Consume `OverlayedChanges` and take committed set. /// - /// The child keyspace is only added if it was change in the - /// commit set. + /// TODO EMCH this one is use to feed storage cache: but that means + /// we miss the child trie root update in storage cache and the + /// keyspace updates to (good thing is that kv store can be cache from + /// transaction, a cache update should also be send for the root from full + /// storage root). (both are not in the overlay but change on fsr call). /// /// Panics: /// Will panic if there are any uncommitted prospective changes. pub fn into_committed(self) -> ( impl Iterator, Option>)>, impl Iterator, impl Iterator, Option>)>)>, - HashMap, Option>>, + impl Iterator, Option>)>, ){ assert!(self.prospective.is_empty()); (self.committed.top.into_iter().map(|(k, v)| (k, v.value)), self.committed.children.into_iter() .map(|(sk, v)| (sk, v.into_iter().map(|(k, v)| (k, v.value)))), - self.committed.kv) + self.committed.offstate.into_iter()) } /// Inserts storage entry responsible for current extrinsic index. From 1209ac7134fe659bef07afc018f66a588d074e5f Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Oct 2019 14:19:23 +0200 Subject: [PATCH 65/68] apply renaming --- core/state-machine/src/overlayed_changes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 1277a00bfc3f1..b0991440fa15d 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -343,7 +343,7 @@ impl OverlayedChanges { (self.committed.top.into_iter().map(|(k, v)| (k, v.value)), self.committed.children.into_iter() .map(|(sk, v)| (sk, v.into_iter().map(|(k, v)| (k, v.value)))), - self.committed.offstate.into_iter()) + self.committed.kv.into_iter()) } /// Inserts storage entry responsible for current extrinsic index. From 0eed4114d312da2f3036754939108d043e738ea4 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Oct 2019 17:00:41 +0200 Subject: [PATCH 66/68] Get and modify previous transaction test. Fix incorrect full storage root. --- core/state-machine/src/backend.rs | 4 ++-- core/state-machine/src/lib.rs | 35 +++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs index a46f72415de6f..1eb829d1dda4f 100644 --- a/core/state-machine/src/backend.rs +++ b/core/state-machine/src/backend.rs @@ -194,8 +194,8 @@ pub trait Backend: std::fmt::Debug { counter_keyspace = Some(keyspace); } // increment counter - counter_keyspace.map(|c| { - c + 1 + counter_keyspace.as_mut().map(|c| { + *c += 1; }); let enc_counter_keyspace = produce_keyspace( *counter_keyspace.as_ref().expect("lazy init at start of this block") diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index bfa5268e82ead..db5f2e1283b0f 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -1091,6 +1091,40 @@ mod tests { assert!(state_machine.execute(ExecutionStrategy::NativeWhenPossible).is_err()); } + #[test] + fn child_storage_keyspace() { + use crate::trie_backend::tests::test_trie; + let backend = test_trie(); + let mut overlay = OverlayedChanges::default(); + + let subtrie1 = ChildStorageKey::from_slice(b":child_storage:default:sub_test1").unwrap(); + let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub_test2").unwrap(); + let mut tr1 = { + let backend = test_trie(); + let changes_trie_storage = InMemoryChangesTrieStorage::::new(); + let mut ext = Ext::new( + &mut overlay, + &backend, + Some(&changes_trie_storage), + NeverOffchainExt::new(), + None, + ); + ext.set_child_storage(subtrie1, b"abc".to_vec(), b"def".to_vec()); + ext.set_child_storage(subtrie2, b"abc".to_vec(), b"def".to_vec()); + ext.storage_root(); + (ext.transaction().0).0 + }; + let mut duplicate = false; + for (k, (value, rc)) in tr1.0.drain().iter() { + // look for a key inserted twice: transaction rc is 2 + if *rc == 2 { + duplicate = true; + println!("test duplicate for {:?} {:?}", k, value); + } + } + assert!(!duplicate); + } + #[test] fn cannot_change_changes_trie_config_with_native_else_wasm() { let backend = trie_backend::tests::test_trie(); @@ -1115,4 +1149,5 @@ mod tests { assert!(state_machine.execute(ExecutionStrategy::NativeElseWasm).is_err()); } + } From 42bbd9d825df17246f5cdb233e689c3a8e982db0 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Oct 2019 17:27:25 +0200 Subject: [PATCH 67/68] revert into_committed to previous implementation --- core/client/src/client.rs | 3 +-- core/state-machine/src/overlayed_changes.rs | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/core/client/src/client.rs b/core/client/src/client.rs index d4f6aa76d748a..70ec1abfffcbd 100644 --- a/core/client/src/client.rs +++ b/core/client/src/client.rs @@ -1083,8 +1083,7 @@ impl Client where Ok(( Some(storage_update.0), Some(changes_update), - // switching kv from map to vec for compactness - Some((top.collect(), children, kv.into_iter().collect())), + Some((top.collect(), children, kv.collect())), )) }, None => Ok((None, None, None)) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index ea3fd0138c762..8fe028d18d749 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -330,13 +330,13 @@ impl OverlayedChanges { pub fn into_committed(self) -> ( impl Iterator, Option>)>, impl Iterator, impl Iterator, Option>)>)>, - HashMap, Option>>, + impl Iterator, Option>)>, ){ assert!(self.prospective.is_empty()); (self.committed.top.into_iter().map(|(k, v)| (k, v.value)), self.committed.children.into_iter() .map(|(sk, v)| (sk, v.into_iter().map(|(k, v)| (k, v.value)))), - self.committed.kv) + self.committed.kv.into_iter()) } /// Inserts storage entry responsible for current extrinsic index. From 899e151b697b3f925f8ba66719182e56321255f8 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 16 Oct 2019 09:28:34 +0200 Subject: [PATCH 68/68] Add a place holder for child trie deletion in journals. --- core/state-db/src/noncanonical.rs | 4 ++++ core/state-db/src/pruning.rs | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/core/state-db/src/noncanonical.rs b/core/state-db/src/noncanonical.rs index 73f95964eb135..bd1b44c922b19 100644 --- a/core/state-db/src/noncanonical.rs +++ b/core/state-db/src/noncanonical.rs @@ -30,6 +30,7 @@ use codec::{Encode, Decode}; use log::trace; use crate::branch::{RangeSet, BranchRanges}; use historied_data::tree::History; +use primitives::child_trie::KeySpace; const NON_CANONICAL_JOURNAL: &[u8] = b"noncanonical_journal"; const NON_CANONICAL_OFFSTATE_JOURNAL: &[u8] = b"kv_noncanonical_journal"; @@ -137,6 +138,8 @@ struct JournalRecord { struct KvJournalRecord { inserted: Vec<(KvKey, DBValue)>, deleted: Vec, + // placeholder for deleted keyspace + keyspace_deleted: Vec, } fn to_journal_key(block: BlockNumber, index: u64) -> Vec { @@ -454,6 +457,7 @@ impl NonCanonicalOverlay { let kv_journal_record = KvJournalRecord { inserted: kv_inserted_value, deleted: kv_deleted, + keyspace_deleted: Default::default(), }; commit.meta.inserted.push((kv_journal_key, kv_journal_record.encode())); diff --git a/core/state-db/src/pruning.rs b/core/state-db/src/pruning.rs index c07dcf5b879b7..ce1ea839523b3 100644 --- a/core/state-db/src/pruning.rs +++ b/core/state-db/src/pruning.rs @@ -27,6 +27,7 @@ use codec::{Encode, Decode}; use crate::{CommitSet, CommitSetCanonical, Error, MetaDb, to_meta_key, Hash, KvKey}; use log::{trace, warn}; +use primitives::child_trie::KeySpace; const LAST_PRUNED: &[u8] = b"last_pruned"; const PRUNING_JOURNAL: &[u8] = b"pruning_journal"; @@ -70,6 +71,9 @@ struct JournalRecord { #[derive(Encode, Decode)] struct KvJournalRecord { modified: Vec, + // placeholder for trie deletion + // over a full keyspace. + keyspace_deleted: Vec, } fn to_journal_key(block: u64) -> Vec { @@ -231,6 +235,7 @@ impl RefWindow { }; let kv_journal_record = KvJournalRecord { modified: kv_modified, + keyspace_deleted: Default::default(), }; let block = self.pending_number + self.death_rows.len() as u64; let journal_key = to_journal_key(block);