From e7312cc98f60fe059ce1bf24011fb5ba92ecaa25 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 4 Jul 2018 12:19:10 +0200 Subject: [PATCH 1/3] NodeHandle does not require Hasher --- util/patricia_trie/src/triedbmut.rs | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index f5c28bac32f..a9b1c739726 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -39,20 +39,20 @@ struct StorageHandle(usize); // Handles to nodes in the trie. #[derive(Debug)] -enum NodeHandle { +enum NodeHandle { /// Loaded into memory. InMemory(StorageHandle), /// Either a hash or an inline node - Hash(H::Out), + Hash(H), } -impl From for NodeHandle { +impl From for NodeHandle { fn from(handle: StorageHandle) -> Self { NodeHandle::InMemory(handle) } } -fn empty_children() -> Box<[Option>; 16]> { +fn empty_children() -> Box<[Option>; 16]> { Box::new([ None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, @@ -72,14 +72,14 @@ enum Node { /// The shared portion is encoded from a `NibbleSlice` meaning it contains /// a flag indicating it is an extension. /// The child node is always a branch. - Extension(NodeKey, NodeHandle), + Extension(NodeKey, NodeHandle), /// A branch has up to 16 children and an optional value. - Branch(Box<[Option>; 16]>, Option) + Branch(Box<[Option>; 16]>, Option) } impl Node { // load an inline node into memory or get the hash to do the lookup later. - fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle + fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle where C: NodeCodec { C::try_decode_hash(&node) @@ -128,7 +128,7 @@ impl Node { fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 where C: NodeCodec, - F: FnMut(NodeHandle) -> ChildReference + F: FnMut(NodeHandle) -> ChildReference { match self { Node::Empty => C::empty_node(), @@ -139,7 +139,7 @@ impl Node { // map the `NodeHandle`s from the Branch to `ChildReferences` children.iter_mut() .map(Option::take) - .map(|maybe_child| + .map(|maybe_child| maybe_child.map(|child| child_cb(child)) ), value @@ -287,7 +287,7 @@ where storage: NodeStorage, db: &'a mut HashDB, root: &'a mut H::Out, - root_handle: NodeHandle, + root_handle: NodeHandle, death_row: HashSet, /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. @@ -380,7 +380,7 @@ where } // walk the trie, attempting to find the key's node. - fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> Result, H::Out, C::Error> + fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> Result, H::Out, C::Error> where 'x: 'key { let mut handle = handle; @@ -429,7 +429,7 @@ where } /// insert a key-value pair into the trie, creating new nodes if necessary. - fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out, C::Error> { + fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out, C::Error> { let h = match handle { NodeHandle::InMemory(h) => h, NodeHandle::Hash(h) => self.cache(h)?, @@ -604,7 +604,7 @@ where } /// Remove a node from the trie based on key. - fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { + fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), NodeHandle::Hash(h) => { @@ -845,7 +845,7 @@ where /// case where we can fit the actual data in the `Hasher`s output type, we /// store the data inline. This function is used as the callback to the /// `into_encoded` method of `Node`. - fn commit_child(&mut self, handle: NodeHandle) -> ChildReference { + fn commit_child(&mut self, handle: NodeHandle) -> ChildReference { match handle { NodeHandle::Hash(hash) => ChildReference::Hash(hash), NodeHandle::InMemory(storage_handle) => { @@ -871,7 +871,7 @@ where } // a hack to get the root node's handle - fn root_handle(&self) -> NodeHandle { + fn root_handle(&self) -> NodeHandle { match self.root_handle { NodeHandle::Hash(h) => NodeHandle::Hash(h), NodeHandle::InMemory(StorageHandle(x)) => NodeHandle::InMemory(StorageHandle(x)), @@ -880,7 +880,7 @@ where } impl<'a, H, C> TrieMut for TrieDBMut<'a, H, C> -where +where H: Hasher, C: NodeCodec { From 48cc26411383ca80e96f9dc522347677d3858458 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 4 Jul 2018 12:43:55 +0200 Subject: [PATCH 2/3] Node does not require Hasher --- Cargo.lock | 1 + util/hashdb/src/lib.rs | 2 +- util/patricia_trie/Cargo.toml | 5 ++- util/patricia_trie/src/lib.rs | 5 ++- util/patricia_trie/src/triedbmut.rs | 65 ++++++++++++++++------------- 5 files changed, 43 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a54711fba8d..535d20408b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2468,6 +2468,7 @@ dependencies = [ "ethcore-bytes 0.1.0", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", + "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2", "keccak-hasher 0.1.0", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 5961d90f98a..4f5bed48c41 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -23,7 +23,7 @@ use heapsize::HeapSizeOf; use std::collections::HashMap; use std::{fmt::Debug, hash::Hash}; -/// Trait describing an object that can hash a slice of bytes. Used to abstract +/// Trait describing an object that can hash a slice of bytes. Used to abstract /// other types over the hashing algorithm. Defines a single `hash` method and an /// `Out` associated type with the necessary bounds. pub trait Hasher: Sync + Send { diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index 5324656785b..ebd66504546 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -7,10 +7,11 @@ license = "GPL-3.0" [dependencies] elastic-array = "0.10" +ethcore-bytes = { version = "0.1.0", path = "../bytes" } +hashdb = { version = "0.2", path = "../hashdb" } +heapsize = "0.4" log = "0.3" rand = "0.4" -hashdb = { version = "0.2", path = "../hashdb" } -ethcore-bytes = { version = "0.1.0", path = "../bytes" } [dev-dependencies] env_logger = "0.5" diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 7cc623b1a14..a028be87ac9 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -18,6 +18,7 @@ extern crate elastic_array; extern crate ethcore_bytes as bytes; extern crate hashdb; +extern crate heapsize; extern crate rand; #[macro_use] extern crate log; @@ -277,8 +278,8 @@ impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> { } impl<'db, H, C> TrieFactory -where - H: Hasher, +where + H: Hasher, C: NodeCodec + 'db { /// Creates new factory. diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index a9b1c739726..6058597c104 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -31,6 +31,8 @@ use std::collections::{HashSet, VecDeque}; use std::marker::PhantomData; use std::mem; use std::ops::Index; +use heapsize::HeapSizeOf; +use std::{fmt::Debug, hash::Hash}; // For lookups into the Node storage buffer. // This is deliberately non-copyable. @@ -61,7 +63,7 @@ fn empty_children() -> Box<[Option>; 16]> { /// Node types in the Trie. #[derive(Debug)] -enum Node { +enum Node { /// Empty node. Empty, /// A leaf node contains the end of a key and a value. @@ -72,27 +74,29 @@ enum Node { /// The shared portion is encoded from a `NibbleSlice` meaning it contains /// a flag indicating it is an extension. /// The child node is always a branch. - Extension(NodeKey, NodeHandle), + Extension(NodeKey, NodeHandle), /// A branch has up to 16 children and an optional value. - Branch(Box<[Option>; 16]>, Option) + Branch(Box<[Option>; 16]>, Option) } -impl Node { +impl Node where I: AsRef<[u8]> + AsMut<[u8]> + Default + HeapSizeOf + Debug + PartialEq + Eq + Hash + Send + Sync + Clone + Copy { // load an inline node into memory or get the hash to do the lookup later. - fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle - where C: NodeCodec + fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle + where C: NodeCodec, + H: Hasher, { C::try_decode_hash(&node) .map(NodeHandle::Hash) .unwrap_or_else(|| { - let child = Node::from_encoded::(node, db, storage); + let child = Node::from_encoded::(node, db, storage); NodeHandle::InMemory(storage.alloc(Stored::New(child))) }) } // decode a node from encoded bytes without getting its children. - fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self - where C: NodeCodec + fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self + where C: NodeCodec, + H: Hasher, { match C::decode(data).expect("encoded bytes read from db; qed") { EncodedNode::Empty => Node::Empty, @@ -100,13 +104,13 @@ impl Node { EncodedNode::Extension(key, cb) => { Node::Extension( key.encoded(false), - Self::inline_or_hash::(cb, db, storage)) + Self::inline_or_hash::(cb, db, storage)) } EncodedNode::Branch(ref encoded_children, val) => { let mut child = |i:usize| { let raw = encoded_children[i]; if !C::is_empty_node(raw) { - Some(Self::inline_or_hash::(raw, db, storage)) + Some(Self::inline_or_hash::(raw, db, storage)) } else { None } @@ -125,10 +129,11 @@ impl Node { } // TODO: parallelize - fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 + fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 where C: NodeCodec, - F: FnMut(NodeHandle) -> ChildReference + F: FnMut(NodeHandle) -> ChildReference, + H: Hasher, { match self { Node::Empty => C::empty_node(), @@ -150,7 +155,7 @@ impl Node { } // post-inspect action. -enum Action { +enum Action { // Replace a node with a new one. Replace(Node), // Restore the original node. This trusts that the node is actually the original. @@ -160,14 +165,14 @@ enum Action { } // post-insert action. Same as action without delete -enum InsertAction { +enum InsertAction { // Replace a node with a new one. Replace(Node), // Restore the original node. Restore(Node), } -impl InsertAction { +impl InsertAction { fn into_action(self) -> Action { match self { InsertAction::Replace(n) => Action::Replace(n), @@ -184,11 +189,11 @@ impl InsertAction { } // What kind of node is stored here. -enum Stored { +enum Stored { // A new node. New(Node), // A cached node, loaded from the DB. - Cached(Node, H::Out), + Cached(Node, H), } /// Used to build a collection of child nodes from a collection of `NodeHandle`s @@ -198,12 +203,12 @@ pub enum ChildReference { // `HO` is e.g. `H256`, i.e. the output of a `Hash } /// Compact and cache-friendly storage for Trie nodes. -struct NodeStorage { +struct NodeStorage { nodes: Vec>, free_indices: VecDeque, } -impl NodeStorage { +impl NodeStorage { /// Create a new storage. fn empty() -> Self { NodeStorage { @@ -232,7 +237,7 @@ impl NodeStorage { } } -impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { +impl<'a, H> Index<&'a StorageHandle> for NodeStorage { type Output = Node; fn index(&self, handle: &'a StorageHandle) -> &Node { @@ -284,7 +289,7 @@ where H: Hasher + 'a, C: NodeCodec { - storage: NodeStorage, + storage: NodeStorage, db: &'a mut HashDB, root: &'a mut H::Out, root_handle: NodeHandle, @@ -347,7 +352,7 @@ where // cache a node by hash fn cache(&mut self, hash: H::Out) -> Result { let node_encoded = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; - let node = Node::from_encoded::( + let node = Node::from_encoded::( &node_encoded, &*self.db, &mut self.storage @@ -357,8 +362,8 @@ where // inspect a node, choosing either to replace, restore, or delete it. // if restored or replaced, returns the new node along with a flag of whether it was changed. - fn inspect(&mut self, stored: Stored, inspector: F) -> Result, bool)>, H::Out, C::Error> - where F: FnOnce(&mut Self, Node) -> Result, H::Out, C::Error> { + fn inspect(&mut self, stored: Stored, inspector: F) -> Result, bool)>, H::Out, C::Error> + where F: FnOnce(&mut Self, Node) -> Result, H::Out, C::Error> { Ok(match stored { Stored::New(node) => match inspector(self, node)? { Action::Restore(node) => Some((Stored::New(node), false)), @@ -443,7 +448,7 @@ where } /// the insertion inspector. - fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result, H::Out, C::Error> { + fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result, H::Out, C::Error> { trace!(target: "trie", "augmented (partial: {:?}, value: {:?})", partial, value.pretty()); Ok(match node { @@ -619,7 +624,7 @@ where } /// the removal inspector - fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { + fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { Ok(match (node, partial.is_empty()) { (Node::Empty, _) => Action::Delete, (Node::Branch(c, None), true) => Action::Restore(Node::Branch(c, None)), @@ -705,7 +710,7 @@ where /// _invalid state_ means: /// - Branch node where there is only a single entry; /// - Extension node followed by anything other than a Branch node. - fn fix(&mut self, node: Node) -> Result, H::Out, C::Error> { + fn fix(&mut self, node: Node) -> Result, H::Out, C::Error> { match node { Node::Branch(mut children, value) => { // if only a single value, transmute to leaf/extension and feed through fixed. @@ -825,7 +830,7 @@ where match self.storage.destroy(handle) { Stored::New(node) => { - let encoded_root = node.into_encoded::<_, C>(|child| self.commit_child(child) ); + let encoded_root = node.into_encoded::<_, C, H>(|child| self.commit_child(child) ); *self.root = self.db.insert(&encoded_root[..]); self.hash_count += 1; @@ -852,7 +857,7 @@ where match self.storage.destroy(storage_handle) { Stored::Cached(_, hash) => ChildReference::Hash(hash), Stored::New(node) => { - let encoded = node.into_encoded::<_, C>(|node_handle| self.commit_child(node_handle) ); + let encoded = node.into_encoded::<_, C, H>(|node_handle| self.commit_child(node_handle) ); if encoded.len() >= H::LENGTH { let hash = self.db.insert(&encoded[..]); self.hash_count +=1; From 9a5870fb717b3c6cb923b2de2893768d082af359 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 5 Jul 2018 16:04:53 +0200 Subject: [PATCH 3/3] change name of the template typo from I to O --- util/patricia_trie/src/triedbmut.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 6058597c104..4490285d51d 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -79,11 +79,11 @@ enum Node { Branch(Box<[Option>; 16]>, Option) } -impl Node where I: AsRef<[u8]> + AsMut<[u8]> + Default + HeapSizeOf + Debug + PartialEq + Eq + Hash + Send + Sync + Clone + Copy { +impl Node where O: AsRef<[u8]> + AsMut<[u8]> + Default + HeapSizeOf + Debug + PartialEq + Eq + Hash + Send + Sync + Clone + Copy { // load an inline node into memory or get the hash to do the lookup later. fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle where C: NodeCodec, - H: Hasher, + H: Hasher, { C::try_decode_hash(&node) .map(NodeHandle::Hash) @@ -96,7 +96,7 @@ impl Node where I: AsRef<[u8]> + AsMut<[u8]> + Default + HeapSizeOf + Debu // decode a node from encoded bytes without getting its children. fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self where C: NodeCodec, - H: Hasher, + H: Hasher, { match C::decode(data).expect("encoded bytes read from db; qed") { EncodedNode::Empty => Node::Empty, @@ -133,7 +133,7 @@ impl Node where I: AsRef<[u8]> + AsMut<[u8]> + Default + HeapSizeOf + Debu where C: NodeCodec, F: FnMut(NodeHandle) -> ChildReference, - H: Hasher, + H: Hasher, { match self { Node::Empty => C::empty_node(),