From e2977d89df3419cfe505849cfbcd5a6dba00404f Mon Sep 17 00:00:00 2001 From: clabby Date: Mon, 22 Apr 2024 02:39:46 -0400 Subject: [PATCH] feat(mpt): Refactor `TrieNode` Refactors the `TrieNode` type to not contain the intermediate `NodeElement`, but rather be recursive upon itself. This allows us to create openings on a root `TrieNode` within the cache database and cache all opened nodes throughout the execution of each block. --- Cargo.lock | 130 ++++++++++++- crates/mpt/Cargo.toml | 7 +- crates/mpt/README.md | 2 +- crates/mpt/src/lib.rs | 2 +- crates/mpt/src/list_walker.rs | 64 ++++--- crates/mpt/src/node.rs | 335 ++++++++++++++++++++++++---------- crates/mpt/src/test_util.rs | 92 +++++----- 7 files changed, 465 insertions(+), 167 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6139bf4cf..01ce5c5f48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -599,6 +599,16 @@ dependencies = [ "syn 2.0.60", ] +[[package]] +name = "aurora-engine-modexp" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aef7712851e524f35fbbb74fa6599c5cd8692056a1c36f9ca0d2001b670e7e5" +dependencies = [ + "hex", + "num", +] + [[package]] name = "auto_impl" version = "1.2.0" @@ -1566,7 +1576,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "spin", + "spin 0.9.8", "tokio", "tracing", "tracing-subscriber", @@ -1598,7 +1608,11 @@ dependencies = [ "alloy-transport-http", "alloy-trie", "anyhow", + "futures", "reqwest", + "revm", + "revm-primitives", + "smallvec", "tokio", "tracing", "tracing-subscriber", @@ -1653,6 +1667,9 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "libc" @@ -1766,6 +1783,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "num" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.4" @@ -1777,6 +1808,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +dependencies = [ + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -1786,6 +1826,29 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.18" @@ -2211,6 +2274,43 @@ dependencies = [ "winreg", ] +[[package]] +name = "revm" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a454c1c650b2b2e23f0c461af09e6c31e1d15e1cbebe905a701c46b8a50afc" +dependencies = [ + "auto_impl", + "cfg-if", + "dyn-clone", + "revm-interpreter", + "revm-precompile", +] + +[[package]] +name = "revm-interpreter" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d322f2730cd300e99d271a1704a2dfb8973d832428f5aa282aaa40e2473b5eec" +dependencies = [ + "revm-primitives", +] + +[[package]] +name = "revm-precompile" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "931f692f3f4fc72ec39d5d270f8e9d208c4a6008de7590ee96cf948e3b6d3f8d" +dependencies = [ + "aurora-engine-modexp", + "k256", + "once_cell", + "revm-primitives", + "ripemd", + "sha2", + "substrate-bn", +] + [[package]] name = "revm-primitives" version = "3.1.1" @@ -2242,6 +2342,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "ripemd" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "rlp" version = "0.5.2" @@ -2552,6 +2661,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.8" @@ -2592,6 +2707,19 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "substrate-bn" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b5bbfa79abbae15dd642ea8176a21a635ff3c00059961d1ea27ad04e5b441c" +dependencies = [ + "byteorder", + "crunchy", + "lazy_static", + "rand", + "rustc-hex", +] + [[package]] name = "subtle" version = "2.5.0" diff --git a/crates/mpt/Cargo.toml b/crates/mpt/Cargo.toml index f2c075d091..e85031d8d4 100644 --- a/crates/mpt/Cargo.toml +++ b/crates/mpt/Cargo.toml @@ -13,10 +13,14 @@ homepage.workspace = true anyhow.workspace = true tracing.workspace = true alloy-primitives = { workspace = true, features = ["rlp"] } -alloy-rlp = { workspace = true, default-features = false } +alloy-rlp.workspace = true +alloy-consensus.workspace = true # External alloy-trie = { version = "0.3.1", default-features = false } +smallvec = "1.13" +revm-primitives = { version = "3.1.1", default-features = false } +revm = { version = "8.0.0", default-features = false } [dev-dependencies] alloy-consensus.workspace = true @@ -26,3 +30,4 @@ alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", rev = "e3f2f07" } alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", rev = "e3f2f07" } reqwest = "0.12" tracing-subscriber = "0.3.18" +futures = { version = "0.3.30", default-features = false } diff --git a/crates/mpt/README.md b/crates/mpt/README.md index 44e4889ec0..1d1f8ac5dc 100644 --- a/crates/mpt/README.md +++ b/crates/mpt/README.md @@ -1,3 +1,3 @@ # `kona-mpt` -Utilities for interacting with and iterating through a merkle patricia trie +Utilities for interacting with a merkle patricia trie in the client program. diff --git a/crates/mpt/src/lib.rs b/crates/mpt/src/lib.rs index a0b59ff1ae..d76280dcb7 100644 --- a/crates/mpt/src/lib.rs +++ b/crates/mpt/src/lib.rs @@ -7,7 +7,7 @@ extern crate alloc; mod node; -pub use node::{NodeElement, TrieNode}; +pub use node::TrieNode; mod list_walker; pub use list_walker::OrderedListWalker; diff --git a/crates/mpt/src/list_walker.rs b/crates/mpt/src/list_walker.rs index 3aa0f8cbf7..68bd676a52 100644 --- a/crates/mpt/src/list_walker.rs +++ b/crates/mpt/src/list_walker.rs @@ -1,12 +1,12 @@ //! This module contains the [OrderedListWalker] struct, which allows for traversing an MPT root of //! a derivable ordered list. -use crate::{NodeElement, TrieNode}; +use crate::TrieNode; use alloc::{collections::VecDeque, vec}; use alloy_primitives::{Bytes, B256}; use alloy_rlp::{Decodable, EMPTY_STRING_CODE}; use anyhow::{anyhow, Result}; -use core::{fmt::Display, marker::PhantomData}; +use core::marker::PhantomData; /// A [OrderedListWalker] allows for traversing over a Merkle Patricia Trie containing a derivable /// ordered list. @@ -19,7 +19,7 @@ pub struct OrderedListWalker { root: B256, /// The leaf nodes of the derived list, in order. [None] if the tree has yet to be fully /// traversed with [Self::hydrate]. - inner: Option>, + inner: Option>, /// Phantom data _phantom: PhantomData, } @@ -55,7 +55,7 @@ where // With small lists the iterator seems to use 0x80 (RLP empty string, unlike the others) // as key for item 0, causing it to come last. We need to account for this, pulling the // first element into its proper position. - let mut ordered_list = Self::fetch_leaves(root_trie_node, fetcher)?; + let mut ordered_list = Self::fetch_leaves(&root_trie_node, fetcher)?; if !ordered_list.is_empty() { if ordered_list.len() <= EMPTY_STRING_CODE as usize { // If the list length is < 0x80, the final element is the first element. @@ -74,35 +74,51 @@ where Ok(()) } + /// Takes the inner list of the [OrderedListWalker], returning it and setting the inner list to + /// [None]. + pub fn take_inner(&mut self) -> Option> { + self.inner.take() + } + /// Traverses a [TrieNode], returning all values of child [TrieNode::Leaf] variants. - fn fetch_leaves(trie_node: TrieNode, fetcher: PreimageFetcher) -> Result> { + fn fetch_leaves( + trie_node: &TrieNode, + fetcher: PreimageFetcher, + ) -> Result> { match trie_node { TrieNode::Branch { stack } => { let mut leaf_values = VecDeque::with_capacity(stack.len()); - for item in stack.into_iter() { + for item in stack.iter() { match item { - NodeElement::String(s) => { + TrieNode::Blinded { commitment } => { // If the string is a hash, we need to grab the preimage for it and // continue recursing. - let trie_node = Self::get_trie_node(s.as_ref(), fetcher)?; - leaf_values.append(&mut Self::fetch_leaves(trie_node, fetcher)?); + let trie_node = Self::get_trie_node(commitment.as_ref(), fetcher)?; + leaf_values.append(&mut Self::fetch_leaves(&trie_node, fetcher)?); + } + TrieNode::Empty => { /* Skip over empty nodes, we're looking for values. */ } - list @ NodeElement::List(_) => { - let trie_node = list.try_list_into_node()?; - leaf_values.append(&mut Self::fetch_leaves(trie_node, fetcher)?); + item => { + // If the item is already retrieved, recurse on it. + leaf_values.append(&mut Self::fetch_leaves(item, fetcher)?); } - _ => { /* Skip over empty lists and strings; We're looking for leaves */ } } } Ok(leaf_values) } - TrieNode::Leaf { value, .. } => Ok(vec![value].into()), + TrieNode::Leaf { key, value } => Ok(vec![(key.clone(), value.clone())].into()), TrieNode::Extension { node, .. } => { // If the node is a hash, we need to grab the preimage for it and continue - // recursing. - let trie_node = Self::get_trie_node(node.as_ref(), fetcher)?; - Ok(Self::fetch_leaves(trie_node, fetcher)?) + // recursing. If it is already retrieved, recurse on it. + match node.as_ref() { + TrieNode::Blinded { commitment } => { + let trie_node = Self::get_trie_node(commitment.as_ref(), fetcher)?; + Ok(Self::fetch_leaves(&trie_node, fetcher)?) + } + node => Ok(Self::fetch_leaves(node, fetcher)?), + } } + _ => anyhow::bail!("Invalid trie node type encountered"), } } @@ -110,17 +126,15 @@ where /// a [TrieNode]. Will error if the conversion of `T` into [B256] fails. fn get_trie_node(hash: T, fetcher: PreimageFetcher) -> Result where - T: TryInto, - >::Error: Display, + T: Into, { - let hash = hash.try_into().map_err(|e| anyhow!("Error in conversion: {e}"))?; - let preimage = fetcher(hash)?; + let preimage = fetcher(hash.into())?; TrieNode::decode(&mut preimage.as_ref()).map_err(|e| anyhow!(e)) } } impl Iterator for OrderedListWalker { - type Item = Bytes; + type Item = (Bytes, Bytes); fn next(&mut self) -> Option { match self.inner { @@ -152,7 +166,7 @@ mod test { assert_eq!( list.into_iter() - .map(|rlp| ReceiptEnvelope::decode_2718(&mut rlp.as_ref()).unwrap()) + .map(|(_, rlp)| ReceiptEnvelope::decode_2718(&mut rlp.as_ref()).unwrap()) .collect::>(), envelopes ); @@ -167,7 +181,7 @@ mod test { assert_eq!( list.into_iter() - .map(|rlp| TxEnvelope::decode(&mut rlp.as_ref()).unwrap()) + .map(|(_, rlp)| TxEnvelope::decode(&mut rlp.as_ref()).unwrap()) .collect::>(), envelopes ); @@ -194,7 +208,7 @@ mod test { list.inner .unwrap() .iter() - .map(|v| String::decode(&mut v.as_ref()).unwrap()) + .map(|(_, v)| { String::decode(&mut v.as_ref()).unwrap() }) .collect::>(), VALUES ); diff --git a/crates/mpt/src/node.rs b/crates/mpt/src/node.rs index 105ae1fc82..840dc9668d 100644 --- a/crates/mpt/src/node.rs +++ b/crates/mpt/src/node.rs @@ -1,8 +1,9 @@ -//! This module contains the [TrieNode] type, which allows for decoding the RLP +//! This module contains the [TrieNode] type, which represents a node within a standard Merkle +//! Patricia Trie. -use alloc::{collections::VecDeque, vec::Vec}; -use alloy_primitives::Bytes; -use alloy_rlp::{Buf, Decodable, EMPTY_LIST_CODE, EMPTY_STRING_CODE}; +use alloc::{boxed::Box, vec, vec::Vec}; +use alloy_primitives::{keccak256, Bytes, B256}; +use alloy_rlp::{Buf, BufMut, Decodable, Encodable, Header, EMPTY_STRING_CODE}; use anyhow::{anyhow, Result}; /// The length of the branch list when RLP encoded @@ -23,14 +24,33 @@ const PREFIX_LEAF_EVEN: u8 = 2; /// Prefix for odd-nibbled leaf node paths. const PREFIX_LEAF_ODD: u8 = 3; -/// A [TrieNode] is a node within a standard Merkle Patricia Trie. +/// A [TrieNode] is a node within a standard Ethereum Merkle Patricia Trie. +/// +/// The [TrieNode] has several variants: +/// - [TrieNode::Empty] represents an empty node. +/// - [TrieNode::Blinded] represents a node that has been blinded by a commitment. +/// - [TrieNode::Leaf] represents a 2-item node with the encoding `rlp([encoded_path, value])`. +/// - [TrieNode::Extension] represents a 2-item pointer node with the encoding `rlp([encoded_path, +/// key])`. +/// - [TrieNode::Branch] represents a node that refers to up to 16 child nodes with the encoding +/// `rlp([ v0, ..., v15, value ])`. +/// +/// In the Ethereum Merkle Patricia Trie, nodes longer than an encoded 32 byte string (33 total +/// bytes) are blinded with [keccak256] hashes. When a node is "opened", it is replaced with the +/// [TrieNode] that is decoded from to the preimage of the hash. +/// +/// The [alloy_rlp::Encodable] and [alloy_rlp::Decodable] traits are implemented for [TrieNode], +/// allowing for RLP encoding and decoding of the types for storage and retrieval. The +/// implementation of these traits will implicitly blind nodes that are longer than 32 bytes in +/// length when encoding. When decoding, the implementation will leave blinded nodes in place. #[derive(Debug, Clone, Eq, PartialEq)] pub enum TrieNode { - /// A branch node refers to up to 16 child nodes with the encoding `rlp([ v0, ..., v15, value - /// ])` - Branch { - /// The 16 child nodes and value of the branch. - stack: VecDeque, + /// An empty [TrieNode] is represented as an [EMPTY_STRING_CODE] (0x80). + Empty, + /// A blinded node is a node that has been blinded by a [keccak256] commitment. + Blinded { + /// The commitment that blinds the node. + commitment: B256, }, /// A leaf node is a 2-item node with the encoding `rlp([encoded_path, value])` Leaf { @@ -44,21 +64,37 @@ pub enum TrieNode { /// The path prefix of the extension prefix: Bytes, /// The pointer to the child node - node: Bytes, + node: Box, + }, + /// A branch node refers to up to 16 child nodes with the encoding + /// `rlp([ v0, ..., v15, value ])` + Branch { + /// The 16 child nodes and value of the branch. + stack: Vec, }, } impl TrieNode { /// Attempts to convert a `path` and `value` into a [TrieNode], if they correspond to a /// [TrieNode::Leaf] or [TrieNode::Extension]. - pub fn try_from_path_and_value(path: Bytes, value: Bytes) -> Result { + /// + /// **Note:** This function assumes that the passed reader has already consumed the RLP header + /// of the [TrieNode::Leaf] or [TrieNode::Extension] node. + pub fn try_decode_leaf_or_extension_payload(buf: &mut &[u8]) -> Result { + // Decode the path and value of the leaf or extension node. + let path = Bytes::decode(buf).map_err(|e| anyhow!("Failed to decode: {e}"))?; + + // Check the high-order nibble of the path to determine the type of node. match path[0] >> 4 { PREFIX_EXTENSION_EVEN | PREFIX_EXTENSION_ODD => { // extension node - Ok(TrieNode::Extension { prefix: path, node: value }) + let extension_node_value = + TrieNode::decode(buf).map_err(|e| anyhow!("Failed to decode: {e}"))?; + Ok(TrieNode::Extension { prefix: path, node: Box::new(extension_node_value) }) } PREFIX_LEAF_EVEN | PREFIX_LEAF_ODD => { // leaf node + let value = Bytes::decode(buf).map_err(|e| anyhow!("Failed to decode: {e}"))?; Ok(TrieNode::Leaf { key: path, value }) } _ => { @@ -66,119 +102,234 @@ impl TrieNode { } } } -} -impl Decodable for TrieNode { - /// Attempts to decode the [TrieNode]. - fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { - let mut list: VecDeque<_> = Vec::::decode(buf)?.into(); - - match list.len() { - BRANCH_LIST_LENGTH => Ok(Self::Branch { stack: list }), - LEAF_OR_EXTENSION_LIST_LENGTH => { - let Some(NodeElement::String(path)) = list.pop_front() else { - return Err(alloy_rlp::Error::UnexpectedList); - }; - let Some(NodeElement::String(value)) = list.pop_front() else { - return Err(alloy_rlp::Error::UnexpectedList); - }; - - Self::try_from_path_and_value(path, value) - .map_err(|_| alloy_rlp::Error::UnexpectedList) - } - _ => Err(alloy_rlp::Error::UnexpectedLength), + /// Blinds the [TrieNode] if it is longer than an encoded [B256] string in length, and returns + /// the mutated node. + pub fn blind(self) -> Self { + if self.length() > B256::ZERO.length() { + let mut rlp_buf = Vec::with_capacity(self.length()); + self.encode(&mut rlp_buf); + TrieNode::Blinded { commitment: keccak256(rlp_buf) } + } else { + self } } } -/// A [NodeElement] is an element within a MPT node's RLP array -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum NodeElement { - /// An RLP String - String(Bytes), - /// An empty RLP string (0x80) - EmptyString, - /// An RLP List - List(VecDeque), - /// An empty RLP list (0xC0) - EmptyList, -} +impl Encodable for TrieNode { + fn encode(&self, out: &mut dyn alloy_rlp::BufMut) { + match self { + Self::Empty => out.put_u8(EMPTY_STRING_CODE), + Self::Blinded { commitment } => commitment.encode(out), + Self::Leaf { key, value } => { + // Encode the leaf node's header and key-value pair. + let leaf_list = vec![key, value]; + leaf_list.encode(out); + } + Self::Extension { prefix, node } => { + // Encode the extension node's header, prefix, and pointer node. + Header { list: true, payload_length: prefix.length() + node.length() }.encode(out); + prefix.encode(out); + encode_blinded(node.as_ref(), out); + } + Self::Branch { stack } => { + // In branch nodes, if an element is longer than 32 bytes in length, it is blinded. + // Assuming we have an open trie node, we must re-hash the elements + // that are longer than 32 bytes in length. + let blinded_nodes = + stack.iter().cloned().map(|node| node.blind()).collect::>(); + blinded_nodes.encode(out); + } + } + } -impl NodeElement { - /// Attempts to convert `Self` into a [TrieNode::Leaf] or [TrieNode::Extension], if `Self` is a - /// [NodeElement::List] variant. - pub fn try_list_into_node(self) -> Result { - if let NodeElement::List(mut list) = self { - if list.len() != LEAF_OR_EXTENSION_LIST_LENGTH { - anyhow::bail!("Invalid length"); + fn length(&self) -> usize { + match self { + Self::Empty => 1, + Self::Blinded { commitment } => commitment.length(), + Self::Leaf { key, value } => { + let leaf_list = vec![key, value]; + leaf_list.length() + } + Self::Extension { prefix, node } => { + let prefix_length = prefix.length(); + let node_length = blinded_length(node.as_ref()); + Header { list: true, payload_length: prefix_length + node_length }.length() + + prefix_length + + node_length } + Self::Branch { stack } => { + // In branch nodes, if an element is longer than an encoded 32 byte string, it is + // blinded. Assuming we have an open trie node, we must re-hash the + // elements that are longer than an encoded 32 byte string + // in length. + let inner_length = stack.iter().fold(0, |mut acc, node| { + acc += blinded_length(node); + acc + }); - let path = list.pop_front().ok_or(anyhow!("List is empty; Impossible case"))?; - let value = list.pop_front().ok_or(anyhow!("List is empty; Impossible case"))?; - TrieNode::try_from_path_and_value(path, value) - } else { - anyhow::bail!("Self is not a list") + inner_length + Header { list: true, payload_length: inner_length }.length() + } } } } -impl Decodable for NodeElement { +impl Decodable for TrieNode { + /// Attempts to decode the [TrieNode]. fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { - match buf[0] { - EMPTY_STRING_CODE => { - buf.advance(1); - Ok(Self::EmptyString) + // Peek at the header to determine the type of Trie node we're currently decoding. + let header = Header::decode(&mut (**buf).as_ref())?; + + if header.list { + // Peek at the RLP stream to determine the number of elements in the list. + let list_length = rlp_list_element_length(&mut (**buf).as_ref())?; + + match list_length { + BRANCH_LIST_LENGTH => { + let list = Vec::::decode(buf)?; + Ok(Self::Branch { stack: list }) + } + LEAF_OR_EXTENSION_LIST_LENGTH => { + // Advance the buffer to the start of the list payload. + buf.advance(header.length()); + // Decode the leaf or extension node's raw payload. + Self::try_decode_leaf_or_extension_payload(buf) + .map_err(|_| alloy_rlp::Error::UnexpectedList) + } + _ => Err(alloy_rlp::Error::UnexpectedLength), } - EMPTY_LIST_CODE => { - buf.advance(1); - Ok(Self::EmptyList) + } else { + match header.payload_length { + 0 => { + buf.advance(header.length()); + Ok(Self::Empty) + } + _ => { + if header.payload_length != B256::len_bytes() { + return Err(alloy_rlp::Error::UnexpectedLength); + } + let commitment = B256::decode(buf)?; + + Ok(Self::Blinded { commitment }) + } } - EMPTY_LIST_CODE.. => Ok(Self::List(Vec::::decode(buf)?.into())), - _ => Ok(Self::String(Bytes::decode(buf)?)), } } } +/// Returns the encoded length of an [Encodable] value, blinding it if it is longer than an encoded +/// [B256] string in length. +fn blinded_length(value: T) -> usize { + if value.length() > B256::ZERO.length() { + B256::ZERO.length() + } else { + value.length() + } +} + +/// Encodes a value into an RLP stream, blidning it with a [keccak256] commitment if it is longer +/// than an encoded [B256] string in length. +fn encode_blinded(value: T, out: &mut dyn BufMut) { + if value.length() > B256::ZERO.length() { + let mut rlp_buf = Vec::with_capacity(value.length()); + value.encode(&mut rlp_buf); + TrieNode::Blinded { commitment: keccak256(rlp_buf) }.encode(out); + } else { + value.encode(out); + } +} + +/// Walks through a RLP list's elements and returns the total number of elements in the list. +/// Returns [alloy_rlp::Error::UnexpectedString] if the RLP stream is not a list. +fn rlp_list_element_length(buf: &mut &[u8]) -> alloy_rlp::Result { + let header = Header::decode(buf)?; + if !header.list { + return Err(alloy_rlp::Error::UnexpectedString); + } + let len_after_consume = buf.len() - header.payload_length; + + let mut list_element_length = 0; + while buf.len() > len_after_consume { + let header = Header::decode(buf)?; + buf.advance(header.payload_length); + list_element_length += 1; + } + Ok(list_element_length) +} + #[cfg(test)] mod test { use super::*; use alloc::vec; - use alloy_primitives::{bytes, hex}; + use alloy_primitives::{b256, bytes, hex}; #[test] fn test_decode_branch() { const BRANCH_RLP: [u8; 64] = hex!("f83ea0eb08a66a94882454bec899d3e82952dcc918ba4b35a09a84acd98019aef4345080808080808080cd308b8a746573742074687265658080808080808080"); let expected = TrieNode::Branch { stack: vec![ - NodeElement::String(bytes!( - "eb08a66a94882454bec899d3e82952dcc918ba4b35a09a84acd98019aef43450" - )), - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::List(vec![bytes!("30"), bytes!("8a74657374207468726565")].into()), - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::EmptyString, - NodeElement::EmptyString, - ] - .into(), + TrieNode::Blinded { + commitment: b256!( + "eb08a66a94882454bec899d3e82952dcc918ba4b35a09a84acd98019aef43450" + ), + }, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Leaf { key: bytes!("30"), value: bytes!("8a74657374207468726565") }, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Empty, + TrieNode::Empty, + ], }; + + let mut rlp_buf = Vec::with_capacity(expected.length()); + expected.encode(&mut rlp_buf); + assert_eq!(rlp_buf.len(), BRANCH_RLP.len()); + assert_eq!(expected.length(), BRANCH_RLP.len()); + assert_eq!(expected, TrieNode::decode(&mut BRANCH_RLP.as_slice()).unwrap()); + assert_eq!(rlp_buf.as_slice(), &BRANCH_RLP[..]); + } + + #[test] + fn test_encode_decode_extension_open_short() { + const EXTENSION_RLP: [u8; 19] = hex!("d28300646fcd308b8a74657374207468726565"); + + let opened = TrieNode::Leaf { key: bytes!("30"), value: bytes!("8a74657374207468726565") }; + let expected = TrieNode::Extension { prefix: bytes!("00646f"), node: Box::new(opened) }; + + let mut rlp_buf = Vec::with_capacity(expected.length()); + expected.encode(&mut rlp_buf); + + assert_eq!(expected, TrieNode::decode(&mut EXTENSION_RLP.as_slice()).unwrap()); } #[test] - fn test_decode_extension() { - const EXTENSION_RLP: [u8; 10] = hex!("c98300646f8476657262"); - let expected = TrieNode::Extension { prefix: bytes!("00646f"), node: bytes!("76657262") }; + fn test_encode_decode_extension_blinded_long() { + const EXTENSION_RLP: [u8; 38] = + hex!("e58300646fa0f3fe8b3c5b21d3e52860f1e4a5825a6100bb341069c1e88f4ebf6bd98de0c190"); + let mut rlp_buf = Vec::new(); + + let opened = TrieNode::Leaf { key: bytes!("30"), value: bytes!("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF") }; + opened.encode(&mut rlp_buf); + let blinded = TrieNode::Blinded { commitment: keccak256(&rlp_buf) }; + + rlp_buf.clear(); + let opened_extension = + TrieNode::Extension { prefix: bytes!("00646f"), node: Box::new(opened) }; + opened_extension.encode(&mut rlp_buf); + + let expected = TrieNode::Extension { prefix: bytes!("00646f"), node: Box::new(blinded) }; assert_eq!(expected, TrieNode::decode(&mut EXTENSION_RLP.as_slice()).unwrap()); } diff --git a/crates/mpt/src/test_util.rs b/crates/mpt/src/test_util.rs index 49194d0e81..454dc59ed3 100644 --- a/crates/mpt/src/test_util.rs +++ b/crates/mpt/src/test_util.rs @@ -14,6 +14,52 @@ use reqwest::Url; const RPC_URL: &str = "https://docs-demo.quiknode.pro/"; +/// Compute a trie root of the collection of items with a custom encoder. +pub(crate) fn ordered_trie_with_encoder(items: &[T], mut encode: F) -> HashBuilder +where + F: FnMut(&T, &mut dyn BufMut), +{ + let mut index_buffer = Vec::new(); + let mut value_buffer = Vec::new(); + let items_len = items.len(); + + // Store preimages for all intermediates + let path_nibbles = (0..items_len) + .map(|i| { + let i = adjust_index_for_rlp(i, items_len); + index_buffer.clear(); + i.encode(&mut index_buffer); + Nibbles::unpack(&index_buffer) + }) + .collect::>(); + + let mut hb = HashBuilder::default().with_proof_retainer(path_nibbles); + for i in 0..items_len { + let index = adjust_index_for_rlp(i, items_len); + + index_buffer.clear(); + index.encode(&mut index_buffer); + + value_buffer.clear(); + encode(&items[index], &mut value_buffer); + + hb.add_leaf(Nibbles::unpack(&index_buffer), &value_buffer); + } + + hb +} + +/// Adjust the index of an item for rlp encoding. +pub(crate) const fn adjust_index_for_rlp(i: usize, len: usize) -> usize { + if i > 0x7f { + i + } else if i == 0x7f || i + 1 == len { + 0 + } else { + i + 1 + } +} + /// Grabs a live merkleized receipts list within a block header. pub(crate) async fn get_live_derivable_receipts_list( ) -> Result<(B256, BTreeMap, Vec)> { @@ -119,49 +165,3 @@ pub(crate) async fn get_live_derivable_transactions_list( Ok((root, preimages, consensus_txs)) } - -/// Compute a trie root of the collection of items with a custom encoder. -pub(crate) fn ordered_trie_with_encoder(items: &[T], mut encode: F) -> HashBuilder -where - F: FnMut(&T, &mut dyn BufMut), -{ - let mut index_buffer = Vec::new(); - let mut value_buffer = Vec::new(); - let items_len = items.len(); - - // Store preimages for all intermediates - let path_nibbles = (0..items_len) - .map(|i| { - let i = adjust_index_for_rlp(i, items_len); - index_buffer.clear(); - i.encode(&mut index_buffer); - Nibbles::unpack(&index_buffer) - }) - .collect::>(); - - let mut hb = HashBuilder::default().with_proof_retainer(path_nibbles); - for i in 0..items_len { - let index = adjust_index_for_rlp(i, items_len); - - index_buffer.clear(); - index.encode(&mut index_buffer); - - value_buffer.clear(); - encode(&items[index], &mut value_buffer); - - hb.add_leaf(Nibbles::unpack(&index_buffer), &value_buffer); - } - - hb -} - -/// Adjust the index of an item for rlp encoding. -pub(crate) const fn adjust_index_for_rlp(i: usize, len: usize) -> usize { - if i > 0x7f { - i - } else if i == 0x7f || i + 1 == len { - 0 - } else { - i + 1 - } -}