From e99df72e0bf7e7d4efeac3559a40320e43ecb77a Mon Sep 17 00:00:00 2001 From: Eason Date: Wed, 16 Aug 2023 15:25:01 +0800 Subject: [PATCH 1/4] chore: upgrade some crates --- Cargo.lock | 59 +++++++++++++++++++++++++++------- core/storage/Cargo.toml | 2 +- protocol/Cargo.toml | 2 +- protocol/src/traits/storage.rs | 3 -- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49d3c86e7..e137ee66b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,6 +108,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -1815,7 +1821,7 @@ dependencies = [ "serde", "serde_json", "snap", - "socket2", + "socket2 0.4.9", "tentacle", "tokio-util", ] @@ -1883,7 +1889,7 @@ dependencies = [ "futures", "lazy_static", "log", - "lru 0.10.0", + "lru 0.11.0", "num-traits", "parking_lot 0.12.1", ] @@ -3360,6 +3366,16 @@ dependencies = [ "ahash 0.8.3", ] +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +dependencies = [ + "ahash 0.8.3", + "allocator-api2", +] + [[package]] name = "hasher" version = "0.1.4" @@ -3518,7 +3534,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -3949,9 +3965,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.144" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libloading" @@ -4057,6 +4073,15 @@ dependencies = [ "hashbrown 0.13.2", ] +[[package]] +name = "lru" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eedb2bdbad7e0634f83989bf596f497b070130daaa398ab22d84c39e266deec5" +dependencies = [ + "hashbrown 0.14.0", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -4915,9 +4940,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" [[package]] name = "pin-utils" @@ -6155,6 +6180,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "soketto" version = "0.7.1" @@ -6421,7 +6456,7 @@ dependencies = [ "once_cell", "parking_lot 0.12.1", "rand 0.8.5", - "socket2", + "socket2 0.4.9", "tentacle-multiaddr", "tentacle-secio", "thiserror", @@ -6663,11 +6698,11 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.1" +version = "1.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" +checksum = "40de3a2ba249dcb097e01be5e67a5ff53cf250397715a071a81543e8a832a920" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", "mio", @@ -6675,7 +6710,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.3", "tokio-macros", "windows-sys 0.48.0", ] diff --git a/core/storage/Cargo.toml b/core/storage/Cargo.toml index 45c5be342..22c3854ce 100644 --- a/core/storage/Cargo.toml +++ b/core/storage/Cargo.toml @@ -9,7 +9,7 @@ arc-swap = "1.6" futures = "0.3" lazy_static = "1.4" log = "0.4" -lru = "0.10" +lru = "0.11" parking_lot = "0.12" rocksdb = { version = "0.20", package = "ckb-rocksdb" } diff --git a/protocol/Cargo.toml b/protocol/Cargo.toml index 89ff149c4..ad83758da 100644 --- a/protocol/Cargo.toml +++ b/protocol/Cargo.toml @@ -29,7 +29,7 @@ rand = "0.7" rlp = "0.5" rlp-derive = "0.1" serde = { version = "1.0", features = ["derive"] } -tokio = { version = "1.27", features = ["full"] } +tokio = { version = "1.31", features = ["full"] } trie = { package = "cita_trie", version = "4.0" } common-crypto = { path = "../common/crypto" } diff --git a/protocol/src/traits/storage.rs b/protocol/src/traits/storage.rs index a28b0b37d..e55718160 100644 --- a/protocol/src/traits/storage.rs +++ b/protocol/src/traits/storage.rs @@ -122,9 +122,6 @@ pub trait Storage: CommonStorage { async fn get_latest_proof(&self, ctx: Context) -> ProtocolResult; } -#[async_trait] -pub trait MaintenanceStorage: CommonStorage {} - pub enum StorageBatchModify { Remove, Insert(::Value), From 790909274d9be15eae7cb1effad0f5d30a464378 Mon Sep 17 00:00:00 2001 From: Eason Date: Wed, 16 Aug 2023 20:12:06 +0800 Subject: [PATCH 2/4] change storage and executor to readonly and write --- core/api/src/adapter.rs | 20 +- core/cli/src/lib.rs | 10 +- core/consensus/src/adapter.rs | 6 +- core/executor/benches/bench_transfer.rs | 7 +- core/executor/benches/bench_vm.rs | 8 +- core/executor/benches/mock.rs | 1 - core/executor/src/adapter/apply.rs | 286 ++++++++++++++ core/executor/src/adapter/mod.rs | 366 +----------------- core/executor/src/adapter/read_only.rs | 234 +++++++++++ core/executor/src/debugger/mod.rs | 10 +- core/executor/src/lib.rs | 42 +- core/executor/src/precompiles/get_cell.rs | 6 +- core/executor/src/precompiles/get_header.rs | 4 +- .../system_contract/ckb_light_client/mod.rs | 24 +- .../src/system_contract/image_cell/mod.rs | 26 +- .../src/system_contract/image_cell/store.rs | 1 + .../src/system_contract/metadata/mod.rs | 19 +- core/executor/src/system_contract/mod.rs | 59 +-- .../src/system_contract/native_token.rs | 12 +- core/executor/src/system_contract/utils.rs | 22 +- core/executor/src/tests/mod.rs | 6 +- .../tests/system_script/ckb_light_client.rs | 69 ++-- .../src/tests/system_script/image_cell.rs | 79 ++-- .../src/tests/system_script/metadata.rs | 24 +- .../src/tests/system_script/native_token.rs | 8 +- core/interoperation/src/tests/mod.rs | 12 +- core/mempool/src/adapter/mod.rs | 17 +- core/run/src/lib.rs | 13 +- core/storage/src/lib.rs | 168 ++++---- core/storage/src/tests/storage.rs | 2 +- protocol/src/traits/executor.rs | 42 +- protocol/src/traits/mod.rs | 6 +- protocol/src/traits/storage.rs | 72 ++-- protocol/src/types/executor.rs | 3 - 34 files changed, 966 insertions(+), 718 deletions(-) create mode 100644 core/executor/src/adapter/apply.rs create mode 100644 core/executor/src/adapter/read_only.rs diff --git a/core/api/src/adapter.rs b/core/api/src/adapter.rs index 711584a11..7239c226e 100644 --- a/core/api/src/adapter.rs +++ b/core/api/src/adapter.rs @@ -1,6 +1,8 @@ use std::sync::Arc; -use protocol::traits::{APIAdapter, Context, Executor, ExecutorAdapter, MemPool, Network, Storage}; +use protocol::traits::{ + APIAdapter, Context, Executor, ExecutorReadOnlyAdapter, MemPool, Network, ReadOnlyStorage, +}; use protocol::types::{ Account, BigEndianHash, Block, BlockNumber, Bytes, CkbRelatedInfo, ExecutorContext, Hash, Header, Metadata, Proposal, Receipt, SignedTransaction, TxResp, H160, H256, @@ -9,7 +11,7 @@ use protocol::types::{ use protocol::{async_trait, codec::ProtocolCodec, trie, ProtocolResult}; use core_executor::{ - system_contract::metadata::MetadataHandle, AxonExecutor, AxonExecutorAdapter, MPTTrie, + system_contract::metadata::MetadataHandle, AxonExecutor, AxonExecutorReadOnlyAdapter, MPTTrie, }; use crate::APIError; @@ -25,7 +27,7 @@ pub struct DefaultAPIAdapter { impl DefaultAPIAdapter where M: MemPool + 'static, - S: Storage + 'static, + S: ReadOnlyStorage + 'static, DB: trie::DB + 'static, Net: Network + 'static, { @@ -41,13 +43,13 @@ where pub async fn evm_backend( &self, number: Option, - ) -> ProtocolResult> { + ) -> ProtocolResult> { let block = self .get_block_by_number(Context::new(), number) .await? .ok_or_else(|| APIError::Adapter(format!("Cannot get {:?} block", number)))?; - AxonExecutorAdapter::from_root( + AxonExecutorReadOnlyAdapter::from_root( block.header.state_root, Arc::clone(&self.trie_db), Arc::clone(&self.storage), @@ -60,7 +62,7 @@ where impl APIAdapter for DefaultAPIAdapter where M: MemPool + 'static, - S: Storage + 'static, + S: ReadOnlyStorage + 'static, DB: trie::DB + 'static, Net: Network + 'static, { @@ -188,7 +190,7 @@ where exec_ctx.origin = from.unwrap_or_default(); exec_ctx.gas_price = gas_price.unwrap_or_else(U256::one); - let backend = AxonExecutorAdapter::from_root( + let backend = AxonExecutorReadOnlyAdapter::from_root( state_root, Arc::clone(&self.trie_db), Arc::clone(&self.storage), @@ -257,7 +259,7 @@ where async fn get_image_cell_root(&self, ctx: Context) -> ProtocolResult { let state_root = self.storage.get_latest_block_header(ctx).await?.state_root; - Ok(AxonExecutorAdapter::from_root( + Ok(AxonExecutorReadOnlyAdapter::from_root( state_root, Arc::clone(&self.trie_db), Arc::clone(&self.storage), @@ -269,7 +271,7 @@ where async fn get_metadata_root(&self, ctx: Context) -> ProtocolResult { let state_root = self.storage.get_latest_block_header(ctx).await?.state_root; - Ok(AxonExecutorAdapter::from_root( + Ok(AxonExecutorReadOnlyAdapter::from_root( state_root, Arc::clone(&self.trie_db), Arc::clone(&self.storage), diff --git a/core/cli/src/lib.rs b/core/cli/src/lib.rs index 5a86368b2..e2a250188 100644 --- a/core/cli/src/lib.rs +++ b/core/cli/src/lib.rs @@ -1,8 +1,3 @@ -use clap::{CommandFactory as _, FromArgMatches as _, Parser, Subcommand}; - -use common_version::Version; -use core_run::{KeyProvider, SecioKeyPair}; - mod args; mod error; pub(crate) mod utils; @@ -10,6 +5,11 @@ pub(crate) mod utils; pub use args::run::RunArgs; pub use error::{CheckingVersionError, Error, Result}; +use clap::{CommandFactory as _, FromArgMatches as _, Parser, Subcommand}; + +use common_version::Version; +use core_run::{KeyProvider, SecioKeyPair}; + #[derive(Parser, Debug)] #[command(name = "axon")] struct Cli { diff --git a/core/consensus/src/adapter.rs b/core/consensus/src/adapter.rs index 60af3f25b..ed6b4c208 100644 --- a/core/consensus/src/adapter.rs +++ b/core/consensus/src/adapter.rs @@ -8,7 +8,7 @@ use parking_lot::RwLock; use common_apm::Instant; use common_apm_derive::trace_span; use core_executor::system_contract::metadata::MetadataHandle; -use core_executor::{AxonExecutor, AxonExecutorAdapter}; +use core_executor::{AxonExecutor, AxonExecutorApplyAdapter, AxonExecutorReadOnlyAdapter}; use core_network::{PeerId, PeerIdExt}; use protocol::traits::{ CommonConsensusAdapter, ConsensusAdapter, Context, Executor, Gossip, MemPool, MessageTarget, @@ -332,7 +332,7 @@ where proposal: &Proposal, signed_txs: &[SignedTransaction], ) -> ProtocolResult { - let mut backend = AxonExecutorAdapter::from_root( + let mut backend = AxonExecutorApplyAdapter::from_root( last_state_root, Arc::clone(&self.trie_db), Arc::clone(&self.storage), @@ -655,7 +655,7 @@ where async fn get_metadata_handle(&self, ctx: Context) -> ProtocolResult { let current_state_root = self.storage.get_latest_block_header(ctx).await?.state_root; - let root = AxonExecutorAdapter::from_root( + let root = AxonExecutorReadOnlyAdapter::from_root( current_state_root, Arc::clone(&self.trie_db), Arc::clone(&self.storage), diff --git a/core/executor/benches/bench_transfer.rs b/core/executor/benches/bench_transfer.rs index e77e0a28e..71d5175c9 100644 --- a/core/executor/benches/bench_transfer.rs +++ b/core/executor/benches/bench_transfer.rs @@ -7,7 +7,7 @@ use common_crypto::{ Crypto, PrivateKey, Secp256k1Recoverable, Secp256k1RecoverablePrivateKey, Signature, ToPublicKey, UncompressedPublicKey, }; -use core_executor::{AxonExecutor, AxonExecutorAdapter, MPTTrie, RocksTrieDB}; +use core_executor::{AxonExecutor, AxonExecutorApplyAdapter, MPTTrie, RocksTrieDB}; use core_storage::{adapter::rocks::RocksAdapter, ImplStorage}; use protocol::codec::{hex_decode, ProtocolCodec}; use protocol::traits::Executor; @@ -60,8 +60,8 @@ impl BenchAdapter { mpt.commit().unwrap() } - fn init_backend(&self) -> AxonExecutorAdapter, RocksTrieDB> { - AxonExecutorAdapter::from_root( + fn init_backend(&self) -> AxonExecutorApplyAdapter, RocksTrieDB> { + AxonExecutorApplyAdapter::from_root( self.init_mpt(), Arc::clone(&self.trie_db), Arc::clone(&self.storage), @@ -74,7 +74,6 @@ impl BenchAdapter { gas_price: 85u64.into(), block_gas_limit: 100_000_000_000u64.into(), block_base_fee_per_gas: Default::default(), - logs: vec![], }, ) .unwrap() diff --git a/core/executor/benches/bench_vm.rs b/core/executor/benches/bench_vm.rs index 88a268f05..f0d67deee 100644 --- a/core/executor/benches/bench_vm.rs +++ b/core/executor/benches/bench_vm.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use criterion::{criterion_group, criterion_main, Criterion}; -use core_executor::{AxonExecutor, AxonExecutorAdapter, MPTTrie}; +use core_executor::{AxonExecutor, AxonExecutorApplyAdapter, MPTTrie}; use protocol::{ codec::ProtocolCodec, traits::{Executor, Storage}, @@ -46,7 +46,7 @@ where } } -impl BackendInit for AxonExecutorAdapter +impl BackendInit for AxonExecutorApplyAdapter where S: Storage + 'static, DB: trie::DB + 'static, @@ -65,7 +65,7 @@ where .unwrap(); let state_root = mpt.commit().unwrap(); - AxonExecutorAdapter::from_root(state_root, db, Arc::new(storage), exec_ctx).unwrap() + AxonExecutorApplyAdapter::from_root(state_root, db, Arc::new(storage), exec_ctx).unwrap() } } @@ -90,7 +90,7 @@ fn criterion_10000_txs(c: &mut Criterion) { let db = new_rocks_trie_db(); let exec_ctx = mock_executor_context(); let (account, addr) = init_account(); - let mut axon_adapter = AxonExecutorAdapter::init(storage, db, exec_ctx, account, addr); + let mut axon_adapter = AxonExecutorApplyAdapter::init(storage, db, exec_ctx, account, addr); let executor = AxonExecutor::default(); b.iter(|| { executor.exec(&mut axon_adapter, &txs, &[]); diff --git a/core/executor/benches/mock.rs b/core/executor/benches/mock.rs index e432b5a6b..3203028ee 100644 --- a/core/executor/benches/mock.rs +++ b/core/executor/benches/mock.rs @@ -56,7 +56,6 @@ pub fn mock_executor_context() -> ExecutorContext { gas_price: 85u64.into(), block_gas_limit: 100_000_000_000u64.into(), block_base_fee_per_gas: Default::default(), - logs: vec![], } } diff --git a/core/executor/src/adapter/apply.rs b/core/executor/src/adapter/apply.rs new file mode 100644 index 000000000..c52e4e0e9 --- /dev/null +++ b/core/executor/src/adapter/apply.rs @@ -0,0 +1,286 @@ +use std::sync::Arc; + +use evm::backend::{Apply, Basic}; + +use protocol::traits::{ + ApplyBackend, Backend, Context, ExecutorAdapter, ExecutorReadOnlyAdapter, ReadOnlyStorage, + Storage, +}; +use protocol::types::{ + Account, Bytes, ExecutorContext, Hasher, Log, MerkleRoot, H160, H256, NIL_DATA, RLP_NULL, U256, +}; +use protocol::{codec::ProtocolCodec, trie, ProtocolResult}; + +use crate::blocking_async; +use crate::system_contract::{METADATA_CONTRACT_ADDRESS, METADATA_ROOT_KEY}; +use crate::{adapter::AxonExecutorReadOnlyAdapter, MPTTrie}; + +pub struct AxonExecutorApplyAdapter { + inner: AxonExecutorReadOnlyAdapter, + logs: Vec, +} + +impl ExecutorReadOnlyAdapter for AxonExecutorApplyAdapter +where + S: ReadOnlyStorage + 'static, + DB: trie::DB + 'static, +{ + fn get_ctx(&self) -> ExecutorContext { + self.inner.get_ctx() + } + + fn get(&self, key: &[u8]) -> Option { + self.inner.get(key) + } + + fn get_account(&self, address: &H160) -> Account { + self.inner.get_account(address) + } +} + +impl Backend for AxonExecutorApplyAdapter +where + S: ReadOnlyStorage + 'static, + DB: trie::DB + 'static, +{ + fn gas_price(&self) -> U256 { + self.inner.gas_price() + } + + fn origin(&self) -> H160 { + self.inner.origin() + } + + fn block_hash(&self, number: U256) -> H256 { + self.inner.block_hash(number) + } + + fn block_number(&self) -> U256 { + self.inner.block_number() + } + + fn block_coinbase(&self) -> H160 { + self.inner.block_coinbase() + } + + fn block_timestamp(&self) -> U256 { + self.inner.block_timestamp() + } + + fn block_difficulty(&self) -> U256 { + self.inner.block_difficulty() + } + + fn block_gas_limit(&self) -> U256 { + self.inner.block_gas_limit() + } + + fn block_base_fee_per_gas(&self) -> U256 { + self.inner.block_base_fee_per_gas() + } + + fn chain_id(&self) -> U256 { + self.inner.chain_id() + } + + fn exists(&self, address: H160) -> bool { + self.inner.exists(address) + } + + fn basic(&self, address: H160) -> Basic { + self.inner.basic(address) + } + + fn code(&self, address: H160) -> Vec { + self.inner.code(address) + } + + fn storage(&self, address: H160, key: H256) -> H256 { + self.inner.storage(address, key) + } + + fn original_storage(&self, address: H160, key: H256) -> Option { + self.inner.original_storage(address, key) + } +} + +impl ExecutorAdapter for AxonExecutorApplyAdapter +where + S: Storage + 'static, + DB: trie::DB + 'static, +{ + fn set_origin(&mut self, origin: H160) { + self.inner.exec_ctx.origin = origin; + } + + fn set_gas_price(&mut self, gas_price: U256) { + self.inner.exec_ctx.gas_price = gas_price; + } + + fn take_logs(&mut self) -> Vec { + let mut ret = Vec::new(); + ret.append(&mut self.logs); + ret + } + + fn commit(&mut self) -> MerkleRoot { + self.inner.trie.commit().unwrap() + } + + fn save_account(&mut self, address: &H160, account: &Account) { + self.inner + .trie + .insert(address.as_bytes(), &account.encode().unwrap()) + .unwrap(); + } +} + +impl AxonExecutorApplyAdapter +where + S: Storage + 'static, + DB: trie::DB + 'static, +{ + fn apply>( + &mut self, + address: H160, + basic: Basic, + code: Option>, + storage: I, + reset_storage: bool, + ) -> bool { + let old_account = match self.inner.trie.get(address.as_bytes()) { + Ok(Some(raw)) => Account::decode(raw).unwrap(), + _ => Account { + nonce: U256::zero(), + balance: U256::zero(), + storage_root: RLP_NULL, + code_hash: NIL_DATA, + }, + }; + + let storage_root = if reset_storage { + RLP_NULL + } else { + old_account.storage_root + }; + + let mut storage_trie = if storage_root == RLP_NULL { + MPTTrie::new(self.inner.db.clone()) + } else { + MPTTrie::from_root(old_account.storage_root, self.inner.db.clone()).unwrap() + }; + + storage.into_iter().for_each(|(k, v)| { + let _ = storage_trie.insert(k.as_bytes(), v.as_bytes()); + }); + + let storage_root = storage_trie + .commit() + .unwrap_or_else(|err| panic!("failed to update the trie storage since {err}")); + + let mut new_account = Account { + nonce: basic.nonce, + balance: basic.balance, + code_hash: old_account.code_hash, + storage_root, + }; + + if let Some(c) = code { + let new_code_hash = Hasher::digest(&c); + if new_code_hash != old_account.code_hash { + blocking_async!( + self, + get_storage, + insert_code, + Context::new(), + address.into(), + new_code_hash, + c.into() + ); + + new_account.code_hash = new_code_hash; + } + } + + let bytes = new_account.encode().unwrap(); + + { + self.inner + .trie + .insert(address.as_bytes(), bytes.as_ref()) + .unwrap(); + } + + new_account.balance == U256::zero() + && new_account.nonce == U256::zero() + && new_account.code_hash.is_zero() + } +} + +impl ApplyBackend for AxonExecutorApplyAdapter +where + S: Storage + 'static, + DB: trie::DB + 'static, +{ + fn apply(&mut self, values: A, logs: L, delete_empty: bool) + where + A: IntoIterator>, + I: IntoIterator, + L: IntoIterator, + { + for apply in values.into_iter() { + match apply { + Apply::Modify { + address, + basic, + code, + storage, + reset_storage, + } => { + let is_empty = self.apply(address, basic, code, storage, reset_storage); + if is_empty && delete_empty { + self.inner.trie.remove(address.as_bytes()).unwrap(); + } + } + Apply::Delete { address } => { + let _ = self.inner.trie.remove(address.as_bytes()); + } + } + } + + self.logs = logs.into_iter().collect::>(); + } +} + +impl AxonExecutorApplyAdapter +where + S: ReadOnlyStorage + 'static, + DB: trie::DB + 'static, +{ + pub fn new(db: Arc, storage: Arc, exec_ctx: ExecutorContext) -> ProtocolResult { + Ok(AxonExecutorApplyAdapter { + inner: AxonExecutorReadOnlyAdapter::new(db, storage, exec_ctx)?, + logs: Vec::new(), + }) + } + + pub fn from_root( + state_root: MerkleRoot, + db: Arc, + storage: Arc, + exec_ctx: ExecutorContext, + ) -> ProtocolResult { + Ok(AxonExecutorApplyAdapter { + inner: AxonExecutorReadOnlyAdapter::from_root(state_root, db, storage, exec_ctx)?, + logs: Vec::new(), + }) + } + + pub fn get_metadata_root(&self) -> H256 { + self.storage(METADATA_CONTRACT_ADDRESS, *METADATA_ROOT_KEY) + } + + fn get_storage(&self) -> Arc { + Arc::clone(&self.inner.storage) + } +} diff --git a/core/executor/src/adapter/mod.rs b/core/executor/src/adapter/mod.rs index a3c1b5050..035ea4759 100644 --- a/core/executor/src/adapter/mod.rs +++ b/core/executor/src/adapter/mod.rs @@ -1,374 +1,20 @@ +mod apply; +mod read_only; mod trie_db; mod wrapped_trie; pub use trie_db::RocksTrieDB; pub use wrapped_trie::MPTTrie; +pub use {apply::AxonExecutorApplyAdapter, read_only::AxonExecutorReadOnlyAdapter}; -use std::sync::Arc; - -use evm::backend::{Apply, Basic}; - -use protocol::traits::{ApplyBackend, Backend, Context, ExecutorAdapter, Storage}; -use protocol::types::{ - Account, Bytes, ExecutorContext, Hasher, Log, MerkleRoot, H160, H256, NIL_DATA, RLP_NULL, U256, -}; -use protocol::{codec::ProtocolCodec, trie, ProtocolResult}; - -use crate::system_contract::{ - HEADER_CELL_ROOT_KEY, IMAGE_CELL_CONTRACT_ADDRESS, METADATA_CONTRACT_ADDRESS, METADATA_ROOT_KEY, -}; - -const GET_BLOCK_HASH_NUMBER_RANGE: u64 = 256; - +#[macro_export] macro_rules! blocking_async { ($self_: ident, $adapter: ident, $method: ident$ (, $args: expr)*) => {{ let rt = protocol::tokio::runtime::Handle::current(); - let adapter = Arc::clone(&$self_.$adapter); + let adapter_clone = $self_.$adapter(); protocol::tokio::task::block_in_place(move || { - rt.block_on(adapter.$method( $($args,)* )).unwrap() + rt.block_on(adapter_clone.$method( $($args,)* )).unwrap() }) }}; } - -pub struct AxonExecutorAdapter { - exec_ctx: ExecutorContext, - trie: MPTTrie, - storage: Arc, - db: Arc, -} - -impl ExecutorAdapter for AxonExecutorAdapter -where - S: Storage + 'static, - DB: trie::DB + 'static, -{ - fn get_ctx(&self) -> ExecutorContext { - self.exec_ctx.clone() - } - - fn set_origin(&mut self, origin: H160) { - self.exec_ctx.origin = origin; - } - - fn set_gas_price(&mut self, gas_price: U256) { - self.exec_ctx.gas_price = gas_price; - } - - fn get_logs(&mut self) -> Vec { - let mut ret = Vec::new(); - ret.append(&mut self.exec_ctx.logs); - ret - } - - fn commit(&mut self) -> MerkleRoot { - self.trie.commit().unwrap() - } - - fn get(&self, key: &[u8]) -> Option { - self.trie.get(key).ok().flatten() - } - - fn get_account(&self, address: &H160) -> Account { - if let Ok(Some(raw)) = self.trie.get(address.as_bytes()) { - return Account::decode(raw).unwrap(); - } - - Account { - nonce: U256::zero(), - balance: U256::zero(), - storage_root: RLP_NULL, - code_hash: NIL_DATA, - } - } - - fn save_account(&mut self, address: &H160, account: &Account) { - self.trie - .insert(address.as_bytes(), &account.encode().unwrap()) - .unwrap(); - } -} - -impl Backend for AxonExecutorAdapter -where - S: Storage + 'static, - DB: trie::DB + 'static, -{ - fn gas_price(&self) -> U256 { - self.exec_ctx.gas_price - } - - fn origin(&self) -> H160 { - self.exec_ctx.origin - } - - fn block_number(&self) -> U256 { - self.exec_ctx.block_number - } - - fn block_hash(&self, number: U256) -> H256 { - let current_number = self.block_number(); - if number >= current_number { - return H256::default(); - } - - if (current_number - number) > U256::from(GET_BLOCK_HASH_NUMBER_RANGE) { - return H256::default(); - } - - let number = number.as_u64(); - blocking_async!(self, storage, get_block, Context::new(), number) - .map(|b| b.hash()) - .unwrap_or_default() - } - - fn block_coinbase(&self) -> H160 { - self.exec_ctx.block_coinbase - } - - fn block_timestamp(&self) -> U256 { - self.exec_ctx.block_timestamp - } - - fn block_difficulty(&self) -> U256 { - U256::one() - } - - fn block_gas_limit(&self) -> U256 { - self.exec_ctx.block_gas_limit - } - - fn block_base_fee_per_gas(&self) -> U256 { - self.exec_ctx.block_base_fee_per_gas - } - - fn chain_id(&self) -> U256 { - self.exec_ctx.chain_id - } - - fn exists(&self, address: H160) -> bool { - self.trie - .contains(&Bytes::from(address.as_bytes().to_vec())) - .unwrap_or(false) - } - - fn basic(&self, address: H160) -> Basic { - self.trie - .get(address.as_bytes()) - .map(|raw| { - if raw.is_none() { - return Basic::default(); - } - Account::decode(raw.unwrap()).map_or_else( - |_| Default::default(), - |account| Basic { - balance: account.balance, - nonce: account.nonce, - }, - ) - }) - .unwrap_or_default() - } - - fn code(&self, address: H160) -> Vec { - let code_hash = if let Some(bytes) = self.trie.get(address.as_bytes()).unwrap() { - Account::decode(bytes).unwrap().code_hash - } else { - return Vec::new(); - }; - - if code_hash == NIL_DATA { - return Vec::new(); - } - - let res = blocking_async!(self, storage, get_code_by_hash, Context::new(), &code_hash); - - res.unwrap_or_default().to_vec() - } - - // ### Notes - // - // - If a MPT tree is empty, the root should be `RLP_NULL`. - // - In this function, when returns `H256::default()`, that means the tree is - // not initialized. - fn storage(&self, address: H160, index: H256) -> H256 { - if let Ok(raw) = self.trie.get(address.as_bytes()) { - if raw.is_none() { - return H256::default(); - } - - Account::decode(raw.unwrap()) - .and_then(|account| { - let storage_root = account.storage_root; - if storage_root == RLP_NULL { - Ok(H256::default()) - } else { - MPTTrie::from_root(storage_root, Arc::clone(&self.db)).map( - |trie| match trie.get(index.as_bytes()) { - Ok(Some(res)) => H256::from_slice(res.as_ref()), - _ => H256::default(), - }, - ) - } - }) - .unwrap_or_default() - } else { - H256::default() - } - } - - fn original_storage(&self, address: H160, index: H256) -> Option { - // Fixme - Some(self.storage(address, index)) - } -} - -impl ApplyBackend for AxonExecutorAdapter -where - S: Storage + 'static, - DB: trie::DB + 'static, -{ - fn apply(&mut self, values: A, logs: L, delete_empty: bool) - where - A: IntoIterator>, - I: IntoIterator, - L: IntoIterator, - { - for apply in values.into_iter() { - match apply { - Apply::Modify { - address, - basic, - code, - storage, - reset_storage, - } => { - let is_empty = self.apply(address, basic, code, storage, reset_storage); - if is_empty && delete_empty { - self.trie.remove(address.as_bytes()).unwrap(); - } - } - Apply::Delete { address } => { - let _ = self.trie.remove(address.as_bytes()); - } - } - } - - self.exec_ctx.logs = logs.into_iter().collect::>(); - } -} - -impl AxonExecutorAdapter -where - S: Storage + 'static, - DB: trie::DB + 'static, -{ - pub fn new(db: Arc, storage: Arc, exec_ctx: ExecutorContext) -> ProtocolResult { - let trie = MPTTrie::new(Arc::clone(&db)); - Ok(AxonExecutorAdapter { - trie, - db, - storage, - exec_ctx, - }) - } - - pub fn from_root( - state_root: MerkleRoot, - db: Arc, - storage: Arc, - exec_ctx: ExecutorContext, - ) -> ProtocolResult { - let trie = MPTTrie::from_root(state_root, Arc::clone(&db))?; - - Ok(AxonExecutorAdapter { - trie, - db, - storage, - exec_ctx, - }) - } - - pub fn get_metadata_root(&self) -> H256 { - self.storage(METADATA_CONTRACT_ADDRESS, *METADATA_ROOT_KEY) - } - - pub fn get_image_cell_root(&self) -> H256 { - self.storage(IMAGE_CELL_CONTRACT_ADDRESS, *HEADER_CELL_ROOT_KEY) - } - - fn apply>( - &mut self, - address: H160, - basic: Basic, - code: Option>, - storage: I, - reset_storage: bool, - ) -> bool { - let old_account = match self.trie.get(address.as_bytes()) { - Ok(Some(raw)) => Account::decode(raw).unwrap(), - _ => Account { - nonce: U256::zero(), - balance: U256::zero(), - storage_root: RLP_NULL, - code_hash: NIL_DATA, - }, - }; - - let storage_root = if reset_storage { - RLP_NULL - } else { - old_account.storage_root - }; - - let mut storage_trie = if storage_root == RLP_NULL { - MPTTrie::new(Arc::clone(&self.db)) - } else { - MPTTrie::from_root(old_account.storage_root, Arc::clone(&self.db)).unwrap() - }; - - storage.into_iter().for_each(|(k, v)| { - let _ = storage_trie.insert(k.as_bytes(), v.as_bytes()); - }); - - let storage_root = storage_trie - .commit() - .unwrap_or_else(|err| panic!("failed to update the trie storage since {err}")); - - let mut new_account = Account { - nonce: basic.nonce, - balance: basic.balance, - code_hash: old_account.code_hash, - storage_root, - }; - - if let Some(c) = code { - let new_code_hash = Hasher::digest(&c); - if new_code_hash != old_account.code_hash { - blocking_async!( - self, - storage, - insert_code, - Context::new(), - address.into(), - new_code_hash, - c.into() - ); - - new_account.code_hash = new_code_hash; - } - } - - let bytes = new_account.encode().unwrap(); - - { - self.trie - .insert(address.as_bytes(), bytes.as_ref()) - .unwrap(); - } - - new_account.balance == U256::zero() - && new_account.nonce == U256::zero() - && new_account.code_hash.is_zero() - } -} diff --git a/core/executor/src/adapter/read_only.rs b/core/executor/src/adapter/read_only.rs new file mode 100644 index 000000000..5159fd8da --- /dev/null +++ b/core/executor/src/adapter/read_only.rs @@ -0,0 +1,234 @@ +use std::sync::Arc; + +use evm::backend::Basic; + +use protocol::traits::{Backend, Context, ExecutorReadOnlyAdapter, ReadOnlyStorage}; +use protocol::types::{ + Account, Bytes, ExecutorContext, MerkleRoot, H160, H256, NIL_DATA, RLP_NULL, U256, +}; +use protocol::{codec::ProtocolCodec, trie, ProtocolResult}; + +use crate::system_contract::{ + HEADER_CELL_ROOT_KEY, IMAGE_CELL_CONTRACT_ADDRESS, METADATA_CONTRACT_ADDRESS, METADATA_ROOT_KEY, +}; +use crate::{blocking_async, MPTTrie}; + +const GET_BLOCK_HASH_NUMBER_RANGE: u64 = 256; + +pub struct AxonExecutorReadOnlyAdapter { + pub(crate) exec_ctx: ExecutorContext, + pub(crate) trie: MPTTrie, + pub(crate) storage: Arc, + pub(crate) db: Arc, +} + +impl ExecutorReadOnlyAdapter for AxonExecutorReadOnlyAdapter +where + S: ReadOnlyStorage + 'static, + DB: trie::DB + 'static, +{ + fn get_ctx(&self) -> ExecutorContext { + self.exec_ctx.clone() + } + + fn get(&self, key: &[u8]) -> Option { + self.trie.get(key).ok().flatten() + } + + fn get_account(&self, address: &H160) -> Account { + if let Ok(Some(raw)) = self.trie.get(address.as_bytes()) { + return Account::decode(raw).unwrap(); + } + + Account { + nonce: U256::zero(), + balance: U256::zero(), + storage_root: RLP_NULL, + code_hash: NIL_DATA, + } + } +} + +impl Backend for AxonExecutorReadOnlyAdapter +where + S: ReadOnlyStorage + 'static, + DB: trie::DB + 'static, +{ + fn gas_price(&self) -> U256 { + self.exec_ctx.gas_price + } + + fn origin(&self) -> H160 { + self.exec_ctx.origin + } + + fn block_number(&self) -> U256 { + self.exec_ctx.block_number + } + + fn block_hash(&self, number: U256) -> H256 { + let current_number = self.block_number(); + if number >= current_number { + return H256::default(); + } + + if (current_number - number) > U256::from(GET_BLOCK_HASH_NUMBER_RANGE) { + return H256::default(); + } + + let number = number.as_u64(); + blocking_async!(self, get_storage, get_block, Context::new(), number) + .map(|b| b.hash()) + .unwrap_or_default() + } + + fn block_coinbase(&self) -> H160 { + self.exec_ctx.block_coinbase + } + + fn block_timestamp(&self) -> U256 { + self.exec_ctx.block_timestamp + } + + fn block_difficulty(&self) -> U256 { + self.exec_ctx.difficulty + } + + fn block_gas_limit(&self) -> U256 { + self.exec_ctx.block_gas_limit + } + + fn block_base_fee_per_gas(&self) -> U256 { + self.exec_ctx.block_base_fee_per_gas + } + + fn chain_id(&self) -> U256 { + self.exec_ctx.chain_id + } + + fn exists(&self, address: H160) -> bool { + self.trie + .contains(&Bytes::from(address.as_bytes().to_vec())) + .unwrap_or(false) + } + + fn basic(&self, address: H160) -> Basic { + self.trie + .get(address.as_bytes()) + .map(|raw| { + if raw.is_none() { + return Basic::default(); + } + Account::decode(raw.unwrap()).map_or_else( + |_| Default::default(), + |account| Basic { + balance: account.balance, + nonce: account.nonce, + }, + ) + }) + .unwrap_or_default() + } + + fn code(&self, address: H160) -> Vec { + let code_hash = if let Some(bytes) = self.trie.get(address.as_bytes()).unwrap() { + Account::decode(bytes).unwrap().code_hash + } else { + return Vec::new(); + }; + + if code_hash == NIL_DATA { + return Vec::new(); + } + + let res = blocking_async!( + self, + get_storage, + get_code_by_hash, + Context::new(), + &code_hash + ); + + res.unwrap_or_default().to_vec() + } + + // ### Notes + // + // - If a MPT tree is empty, the root should be `RLP_NULL`. + // - In this function, when returns `H256::default()`, that means the tree is + // not initialized. + fn storage(&self, address: H160, index: H256) -> H256 { + if let Ok(raw) = self.trie.get(address.as_bytes()) { + if raw.is_none() { + return H256::default(); + } + + Account::decode(raw.unwrap()) + .and_then(|account| { + let storage_root = account.storage_root; + if storage_root == RLP_NULL { + Ok(H256::default()) + } else { + MPTTrie::from_root(storage_root, Arc::clone(&self.db)).map( + |trie| match trie.get(index.as_bytes()) { + Ok(Some(res)) => H256::from_slice(res.as_ref()), + _ => H256::default(), + }, + ) + } + }) + .unwrap_or_default() + } else { + H256::default() + } + } + + fn original_storage(&self, address: H160, index: H256) -> Option { + // Fixme + Some(self.storage(address, index)) + } +} + +impl AxonExecutorReadOnlyAdapter +where + S: ReadOnlyStorage + 'static, + DB: trie::DB + 'static, +{ + pub fn new(db: Arc, storage: Arc, exec_ctx: ExecutorContext) -> ProtocolResult { + let trie = MPTTrie::new(Arc::clone(&db)); + Ok(AxonExecutorReadOnlyAdapter { + trie, + db, + storage, + exec_ctx, + }) + } + + pub fn from_root( + state_root: MerkleRoot, + db: Arc, + storage: Arc, + exec_ctx: ExecutorContext, + ) -> ProtocolResult { + let trie = MPTTrie::from_root(state_root, Arc::clone(&db))?; + + Ok(AxonExecutorReadOnlyAdapter { + trie, + db, + storage, + exec_ctx, + }) + } + + pub fn get_metadata_root(&self) -> H256 { + self.storage(METADATA_CONTRACT_ADDRESS, *METADATA_ROOT_KEY) + } + + pub fn get_image_cell_root(&self) -> H256 { + self.storage(IMAGE_CELL_CONTRACT_ADDRESS, *HEADER_CELL_ROOT_KEY) + } + + fn get_storage(&self) -> Arc { + Arc::clone(&self.storage) + } +} diff --git a/core/executor/src/debugger/mod.rs b/core/executor/src/debugger/mod.rs index f889c5ce0..e00732689 100644 --- a/core/executor/src/debugger/mod.rs +++ b/core/executor/src/debugger/mod.rs @@ -20,7 +20,7 @@ use protocol::types::{ use core_storage::{adapter::rocks::RocksAdapter, ImplStorage}; -use crate::adapter::{AxonExecutorAdapter, MPTTrie}; +use crate::adapter::{AxonExecutorApplyAdapter, MPTTrie}; use crate::{AxonExecutor, RocksTrieDB}; const GENESIS_PATH: &str = "../../tests/data/genesis.json"; @@ -97,7 +97,10 @@ impl EvmDebugger { evm.call(&backend, MAX_BLOCK_GAS_LIMIT, from, to, value, data) } - fn backend(&self, number: u64) -> AxonExecutorAdapter, RocksTrieDB> { + fn backend( + &self, + number: u64, + ) -> AxonExecutorApplyAdapter, RocksTrieDB> { let exec_ctx = ExecutorContext { block_number: number.into(), block_coinbase: H160::random(), @@ -107,10 +110,9 @@ impl EvmDebugger { gas_price: 1u64.into(), block_gas_limit: 4294967295000u64.into(), block_base_fee_per_gas: 1337u64.into(), - logs: vec![], }; - AxonExecutorAdapter::from_root( + AxonExecutorApplyAdapter::from_root( self.state_root, Arc::clone(&self.trie_db), Arc::clone(&self.storage), diff --git a/core/executor/src/lib.rs b/core/executor/src/lib.rs index 4da048d46..68daf3a60 100644 --- a/core/executor/src/lib.rs +++ b/core/executor/src/lib.rs @@ -9,7 +9,9 @@ pub mod system_contract; mod tests; mod utils; -pub use crate::adapter::{AxonExecutorAdapter, MPTTrie, RocksTrieDB}; +pub use crate::adapter::{ + AxonExecutorApplyAdapter, AxonExecutorReadOnlyAdapter, MPTTrie, RocksTrieDB, +}; pub use crate::system_contract::{metadata::MetadataHandle, DataProvider}; pub use crate::utils::{code_address, decode_revert_msg, DefaultFeeAllocator, FeeInlet}; @@ -22,20 +24,18 @@ use evm::executor::stack::{MemoryStackState, PrecompileFn, StackExecutor, StackS use evm::CreateScheme; use common_merkle::TrieMerkle; -use protocol::codec::ProtocolCodec; use protocol::traits::{Backend, Executor, ExecutorAdapter}; use protocol::types::{ - data_gas_cost, logs_bloom, Account, Config, ExecResp, Hasher, SignedTransaction, - TransactionAction, TxResp, ValidatorExtend, GAS_CALL_TRANSACTION, GAS_CREATE_TRANSACTION, H160, - H256, NIL_DATA, RLP_NULL, U256, + data_gas_cost, logs_bloom, Config, ExecResp, Hasher, SignedTransaction, TransactionAction, + TxResp, ValidatorExtend, GAS_CALL_TRANSACTION, GAS_CREATE_TRANSACTION, H160, H256, RLP_NULL, + U256, }; use crate::precompiles::build_precompile_set; use crate::system_contract::{ - after_block_hook, before_block_hook, system_contract_dispatch, CkbLightClientContract, - ImageCellContract, MetadataContract, NativeTokenContract, SystemContract, - CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, HEADER_CELL_ROOT_KEY, METADATA_CONTRACT_ADDRESS, - METADATA_ROOT_KEY, + after_block_hook, before_block_hook, system_contract_dispatch, + CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, HEADER_CELL_ROOT_KEY, IMAGE_CELL_CONTRACT_ADDRESS, + METADATA_CONTRACT_ADDRESS, METADATA_ROOT_KEY, NATIVE_TOKEN_CONTRACT_ADDRESS, }; lazy_static::lazy_static! { @@ -152,7 +152,7 @@ impl Executor for AxonExecutor { let mut r = system_contract_dispatch(adapter, tx) .unwrap_or_else(|| Self::evm_exec(adapter, &config, &precompiles, tx)); - r.logs = adapter.get_logs(); + r.logs = adapter.take_logs(); gas += r.gas_used; fee = fee.checked_add(r.fee_cost).unwrap_or(U256::max_value()); @@ -204,18 +204,6 @@ impl Executor for AxonExecutor { tx_resp: res, } } - - fn get_account(&self, backend: &Adapter, address: &H160) -> Account { - match backend.get(address.as_bytes()) { - Some(bytes) => Account::decode(bytes).unwrap(), - None => Account { - nonce: Default::default(), - balance: Default::default(), - storage_root: RLP_NULL, - code_hash: NIL_DATA, - }, - } - } } impl AxonExecutor { @@ -354,7 +342,7 @@ impl AxonExecutor { let mut r = system_contract_dispatch(adapter, tx) .unwrap_or_else(|| Self::evm_exec(adapter, &config, &precompiles, tx)); - r.logs = adapter.get_logs(); + r.logs = adapter.take_logs(); gas += r.gas_used; fee = fee.checked_add(r.fee_cost).unwrap_or(U256::max_value()); @@ -405,10 +393,10 @@ impl AxonExecutor { pub fn is_call_system_script(action: &TransactionAction) -> bool { let system_contracts = vec![ - NativeTokenContract::ADDRESS, - MetadataContract::ADDRESS, - CkbLightClientContract::ADDRESS, - ImageCellContract::ADDRESS, + NATIVE_TOKEN_CONTRACT_ADDRESS, + METADATA_CONTRACT_ADDRESS, + CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, + IMAGE_CELL_CONTRACT_ADDRESS, ]; match action { diff --git a/core/executor/src/precompiles/get_cell.rs b/core/executor/src/precompiles/get_cell.rs index 7c9fb77d4..ddb44a8ad 100644 --- a/core/executor/src/precompiles/get_cell.rs +++ b/core/executor/src/precompiles/get_cell.rs @@ -8,8 +8,8 @@ use evm::{Context, ExitError, ExitSucceed}; use protocol::types::{H160, H256}; use crate::precompiles::{axon_precompile_address, PrecompileContract}; -use crate::system_contract::image_cell::{image_cell_abi, CellKey}; -use crate::{err, system_contract::image_cell::ImageCellContract, CURRENT_HEADER_CELL_ROOT}; +use crate::system_contract::image_cell::{image_cell_abi, CellKey, ImageCellReader}; +use crate::{err, CURRENT_HEADER_CELL_ROOT}; #[derive(Default, Clone)] pub struct GetCell; @@ -34,7 +34,7 @@ impl PrecompileContract for GetCell { let (tx_hash, index) = parse_input(input)?; let root = CURRENT_HEADER_CELL_ROOT.with(|r| *r.borrow()); - let cell = ImageCellContract::default() + let cell = ImageCellReader::default() .get_cell(root, &CellKey { tx_hash, index }) .map_err(|_| err!(_, "get cell"))? .map(|c| Cell { diff --git a/core/executor/src/precompiles/get_header.rs b/core/executor/src/precompiles/get_header.rs index 1efa20611..abed53311 100644 --- a/core/executor/src/precompiles/get_header.rs +++ b/core/executor/src/precompiles/get_header.rs @@ -5,7 +5,7 @@ use evm::{Context, ExitError, ExitSucceed}; use protocol::types::{H160, H256}; use crate::precompiles::{axon_precompile_address, PrecompileContract}; -use crate::{err, system_contract::CkbLightClientContract, CURRENT_HEADER_CELL_ROOT}; +use crate::{err, system_contract::ckb_light_client::CkbHeaderReader, CURRENT_HEADER_CELL_ROOT}; #[derive(Default, Clone)] pub struct GetHeader; @@ -31,7 +31,7 @@ impl PrecompileContract for GetHeader { H256(<[u8; 32] as AbiDecode>::decode(input).map_err(|_| err!(_, "decode input"))?); let root = CURRENT_HEADER_CELL_ROOT.with(|r| *r.borrow()); - let raw = CkbLightClientContract::default() + let raw = CkbHeaderReader::default() .get_raw(root, &block_hash.0) .map_err(|_| err!(_, "get header"))?; diff --git a/core/executor/src/system_contract/ckb_light_client/mod.rs b/core/executor/src/system_contract/ckb_light_client/mod.rs index 004b035a1..9ee4d95dc 100644 --- a/core/executor/src/system_contract/ckb_light_client/mod.rs +++ b/core/executor/src/system_contract/ckb_light_client/mod.rs @@ -7,28 +7,26 @@ use std::sync::atomic::{AtomicBool, Ordering}; use ethers::abi::AbiDecode; +use protocol::traits::{ApplyBackend, ExecutorAdapter}; use protocol::types::{SignedTransaction, TxResp, H160, H256}; -use protocol::{traits::ExecutorAdapter, ProtocolResult}; +use protocol::ProtocolResult; use crate::system_contract::ckb_light_client::store::CkbLightClientStore; use crate::system_contract::utils::{succeed_resp, update_states}; use crate::system_contract::{system_contract_address, SystemContract}; -use crate::{exec_try, CURRENT_HEADER_CELL_ROOT}; +use crate::{exec_try, system_contract_struct, CURRENT_HEADER_CELL_ROOT}; pub const CKB_LIGHT_CLIENT_CONTRACT_ADDRESS: H160 = system_contract_address(0x2); static ALLOW_READ: AtomicBool = AtomicBool::new(false); -#[derive(Default)] -pub struct CkbLightClientContract; +system_contract_struct!(CkbLightClientContract); -impl SystemContract for CkbLightClientContract { +impl SystemContract + for CkbLightClientContract +{ const ADDRESS: H160 = CKB_LIGHT_CLIENT_CONTRACT_ADDRESS; - fn exec_( - &self, - adapter: &mut Adapter, - tx: &SignedTransaction, - ) -> TxResp { + fn exec_(&self, adapter: &mut Adapter, tx: &SignedTransaction) -> TxResp { let sender = tx.sender; let tx = &tx.transaction.unsigned; let tx_data = tx.data(); @@ -72,8 +70,11 @@ impl SystemContract for CkbLightClientContract { } } +#[derive(Default)] +pub(crate) struct CkbHeaderReader; + /// These methods are provide for interoperation module to get CKB headers. -impl CkbLightClientContract { +impl CkbHeaderReader { pub fn get_header_by_block_hash( &self, root: H256, @@ -89,6 +90,7 @@ impl CkbLightClientContract { Ok(ret) } + #[cfg(test)] pub fn allow_read(&self) -> bool { ALLOW_READ.load(Ordering::Relaxed) } diff --git a/core/executor/src/system_contract/image_cell/mod.rs b/core/executor/src/system_contract/image_cell/mod.rs index 6d85b00e7..e21e012b9 100644 --- a/core/executor/src/system_contract/image_cell/mod.rs +++ b/core/executor/src/system_contract/image_cell/mod.rs @@ -4,32 +4,30 @@ mod store; pub use abi::image_cell_abi; pub use store::{CellInfo, CellKey}; -use ethers::abi::AbiDecode; use std::sync::atomic::{AtomicBool, Ordering}; -use protocol::traits::ExecutorAdapter; +use ethers::abi::AbiDecode; + +use protocol::traits::{ApplyBackend, ExecutorAdapter}; use protocol::types::{SignedTransaction, TxResp, H160, H256}; use protocol::ProtocolResult; use crate::system_contract::image_cell::store::ImageCellStore; use crate::system_contract::utils::{succeed_resp, update_states}; use crate::system_contract::{system_contract_address, SystemContract}; -use crate::{exec_try, MPTTrie, CURRENT_HEADER_CELL_ROOT}; +use crate::{exec_try, system_contract_struct, MPTTrie, CURRENT_HEADER_CELL_ROOT}; pub const IMAGE_CELL_CONTRACT_ADDRESS: H160 = system_contract_address(0x3); static ALLOW_READ: AtomicBool = AtomicBool::new(false); -#[derive(Default)] -pub struct ImageCellContract; +system_contract_struct!(ImageCellContract); -impl SystemContract for ImageCellContract { +impl SystemContract + for ImageCellContract +{ const ADDRESS: H160 = IMAGE_CELL_CONTRACT_ADDRESS; - fn exec_( - &self, - adapter: &mut Adapter, - tx: &SignedTransaction, - ) -> TxResp { + fn exec_(&self, adapter: &mut Adapter, tx: &SignedTransaction) -> TxResp { let sender = tx.sender; let tx = &tx.transaction.unsigned; let tx_data = tx.data(); @@ -69,12 +67,16 @@ impl SystemContract for ImageCellContract { } } +#[derive(Default)] +pub(crate) struct ImageCellReader; + /// These methods are provide for interoperation module to get CKB cells. -impl ImageCellContract { +impl ImageCellReader { pub fn get_cell(&self, root: H256, key: &CellKey) -> ProtocolResult> { ImageCellStore::new(root)?.get_cell(key) } + #[cfg(test)] pub fn allow_read(&self) -> bool { ALLOW_READ.load(Ordering::Relaxed) } diff --git a/core/executor/src/system_contract/image_cell/store.rs b/core/executor/src/system_contract/image_cell/store.rs index 32e995cbc..d91dd8d16 100644 --- a/core/executor/src/system_contract/image_cell/store.rs +++ b/core/executor/src/system_contract/image_cell/store.rs @@ -202,6 +202,7 @@ impl CellKey { CellKey { tx_hash, index } } + #[allow(dead_code)] pub fn decode(input: &[u8]) -> ProtocolResult { if input.len() != Self::ENCODED_LEN { return Err(SystemScriptError::DataLengthMismatch { diff --git a/core/executor/src/system_contract/metadata/mod.rs b/core/executor/src/system_contract/metadata/mod.rs index c85ac43a0..95849d9a9 100644 --- a/core/executor/src/system_contract/metadata/mod.rs +++ b/core/executor/src/system_contract/metadata/mod.rs @@ -13,14 +13,14 @@ use ethers::abi::AbiDecode; use lru::LruCache; use parking_lot::RwLock; -use protocol::traits::ExecutorAdapter; +use protocol::traits::{ApplyBackend, ExecutorAdapter}; use protocol::types::{Hasher, Metadata, SignedTransaction, TxResp, H160, H256}; use crate::system_contract::utils::{ generate_mpt_root_changes, revert_resp, succeed_resp, update_states, }; use crate::system_contract::{system_contract_address, SystemContract}; -use crate::{exec_try, CURRENT_METADATA_ROOT}; +use crate::{exec_try, system_contract_struct, CURRENT_METADATA_ROOT}; type Epoch = u64; @@ -33,17 +33,14 @@ lazy_static::lazy_static! { static ref METADATA_CACHE: RwLock> = RwLock::new(LruCache::new(METADATA_CACHE_SIZE)); } -#[derive(Default)] -pub struct MetadataContract; +system_contract_struct!(MetadataContract); -impl SystemContract for MetadataContract { +impl SystemContract + for MetadataContract +{ const ADDRESS: H160 = METADATA_CONTRACT_ADDRESS; - fn exec_( - &self, - adapter: &mut Adapter, - tx: &SignedTransaction, - ) -> TxResp { + fn exec_(&self, adapter: &mut Adapter, tx: &SignedTransaction) -> TxResp { let sender = tx.sender; let tx = &tx.transaction.unsigned; let tx_data = tx.data(); @@ -106,7 +103,7 @@ impl SystemContract for MetadataContract { succeed_resp(gas_limit) } - fn after_block_hook(&self, adapter: &mut Adapter) { + fn after_block_hook(&self, adapter: &mut Adapter) { let block_number = adapter.block_number(); if block_number.is_zero() { return; diff --git a/core/executor/src/system_contract/mod.rs b/core/executor/src/system_contract/mod.rs index d891162f9..a6a365fff 100644 --- a/core/executor/src/system_contract/mod.rs +++ b/core/executor/src/system_contract/mod.rs @@ -24,6 +24,7 @@ use ckb_traits::{CellDataProvider, HeaderProvider}; use ckb_types::core::cell::{CellProvider, CellStatus}; use ckb_types::core::{HeaderBuilder, HeaderView}; use ckb_types::{packed, prelude::*}; +use evm::backend::ApplyBackend; use parking_lot::RwLock; use common_config_parser::types::ConfigRocksDB; @@ -32,6 +33,7 @@ use protocol::{ckb_blake2b_256, traits::ExecutorAdapter}; use crate::adapter::RocksTrieDB; use crate::system_contract::utils::generate_mpt_root_changes; +use crate::system_contract::{ckb_light_client::CkbHeaderReader, image_cell::ImageCellReader}; pub const fn system_contract_address(addr: u8) -> H160 { H160([ @@ -62,18 +64,29 @@ macro_rules! exec_try { }; } -pub trait SystemContract { +#[macro_export] +macro_rules! system_contract_struct { + ($name: ident) => { + pub struct $name( + std::marker::PhantomData, + ); + + impl Default for $name { + fn default() -> Self { + Self(std::marker::PhantomData) + } + } + }; +} + +pub trait SystemContract { const ADDRESS: H160; - fn exec_( - &self, - adapter: &mut Adapter, - tx: &SignedTransaction, - ) -> TxResp; + fn exec_(&self, adapter: &mut Adapter, tx: &SignedTransaction) -> TxResp; - fn before_block_hook(&self, _adapter: &mut Adapter) {} + fn before_block_hook(&self, _adapter: &mut Adapter) {} - fn after_block_hook(&self, _adapter: &mut Adapter) {} + fn after_block_hook(&self, _adapter: &mut Adapter) {} } pub fn swap_metadata_db(new_db: Arc) -> Arc { @@ -90,12 +103,12 @@ pub fn swap_header_cell_db(new_db: Arc) -> Arc { .unwrap_or_else(|| panic!("header cell db is not initialized")) } -pub fn init, Adapter: ExecutorAdapter>( +pub fn init, Adapter: ExecutorAdapter + ApplyBackend>( path: P, config: ConfigRocksDB, adapter: &mut Adapter, ) -> (H256, H256) { - let current_metadata_root = adapter.storage(MetadataContract::ADDRESS, *METADATA_ROOT_KEY); + let current_metadata_root = adapter.storage(METADATA_CONTRACT_ADDRESS, *METADATA_ROOT_KEY); // Init metadata db. let metadata_db_path = path.as_ref().join("metadata"); @@ -116,46 +129,48 @@ pub fn init, Adapter: ExecutorAdapter>( )); } - let current_cell_root = adapter.storage(CkbLightClientContract::ADDRESS, *HEADER_CELL_ROOT_KEY); + let current_cell_root = + adapter.storage(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, *HEADER_CELL_ROOT_KEY); if current_cell_root.is_zero() { - let changes = generate_mpt_root_changes(adapter, ImageCellContract::ADDRESS); + let changes = generate_mpt_root_changes(adapter, IMAGE_CELL_CONTRACT_ADDRESS); adapter.apply(changes, vec![], false); } (current_metadata_root, current_cell_root) } -pub fn before_block_hook(adapter: &mut Adapter) { +pub fn before_block_hook(adapter: &mut Adapter) { NativeTokenContract::default().before_block_hook(adapter); MetadataContract::default().before_block_hook(adapter); CkbLightClientContract::default().before_block_hook(adapter); ImageCellContract::default().before_block_hook(adapter); } -pub fn after_block_hook(adapter: &mut Adapter) { +pub fn after_block_hook(adapter: &mut Adapter) { NativeTokenContract::default().after_block_hook(adapter); MetadataContract::default().after_block_hook(adapter); CkbLightClientContract::default().after_block_hook(adapter); ImageCellContract::default().after_block_hook(adapter); } -pub fn system_contract_dispatch( +pub fn system_contract_dispatch( adapter: &mut Adapter, tx: &SignedTransaction, ) -> Option { if let Some(addr) = tx.get_to() { log::debug!("execute addr {:}", addr); - if addr == NativeTokenContract::ADDRESS { + if addr == NATIVE_TOKEN_CONTRACT_ADDRESS { return Some(NativeTokenContract::default().exec_(adapter, tx)); - } else if addr == MetadataContract::ADDRESS { + } else if addr == METADATA_CONTRACT_ADDRESS { return Some(MetadataContract::default().exec_(adapter, tx)); - } else if addr == CkbLightClientContract::ADDRESS { + } else if addr == CKB_LIGHT_CLIENT_CONTRACT_ADDRESS { return Some(CkbLightClientContract::default().exec_(adapter, tx)); - } else if addr == ImageCellContract::ADDRESS { + } else if addr == IMAGE_CELL_CONTRACT_ADDRESS { return Some(ImageCellContract::default().exec_(adapter, tx)); } } + None } @@ -166,7 +181,7 @@ pub struct DataProvider { impl CellProvider for DataProvider { fn cell(&self, out_point: &packed::OutPoint, _eager_load: bool) -> CellStatus { - if let Some(c) = ImageCellContract::default() + if let Some(c) = ImageCellReader::default() .get_cell(self.root, &(out_point).into()) .ok() .flatten() @@ -180,7 +195,7 @@ impl CellProvider for DataProvider { impl CellDataProvider for DataProvider { fn get_cell_data(&self, out_point: &packed::OutPoint) -> Option { - ImageCellContract::default() + ImageCellReader::default() .get_cell(self.root, &(out_point.into())) .ok() .flatten() @@ -201,7 +216,7 @@ impl CellDataProvider for DataProvider { impl HeaderProvider for DataProvider { fn get_header(&self, hash: &packed::Byte32) -> Option { let block_hash = hash.unpack(); - CkbLightClientContract::default() + CkbHeaderReader::default() .get_header_by_block_hash(self.root, &H256(block_hash.0)) .ok() .flatten() diff --git a/core/executor/src/system_contract/native_token.rs b/core/executor/src/system_contract/native_token.rs index 77a333dcc..3e8dd07f2 100644 --- a/core/executor/src/system_contract/native_token.rs +++ b/core/executor/src/system_contract/native_token.rs @@ -1,18 +1,20 @@ -use protocol::traits::{ApplyBackend, Backend}; +use protocol::traits::{ApplyBackend, ExecutorAdapter}; use protocol::types::{Apply, Basic, SignedTransaction, TxResp, H160, U256}; use crate::system_contract::utils::{revert_resp, succeed_resp}; use crate::system_contract::{system_contract_address, SystemContract}; +use crate::system_contract_struct; pub const NATIVE_TOKEN_CONTRACT_ADDRESS: H160 = system_contract_address(0x0); -#[derive(Default)] -pub struct NativeTokenContract; +system_contract_struct!(NativeTokenContract); -impl SystemContract for NativeTokenContract { +impl SystemContract + for NativeTokenContract +{ const ADDRESS: H160 = NATIVE_TOKEN_CONTRACT_ADDRESS; - fn exec_(&self, backend: &mut B, tx: &SignedTransaction) -> TxResp { + fn exec_(&self, backend: &mut Adapter, tx: &SignedTransaction) -> TxResp { let tx = &tx.transaction.unsigned; let tx_data = tx.data(); let tx_value = *tx.value(); diff --git a/core/executor/src/system_contract/utils.rs b/core/executor/src/system_contract/utils.rs index 9cac50c13..37bacbe28 100644 --- a/core/executor/src/system_contract/utils.rs +++ b/core/executor/src/system_contract/utils.rs @@ -5,8 +5,8 @@ use protocol::types::{ }; use crate::system_contract::{ - CkbLightClientContract, ImageCellContract, MetadataContract, SystemContract, - HEADER_CELL_ROOT_KEY, METADATA_ROOT_KEY, + CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, HEADER_CELL_ROOT_KEY, IMAGE_CELL_CONTRACT_ADDRESS, + METADATA_CONTRACT_ADDRESS, METADATA_ROOT_KEY, }; use crate::{CURRENT_HEADER_CELL_ROOT, CURRENT_METADATA_ROOT}; @@ -65,32 +65,32 @@ pub fn generate_mpt_root_changes( backend: &mut B, contract_address: H160, ) -> Vec>> { - if contract_address == CkbLightClientContract::ADDRESS - || contract_address == ImageCellContract::ADDRESS + if contract_address == CKB_LIGHT_CLIENT_CONTRACT_ADDRESS + || contract_address == IMAGE_CELL_CONTRACT_ADDRESS { let current_header_cell_root = CURRENT_HEADER_CELL_ROOT.with(|r| *r.borrow()); let storage_changes = vec![(*HEADER_CELL_ROOT_KEY, current_header_cell_root)]; vec![ Apply::Modify { - address: CkbLightClientContract::ADDRESS, - basic: backend.basic(CkbLightClientContract::ADDRESS), + address: CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, + basic: backend.basic(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS), code: None, storage: storage_changes.clone(), reset_storage: false, }, Apply::Modify { - address: ImageCellContract::ADDRESS, - basic: backend.basic(ImageCellContract::ADDRESS), + address: IMAGE_CELL_CONTRACT_ADDRESS, + basic: backend.basic(IMAGE_CELL_CONTRACT_ADDRESS), code: None, storage: storage_changes, reset_storage: false, }, ] - } else if contract_address == MetadataContract::ADDRESS { + } else if contract_address == METADATA_CONTRACT_ADDRESS { let current_metadata_root = CURRENT_METADATA_ROOT.with(|r| *r.borrow()); vec![Apply::Modify { - address: MetadataContract::ADDRESS, - basic: backend.basic(MetadataContract::ADDRESS), + address: METADATA_CONTRACT_ADDRESS, + basic: backend.basic(METADATA_CONTRACT_ADDRESS), code: None, storage: vec![(*METADATA_ROOT_KEY, current_metadata_root)], reset_storage: false, diff --git a/core/executor/src/tests/mod.rs b/core/executor/src/tests/mod.rs index f4e8a9d8a..08be1ab27 100644 --- a/core/executor/src/tests/mod.rs +++ b/core/executor/src/tests/mod.rs @@ -17,10 +17,10 @@ use protocol::{codec::hex_decode, tokio, traits::Executor, trie::MemoryDB}; use core_storage::ImplStorage; -use crate::AxonExecutorAdapter; +use crate::AxonExecutorApplyAdapter; use crate::{precompiles::build_precompile_set, AxonExecutor as EvmExecutor, AxonExecutor}; -fn exec_adapter() -> AxonExecutorAdapter, MemoryDB> { +fn exec_adapter() -> AxonExecutorApplyAdapter, MemoryDB> { let storage = ImplStorage::new(Arc::new(MemoryAdapter::new()), 20); let ctx = ExecutorContext { block_gas_limit: u32::MAX.into(), @@ -28,7 +28,7 @@ fn exec_adapter() -> AxonExecutorAdapter, MemoryDB> { ..Default::default() }; - AxonExecutorAdapter::new(Arc::new(MemoryDB::new(false)), Arc::new(storage), ctx).unwrap() + AxonExecutorApplyAdapter::new(Arc::new(MemoryDB::new(false)), Arc::new(storage), ctx).unwrap() } fn gen_vicinity() -> MemoryVicinity { diff --git a/core/executor/src/tests/system_script/ckb_light_client.rs b/core/executor/src/tests/system_script/ckb_light_client.rs index 59d2c8256..3a698d119 100644 --- a/core/executor/src/tests/system_script/ckb_light_client.rs +++ b/core/executor/src/tests/system_script/ckb_light_client.rs @@ -6,8 +6,13 @@ use ethers::abi::AbiEncode; use common_config_parser::types::ConfigRocksDB; use protocol::types::{Backend, MemoryBackend, TxResp, H160, H256, U256}; -use crate::system_contract::ckb_light_client::{ckb_light_client_abi, CkbLightClientContract}; -use crate::system_contract::{init, ImageCellContract, SystemContract, HEADER_CELL_ROOT_KEY}; +use crate::system_contract::ckb_light_client::{ + ckb_light_client_abi, CkbHeaderReader, CkbLightClientContract, +}; +use crate::system_contract::{ + init, SystemContract, CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, HEADER_CELL_ROOT_KEY, + IMAGE_CELL_CONTRACT_ADDRESS, +}; use crate::tests::{gen_tx, gen_vicinity}; static ROCKSDB_PATH: &str = "./free-space/system-contract/ckb-light-client"; @@ -51,7 +56,10 @@ fn prepare_header_2() -> ckb_light_client_abi::Header { } } -fn test_update_first(backend: &mut MemoryBackend, executor: &CkbLightClientContract) { +fn test_update_first<'a>( + backend: &mut MemoryBackend<'a>, + executor: &CkbLightClientContract>, +) { let header = prepare_header_1(); let data = ckb_light_client_abi::UpdateCall { headers: vec![header.clone()], @@ -62,8 +70,8 @@ fn test_update_first(backend: &mut MemoryBackend, executor: &CkbLightClientContr check_nonce(backend, 1); - let root = backend.storage(CkbLightClientContract::ADDRESS, *HEADER_CELL_ROOT_KEY); - let queried_header = executor + let root = backend.storage(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, *HEADER_CELL_ROOT_KEY); + let queried_header = CkbHeaderReader::default() .get_header_by_block_hash(root, &H256::default()) .unwrap() .unwrap(); @@ -71,7 +79,10 @@ fn test_update_first(backend: &mut MemoryBackend, executor: &CkbLightClientContr assert_eq!(queried_header, header); } -fn test_update_second(backend: &mut MemoryBackend, executor: &CkbLightClientContract) { +fn test_update_second<'a>( + backend: &mut MemoryBackend<'a>, + executor: &CkbLightClientContract>, +) { let header = prepare_header_2(); let data = ckb_light_client_abi::UpdateCall { headers: vec![header.clone()], @@ -82,8 +93,8 @@ fn test_update_second(backend: &mut MemoryBackend, executor: &CkbLightClientCont check_nonce(backend, 2); - let root = backend.storage(CkbLightClientContract::ADDRESS, *HEADER_CELL_ROOT_KEY); - let queried_header = executor + let root = backend.storage(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, *HEADER_CELL_ROOT_KEY); + let queried_header = CkbHeaderReader::default() .get_header_by_block_hash(root, &H256::from_slice(&header.block_hash)) .unwrap() .unwrap(); @@ -91,7 +102,10 @@ fn test_update_second(backend: &mut MemoryBackend, executor: &CkbLightClientCont assert_eq!(queried_header, header); } -fn test_roll_back_first(backend: &mut MemoryBackend, executor: &CkbLightClientContract) { +fn test_roll_back_first<'a>( + backend: &mut MemoryBackend<'a>, + executor: &CkbLightClientContract>, +) { let data = ckb_light_client_abi::RollbackCall { block_hashes: vec![prepare_header_2().block_hash], }; @@ -99,8 +113,8 @@ fn test_roll_back_first(backend: &mut MemoryBackend, executor: &CkbLightClientCo let r = exec(backend, executor, data.encode()); assert!(r.exit_reason.is_succeed()); - let root = backend.storage(CkbLightClientContract::ADDRESS, *HEADER_CELL_ROOT_KEY); - let queried_header = executor + let root = backend.storage(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, *HEADER_CELL_ROOT_KEY); + let queried_header = CkbHeaderReader::default() .get_header_by_block_hash(root, &H256::default()) .unwrap() .unwrap(); @@ -108,7 +122,10 @@ fn test_roll_back_first(backend: &mut MemoryBackend, executor: &CkbLightClientCo assert_eq!(queried_header, prepare_header_1()); } -fn test_roll_back_second(backend: &mut MemoryBackend, executor: &CkbLightClientContract) { +fn test_roll_back_second<'a>( + backend: &mut MemoryBackend<'a>, + executor: &CkbLightClientContract>, +) { let data = ckb_light_client_abi::RollbackCall { block_hashes: vec![prepare_header_1().block_hash], }; @@ -116,37 +133,45 @@ fn test_roll_back_second(backend: &mut MemoryBackend, executor: &CkbLightClientC let r = exec(backend, executor, data.encode()); assert!(r.exit_reason.is_succeed()); - let root = backend.storage(CkbLightClientContract::ADDRESS, *HEADER_CELL_ROOT_KEY); - let queried_header = executor + let root = backend.storage(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, *HEADER_CELL_ROOT_KEY); + let queried_header = CkbHeaderReader::default() .get_header_by_block_hash(root, &H256::default()) .unwrap(); assert!(queried_header.is_none()); } -fn test_set_state(backend: &mut MemoryBackend, executor: &CkbLightClientContract) { +fn test_set_state<'a>( + backend: &mut MemoryBackend<'a>, + executor: &CkbLightClientContract>, +) { let data = ckb_light_client_abi::SetStateCall { allow_read: true }; + let querier = CkbHeaderReader::default(); - assert!(!executor.allow_read()); + assert!(!querier.allow_read()); let r = exec(backend, executor, data.encode()); assert!(r.exit_reason.is_succeed()); - assert!(executor.allow_read()); + assert!(querier.allow_read()); } -fn exec(backend: &mut MemoryBackend, executor: &CkbLightClientContract, data: Vec) -> TxResp { +fn exec<'a>( + backend: &mut MemoryBackend<'a>, + executor: &CkbLightClientContract>, + data: Vec, +) -> TxResp { let addr = H160::from_str("0xf000000000000000000000000000000000000000").unwrap(); - let tx = gen_tx(addr, CkbLightClientContract::ADDRESS, 1000, data); + let tx = gen_tx(addr, CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, 1000, data); executor.exec_(backend, &tx) } -fn check_nonce(backend: &mut MemoryBackend, nonce: u64) { +fn check_nonce(backend: &mut MemoryBackend<'_>, nonce: u64) { assert_eq!( - backend.basic(CkbLightClientContract::ADDRESS).nonce, + backend.basic(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS).nonce, U256::zero() ); assert_eq!( - backend.basic(ImageCellContract::ADDRESS).nonce, + backend.basic(IMAGE_CELL_CONTRACT_ADDRESS).nonce, U256::zero() ); assert_eq!( diff --git a/core/executor/src/tests/system_script/image_cell.rs b/core/executor/src/tests/system_script/image_cell.rs index 3b9485eee..ed12ca9f0 100644 --- a/core/executor/src/tests/system_script/image_cell.rs +++ b/core/executor/src/tests/system_script/image_cell.rs @@ -7,8 +7,13 @@ use ethers::abi::AbiEncode; use common_config_parser::types::ConfigRocksDB; use protocol::types::{Backend, MemoryBackend, TxResp, H160, U256}; -use crate::system_contract::image_cell::{image_cell_abi, CellInfo, CellKey, ImageCellContract}; -use crate::system_contract::{init, CkbLightClientContract, SystemContract, HEADER_CELL_ROOT_KEY}; +use crate::system_contract::image_cell::{ + image_cell_abi, CellInfo, CellKey, ImageCellContract, ImageCellReader, +}; +use crate::system_contract::{ + init, SystemContract, CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, HEADER_CELL_ROOT_KEY, + IMAGE_CELL_CONTRACT_ADDRESS, +}; use crate::tests::{gen_tx, gen_vicinity}; use crate::{CURRENT_HEADER_CELL_ROOT, CURRENT_METADATA_ROOT}; @@ -33,7 +38,10 @@ pub fn test_write_functions() { test_set_state(&mut backend, &executor); } -fn test_update_first(backend: &mut MemoryBackend, executor: &ImageCellContract) { +fn test_update_first<'a>( + backend: &mut MemoryBackend<'a>, + executor: &ImageCellContract>, +) { let data = image_cell_abi::UpdateCall { blocks: vec![image_cell_abi::BlockUpdate { block_number: 0x1, @@ -47,13 +55,19 @@ fn test_update_first(backend: &mut MemoryBackend, executor: &ImageCellContract) check_nonce(backend, 1); - let root = backend.storage(CkbLightClientContract::ADDRESS, *HEADER_CELL_ROOT_KEY); + let root = backend.storage(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, *HEADER_CELL_ROOT_KEY); let cell_key = CellKey::new([7u8; 32], 0x0); - let get_cell = executor.get_cell(root, &cell_key).unwrap().unwrap(); + let get_cell = ImageCellReader::default() + .get_cell(root, &cell_key) + .unwrap() + .unwrap(); check_cell(&get_cell, 0x1, None); } -fn test_update_second(backend: &mut MemoryBackend, executor: &ImageCellContract) { +fn test_update_second<'a>( + backend: &mut MemoryBackend<'a>, + executor: &ImageCellContract>, +) { let data = image_cell_abi::UpdateCall { blocks: vec![image_cell_abi::BlockUpdate { block_number: 0x2, @@ -70,13 +84,19 @@ fn test_update_second(backend: &mut MemoryBackend, executor: &ImageCellContract) check_nonce(backend, 2); - let root = backend.storage(CkbLightClientContract::ADDRESS, *HEADER_CELL_ROOT_KEY); + let root = backend.storage(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, *HEADER_CELL_ROOT_KEY); let cell_key = CellKey::new([7u8; 32], 0x0); - let get_cell = executor.get_cell(root, &cell_key).unwrap().unwrap(); + let get_cell = ImageCellReader::default() + .get_cell(root, &cell_key) + .unwrap() + .unwrap(); check_cell(&get_cell, 0x1, Some(0x2)); } -fn test_rollback_first(backend: &mut MemoryBackend, executor: &ImageCellContract) { +fn test_rollback_first<'a>( + backend: &mut MemoryBackend<'a>, + executor: &ImageCellContract>, +) { let data = image_cell_abi::RollbackCall { blocks: vec![image_cell_abi::BlockRollBlack { tx_inputs: vec![image_cell_abi::OutPoint { @@ -90,13 +110,19 @@ fn test_rollback_first(backend: &mut MemoryBackend, executor: &ImageCellContract let r = exec(backend, executor, data.encode()); assert!(r.exit_reason.is_succeed()); - let root = backend.storage(CkbLightClientContract::ADDRESS, *HEADER_CELL_ROOT_KEY); + let root = backend.storage(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, *HEADER_CELL_ROOT_KEY); let cell_key = CellKey::new([7u8; 32], 0x0); - let get_cell = executor.get_cell(root, &cell_key).unwrap().unwrap(); + let get_cell = ImageCellReader::default() + .get_cell(root, &cell_key) + .unwrap() + .unwrap(); check_cell(&get_cell, 0x1, None); } -fn test_rollback_second(backend: &mut MemoryBackend, executor: &ImageCellContract) { +fn test_rollback_second<'a>( + backend: &mut MemoryBackend<'a>, + executor: &ImageCellContract>, +) { let data = image_cell_abi::RollbackCall { blocks: vec![image_cell_abi::BlockRollBlack { tx_inputs: vec![], @@ -110,26 +136,35 @@ fn test_rollback_second(backend: &mut MemoryBackend, executor: &ImageCellContrac let r = exec(backend, executor, data.encode()); assert!(r.exit_reason.is_succeed()); - let root = backend.storage(CkbLightClientContract::ADDRESS, *HEADER_CELL_ROOT_KEY); + let querier = ImageCellReader::default(); + let root = backend.storage(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS, *HEADER_CELL_ROOT_KEY); let cell_key = CellKey::new([7u8; 32], 0x0); - let get_cell = executor.get_cell(root, &cell_key).unwrap(); + let get_cell = querier.get_cell(root, &cell_key).unwrap(); assert!(get_cell.is_none()); } -fn test_set_state(backend: &mut MemoryBackend, executor: &ImageCellContract) { +fn test_set_state<'a>( + backend: &mut MemoryBackend<'a>, + executor: &ImageCellContract>, +) { let data = image_cell_abi::SetStateCall { allow_read: true }; + let querier = ImageCellReader::default(); - assert!(!executor.allow_read()); + assert!(!querier.allow_read()); let r = exec(backend, executor, data.encode()); assert!(r.exit_reason.is_succeed()); - assert!(executor.allow_read()); + assert!(querier.allow_read()); } -fn exec(backend: &mut MemoryBackend, executor: &ImageCellContract, data: Vec) -> TxResp { +fn exec<'a>( + backend: &mut MemoryBackend<'a>, + executor: &ImageCellContract>, + data: Vec, +) -> TxResp { let addr = H160::from_str("0xf000000000000000000000000000000000000000").unwrap(); - let tx = gen_tx(addr, ImageCellContract::ADDRESS, 1000, data); + let tx = gen_tx(addr, IMAGE_CELL_CONTRACT_ADDRESS, 1000, data); executor.exec_(backend, &tx) } @@ -203,13 +238,13 @@ fn prepare_outputs() -> Vec { }] } -fn check_nonce(backend: &mut MemoryBackend, nonce: u64) { +fn check_nonce(backend: &mut MemoryBackend<'_>, nonce: u64) { assert_eq!( - backend.basic(CkbLightClientContract::ADDRESS).nonce, + backend.basic(CKB_LIGHT_CLIENT_CONTRACT_ADDRESS).nonce, U256::zero() ); assert_eq!( - backend.basic(ImageCellContract::ADDRESS).nonce, + backend.basic(IMAGE_CELL_CONTRACT_ADDRESS).nonce, U256::zero() ); assert_eq!( diff --git a/core/executor/src/tests/system_script/metadata.rs b/core/executor/src/tests/system_script/metadata.rs index b1a6e17c5..d1c6b06f5 100644 --- a/core/executor/src/tests/system_script/metadata.rs +++ b/core/executor/src/tests/system_script/metadata.rs @@ -11,7 +11,7 @@ use crate::{ metadata_abi::{self, Metadata, MetadataVersion, ValidatorExtend}, MetadataContract, }, - SystemContract, + SystemContract, METADATA_CONTRACT_ADDRESS, }, tests::{gen_tx, gen_vicinity}, }; @@ -37,14 +37,17 @@ fn test_write_functions() { test_validator(&mut backend, &executor); } -fn test_init(backend: &mut MemoryBackend, executor: &MetadataContract) { +fn test_init<'a>(backend: &mut MemoryBackend<'a>, executor: &MetadataContract>) { let addr = H160::from_str("0xf000000000000000000000000000000000000000").unwrap(); let tx = prepare_tx_1(&addr); let r = executor.exec_(backend, &tx); assert!(r.exit_reason.is_succeed()); } -fn test_second(backend: &mut MemoryBackend, executor: &MetadataContract) { +fn test_second<'a>( + backend: &mut MemoryBackend<'a>, + executor: &MetadataContract>, +) { let addr = H160::from_str("0xf000000000000000000000000000000000000000").unwrap(); // this transaction will fail because the epoch is not incremental @@ -69,7 +72,10 @@ fn test_second(backend: &mut MemoryBackend, executor: &MetadataContract) { assert!(r.exit_reason.is_revert()); } -fn test_validator(backend: &mut MemoryBackend, executor: &MetadataContract) { +fn test_validator<'a>( + backend: &mut MemoryBackend<'a>, + executor: &MetadataContract>, +) { let addr = H160::from_str("0x0000000000000000000000000000000000000000").unwrap(); // this transaction will fail because the sender is not in validator list @@ -83,7 +89,7 @@ fn prepare_tx_1(addr: &H160) -> SignedTransaction { metadata: prepare_metadata(), }; - gen_tx(*addr, MetadataContract::ADDRESS, 1000, data.encode()) + gen_tx(*addr, METADATA_CONTRACT_ADDRESS, 1000, data.encode()) } fn prepare_tx_2(addr: &H160) -> SignedTransaction { @@ -93,7 +99,7 @@ fn prepare_tx_2(addr: &H160) -> SignedTransaction { data.metadata.version.start = 101; data.metadata.version.end = 200; - gen_tx(*addr, MetadataContract::ADDRESS, 1000, data.encode()) + gen_tx(*addr, METADATA_CONTRACT_ADDRESS, 1000, data.encode()) } fn prepare_tx_3(add: &H160) -> SignedTransaction { @@ -104,7 +110,7 @@ fn prepare_tx_3(add: &H160) -> SignedTransaction { data.metadata.version.start = 1; data.metadata.version.end = 100; - gen_tx(*add, MetadataContract::ADDRESS, 1000, data.encode()) + gen_tx(*add, METADATA_CONTRACT_ADDRESS, 1000, data.encode()) } fn prepare_tx_4(addr: &H160) -> SignedTransaction { @@ -115,7 +121,7 @@ fn prepare_tx_4(addr: &H160) -> SignedTransaction { data.metadata.version.start = 101; data.metadata.version.end = 200; - gen_tx(*addr, MetadataContract::ADDRESS, 1000, data.encode()) + gen_tx(*addr, METADATA_CONTRACT_ADDRESS, 1000, data.encode()) } fn prepare_tx_5(addr: &H160) -> SignedTransaction { @@ -126,7 +132,7 @@ fn prepare_tx_5(addr: &H160) -> SignedTransaction { data.metadata.version.start = 201; data.metadata.version.end = 300; - gen_tx(*addr, MetadataContract::ADDRESS, 1000, data.encode()) + gen_tx(*addr, METADATA_CONTRACT_ADDRESS, 1000, data.encode()) } fn prepare_metadata() -> Metadata { diff --git a/core/executor/src/tests/system_script/native_token.rs b/core/executor/src/tests/system_script/native_token.rs index 0317d3517..951724e1e 100644 --- a/core/executor/src/tests/system_script/native_token.rs +++ b/core/executor/src/tests/system_script/native_token.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, str::FromStr}; use protocol::types::{MemoryAccount, MemoryBackend, H160, U256}; use crate::{ - system_contract::{NativeTokenContract, SystemContract}, + system_contract::{NativeTokenContract, SystemContract, NATIVE_TOKEN_CONTRACT_ADDRESS}, tests::{gen_tx, gen_vicinity}, }; @@ -20,7 +20,7 @@ fn test_issue_token() { let executor = NativeTokenContract::default(); let addr = H160::from_str("0xf000000000000000000000000000000000000000").unwrap(); let data = mock_data(0, addr); - let tx = gen_tx(addr, NativeTokenContract::ADDRESS, 1000, data); + let tx = gen_tx(addr, NATIVE_TOKEN_CONTRACT_ADDRESS, 1000, data); let r = executor.exec_(&mut backend, &tx); assert!(r.exit_reason.is_succeed()); @@ -45,7 +45,7 @@ fn test_burn_token() { let mut backend = MemoryBackend::new(&vicinity, state); let executor = NativeTokenContract::default(); let data = mock_data(1, addr); - let tx = gen_tx(addr, NativeTokenContract::ADDRESS, 1000, data); + let tx = gen_tx(addr, NATIVE_TOKEN_CONTRACT_ADDRESS, 1000, data); let r = executor.exec_(&mut backend, &tx); assert!(r.exit_reason.is_succeed()); @@ -70,7 +70,7 @@ fn test_burn_token_failed() { let mut backend = MemoryBackend::new(&vicinity, state); let executor = NativeTokenContract::default(); let data = mock_data(1, addr); - let tx = gen_tx(addr, NativeTokenContract::ADDRESS, 1000, data); + let tx = gen_tx(addr, NATIVE_TOKEN_CONTRACT_ADDRESS, 1000, data); let r = executor.exec_(&mut backend, &tx); assert!(r.exit_reason.is_revert()); diff --git a/core/interoperation/src/tests/mod.rs b/core/interoperation/src/tests/mod.rs index 5ca39f099..5a0ac45eb 100644 --- a/core/interoperation/src/tests/mod.rs +++ b/core/interoperation/src/tests/mod.rs @@ -4,11 +4,13 @@ use std::io::BufReader; use std::sync::Arc; use core_executor::adapter::{MPTTrie, RocksTrieDB}; -use core_executor::{system_contract::system_contract_address, AxonExecutor, AxonExecutorAdapter}; +use core_executor::{ + system_contract::system_contract_address, AxonExecutor, AxonExecutorApplyAdapter, +}; use core_rpc_client::RpcClient; use core_storage::{adapter::rocks::RocksAdapter, ImplStorage}; use protocol::codec::ProtocolCodec; -use protocol::traits::{CommonStorage, Context, Executor, Storage}; +use protocol::traits::{Context, Executor, Storage}; use protocol::types::{ Account, Address, Bytes, Eip1559Transaction, ExecResp, Proposal, Public, RichBlock, SignatureComponents, SignedTransaction, TransactionAction, UnsignedTransaction, @@ -49,7 +51,7 @@ impl TestHandle { }; handle.load_genesis().await; - let mut backend = AxonExecutorAdapter::from_root( + let mut backend = AxonExecutorApplyAdapter::from_root( handle.state_root, Arc::clone(&handle.trie_db), Arc::clone(&handle.storage), @@ -87,7 +89,7 @@ impl TestHandle { .unwrap(); let executor = AxonExecutor::default(); - let mut backend = AxonExecutorAdapter::from_root( + let mut backend = AxonExecutorApplyAdapter::from_root( mpt.commit().unwrap(), Arc::clone(&self.trie_db), Arc::clone(&self.storage), @@ -117,7 +119,7 @@ impl TestHandle { } pub fn exec(&mut self, txs: Vec) -> ExecResp { - let mut backend = AxonExecutorAdapter::from_root( + let mut backend = AxonExecutorApplyAdapter::from_root( self.state_root, Arc::clone(&self.trie_db), Arc::clone(&self.storage), diff --git a/core/mempool/src/adapter/mod.rs b/core/mempool/src/adapter/mod.rs index 593af09d7..fa8cd0056 100644 --- a/core/mempool/src/adapter/mod.rs +++ b/core/mempool/src/adapter/mod.rs @@ -12,7 +12,7 @@ use log::{debug, error}; use parking_lot::Mutex; use protocol::traits::{ - Context, Gossip, Interoperation, MemPoolAdapter, PeerTrust, Priority, Rpc, Storage, + Context, Gossip, Interoperation, MemPoolAdapter, PeerTrust, Priority, ReadOnlyStorage, Rpc, TrustFeedback, }; use protocol::types::{ @@ -26,7 +26,9 @@ use protocol::{ use common_apm_derive::trace_span; use common_crypto::{Crypto, Secp256k1Recoverable}; -use core_executor::{is_call_system_script, AxonExecutorAdapter, DataProvider, MetadataHandle}; +use core_executor::{ + is_call_system_script, AxonExecutorReadOnlyAdapter, DataProvider, MetadataHandle, +}; use core_interoperation::InteroperationImpl; use crate::adapter::message::{MsgPullTxs, END_GOSSIP_NEW_TXS, RPC_PULL_TXS}; @@ -134,7 +136,7 @@ impl DefaultMemPoolAdapter where C: Crypto, N: Rpc + PeerTrust + Gossip + Clone + Unpin + 'static, - S: Storage + 'static, + S: ReadOnlyStorage + 'static, DB: trie::DB + 'static, I: Interoperation + 'static, { @@ -332,9 +334,12 @@ where Ok(()) } - async fn executor_backend(&self, ctx: Context) -> ProtocolResult> { + async fn executor_backend( + &self, + ctx: Context, + ) -> ProtocolResult> { let current_state_root = self.storage.get_latest_block_header(ctx).await?.state_root; - AxonExecutorAdapter::from_root( + AxonExecutorReadOnlyAdapter::from_root( current_state_root, Arc::clone(&self.trie_db), Arc::clone(&self.storage), @@ -348,7 +353,7 @@ impl MemPoolAdapter for DefaultMemPoolAdapter where C: Crypto + Send + Sync + 'static, N: Rpc + PeerTrust + Gossip + Clone + Unpin + 'static, - S: Storage + 'static, + S: ReadOnlyStorage + 'static, DB: trie::DB + 'static, I: Interoperation + 'static, { diff --git a/core/run/src/lib.rs b/core/run/src/lib.rs index 6047c3096..999549e93 100644 --- a/core/run/src/lib.rs +++ b/core/run/src/lib.rs @@ -32,7 +32,7 @@ use protocol::tokio::{ self, runtime::Builder as RuntimeBuilder, sync::Mutex as AsyncMutex, time::sleep, }; use protocol::traits::{ - CommonStorage, Consensus, Context, Executor, Gossip, MemPool, Network, NodeInfo, PeerTrust, + Consensus, Context, Executor, Gossip, MemPool, Network, NodeInfo, PeerTrust, ReadOnlyStorage, Rpc, Storage, SynchronizationAdapter, }; use protocol::types::{ @@ -59,7 +59,9 @@ use core_consensus::{ OverlordConsensusAdapter, OverlordSynchronization, SignedTxsWAL, }; use core_executor::system_contract::{self, metadata::MetadataHandle}; -use core_executor::{AxonExecutor, AxonExecutorAdapter, MPTTrie, RocksTrieDB}; +use core_executor::{ + AxonExecutor, AxonExecutorApplyAdapter, AxonExecutorReadOnlyAdapter, MPTTrie, RocksTrieDB, +}; use core_interoperation::InteroperationImpl; use core_mempool::{ DefaultMemPoolAdapter, MemPoolImpl, NewTxsHandler, PullTxsHandler, END_GOSSIP_NEW_TXS, @@ -315,7 +317,7 @@ impl Axon { .await; // Get the validator list from current metadata for consensus initialization - let metadata_root = AxonExecutorAdapter::from_root( + let metadata_root = AxonExecutorReadOnlyAdapter::from_root( current_state_root, Arc::clone(&trie_db), Arc::clone(&storage), @@ -518,13 +520,14 @@ impl Axon { storage: &Arc>, ) -> (H256, H256) { let path_system_contract = self.config.data_path_for_system_contract(); - let mut backend = AxonExecutorAdapter::from_root( + let mut backend = AxonExecutorApplyAdapter::from_root( current_block.header.state_root, Arc::clone(trie_db), Arc::clone(storage), Proposal::new_without_state_root(¤t_block.header).into(), ) .unwrap(); + system_contract::init( path_system_contract, self.config.rocksdb.clone(), @@ -1035,7 +1038,7 @@ where DB: TrieDB + 'static, { let executor = AxonExecutor::default(); - let mut backend = AxonExecutorAdapter::from_root( + let mut backend = AxonExecutorApplyAdapter::from_root( state_root, Arc::clone(trie_db), Arc::clone(storage), diff --git a/core/storage/src/lib.rs b/core/storage/src/lib.rs index 648c46ed9..51b423475 100644 --- a/core/storage/src/lib.rs +++ b/core/storage/src/lib.rs @@ -19,7 +19,7 @@ use common_apm::Instant; use common_apm_derive::trace_span; use protocol::codec::ProtocolCodec; use protocol::traits::{ - CommonStorage, Context, Storage, StorageAdapter, StorageBatchModify, StorageCategory, + Context, ReadOnlyStorage, Storage, StorageAdapter, StorageBatchModify, StorageCategory, StorageSchema, }; use protocol::types::{ @@ -179,16 +179,7 @@ impl ImplStorage { } #[async_trait] -impl CommonStorage for ImplStorage { - #[trace_span(kind = "storage")] - async fn insert_block(&self, ctx: Context, block: Block) -> ProtocolResult<()> { - self.set_block(ctx.clone(), block.clone()).await?; - - self.set_latest_block(ctx, block).await?; - - Ok(()) - } - +impl ReadOnlyStorage for ImplStorage { async fn get_block(&self, _ctx: Context, height: u64) -> ProtocolResult> { get_cache!(self, &height, blocks); let ret = self.adapter.get::(BlockKey::new(height))?; @@ -209,22 +200,6 @@ impl CommonStorage for ImplStorage { Ok(self.get_block(ctx, height).await?.map(|b| b.header)) } - async fn set_block(&self, _ctx: Context, block: Block) -> ProtocolResult<()> { - self.adapter - .insert::(BlockKey::new(block.header.number), block.clone())?; - self.adapter.insert::( - BlockKey::new(block.header.number), - block.header.clone(), - )?; - self.adapter - .insert::(block.hash(), block.header.number)?; - Ok(()) - } - - async fn remove_block(&self, _ctx: Context, height: u64) -> ProtocolResult<()> { - self.adapter.remove::(BlockKey::new(height)) - } - async fn get_latest_block(&self, _ctx: Context) -> ProtocolResult { if let Some(block) = self.latest_block.load().as_ref().clone() { Ok(block) @@ -234,15 +209,6 @@ impl CommonStorage for ImplStorage { } } - async fn set_latest_block(&self, _ctx: Context, block: Block) -> ProtocolResult<()> { - self.adapter - .insert::(*LATEST_BLOCK_KEY, block.clone())?; - - self.latest_block.store(Arc::new(Some(block))); - - Ok(()) - } - async fn get_latest_block_header(&self, _ctx: Context) -> ProtocolResult
{ let opt_header = { let guard = self.latest_block.load(); @@ -257,21 +223,6 @@ impl CommonStorage for ImplStorage { Ok(block.header) } } -} - -#[async_trait] -impl Storage for ImplStorage { - #[trace_span(kind = "storage")] - async fn insert_transactions( - &self, - ctx: Context, - block_height: u64, - signed_txs: Vec, - ) -> ProtocolResult<()> { - self.batch_insert_stxs(signed_txs, block_height).await?; - - Ok(()) - } async fn get_block_by_hash( &self, @@ -287,7 +238,7 @@ impl Storage for ImplStorage { async fn get_block_number_by_hash( &self, - _: Context, + _ctx: Context, block_hash: &Hash, ) -> ProtocolResult> { self.get_block_number_by_hash(block_hash).await @@ -374,18 +325,6 @@ impl Storage for ImplStorage { Ok(hashes.iter().map(|h| found.remove(h)).collect::>()) } - async fn insert_code( - &self, - _ctx: Context, - code_address: H256, - code_hash: Hash, - code: Bytes, - ) -> ProtocolResult<()> { - self.adapter.insert::(code_hash, code)?; - self.adapter - .insert::(code_address, code_hash) - } - async fn get_code_by_hash(&self, _ctx: Context, hash: &Hash) -> ProtocolResult> { get_cache!(self, hash, codes); let ret = self.adapter.get::(*hash)?; @@ -427,18 +366,6 @@ impl Storage for ImplStorage { } } - #[trace_span(kind = "storage")] - async fn insert_receipts( - &self, - ctx: Context, - block_height: u64, - receipts: Vec, - ) -> ProtocolResult<()> { - self.batch_insert_receipts(receipts, block_height).await?; - - Ok(()) - } - async fn get_receipt_by_hash( &self, _ctx: Context, @@ -540,15 +467,6 @@ impl Storage for ImplStorage { Ok(hashes.iter().map(|h| found.remove(h)).collect::>()) } - async fn update_latest_proof(&self, _ctx: Context, proof: Proof) -> ProtocolResult<()> { - self.adapter - .insert::(*LATEST_PROOF_KEY, proof.clone())?; - - self.latest_proof.store(Arc::new(Some(proof))); - - Ok(()) - } - async fn get_latest_proof(&self, _ctx: Context) -> ProtocolResult { if let Some(proof) = self.latest_proof.load().as_ref().clone() { Ok(proof) @@ -559,6 +477,86 @@ impl Storage for ImplStorage { } } +#[async_trait] +impl Storage for ImplStorage { + async fn insert_block(&self, ctx: Context, block: Block) -> ProtocolResult<()> { + self.set_block(ctx.clone(), block.clone()).await?; + + self.set_latest_block(ctx, block).await?; + + Ok(()) + } + + async fn set_block(&self, _ctx: Context, block: Block) -> ProtocolResult<()> { + self.adapter + .insert::(BlockKey::new(block.header.number), block.clone())?; + self.adapter.insert::( + BlockKey::new(block.header.number), + block.header.clone(), + )?; + self.adapter + .insert::(block.hash(), block.header.number)?; + Ok(()) + } + + async fn remove_block(&self, _ctx: Context, height: u64) -> ProtocolResult<()> { + self.adapter.remove::(BlockKey::new(height)) + } + + async fn set_latest_block(&self, _ctx: Context, block: Block) -> ProtocolResult<()> { + self.adapter + .insert::(*LATEST_BLOCK_KEY, block.clone())?; + + self.latest_block.store(Arc::new(Some(block))); + + Ok(()) + } + + async fn insert_transactions( + &self, + _ctx: Context, + block_height: u64, + signed_txs: Vec, + ) -> ProtocolResult<()> { + self.batch_insert_stxs(signed_txs, block_height).await?; + + Ok(()) + } + + async fn insert_code( + &self, + _ctx: Context, + code_address: H256, + code_hash: Hash, + code: Bytes, + ) -> ProtocolResult<()> { + self.adapter.insert::(code_hash, code)?; + self.adapter + .insert::(code_address, code_hash) + } + + #[trace_span(kind = "storage")] + async fn insert_receipts( + &self, + ctx: Context, + block_height: u64, + receipts: Vec, + ) -> ProtocolResult<()> { + self.batch_insert_receipts(receipts, block_height).await?; + + Ok(()) + } + + async fn update_latest_proof(&self, _ctx: Context, proof: Proof) -> ProtocolResult<()> { + self.adapter + .insert::(*LATEST_PROOF_KEY, proof.clone())?; + + self.latest_proof.store(Arc::new(Some(proof))); + + Ok(()) + } +} + #[derive(Debug, Display, From)] pub enum StorageError { #[display(fmt = "get none {:?}", _0)] diff --git a/core/storage/src/tests/storage.rs b/core/storage/src/tests/storage.rs index f472d92cb..d04d0ccab 100644 --- a/core/storage/src/tests/storage.rs +++ b/core/storage/src/tests/storage.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use protocol::traits::{CommonStorage, Context, Storage}; +use protocol::traits::{Context, ReadOnlyStorage, Storage}; use protocol::types::Hasher; use crate::adapter::memory::MemoryAdapter; diff --git a/protocol/src/traits/executor.rs b/protocol/src/traits/executor.rs index 07c84cdfc..0adb808ba 100644 --- a/protocol/src/traits/executor.rs +++ b/protocol/src/traits/executor.rs @@ -5,22 +5,24 @@ use crate::types::{ ValidatorExtend, H160, U256, }; -pub trait ExecutorAdapter: ApplyBackend + Backend { - fn set_origin(&mut self, origin: H160); - - fn set_gas_price(&mut self, gas_price: U256); - - fn get_logs(&mut self) -> Vec; - - fn commit(&mut self) -> MerkleRoot; - +pub trait ExecutorReadOnlyAdapter: Backend { fn get(&self, key: &[u8]) -> Option; fn get_ctx(&self) -> ExecutorContext; fn get_account(&self, address: &H160) -> Account; +} + +pub trait ExecutorAdapter: ExecutorReadOnlyAdapter + ApplyBackend { + fn set_origin(&mut self, origin: H160); + + fn set_gas_price(&mut self, gas_price: U256); fn save_account(&mut self, address: &H160, account: &Account); + + fn commit(&mut self) -> MerkleRoot; + + fn take_logs(&mut self) -> Vec; } pub trait Executor: Send + Sync { @@ -34,43 +36,43 @@ pub trait Executor: Send + Sync { data: Vec, ) -> TxResp; - fn exec( + fn exec( &self, adapter: &mut Adapter, txs: &[SignedTransaction], validators: &[ValidatorExtend], ) -> ExecResp; - - fn get_account(&self, adapter: &Adapter, address: &H160) -> Account; } /// This implementation is only used for test. -impl<'a> ExecutorAdapter for MemoryBackend<'a> { - fn set_origin(&mut self, _origin: H160) { +impl<'a> ExecutorReadOnlyAdapter for MemoryBackend<'a> { + fn get(&self, _key: &[u8]) -> Option { unreachable!() } - fn set_gas_price(&mut self, _gas_price: U256) { + fn get_ctx(&self) -> ExecutorContext { unreachable!() } - fn get_logs(&mut self) -> Vec { + fn get_account(&self, _address: &H160) -> Account { unreachable!() } +} - fn commit(&mut self) -> MerkleRoot { +impl<'a> ExecutorAdapter for MemoryBackend<'a> { + fn set_origin(&mut self, _origin: H160) { unreachable!() } - fn get(&self, _key: &[u8]) -> Option { + fn set_gas_price(&mut self, _gas_price: U256) { unreachable!() } - fn get_ctx(&self) -> ExecutorContext { + fn take_logs(&mut self) -> Vec { unreachable!() } - fn get_account(&self, _address: &H160) -> Account { + fn commit(&mut self) -> MerkleRoot { unreachable!() } diff --git a/protocol/src/traits/mod.rs b/protocol/src/traits/mod.rs index bec774c5e..e3cdd5abd 100644 --- a/protocol/src/traits/mod.rs +++ b/protocol/src/traits/mod.rs @@ -14,13 +14,13 @@ pub use consensus::{ SynchronizationAdapter, }; pub use creep::{Cloneable, Context}; -pub use executor::{ApplyBackend, Backend, Executor, ExecutorAdapter}; +pub use executor::{ApplyBackend, Backend, Executor, ExecutorAdapter, ExecutorReadOnlyAdapter}; pub use interoperation::{Interoperation, BYTE_SHANNONS, SIGNATURE_HASH_CELL_OCCUPIED_CAPACITY}; pub use mempool::{MemPool, MemPoolAdapter}; pub use network::{ Gossip, MessageCodec, MessageHandler, Network, PeerTag, PeerTrust, Priority, Rpc, TrustFeedback, }; pub use storage::{ - CommonStorage, IntoIteratorByRef, Storage, StorageAdapter, StorageBatchModify, StorageCategory, - StorageIterator, StorageSchema, + IntoIteratorByRef, ReadOnlyStorage, Storage, StorageAdapter, StorageBatchModify, + StorageCategory, StorageIterator, StorageSchema, }; diff --git a/protocol/src/traits/storage.rs b/protocol/src/traits/storage.rs index e55718160..bf9debd11 100644 --- a/protocol/src/traits/storage.rs +++ b/protocol/src/traits/storage.rs @@ -29,32 +29,14 @@ pub trait IntoIteratorByRef { } #[async_trait] -pub trait CommonStorage: Send + Sync { - async fn insert_block(&self, ctx: Context, block: Block) -> ProtocolResult<()>; - +pub trait ReadOnlyStorage: Sync + Send { async fn get_block(&self, ctx: Context, height: u64) -> ProtocolResult>; async fn get_block_header(&self, ctx: Context, height: u64) -> ProtocolResult>; - async fn set_block(&self, _ctx: Context, block: Block) -> ProtocolResult<()>; - - async fn remove_block(&self, ctx: Context, height: u64) -> ProtocolResult<()>; - async fn get_latest_block(&self, ctx: Context) -> ProtocolResult; - async fn set_latest_block(&self, ctx: Context, block: Block) -> ProtocolResult<()>; - async fn get_latest_block_header(&self, ctx: Context) -> ProtocolResult
; -} - -#[async_trait] -pub trait Storage: CommonStorage { - async fn insert_transactions( - &self, - ctx: Context, - block_height: u64, - signed_txs: Vec, - ) -> ProtocolResult<()>; async fn get_block_by_hash( &self, @@ -81,21 +63,6 @@ pub trait Storage: CommonStorage { hash: &Hash, ) -> ProtocolResult>; - async fn insert_receipts( - &self, - ctx: Context, - block_height: u64, - receipts: Vec, - ) -> ProtocolResult<()>; - - async fn insert_code( - &self, - ctx: Context, - code_address: H256, - code_hash: Hash, - code: Bytes, - ) -> ProtocolResult<()>; - async fn get_code_by_hash(&self, ctx: Context, hash: &Hash) -> ProtocolResult>; async fn get_code_by_address( @@ -117,11 +84,44 @@ pub trait Storage: CommonStorage { hashes: &[Hash], ) -> ProtocolResult>>; - async fn update_latest_proof(&self, ctx: Context, proof: Proof) -> ProtocolResult<()>; - async fn get_latest_proof(&self, ctx: Context) -> ProtocolResult; } +#[async_trait] +pub trait Storage: ReadOnlyStorage { + async fn insert_block(&self, ctx: Context, block: Block) -> ProtocolResult<()>; + + async fn set_block(&self, ctx: Context, block: Block) -> ProtocolResult<()>; + + async fn remove_block(&self, ctx: Context, height: u64) -> ProtocolResult<()>; + + async fn set_latest_block(&self, ctx: Context, block: Block) -> ProtocolResult<()>; + + async fn insert_transactions( + &self, + ctx: Context, + block_height: u64, + signed_txs: Vec, + ) -> ProtocolResult<()>; + + async fn insert_receipts( + &self, + ctx: Context, + block_height: u64, + receipts: Vec, + ) -> ProtocolResult<()>; + + async fn insert_code( + &self, + ctx: Context, + code_address: H256, + code_hash: Hash, + code: Bytes, + ) -> ProtocolResult<()>; + + async fn update_latest_proof(&self, ctx: Context, proof: Proof) -> ProtocolResult<()>; +} + pub enum StorageBatchModify { Remove, Insert(::Value), diff --git a/protocol/src/types/executor.rs b/protocol/src/types/executor.rs index af0af70af..05d1ef556 100644 --- a/protocol/src/types/executor.rs +++ b/protocol/src/types/executor.rs @@ -52,7 +52,6 @@ pub struct ExecutorContext { pub gas_price: U256, pub block_gas_limit: U256, pub block_base_fee_per_gas: U256, - pub logs: Vec, } impl From for ExecutorContext { @@ -66,7 +65,6 @@ impl From for ExecutorContext { gas_price: U256::one(), block_gas_limit: h.gas_limit, block_base_fee_per_gas: h.base_fee_per_gas, - logs: Vec::new(), } } } @@ -82,7 +80,6 @@ impl From<&Header> for ExecutorContext { gas_price: U256::one(), block_gas_limit: h.gas_limit, block_base_fee_per_gas: h.base_fee_per_gas, - logs: Vec::new(), } } } From af2427207a2ce2e1233ea38212af55e2b85359ea Mon Sep 17 00:00:00 2001 From: Eason Date: Wed, 16 Aug 2023 21:22:21 +0800 Subject: [PATCH 3/4] refactor: merge trie db to main db --- Cargo.lock | 29 +- Cargo.toml | 1 + common/config-parser/src/types/config.rs | 17 +- core/api/src/adapter.rs | 4 +- core/cli/src/args/run.rs | 2 + core/consensus/src/adapter.rs | 6 +- core/db/Cargo.toml | 15 + core/db/src/lib.rs | 6 + .../{storage/src/adapter => db/src}/memory.rs | 27 +- core/{storage/src/adapter => db/src}/rocks.rs | 64 ++-- core/executor/Cargo.toml | 1 + core/executor/benches/bench_transfer.rs | 16 +- core/executor/benches/bench_vm.rs | 10 +- core/executor/benches/mock.rs | 20 +- .../src/adapter/{ => backend}/apply.rs | 4 +- core/executor/src/adapter/backend/mod.rs | 2 + .../src/adapter/{ => backend}/read_only.rs | 0 core/executor/src/adapter/mod.rs | 11 +- core/executor/src/adapter/trie/db.rs | 192 ++++++++++ core/executor/src/adapter/trie/mod.rs | 2 + .../{wrapped_trie.rs => trie/wrapped.rs} | 9 +- core/executor/src/adapter/trie_db.rs | 332 ------------------ core/executor/src/debugger/mod.rs | 6 +- core/executor/src/system_contract/mod.rs | 36 +- core/executor/src/tests/mod.rs | 2 +- .../tests/system_script/ckb_light_client.rs | 7 +- .../src/tests/system_script/image_cell.rs | 7 +- .../src/tests/system_script/metadata.rs | 9 +- core/interoperation/Cargo.toml | 1 + core/interoperation/src/tests/mod.rs | 30 +- core/mempool/src/adapter/mod.rs | 2 +- core/run/Cargo.toml | 1 + core/run/src/lib.rs | 156 ++++---- core/run/src/tests.rs | 41 +-- core/storage/Cargo.toml | 3 +- core/storage/benches/bench.rs | 4 +- core/storage/src/adapter/mod.rs | 2 - core/storage/src/lib.rs | 1 - core/storage/src/tests/adapter.rs | 6 +- core/storage/src/tests/storage.rs | 3 +- protocol/Cargo.toml | 6 +- protocol/src/lib.rs | 8 +- protocol/src/traits/mod.rs | 4 +- protocol/src/traits/storage.rs | 28 +- protocol/src/types/primitive.rs | 6 + 45 files changed, 517 insertions(+), 622 deletions(-) create mode 100644 core/db/Cargo.toml create mode 100644 core/db/src/lib.rs rename core/{storage/src/adapter => db/src}/memory.rs (89%) rename core/{storage/src/adapter => db/src}/rocks.rs (81%) rename core/executor/src/adapter/{ => backend}/apply.rs (98%) create mode 100644 core/executor/src/adapter/backend/mod.rs rename core/executor/src/adapter/{ => backend}/read_only.rs (100%) create mode 100644 core/executor/src/adapter/trie/db.rs create mode 100644 core/executor/src/adapter/trie/mod.rs rename core/executor/src/adapter/{wrapped_trie.rs => trie/wrapped.rs} (93%) delete mode 100644 core/executor/src/adapter/trie_db.rs delete mode 100644 core/storage/src/adapter/mod.rs diff --git a/Cargo.lock b/Cargo.lock index e137ee66b..3e0672cbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -333,12 +333,12 @@ dependencies = [ "ophelia", "ophelia-secp256k1", "overlord", - "prost", "rand 0.7.3", "rlp", "rlp-derive", "serde", "serde_json", + "thiserror", "tokio", "toml 0.7.3", ] @@ -873,9 +873,8 @@ dependencies = [ [[package]] name = "cita_trie" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afe7baab47f510f52ca8dc9c0eb9082020c627c7f22285bea30edc3511f7ee29" +version = "4.2.0" +source = "git+https://github.com/KaoImin/cita-trie.git?rev=eea569c#eea569c06780d0b38b5c03d6a6cb37b7ac4ea7af" dependencies = [ "hasher", "parking_lot 0.12.1", @@ -1692,6 +1691,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "core-db" +version = "0.1.0" +dependencies = [ + "axon-protocol", + "ckb-rocksdb", + "common-apm", + "common-apm-derive", + "common-config-parser", + "parking_lot 0.12.1", +] + [[package]] name = "core-executor" version = "0.1.0" @@ -1708,6 +1719,7 @@ dependencies = [ "common-config-parser", "common-crypto", "common-merkle", + "core-db", "core-interoperation", "core-rpc-client", "core-storage", @@ -1767,6 +1779,7 @@ dependencies = [ "ckb-traits", "ckb-types", "ckb-vm", + "core-db", "core-executor", "core-rpc-client", "core-storage", @@ -1856,6 +1869,7 @@ dependencies = [ "common-memory-tracker", "core-api", "core-consensus", + "core-db", "core-executor", "core-interoperation", "core-mempool", @@ -1880,11 +1894,10 @@ version = "0.1.0" dependencies = [ "arc-swap", "axon-protocol", - "ckb-rocksdb", "common-apm", "common-apm-derive", - "common-config-parser", "common-crypto", + "core-db", "criterion", "futures", "lazy_static", @@ -6698,9 +6711,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.31.0" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40de3a2ba249dcb097e01be5e67a5ff53cf250397715a071a81543e8a832a920" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 71e15ad56..3f488bf9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ members = [ "core/api", "core/cli", "core/consensus", + "core/db", "core/executor", "core/mempool", "core/network", diff --git a/common/config-parser/src/types/config.rs b/common/config-parser/src/types/config.rs index 30c4dfe38..10d316bd8 100644 --- a/common/config-parser/src/types/config.rs +++ b/common/config-parser/src/types/config.rs @@ -43,24 +43,9 @@ pub struct Config { } impl Config { - pub fn data_path_for_system_contract(&self) -> PathBuf { + pub fn data_path_for_rocksdb(&self) -> PathBuf { let mut path_state = self.data_path.clone(); path_state.push("rocksdb"); - path_state.push("system_contract"); - path_state - } - - pub fn data_path_for_state(&self) -> PathBuf { - let mut path_state = self.data_path.clone(); - path_state.push("rocksdb"); - path_state.push("state_data"); - path_state - } - - pub fn data_path_for_block(&self) -> PathBuf { - let mut path_state = self.data_path.clone(); - path_state.push("rocksdb"); - path_state.push("block_data"); path_state } diff --git a/core/api/src/adapter.rs b/core/api/src/adapter.rs index 7239c226e..400ef7cf5 100644 --- a/core/api/src/adapter.rs +++ b/core/api/src/adapter.rs @@ -28,7 +28,7 @@ impl DefaultAPIAdapter where M: MemPool + 'static, S: ReadOnlyStorage + 'static, - DB: trie::DB + 'static, + DB: trie::DB + Send + Sync + 'static, Net: Network + 'static, { pub fn new(mempool: Arc, storage: Arc, trie_db: Arc, net: Arc) -> Self { @@ -63,7 +63,7 @@ impl APIAdapter for DefaultAPIAdapter where M: MemPool + 'static, S: ReadOnlyStorage + 'static, - DB: trie::DB + 'static, + DB: trie::DB + Send + Sync + 'static, Net: Network + 'static, { async fn insert_signed_txs( diff --git a/core/cli/src/args/run.rs b/core/cli/src/args/run.rs index aa37373da..eb37b7189 100644 --- a/core/cli/src/args/run.rs +++ b/core/cli/src/args/run.rs @@ -37,12 +37,14 @@ impl RunArgs { ) -> Result<()> { let Self { config, spec } = self; let genesis = spec.genesis.build_rich_block(); + utils::check_version( &config.data_path_for_version(), &kernel_version, utils::latest_compatible_version(), )?; utils::register_log(&config); + Axon::new(application_version.to_string(), config, spec, genesis) .run(key_provider) .map_err(Error::Running) diff --git a/core/consensus/src/adapter.rs b/core/consensus/src/adapter.rs index ed6b4c208..81a870126 100644 --- a/core/consensus/src/adapter.rs +++ b/core/consensus/src/adapter.rs @@ -51,7 +51,7 @@ where M: MemPool + 'static, N: Network + Rpc + PeerTrust + Gossip + 'static, S: Storage + 'static, - DB: trie::DB + 'static, + DB: trie::DB + Send + Sync + 'static, { #[trace_span(kind = "consensus.adapter")] async fn get_txs_from_mempool( @@ -128,7 +128,7 @@ where M: MemPool + 'static, N: Network + Rpc + PeerTrust + Gossip + 'static, S: Storage + 'static, - DB: trie::DB + 'static, + DB: trie::DB + Send + Sync + 'static, { #[trace_span(kind = "consensus.adapter")] fn update_status( @@ -229,7 +229,7 @@ where M: MemPool + 'static, N: Network + Rpc + PeerTrust + Gossip + 'static, S: Storage + 'static, - DB: trie::DB + 'static, + DB: trie::DB + Send + Sync + 'static, { /// Save a block to the database. #[trace_span(kind = "consensus.adapter", logs = "{txs_len: block.tx_hashes.len()}")] diff --git a/core/db/Cargo.toml b/core/db/Cargo.toml new file mode 100644 index 000000000..e2caa1522 --- /dev/null +++ b/core/db/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "core-db" +version = "0.1.0" +edition = "2021" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +parking_lot = "0.12" +rocksdb = { package = "ckb-rocksdb", version = "0.20" } + +common-apm = { path = "../../common/apm" } +common-apm-derive = { path = "../../common/apm-derive" } +common-config-parser = { path = "../../common/config-parser" } +protocol = { path = "../../protocol", package = "axon-protocol" } + diff --git a/core/db/src/lib.rs b/core/db/src/lib.rs new file mode 100644 index 000000000..84861013f --- /dev/null +++ b/core/db/src/lib.rs @@ -0,0 +1,6 @@ +mod memory; +mod rocks; + +pub use crate::memory::MemoryAdapter; +pub use crate::rocks::{get_column, map_category, RocksAdapter}; +pub use rocksdb::DB as RocksDB; diff --git a/core/storage/src/adapter/memory.rs b/core/db/src/memory.rs similarity index 89% rename from core/storage/src/adapter/memory.rs rename to core/db/src/memory.rs index 15b1c4f5d..3db9af881 100644 --- a/core/storage/src/adapter/memory.rs +++ b/core/db/src/memory.rs @@ -1,16 +1,15 @@ use std::collections::{hash_map, HashMap}; -use std::error::Error; -use std::marker::PhantomData; -use std::sync::Arc; +use std::{error::Error, marker::PhantomData, sync::Arc}; use parking_lot::RwLock; -use protocol::codec::ProtocolCodec; use protocol::traits::{ IntoIteratorByRef, StorageAdapter, StorageBatchModify, StorageIterator, StorageSchema, }; -use protocol::types::Bytes; -use protocol::{Display, From, ProtocolError, ProtocolErrorKind, ProtocolResult}; +use protocol::{ + codec::ProtocolCodec, types::Bytes, Display, From, ProtocolError, ProtocolErrorKind, + ProtocolResult, +}; type Category = HashMap, Vec>; @@ -29,9 +28,7 @@ impl MemoryAdapter { impl Default for MemoryAdapter { fn default() -> Self { - MemoryAdapter { - db: Arc::new(RwLock::new(HashMap::new())), - } + Self::new() } } @@ -155,7 +152,7 @@ impl StorageAdapter for MemoryAdapter { vals: Vec>, ) -> ProtocolResult<()> { if keys.len() != vals.len() { - return Err(MemoryAdapterError::BatchLengthMismatch.into()); + return Err(MemoryDBError::BatchLengthMismatch.into()); } let mut pairs: Vec<(Bytes, Option)> = Vec::with_capacity(keys.len()); @@ -205,15 +202,15 @@ impl StorageAdapter for MemoryAdapter { } #[derive(Debug, Display, From)] -pub enum MemoryAdapterError { +pub enum MemoryDBError { #[display(fmt = "batch length do not match")] BatchLengthMismatch, } -impl Error for MemoryAdapterError {} +impl Error for MemoryDBError {} -impl From for ProtocolError { - fn from(err: MemoryAdapterError) -> ProtocolError { - ProtocolError::new(ProtocolErrorKind::Storage, Box::new(err)) +impl From for ProtocolError { + fn from(err: MemoryDBError) -> ProtocolError { + ProtocolError::new(ProtocolErrorKind::DB, Box::new(err)) } } diff --git a/core/storage/src/adapter/rocks.rs b/core/db/src/rocks.rs similarity index 81% rename from core/storage/src/adapter/rocks.rs rename to core/db/src/rocks.rs index ad58d970c..28351b35e 100644 --- a/core/storage/src/adapter/rocks.rs +++ b/core/db/src/rocks.rs @@ -1,8 +1,4 @@ -use std::error::Error; -use std::marker::PhantomData; -use std::path::Path; -use std::sync::Arc; -use std::{fs, io}; +use std::{error::Error, fs, io, marker::PhantomData, path::Path, sync::Arc}; use rocksdb::ops::{DeleteCF, GetCF, GetColumnFamilys, IterateCF, OpenCF, PutCF, WriteOps}; use rocksdb::{ @@ -27,8 +23,9 @@ pub struct RocksAdapter { impl RocksAdapter { pub fn new>(path: P, config: ConfigRocksDB) -> ProtocolResult { if !path.as_ref().is_dir() { - fs::create_dir_all(&path).map_err(RocksAdapterError::CreateDB)?; + fs::create_dir_all(&path).map_err(RocksDBError::CreateDB)?; } + let categories = [ map_category(StorageCategory::Block), map_category(StorageCategory::BlockHeader), @@ -37,6 +34,9 @@ impl RocksAdapter { map_category(StorageCategory::Wal), map_category(StorageCategory::HashHeight), map_category(StorageCategory::Code), + map_category(StorageCategory::EvmState), + map_category(StorageCategory::MetadataState), + map_category(StorageCategory::CkbLightClientState), ]; let (mut opts, cf_descriptors) = if let Some(ref file) = config.options_file { @@ -45,12 +45,12 @@ impl RocksAdapter { size => Some(size), }; - let mut full_opts = FullOptions::load_from_file(file, cache_size, false) - .map_err(RocksAdapterError::from)?; + let mut full_opts = + FullOptions::load_from_file(file, cache_size, false).map_err(RocksDBError::from)?; full_opts .complete_column_families(&categories, false) - .map_err(RocksAdapterError::from)?; + .map_err(RocksDBError::from)?; let FullOptions { db_opts, cf_descriptors, @@ -69,8 +69,13 @@ impl RocksAdapter { opts.create_missing_column_families(true); opts.set_max_open_files(config.max_open_files); - let db = DB::open_cf_descriptors(&opts, path, cf_descriptors) - .map_err(RocksAdapterError::from)?; + // let tmp_db = DB::list_cf(&opts, path).map_err(RocksDBError::from)?; + // if tmp_db.len() != cf_descriptors.len() { + // opts.create_missing_column_families(true); + // } + + let db = + DB::open_cf_descriptors(&opts, path, cf_descriptors).map_err(RocksDBError::from)?; Ok(RocksAdapter { db: Arc::new(db) }) } @@ -81,12 +86,8 @@ impl RocksAdapter { } macro_rules! db { - ($db:expr, $op:ident, $column:expr, $key:expr) => { - $db.$op($column, $key).map_err(RocksAdapterError::from) - }; - ($db:expr, $op:ident, $column:expr, $key:expr, $val:expr) => { - $db.$op($column, $key, $val) - .map_err(RocksAdapterError::from) + ($db:expr, $op:ident, $column:expr$ (, $args: expr)*) => { + $db.$op($column, $($args,)*).map_err(RocksDBError::from) }; } @@ -198,7 +199,7 @@ impl StorageAdapter for RocksAdapter { vals: Vec>, ) -> ProtocolResult<()> { if keys.len() != vals.len() { - return Err(RocksAdapterError::BatchLengthMismatch.into()); + return Err(RocksDBError::BatchLengthMismatch.into()); } let column = get_column::(&self.db)?; @@ -226,12 +227,12 @@ impl StorageAdapter for RocksAdapter { } None => batch.delete_cf(column, key), } - .map_err(RocksAdapterError::from)?; + .map_err(RocksDBError::from)?; } on_storage_put_cf(S::category(), inst.elapsed(), insert_size as f64); - self.db.write(&batch).map_err(RocksAdapterError::from)?; + self.db.write(&batch).map_err(RocksDBError::from)?; Ok(()) } @@ -252,7 +253,7 @@ impl StorageAdapter for RocksAdapter { } #[derive(Debug, Display, From)] -pub enum RocksAdapterError { +pub enum RocksDBError { #[display(fmt = "category {} not found", _0)] CategoryNotFound(&'static str), @@ -269,14 +270,15 @@ pub enum RocksAdapterError { CreateDB(io::Error), } -impl Error for RocksAdapterError {} +impl Error for RocksDBError {} -impl From for ProtocolError { - fn from(err: RocksAdapterError) -> ProtocolError { - ProtocolError::new(ProtocolErrorKind::Storage, Box::new(err)) +impl From for ProtocolError { + fn from(err: RocksDBError) -> ProtocolError { + ProtocolError::new(ProtocolErrorKind::DB, Box::new(err)) } } +// Todo: column family "c0" is reserved for store version const C_BLOCKS: &str = "c1"; const C_BLOCK_HEADER: &str = "c2"; const C_SIGNED_TRANSACTIONS: &str = "c3"; @@ -284,8 +286,11 @@ const C_RECEIPTS: &str = "c4"; const C_WALS: &str = "c5"; const C_HASH_HEIGHT_MAP: &str = "c6"; const C_EVM_CODE_MAP: &str = "c7"; +const C_EVM_STATE: &str = "c8"; +const C_METADATA_STATE: &str = "c9"; +const C_CKB_LIGHT_CLIENT_STATE: &str = "c10"; -fn map_category(c: StorageCategory) -> &'static str { +pub fn map_category(c: StorageCategory) -> &'static str { match c { StorageCategory::Block => C_BLOCKS, StorageCategory::BlockHeader => C_BLOCK_HEADER, @@ -294,15 +299,18 @@ fn map_category(c: StorageCategory) -> &'static str { StorageCategory::Wal => C_WALS, StorageCategory::HashHeight => C_HASH_HEIGHT_MAP, StorageCategory::Code => C_EVM_CODE_MAP, + StorageCategory::EvmState => C_EVM_STATE, + StorageCategory::MetadataState => C_METADATA_STATE, + StorageCategory::CkbLightClientState => C_CKB_LIGHT_CLIENT_STATE, } } -fn get_column(db: &DB) -> Result<&ColumnFamily, RocksAdapterError> { +pub fn get_column(db: &DB) -> Result<&ColumnFamily, RocksDBError> { let category = map_category(S::category()); let column = db .cf_handle(category) - .ok_or(RocksAdapterError::CategoryNotFound(category))?; + .ok_or(RocksDBError::CategoryNotFound(category))?; Ok(column) } diff --git a/core/executor/Cargo.toml b/core/executor/Cargo.toml index eb358bc9f..f83aadd0d 100644 --- a/core/executor/Cargo.toml +++ b/core/executor/Cargo.toml @@ -14,6 +14,7 @@ common-apm = { path = "../../common/apm" } common-config-parser = { path = "../../common/config-parser" } common-crypto = { path = "../../common/crypto" } common-merkle = { path = "../../common/merkle" } +core-db = { path = "../db" } core-interoperation = { path = "../interoperation" } ethers = "2.0" evm = { version = "0.37", features = ["tracing"] } diff --git a/core/executor/benches/bench_transfer.rs b/core/executor/benches/bench_transfer.rs index 71d5175c9..46cad9336 100644 --- a/core/executor/benches/bench_transfer.rs +++ b/core/executor/benches/bench_transfer.rs @@ -7,8 +7,6 @@ use common_crypto::{ Crypto, PrivateKey, Secp256k1Recoverable, Secp256k1RecoverablePrivateKey, Signature, ToPublicKey, UncompressedPublicKey, }; -use core_executor::{AxonExecutor, AxonExecutorApplyAdapter, MPTTrie, RocksTrieDB}; -use core_storage::{adapter::rocks::RocksAdapter, ImplStorage}; use protocol::codec::{hex_decode, ProtocolCodec}; use protocol::traits::Executor; use protocol::types::{ @@ -17,13 +15,16 @@ use protocol::types::{ RLP_NULL, U256, }; +use core_db::RocksAdapter; +use core_executor::{AxonExecutor, AxonExecutorApplyAdapter, MPTTrie, RocksTrieDB}; +use core_storage::ImplStorage; + lazy_static::lazy_static! { static ref PRIVATE_KEY: Secp256k1RecoverablePrivateKey = Secp256k1RecoverablePrivateKey::try_from(hex_decode("95500289866f83502cc1fb894ef5e2b840ca5f867cc9e84ab32fb8872b5dd36c").unwrap().as_ref()).unwrap(); static ref DISTRIBUTE_ADDRESS: Address = Address::from_hex("0x35e70c3f5a794a77efc2ec5ba964bffcc7fd2c0a").unwrap(); } -const STATE_PATH: &str = "../../free-space/rocks/state"; const DATA_PATH: &str = "../../free-space/rocks/data"; struct BenchAdapter { @@ -33,12 +34,11 @@ struct BenchAdapter { impl BenchAdapter { fn new() -> Self { + let db = RocksAdapter::new(DATA_PATH, Default::default()).unwrap(); + BenchAdapter { - trie_db: Arc::new(RocksTrieDB::new(STATE_PATH, Default::default(), 1000).unwrap()), - storage: Arc::new(ImplStorage::new( - Arc::new(RocksAdapter::new(DATA_PATH, Default::default()).unwrap()), - 100, - )), + trie_db: Arc::new(RocksTrieDB::new_evm(db.inner_db(), 1000)), + storage: Arc::new(ImplStorage::new(Arc::new(db), 100)), } } diff --git a/core/executor/benches/bench_vm.rs b/core/executor/benches/bench_vm.rs index f0d67deee..7022496ae 100644 --- a/core/executor/benches/bench_vm.rs +++ b/core/executor/benches/bench_vm.rs @@ -13,9 +13,7 @@ use protocol::{ types::{Account, Address, ExecutorContext}, }; -use crate::mock::{ - init_account, mock_executor_context, mock_transactions, new_rocks_trie_db, new_storage, -}; +use crate::mock::{init_account, mock_executor_context, mock_transactions, new_storage}; use crate::revm_adapter::{revm_exec, RevmAdapter}; trait BackendInit { @@ -73,8 +71,7 @@ fn criterion_10000_txs(c: &mut Criterion) { let txs = mock_transactions(10000); // MacOS M1 Pro, 16GB: time: 20.098ms c.bench_function("revm 10000 tx", |b| { - let storage = new_storage(); - let db = new_rocks_trie_db(); + let (db, storage) = new_storage(); let exec_ctx = mock_executor_context(); let (account, addr) = init_account(); let revm_adapter = RevmAdapter::init(storage, db, exec_ctx, account, addr); @@ -86,8 +83,7 @@ fn criterion_10000_txs(c: &mut Criterion) { }); // MacOS M1 Pro, 16GB: time:54.987ms c.bench_function("evm 10000 tx", |b| { - let storage = new_storage(); - let db = new_rocks_trie_db(); + let (db, storage) = new_storage(); let exec_ctx = mock_executor_context(); let (account, addr) = init_account(); let mut axon_adapter = AxonExecutorApplyAdapter::init(storage, db, exec_ctx, account, addr); diff --git a/core/executor/benches/mock.rs b/core/executor/benches/mock.rs index 3203028ee..acb4d4e0b 100644 --- a/core/executor/benches/mock.rs +++ b/core/executor/benches/mock.rs @@ -4,9 +4,6 @@ use common_crypto::{ Crypto, PrivateKey, Secp256k1Recoverable, Secp256k1RecoverablePrivateKey, Signature, ToPublicKey, UncompressedPublicKey, }; -use core_executor::RocksTrieDB; -use core_storage::adapter::rocks::RocksAdapter; -use core_storage::ImplStorage; use protocol::{ codec::hex_decode, types::{ @@ -16,23 +13,24 @@ use protocol::{ }, }; +use core_db::RocksAdapter; +use core_executor::RocksTrieDB; +use core_storage::ImplStorage; + lazy_static::lazy_static! { static ref PRIVATE_KEY: Secp256k1RecoverablePrivateKey = Secp256k1RecoverablePrivateKey::try_from(hex_decode("95500289866f83502cc1fb894ef5e2b840ca5f867cc9e84ab32fb8872b5dd36c").unwrap().as_ref()).unwrap(); static ref DISTRIBUTE_ADDRESS: Address = Address::from_hex("0x35e70c3f5a794a77efc2ec5ba964bffcc7fd2c0a").unwrap(); } -const STATE_PATH: &str = "../../free-space/rocks/state"; const DATA_PATH: &str = "../../free-space/rocks/data"; -pub fn new_rocks_trie_db() -> RocksTrieDB { - RocksTrieDB::new(STATE_PATH, Default::default(), 1000).unwrap() -} +pub fn new_storage() -> (RocksTrieDB, ImplStorage) { + let db = RocksAdapter::new(DATA_PATH, Default::default()).unwrap(); -pub fn new_storage() -> ImplStorage { - ImplStorage::new( - Arc::new(RocksAdapter::new(DATA_PATH, Default::default()).unwrap()), - 100, + ( + RocksTrieDB::new_evm(db.inner_db(), 1000), + ImplStorage::new(Arc::new(db), 100), ) } diff --git a/core/executor/src/adapter/apply.rs b/core/executor/src/adapter/backend/apply.rs similarity index 98% rename from core/executor/src/adapter/apply.rs rename to core/executor/src/adapter/backend/apply.rs index c52e4e0e9..7f542d6ef 100644 --- a/core/executor/src/adapter/apply.rs +++ b/core/executor/src/adapter/backend/apply.rs @@ -165,9 +165,9 @@ where }; let mut storage_trie = if storage_root == RLP_NULL { - MPTTrie::new(self.inner.db.clone()) + MPTTrie::new(Arc::clone(&self.inner.db)) } else { - MPTTrie::from_root(old_account.storage_root, self.inner.db.clone()).unwrap() + MPTTrie::from_root(old_account.storage_root, Arc::clone(&self.inner.db)).unwrap() }; storage.into_iter().for_each(|(k, v)| { diff --git a/core/executor/src/adapter/backend/mod.rs b/core/executor/src/adapter/backend/mod.rs new file mode 100644 index 000000000..81d0ca89e --- /dev/null +++ b/core/executor/src/adapter/backend/mod.rs @@ -0,0 +1,2 @@ +pub mod apply; +pub mod read_only; diff --git a/core/executor/src/adapter/read_only.rs b/core/executor/src/adapter/backend/read_only.rs similarity index 100% rename from core/executor/src/adapter/read_only.rs rename to core/executor/src/adapter/backend/read_only.rs diff --git a/core/executor/src/adapter/mod.rs b/core/executor/src/adapter/mod.rs index 035ea4759..cbd087b45 100644 --- a/core/executor/src/adapter/mod.rs +++ b/core/executor/src/adapter/mod.rs @@ -1,11 +1,8 @@ -mod apply; -mod read_only; -mod trie_db; -mod wrapped_trie; +mod backend; +mod trie; -pub use trie_db::RocksTrieDB; -pub use wrapped_trie::MPTTrie; -pub use {apply::AxonExecutorApplyAdapter, read_only::AxonExecutorReadOnlyAdapter}; +pub use backend::{apply::AxonExecutorApplyAdapter, read_only::AxonExecutorReadOnlyAdapter}; +pub use trie::{db::RocksTrieDB, wrapped::MPTTrie}; #[macro_export] macro_rules! blocking_async { diff --git a/core/executor/src/adapter/trie/db.rs b/core/executor/src/adapter/trie/db.rs new file mode 100644 index 000000000..3bfaa1f82 --- /dev/null +++ b/core/executor/src/adapter/trie/db.rs @@ -0,0 +1,192 @@ +use std::{collections::HashMap, io, sync::Arc}; + +use parking_lot::RwLock; +use rocksdb::ops::{GetCF, GetColumnFamilys, PutCF, WriteOps}; +use rocksdb::{ColumnFamily, WriteBatch, DB}; + +use common_apm::metrics::storage::{on_storage_get_state, on_storage_put_state}; +use common_apm::Instant; +use protocol::rand::{rngs::SmallRng, Rng, SeedableRng}; +use protocol::traits::StateStorageCategory; +use protocol::trie; + +use core_db::map_category; + +// 49999 is the largest prime number within 50000. +const RAND_SEED: u64 = 49999; + +macro_rules! db { + ($db:expr, $op:ident, $column:expr$ (, $args: expr)*) => { + $db.$op($column, $($args,)*).map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("rocksdb error: {:?}", e), + ) + })? + }; +} + +pub struct RocksTrieDB { + db: Arc, + category: StateStorageCategory, + cache: RwLock, Vec>>, + cache_size: usize, +} + +impl trie::DB for RocksTrieDB { + fn get(&self, key: &[u8]) -> Result>, io::Error> { + let res = { self.cache.read().get(key).cloned() }; + + if res.is_none() { + let inst = Instant::now(); + let ret = db!(self.db, get_cf, self.get_column(), key); + on_storage_get_state(inst.elapsed(), 1.0); + + if let Some(val) = &ret { + { + self.cache.write().insert(key.to_owned(), val.to_vec()); + } + self.flush()?; + } + + return Ok(ret.map(|r| r.to_vec())); + } + + Ok(res) + } + + fn contains(&self, key: &[u8]) -> Result { + let res = { self.cache.read().contains_key(key) }; + + if res { + Ok(true) + } else if let Some(val) = db!(self.db, get_cf, self.get_column(), key) { + self.cache.write().insert(key.to_owned(), val.to_vec()); + Ok(true) + } else { + Ok(false) + } + } + + fn insert(&self, key: Vec, value: Vec) -> Result<(), io::Error> { + let inst = Instant::now(); + let size = key.len() + value.len(); + + db!(self.db, put_cf, self.get_column(), &key, &value); + + { + self.cache.write().insert(key, value); + } + + on_storage_put_state(inst.elapsed(), size as f64); + self.flush() + } + + fn insert_batch(&self, keys: Vec>, values: Vec>) -> Result<(), io::Error> { + if keys.len() != values.len() { + return Err(io::Error::new( + io::ErrorKind::Other, + "keys and values length not match", + )); + } + + let mut total_size = 0; + let mut batch = WriteBatch::default(); + + { + let mut cache = self.cache.write(); + for (key, val) in keys.into_iter().zip(values.into_iter()) { + total_size += key.len(); + total_size += val.len(); + + let column = self.get_column(); + db!(batch, put_cf, column, &key, &val); + cache.insert(key, val); + } + } + + let inst = Instant::now(); + self.db + .write(&batch) + .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("rocksdb error: {:?}", e)))?; + on_storage_put_state(inst.elapsed(), total_size as f64); + + self.flush() + } + + fn remove(&self, _key: &[u8]) -> Result<(), io::Error> { + Ok(()) + } + + fn remove_batch(&self, _keys: &[Vec]) -> Result<(), io::Error> { + Ok(()) + } + + fn flush(&self) -> Result<(), io::Error> { + let mut cache = self.cache.write(); + + let len = cache.len(); + + if len <= self.cache_size * 2 { + return Ok(()); + } + + let remove_list = { + let keys = cache.iter().map(|(k, _)| k).collect::>(); + rand_remove_list(keys, len - self.cache_size) + }; + + for item in remove_list { + cache.remove(&item); + } + + Ok(()) + } +} + +impl RocksTrieDB { + pub fn new_evm(db: Arc, cache_size: usize) -> Self { + Self::new(db, StateStorageCategory::EvmState, cache_size) + } + + pub fn new_metadata(db: Arc, cache_size: usize) -> Self { + Self::new(db, StateStorageCategory::MetadataState, cache_size) + } + + pub fn new_ckb_light_client(db: Arc, cache_size: usize) -> Self { + Self::new(db, StateStorageCategory::CkbLightClientState, cache_size) + } + + fn new(db: Arc, category: StateStorageCategory, cache_size: usize) -> Self { + let cache = RwLock::new(HashMap::with_capacity(cache_size)); + RocksTrieDB { + db, + category, + cache, + cache_size, + } + } + + fn get_column(&self) -> &ColumnFamily { + let category = map_category(self.category.into()); + self.db + .cf_handle(category) + .unwrap_or_else(|| panic!("Column Family {:?} not found", category)) + } +} + +fn rand_remove_list(keys: Vec<&T>, num: usize) -> impl Iterator { + let mut len = keys.len() - 1; + let mut idx_list = (0..len).collect::>(); + let mut rng = SmallRng::seed_from_u64(RAND_SEED); + let mut ret = Vec::with_capacity(num); + + for _ in 0..num { + let tmp = rng.gen_range(0, len); + let idx = idx_list.remove(tmp); + ret.push(keys[idx].clone()); + len -= 1; + } + + ret.into_iter() +} diff --git a/core/executor/src/adapter/trie/mod.rs b/core/executor/src/adapter/trie/mod.rs new file mode 100644 index 000000000..5a5819d5d --- /dev/null +++ b/core/executor/src/adapter/trie/mod.rs @@ -0,0 +1,2 @@ +pub mod db; +pub mod wrapped; diff --git a/core/executor/src/adapter/wrapped_trie.rs b/core/executor/src/adapter/trie/wrapped.rs similarity index 93% rename from core/executor/src/adapter/wrapped_trie.rs rename to core/executor/src/adapter/trie/wrapped.rs index 1aec4279f..1c07a1d4a 100644 --- a/core/executor/src/adapter/wrapped_trie.rs +++ b/core/executor/src/adapter/trie/wrapped.rs @@ -76,9 +76,12 @@ impl From for ProtocolError { #[cfg(test)] mod tests { use super::*; - use crate::adapter::RocksTrieDB; + + use core_db::RocksAdapter; use protocol::rand::random; + use crate::adapter::RocksTrieDB; + fn rand_bytes(len: usize) -> Vec { (0..len).map(|_| random()).collect() } @@ -86,7 +89,9 @@ mod tests { #[test] fn test_mpt_cache() { let dir = tempfile::tempdir().unwrap(); - let db = RocksTrieDB::new(dir.path(), Default::default(), 100).unwrap(); + let inner_db = + Arc::new(RocksAdapter::new(dir.path(), Default::default()).unwrap()).inner_db(); + let db = RocksTrieDB::new_evm(inner_db, 100); let mut mpt = MPTTrie::new(Arc::new(db)); let key_1 = rand_bytes(5); diff --git a/core/executor/src/adapter/trie_db.rs b/core/executor/src/adapter/trie_db.rs deleted file mode 100644 index 20d75b0fb..000000000 --- a/core/executor/src/adapter/trie_db.rs +++ /dev/null @@ -1,332 +0,0 @@ -use std::{collections::HashMap, fs, io, path::Path, sync::Arc}; - -use parking_lot::RwLock; -use rocksdb::ops::{Get, Open, Put, WriteOps}; -use rocksdb::{FullOptions, Options, WriteBatch, DB}; - -use common_apm::metrics::storage::{on_storage_get_state, on_storage_put_state}; -use common_apm::Instant; -use common_config_parser::types::ConfigRocksDB; -use protocol::rand::{rngs::SmallRng, Rng, SeedableRng}; -use protocol::{trie, Display, From, ProtocolError, ProtocolErrorKind, ProtocolResult}; - -// 49999 is the largest prime number within 50000. -const RAND_SEED: u64 = 49999; - -pub struct RocksTrieDB { - db: Arc, - cache: RwLock, Vec>>, - cache_size: usize, -} - -impl RocksTrieDB { - pub fn new>( - path: P, - config: ConfigRocksDB, - cache_size: usize, - ) -> ProtocolResult { - if !path.as_ref().is_dir() { - fs::create_dir_all(&path).map_err(RocksTrieDBError::CreateDB)?; - } - - let opts = rocksdb_opts(config)?; - let db = Arc::new(DB::open(&opts, path).map_err(RocksTrieDBError::from)?); - - // Init HashMap with capacity 2 * cache_size to avoid reallocate memory. - Ok(RocksTrieDB { - db, - cache: RwLock::new(HashMap::with_capacity(cache_size + cache_size)), - cache_size, - }) - } - - pub fn inner_db(&self) -> Arc { - Arc::clone(&self.db) - } - - fn inner_get(&self, key: &[u8]) -> Result>, RocksTrieDBError> { - use trie::DB; - - let res = { self.cache.read().get(key).cloned() }; - - if res.is_none() { - let inst = Instant::now(); - let ret = self.db.get(key).map_err(to_store_err)?.map(|r| r.to_vec()); - on_storage_get_state(inst.elapsed(), 1.0); - - if let Some(val) = &ret { - { - self.cache.write().insert(key.to_owned(), val.clone()); - } - self.flush()?; - } - - return Ok(ret); - } - - Ok(res) - } - - #[cfg(test)] - fn cache_get(&self, key: &[u8]) -> Option> { - self.cache.read().get(key).cloned() - } - - #[cfg(test)] - fn cache_len(&self) -> usize { - self.cache.read().len() - } -} - -impl trie::DB for RocksTrieDB { - type Error = RocksTrieDBError; - - fn get(&self, key: &[u8]) -> Result>, Self::Error> { - self.inner_get(key) - } - - fn contains(&self, key: &[u8]) -> Result { - let res = { self.cache.read().contains_key(key) }; - - if res { - Ok(true) - } else { - if let Some(val) = self.db.get(key).map_err(to_store_err)?.map(|r| r.to_vec()) { - self.cache.write().insert(key.to_owned(), val); - return Ok(true); - } - Ok(false) - } - } - - fn insert(&self, key: Vec, value: Vec) -> Result<(), Self::Error> { - let inst = Instant::now(); - let size = key.len() + value.len(); - - self.db.put(&key, &value).map_err(to_store_err)?; - - { - self.cache.write().insert(key, value); - } - - on_storage_put_state(inst.elapsed(), size as f64); - - self.flush() - } - - fn insert_batch(&self, keys: Vec>, values: Vec>) -> Result<(), Self::Error> { - if keys.len() != values.len() { - return Err(RocksTrieDBError::BatchLengthMismatch); - } - - let mut total_size = 0; - let mut batch = WriteBatch::default(); - - { - let mut cache = self.cache.write(); - for (key, val) in keys.into_iter().zip(values.into_iter()) { - total_size += key.len(); - total_size += val.len(); - batch.put(&key, &val)?; - cache.insert(key, val); - } - } - - let inst = Instant::now(); - self.db.write(&batch).map_err(to_store_err)?; - on_storage_put_state(inst.elapsed(), total_size as f64); - - self.flush() - } - - fn remove(&self, _key: &[u8]) -> Result<(), Self::Error> { - Ok(()) - } - - fn remove_batch(&self, _keys: &[Vec]) -> Result<(), Self::Error> { - Ok(()) - } - - fn flush(&self) -> Result<(), Self::Error> { - let mut cache = self.cache.write(); - - let len = cache.len(); - - if len <= self.cache_size * 2 { - return Ok(()); - } - - let remove_list = { - let keys = cache.iter().map(|(k, _)| k).collect::>(); - rand_remove_list(keys, len - self.cache_size) - }; - - for item in remove_list { - cache.remove(&item); - } - - Ok(()) - } -} - -fn rocksdb_opts(config: ConfigRocksDB) -> ProtocolResult { - let mut opts = if let Some(ref file) = config.options_file { - let cache_size = match config.cache_size { - 0 => None, - size => Some(size), - }; - let full_opts = - FullOptions::load_from_file(file, cache_size, false).map_err(RocksTrieDBError::from)?; - let FullOptions { db_opts, .. } = full_opts; - db_opts - } else { - Options::default() - }; - - opts.create_if_missing(true); - opts.create_missing_column_families(true); - opts.set_max_open_files(config.max_open_files); - - Ok(opts) -} - -fn rand_remove_list(keys: Vec<&T>, num: usize) -> impl Iterator { - let mut len = keys.len() - 1; - let mut idx_list = (0..len).collect::>(); - let mut rng = SmallRng::seed_from_u64(RAND_SEED); - let mut ret = Vec::with_capacity(num); - - for _ in 0..num { - let tmp = rng.gen_range(0, len); - let idx = idx_list.remove(tmp); - ret.push(keys[idx].clone()); - len -= 1; - } - - ret.into_iter() -} - -#[derive(Debug, Display, From)] -pub enum RocksTrieDBError { - #[display(fmt = "store error")] - Store, - - #[display(fmt = "rocksdb {}", _0)] - RocksDB(rocksdb::Error), - - #[display(fmt = "parameters do not match")] - InsertParameter, - - #[display(fmt = "batch length do not match")] - BatchLengthMismatch, - - #[display(fmt = "Create DB path {}", _0)] - CreateDB(io::Error), -} - -impl std::error::Error for RocksTrieDBError {} - -impl From for ProtocolError { - fn from(err: RocksTrieDBError) -> ProtocolError { - ProtocolError::new(ProtocolErrorKind::Executor, Box::new(err)) - } -} - -fn to_store_err(e: rocksdb::Error) -> RocksTrieDBError { - log::error!("[executor] trie db {:?}", e); - RocksTrieDBError::Store -} - -#[cfg(test)] -mod tests { - use protocol::rand::random; - use trie::DB; - - use super::*; - - fn rand_bytes(len: usize) -> Vec { - (0..len).map(|_| random()).collect() - } - - #[test] - fn test_rand_remove() { - let list = (0..10).collect::>(); - let keys = list.iter().collect::>(); - - for num in 1..10 { - let res = rand_remove_list(keys.clone(), num); - assert_eq!(res.count(), num); - } - } - - #[test] - fn test_trie_insert() { - let key_1 = rand_bytes(32); - let val_1 = rand_bytes(128); - let key_2 = rand_bytes(32); - let val_2 = rand_bytes(256); - - let dir = tempfile::tempdir().unwrap(); - let trie = RocksTrieDB::new(dir.path(), Default::default(), 100).unwrap(); - - trie.insert(key_1.clone(), val_1.clone()).unwrap(); - trie.insert(key_2.clone(), val_2.clone()).unwrap(); - - let get_1 = trie.get(&key_1).unwrap(); - assert_eq!(val_1, get_1.unwrap()); - - let get_2 = trie.get(&key_2).unwrap(); - assert_eq!(val_2, get_2.unwrap()); - - let val_3 = rand_bytes(256); - trie.insert(key_1.clone(), val_3.clone()).unwrap(); - let get_3 = trie.get(&key_1).unwrap(); - assert_eq!(val_3, get_3.unwrap()); - - dir.close().unwrap(); - } - - #[test] - fn test_trie_cache() { - let key_1 = rand_bytes(32); - let val_1 = rand_bytes(128); - let key_2 = rand_bytes(32); - let val_2 = rand_bytes(256); - - let dir = tempfile::tempdir().unwrap(); - let trie = RocksTrieDB::new(dir.path(), Default::default(), 100).unwrap(); - - trie.insert(key_1.clone(), val_1.clone()).unwrap(); - trie.insert(key_2.clone(), val_2.clone()).unwrap(); - - let get_1 = trie.get(&key_1).unwrap(); - assert_eq!(val_1, get_1.unwrap()); - assert_eq!(trie.cache_len(), 2); - - let get_2 = trie.get(&key_2).unwrap(); - assert_eq!(val_2, get_2.unwrap()); - assert_eq!(trie.cache_len(), 2); - - let get_1 = trie.cache_get(&key_1).unwrap(); - assert_eq!(val_1, get_1); - - let val_3 = rand_bytes(256); - trie.insert(key_1.clone(), val_3.clone()).unwrap(); - let get_3 = trie.cache_get(&key_1).unwrap(); - assert_eq!(val_3, get_3); - assert_eq!(trie.cache_len(), 2); - - let val = rand_bytes(16); - for _i in 0..198 { - let key = rand_bytes(128); - trie.insert(key, val.clone()).unwrap(); - } - - assert_eq!(trie.cache_len(), 200); - - trie.insert(rand_bytes(256), rand_bytes(8)).unwrap(); - assert_eq!(trie.cache_len(), 100); - - dir.close().unwrap(); - } -} diff --git a/core/executor/src/debugger/mod.rs b/core/executor/src/debugger/mod.rs index e00732689..1359c3358 100644 --- a/core/executor/src/debugger/mod.rs +++ b/core/executor/src/debugger/mod.rs @@ -18,7 +18,8 @@ use protocol::types::{ MAX_BLOCK_GAS_LIMIT, NIL_DATA, RLP_NULL, U256, }; -use core_storage::{adapter::rocks::RocksAdapter, ImplStorage}; +use core_db::RocksAdapter; +use core_storage::ImplStorage; use crate::adapter::{AxonExecutorApplyAdapter, MPTTrie}; use crate::{AxonExecutor, RocksTrieDB}; @@ -44,7 +45,8 @@ impl EvmDebugger { let mut db_state_path = db_path.to_string(); db_state_path.push_str("/state"); let _ = std::fs::create_dir_all(&db_state_path); - let trie = Arc::new(RocksTrieDB::new(db_state_path, Default::default(), 1000).unwrap()); + let inner_db = rocks_adapter.inner_db(); + let trie = Arc::new(RocksTrieDB::new_evm(inner_db, 1000)); let mut mpt = MPTTrie::new(Arc::clone(&trie)); diff --git a/core/executor/src/system_contract/mod.rs b/core/executor/src/system_contract/mod.rs index a6a365fff..e6433a203 100644 --- a/core/executor/src/system_contract/mod.rs +++ b/core/executor/src/system_contract/mod.rs @@ -17,7 +17,6 @@ pub use crate::system_contract::native_token::{ NativeTokenContract, NATIVE_TOKEN_CONTRACT_ADDRESS, }; -use std::path::Path; use std::sync::Arc; use ckb_traits::{CellDataProvider, HeaderProvider}; @@ -26,14 +25,16 @@ use ckb_types::core::{HeaderBuilder, HeaderView}; use ckb_types::{packed, prelude::*}; use evm::backend::ApplyBackend; use parking_lot::RwLock; +use rocksdb::DB; -use common_config_parser::types::ConfigRocksDB; use protocol::types::{Bytes, Hasher, SignedTransaction, TxResp, H160, H256}; use protocol::{ckb_blake2b_256, traits::ExecutorAdapter}; use crate::adapter::RocksTrieDB; -use crate::system_contract::utils::generate_mpt_root_changes; -use crate::system_contract::{ckb_light_client::CkbHeaderReader, image_cell::ImageCellReader}; +use crate::system_contract::{ + ckb_light_client::CkbHeaderReader, image_cell::ImageCellReader, + utils::generate_mpt_root_changes, +}; pub const fn system_contract_address(addr: u8) -> H160 { H160([ @@ -103,30 +104,27 @@ pub fn swap_header_cell_db(new_db: Arc) -> Arc { .unwrap_or_else(|| panic!("header cell db is not initialized")) } -pub fn init, Adapter: ExecutorAdapter + ApplyBackend>( - path: P, - config: ConfigRocksDB, +pub fn init( + db: Arc, adapter: &mut Adapter, ) -> (H256, H256) { let current_metadata_root = adapter.storage(METADATA_CONTRACT_ADDRESS, *METADATA_ROOT_KEY); // Init metadata db. - let metadata_db_path = path.as_ref().join("metadata"); { - let mut db = METADATA_DB.write(); - db.replace(Arc::new( - RocksTrieDB::new(metadata_db_path, config.clone(), METADATA_DB_CACHE_SIZE) - .expect("[system contract] metadata new rocksdb error"), - )); + let mut _db = METADATA_DB.write(); + _db.replace(Arc::new(RocksTrieDB::new_metadata( + Arc::clone(&db), + METADATA_DB_CACHE_SIZE, + ))); } - let header_cell_db_path = path.as_ref().join("header_cell"); { - let mut db = HEADER_CELL_DB.write(); - db.replace(Arc::new( - RocksTrieDB::new(header_cell_db_path, config, HEADER_CELL_DB_CACHE_SIZE) - .expect("[system contract] header&cell new rocksdb error"), - )); + let mut _db = HEADER_CELL_DB.write(); + _db.replace(Arc::new(RocksTrieDB::new_ckb_light_client( + db, + HEADER_CELL_DB_CACHE_SIZE, + ))); } let current_cell_root = diff --git a/core/executor/src/tests/mod.rs b/core/executor/src/tests/mod.rs index 08be1ab27..ae926a2d5 100644 --- a/core/executor/src/tests/mod.rs +++ b/core/executor/src/tests/mod.rs @@ -4,7 +4,6 @@ use std::collections::BTreeMap; use std::str::FromStr; use std::sync::Arc; -use core_storage::adapter::memory::MemoryAdapter; use evm::backend::{MemoryAccount, MemoryVicinity}; use evm::Config; @@ -15,6 +14,7 @@ use protocol::types::{ }; use protocol::{codec::hex_decode, tokio, traits::Executor, trie::MemoryDB}; +use core_db::MemoryAdapter; use core_storage::ImplStorage; use crate::AxonExecutorApplyAdapter; diff --git a/core/executor/src/tests/system_script/ckb_light_client.rs b/core/executor/src/tests/system_script/ckb_light_client.rs index 3a698d119..eef11607a 100644 --- a/core/executor/src/tests/system_script/ckb_light_client.rs +++ b/core/executor/src/tests/system_script/ckb_light_client.rs @@ -3,7 +3,7 @@ use std::str::FromStr; use ethers::abi::AbiEncode; -use common_config_parser::types::ConfigRocksDB; +use core_db::RocksAdapter; use protocol::types::{Backend, MemoryBackend, TxResp, H160, H256, U256}; use crate::system_contract::ckb_light_client::{ @@ -22,7 +22,10 @@ pub fn test_write_functions() { let mut backend = MemoryBackend::new(&vicinity, BTreeMap::new()); let executor = CkbLightClientContract::default(); - init(ROCKSDB_PATH, ConfigRocksDB::default(), &mut backend); + let inner_db = RocksAdapter::new(ROCKSDB_PATH, Default::default()) + .unwrap() + .inner_db(); + init(inner_db, &mut backend); // need to refactor to be OO test_update_first(&mut backend, &executor); diff --git a/core/executor/src/tests/system_script/image_cell.rs b/core/executor/src/tests/system_script/image_cell.rs index ed12ca9f0..fe1488ce1 100644 --- a/core/executor/src/tests/system_script/image_cell.rs +++ b/core/executor/src/tests/system_script/image_cell.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use ckb_types::{bytes::Bytes, packed, prelude::*}; use ethers::abi::AbiEncode; -use common_config_parser::types::ConfigRocksDB; +use core_db::RocksAdapter; use protocol::types::{Backend, MemoryBackend, TxResp, H160, U256}; use crate::system_contract::image_cell::{ @@ -24,7 +24,10 @@ pub fn test_write_functions() { let mut backend = MemoryBackend::new(&vicinity, BTreeMap::new()); let executor = ImageCellContract::default(); - let (m_root, h_root) = init(ROCKSDB_PATH, ConfigRocksDB::default(), &mut backend); + let inner_db = RocksAdapter::new(ROCKSDB_PATH, Default::default()) + .unwrap() + .inner_db(); + let (m_root, h_root) = init(inner_db, &mut backend); CURRENT_METADATA_ROOT.with(|r| *r.borrow_mut() = m_root); CURRENT_HEADER_CELL_ROOT.with(|r| *r.borrow_mut() = h_root); diff --git a/core/executor/src/tests/system_script/metadata.rs b/core/executor/src/tests/system_script/metadata.rs index d1c6b06f5..d67f90ee2 100644 --- a/core/executor/src/tests/system_script/metadata.rs +++ b/core/executor/src/tests/system_script/metadata.rs @@ -1,7 +1,8 @@ use std::{collections::BTreeMap, str::FromStr}; -use common_config_parser::types::ConfigRocksDB; use ethers::abi::AbiEncode; + +use core_db::RocksAdapter; use protocol::types::{MemoryBackend, SignedTransaction, H160, U256}; use crate::{ @@ -24,8 +25,10 @@ fn test_write_functions() { let mut backend = MemoryBackend::new(&vicinity, BTreeMap::new()); let executor = MetadataContract::default(); - - init(ROCKSDB_PATH, ConfigRocksDB::default(), &mut backend); + let inner_db = RocksAdapter::new(ROCKSDB_PATH, Default::default()) + .unwrap() + .inner_db(); + init(inner_db, &mut backend); test_init(&mut backend, &executor); diff --git a/core/interoperation/Cargo.toml b/core/interoperation/Cargo.toml index 49883ab77..40574f3fa 100644 --- a/core/interoperation/Cargo.toml +++ b/core/interoperation/Cargo.toml @@ -28,3 +28,4 @@ serde_json = "1.0" core-rpc-client = { path = "../../core/rpc-client" } core-executor = { path = "../../core/executor" } core-storage = { path = "../../core/storage" } +core-db = { path = "../../core/db" } diff --git a/core/interoperation/src/tests/mod.rs b/core/interoperation/src/tests/mod.rs index 5a0ac45eb..3832c2a73 100644 --- a/core/interoperation/src/tests/mod.rs +++ b/core/interoperation/src/tests/mod.rs @@ -1,14 +1,9 @@ mod verify_in_mempool; + use std::fs::File; use std::io::BufReader; use std::sync::Arc; -use core_executor::adapter::{MPTTrie, RocksTrieDB}; -use core_executor::{ - system_contract::system_contract_address, AxonExecutor, AxonExecutorApplyAdapter, -}; -use core_rpc_client::RpcClient; -use core_storage::{adapter::rocks::RocksAdapter, ImplStorage}; use protocol::codec::ProtocolCodec; use protocol::traits::{Context, Executor, Storage}; use protocol::types::{ @@ -17,6 +12,14 @@ use protocol::types::{ UnverifiedTransaction, H256, NIL_DATA, RLP_NULL, U256, }; +use core_db::RocksAdapter; +use core_executor::adapter::{MPTTrie, RocksTrieDB}; +use core_executor::{ + system_contract::system_contract_address, AxonExecutor, AxonExecutorApplyAdapter, +}; +use core_rpc_client::RpcClient; +use core_storage::ImplStorage; + const GENESIS_PATH: &str = "../../tests/data/genesis.json"; lazy_static::lazy_static! { @@ -37,12 +40,9 @@ impl TestHandle { Default::default(), ) .unwrap(); - let trie_db = RocksTrieDB::new( - path.clone() + &salt.to_string() + "/trie", - Default::default(), - 50, - ) - .unwrap(); + let inner_db = storage_adapter.inner_db(); + + let trie_db = RocksTrieDB::new_evm(Arc::clone(&inner_db), 100); let mut handle = TestHandle { storage: Arc::new(ImplStorage::new(Arc::new(storage_adapter), 10)), @@ -59,11 +59,7 @@ impl TestHandle { ) .unwrap(); - core_executor::system_contract::init( - path + &salt.to_string() + "/sc", - Default::default(), - &mut backend, - ); + core_executor::system_contract::init(inner_db, &mut backend); handle } diff --git a/core/mempool/src/adapter/mod.rs b/core/mempool/src/adapter/mod.rs index fa8cd0056..de5e12f98 100644 --- a/core/mempool/src/adapter/mod.rs +++ b/core/mempool/src/adapter/mod.rs @@ -354,7 +354,7 @@ where C: Crypto + Send + Sync + 'static, N: Rpc + PeerTrust + Gossip + Clone + Unpin + 'static, S: ReadOnlyStorage + 'static, - DB: trie::DB + 'static, + DB: trie::DB + Send + Sync + 'static, I: Interoperation + 'static, { #[trace_span(kind = "mempool.adapter", logs = "{txs_len: tx_hashes.len()}")] diff --git a/core/run/Cargo.toml b/core/run/Cargo.toml index 92f73745c..6e21037f2 100644 --- a/core/run/Cargo.toml +++ b/core/run/Cargo.toml @@ -23,6 +23,7 @@ common-logger = { path = "../../common/logger" } common-memory-tracker = { path = "../../common/memory-tracker", optional = true } core-api = { path = "../../core/api" } core-consensus = { path = "../../core/consensus" } +core-db = { path = "../../core/db" } core-executor = { path = "../../core/executor" } core-interoperation = { path = "../../core/interoperation" } core-mempool = { path = "../../core/mempool" } diff --git a/core/run/src/lib.rs b/core/run/src/lib.rs index 999549e93..c5dd3c99f 100644 --- a/core/run/src/lib.rs +++ b/core/run/src/lib.rs @@ -1,6 +1,6 @@ #![allow(clippy::uninlined_format_args, clippy::mutable_key_type)] -use std::{collections::HashMap, panic::PanicInfo, path::PathBuf, sync::Arc, time::Duration}; +use std::{collections::HashMap, panic::PanicInfo, sync::Arc, time::Duration}; use backtrace::Backtrace; #[cfg(all( @@ -15,17 +15,14 @@ use { use common_apm::metrics::mempool::{MEMPOOL_CO_QUEUE_LEN, MEMPOOL_LEN_GAUGE}; use common_apm::{server::run_prometheus_server, tracing::global_tracer_register}; -use common_config_parser::types::{ - spec::{ChainSpec, InitialAccount}, - Config, ConfigJaeger, ConfigPrometheus, ConfigRocksDB, -}; +use common_config_parser::types::spec::{ChainSpec, InitialAccount}; +use common_config_parser::types::{Config, ConfigJaeger, ConfigPrometheus}; use common_crypto::{ BlsPrivateKey, BlsPublicKey, PublicKey, Secp256k1, Secp256k1PrivateKey, Secp256k1PublicKey, ToPublicKey, UncompressedPublicKey, }; use protocol::codec::{hex_decode, ProtocolCodec}; -use protocol::lazy::CHAIN_ID; #[cfg(unix)] use protocol::tokio::signal::unix as os_impl; use protocol::tokio::{ @@ -40,8 +37,8 @@ use protocol::types::{ SignedTransaction, Validator, ValidatorExtend, H256, NIL_DATA, RLP_NULL, }; use protocol::{ - async_trait, trie::DB as TrieDB, Display, From, ProtocolError, ProtocolErrorKind, - ProtocolResult, + async_trait, lazy::CHAIN_ID, trie::DB as TrieDB, Display, From, ProtocolError, + ProtocolErrorKind, ProtocolResult, }; use core_api::{jsonrpc::run_jsonrpc_server, DefaultAPIAdapter}; @@ -58,6 +55,7 @@ use core_consensus::{ util::OverlordCrypto, ConsensusWal, DurationConfig, OverlordConsensus, OverlordConsensusAdapter, OverlordSynchronization, SignedTxsWAL, }; +use core_db::{RocksAdapter, RocksDB}; use core_executor::system_contract::{self, metadata::MetadataHandle}; use core_executor::{ AxonExecutor, AxonExecutorApplyAdapter, AxonExecutorReadOnlyAdapter, MPTTrie, RocksTrieDB, @@ -70,7 +68,7 @@ use core_mempool::{ use core_network::{ observe_listen_port_occupancy, NetworkConfig, NetworkService, PeerId, PeerIdExt, SecioError, }; -use core_storage::{adapter::rocks::RocksAdapter, ImplStorage}; +use core_storage::ImplStorage; pub use core_network::{KeyProvider, SecioKeyPair}; @@ -120,15 +118,15 @@ impl Axon { rt.block_on(async move { // TODO Introduce a read-only rocksdb handler for checks. - let storage = self.init_storage(false).await?; + let (storage, inner_db) = self.init_storage(false).await?; if let Some(genesis) = self.try_load_genesis(&storage).await? { log::info!("The Genesis block has been initialized."); self.apply_genesis_after_checks(&genesis).await?; } else { - self.create_genesis(&storage).await?; + self.create_genesis(Arc::clone(&inner_db), &storage).await?; } - drop(storage); - self.start(key_provider).await + + self.start(key_provider, storage, inner_db).await })?; rt.shutdown_timeout(std::time::Duration::from_secs(1)); @@ -150,23 +148,22 @@ impl Axon { async fn create_genesis( &mut self, + inner_db: Arc, storage: &Arc>, ) -> ProtocolResult<()> { - let trie_db = self.init_trie_db(false).await?; + let trie_db = self.init_evm_trie_db(Arc::clone(&inner_db)); let state_root = { let mut mpt = MPTTrie::new(Arc::clone(&trie_db)); insert_accounts(&mut mpt, &self.spec.accounts).expect("insert accounts"); mpt.commit()? }; - let path_metadata = self.config.data_path_for_system_contract(); let resp = execute_transactions( &self.genesis, state_root, - &trie_db, + inner_db, storage, - path_metadata, - &self.config.rocksdb.clone(), + self.config.executor.triedb_cache_size, )?; log::info!( @@ -187,54 +184,44 @@ impl Axon { async fn apply_genesis_after_checks(&mut self, loaded_genesis: &Block) -> ProtocolResult<()> { let tmp_dir = tempfile::tempdir().map_err(|err| { - let errmsg = format!("failed to create temporary directory since {err:?}"); - MainError::Other(errmsg) + let err_msg = format!("failed to create temporary directory since {err:?}"); + MainError::Other(err_msg) })?; + let path_block = tmp_dir.path().join("block"); + let rocks_adapter = Arc::new(RocksAdapter::new(path_block, self.config.rocksdb.clone())?); + let inner_db = rocks_adapter.inner_db(); - let storage = { - let path_block = tmp_dir.path().join("block"); - let rocks_adapter = - Arc::new(RocksAdapter::new(path_block, self.config.rocksdb.clone())?); - let impl_storage = ImplStorage::new(rocks_adapter, self.config.rocksdb.cache_size); - Arc::new(impl_storage) - }; + let storage = Arc::new(ImplStorage::new( + rocks_adapter, + self.config.rocksdb.cache_size, + )); + let trie_db = self.init_evm_trie_db(Arc::clone(&inner_db)); - let trie_db = { - let path_state = tmp_dir.path().join("state"); - let trie_db = RocksTrieDB::new( - path_state, - self.config.rocksdb.clone(), - self.config.executor.triedb_cache_size, - )?; - Arc::new(trie_db) - }; let state_root = { let mut mpt = MPTTrie::new(Arc::clone(&trie_db)); insert_accounts(&mut mpt, &self.spec.accounts).expect("insert accounts"); mpt.commit()? }; - let path_metadata = tmp_dir.path().join("metadata"); let resp = execute_transactions( &self.genesis, state_root, - &trie_db, + inner_db, &storage, - path_metadata, - &self.config.rocksdb.clone(), + self.config.executor.triedb_cache_size, )?; self.apply_genesis_data(resp.state_root, resp.receipt_root)?; let user_provided_genesis = &self.genesis.block; if user_provided_genesis != loaded_genesis { - let errmsg = format!( + let err_msg = format!( "The user provided genesis (hash: {:#x}) is NOT \ the same as the genesis in storage (hash: {:#x})", user_provided_genesis.hash(), loaded_genesis.hash() ); - return Err(MainError::Other(errmsg).into()); + return Err(MainError::Other(err_msg).into()); } Ok(()) @@ -264,7 +251,12 @@ impl Axon { Ok(()) } - pub async fn start(self, key_provider: Option) -> ProtocolResult<()> { + pub async fn start( + self, + key_provider: Option, + storage: Arc>, + inner_db: Arc, + ) -> ProtocolResult<()> { // Start jaeger Self::run_jaeger(self.config.jaeger.clone()); @@ -282,7 +274,6 @@ impl Axon { observe_listen_port_occupancy(&[self.config.network.listening_address.clone()]).await?; // Init Block db and get the current block - let storage = self.init_storage(true).await?; let current_block = storage.get_latest_block(Context::new()).await?; let current_state_root = current_block.header.state_root; @@ -292,7 +283,7 @@ impl Axon { let mut network_service = self.init_network_service(key_provider); // Init trie db - let trie_db = self.init_trie_db(true).await?; + let trie_db = self.init_evm_trie_db(Arc::clone(&inner_db)); // Init full transactions wal let txs_wal_path = self @@ -305,7 +296,7 @@ impl Axon { // Init system contract if current_block.header.number != 0 { - self.init_system_contract(&trie_db, ¤t_block, &storage); + self.init_system_contract(Arc::clone(&inner_db), ¤t_block, &storage); } // Init mempool and recover signed transactions with the current block number @@ -413,9 +404,18 @@ impl Axon { async fn init_storage( &self, _run_service: bool, - ) -> ProtocolResult>> { - let path_block = self.config.data_path_for_block(); - let rocks_adapter = Arc::new(RocksAdapter::new(path_block, self.config.rocksdb.clone())?); + ) -> ProtocolResult<(Arc>, Arc)> { + let rocksdb_path = self.config.data_path_for_rocksdb(); + let rocks_adapter = Arc::new(RocksAdapter::new( + rocksdb_path, + self.config.rocksdb.clone(), + )?); + let inner_db = rocks_adapter.inner_db(); + let storage = Arc::new(ImplStorage::new( + rocks_adapter, + self.config.rocksdb.cache_size, + )); + #[cfg(all( not(target_env = "msvc"), not(target_os = "macos"), @@ -424,11 +424,11 @@ impl Axon { if _run_service { tokio::spawn(common_memory_tracker::track_db_process( "blockdb", - rocks_adapter.inner_db(), + Arc::clone(&inner_db), )); } - let impl_storage = ImplStorage::new(rocks_adapter, self.config.rocksdb.cache_size); - Ok(Arc::new(impl_storage)) + + Ok((storage, inner_db)) } fn init_network_service( @@ -446,25 +446,11 @@ impl Axon { network_service } - async fn init_trie_db(&self, _run_service: bool) -> ProtocolResult> { - let path_state = self.config.data_path_for_state(); - let trie_db = Arc::new(RocksTrieDB::new( - path_state, - self.config.rocksdb.clone(), + fn init_evm_trie_db(&self, inner_db: Arc) -> Arc { + Arc::new(RocksTrieDB::new_evm( + inner_db, self.config.executor.triedb_cache_size, - )?); - #[cfg(all( - not(target_env = "msvc"), - not(target_os = "macos"), - feature = "jemalloc" - ))] - if _run_service { - tokio::spawn(common_memory_tracker::track_db_process( - "triedb", - trie_db.inner_db(), - )); - } - Ok(trie_db) + )) } async fn init_mempool( @@ -477,7 +463,7 @@ impl Axon { where N: Rpc + PeerTrust + Gossip + Clone + Unpin + 'static, S: Storage + 'static, - DB: TrieDB + 'static, + DB: TrieDB + Send + Sync + 'static, { let mempool_adapter = DefaultMemPoolAdapter::::new( network_service.clone(), @@ -515,24 +501,19 @@ impl Axon { fn init_system_contract( &self, - trie_db: &Arc, + inner_db: Arc, current_block: &Block, storage: &Arc>, ) -> (H256, H256) { - let path_system_contract = self.config.data_path_for_system_contract(); let mut backend = AxonExecutorApplyAdapter::from_root( current_block.header.state_root, - Arc::clone(trie_db), + self.init_evm_trie_db(Arc::clone(&inner_db)), Arc::clone(storage), Proposal::new_without_state_root(¤t_block.header).into(), ) .unwrap(); - system_contract::init( - path_system_contract, - self.config.rocksdb.clone(), - &mut backend, - ) + system_contract::init(inner_db, &mut backend) } fn get_my_pubkey(&self, hex_privkey: Vec) -> Secp256k1PublicKey { @@ -615,7 +596,7 @@ impl Axon { M: MemPool + 'static, N: Rpc + PeerTrust + Gossip + Network + 'static, S: Storage + 'static, - DB: TrieDB + 'static, + DB: TrieDB + Send + Sync + 'static, { let consensus_wal_path = self .config @@ -811,7 +792,7 @@ impl Axon { M: MemPool, N: Rpc + PeerTrust + Gossip + Network + 'static, S: Storage, - DB: TrieDB, + DB: TrieDB + Send + Sync, { let timer_config = DurationConfig { propose_ratio: metadata.propose_ratio, @@ -1025,27 +1006,26 @@ fn insert_accounts( Ok(()) } -fn execute_transactions( +fn execute_transactions( rich: &RichBlock, state_root: H256, - trie_db: &Arc, + inner_db: Arc, storage: &Arc, - path_metadata: PathBuf, - rocksdb: &ConfigRocksDB, + cache_size: usize, ) -> ProtocolResult where S: Storage + 'static, - DB: TrieDB + 'static, { let executor = AxonExecutor::default(); + let trie_db = Arc::new(RocksTrieDB::new_evm(Arc::clone(&inner_db), cache_size)); let mut backend = AxonExecutorApplyAdapter::from_root( state_root, - Arc::clone(trie_db), + trie_db, Arc::clone(storage), Proposal::new_without_state_root(&rich.block.header).into(), )?; - system_contract::init(path_metadata, rocksdb.clone(), &mut backend); + system_contract::init(inner_db, &mut backend); let resp = executor.exec(&mut backend, &rich.txs, &[]); diff --git a/core/run/src/tests.rs b/core/run/src/tests.rs index 9a35f307f..4db5d5de2 100644 --- a/core/run/src/tests.rs +++ b/core/run/src/tests.rs @@ -13,14 +13,16 @@ use common_config_parser::types::{ spec::{ChainSpec, ChainSpecValueParser}, Config, ConfigValueParser, }; -use core_executor::{MPTTrie, RocksTrieDB}; -use core_storage::{adapter::rocks::RocksAdapter, ImplStorage}; use protocol::{ codec::hex_decode, tokio, types::{RichBlock, H256}, }; +use core_db::RocksAdapter; +use core_executor::{MPTTrie, RocksTrieDB}; +use core_storage::ImplStorage; + use crate::{execute_transactions, insert_accounts}; const DEV_CONFIG_DIR: &str = "../../devtools/chain"; @@ -131,26 +133,17 @@ async fn check_genesis_data<'a>(case: &TestCase<'a>) { "check hash of tx[{i}], cached: {tx_cached:#x}, calculated: {calculated:#x}", ); } + let path_block = tmp_dir.path().join("block"); + let rocks_adapter = Arc::new( + RocksAdapter::new(path_block, config.rocksdb.clone()).expect("temporary block storage"), + ); + let inner_db = rocks_adapter.inner_db(); - let storage = { - let path_block = tmp_dir.path().join("block"); - let rocks_adapter = Arc::new( - RocksAdapter::new(path_block, config.rocksdb.clone()).expect("temporary block storage"), - ); - let impl_storage = ImplStorage::new(rocks_adapter, config.rocksdb.cache_size); - Arc::new(impl_storage) - }; - - let trie_db = { - let path_state = tmp_dir.path().join("state"); - let trie_db = RocksTrieDB::new( - path_state, - config.rocksdb.clone(), - config.executor.triedb_cache_size, - ) - .expect("temporary trie db"); - Arc::new(trie_db) - }; + let storage = Arc::new(ImplStorage::new(rocks_adapter, config.rocksdb.cache_size)); + let trie_db = Arc::new(RocksTrieDB::new_evm( + Arc::clone(&inner_db), + config.executor.triedb_cache_size, + )); let state_root = { let mut mpt = MPTTrie::new(Arc::clone(&trie_db)); @@ -158,14 +151,12 @@ async fn check_genesis_data<'a>(case: &TestCase<'a>) { mpt.commit().expect("mpt commit") }; - let path_metadata = tmp_dir_path.join("metadata"); let resp = execute_transactions( &genesis, state_root, - &trie_db, + inner_db, &storage, - path_metadata, - &config.rocksdb, + config.executor.triedb_cache_size, ) .expect("execute transactions"); diff --git a/core/storage/Cargo.toml b/core/storage/Cargo.toml index 22c3854ce..08778bfae 100644 --- a/core/storage/Cargo.toml +++ b/core/storage/Cargo.toml @@ -11,11 +11,10 @@ lazy_static = "1.4" log = "0.4" lru = "0.11" parking_lot = "0.12" -rocksdb = { version = "0.20", package = "ckb-rocksdb" } common-apm = { path = "../../common/apm" } common-apm-derive = { path = "../../common/apm-derive" } -common-config-parser = { path = "../../common/config-parser" } +core-db ={ path = "../../core/db" } protocol = { path = "../../protocol", package = "axon-protocol" } [dev-dependencies] diff --git a/core/storage/benches/bench.rs b/core/storage/benches/bench.rs index 4e101a452..6e7598863 100644 --- a/core/storage/benches/bench.rs +++ b/core/storage/benches/bench.rs @@ -5,7 +5,6 @@ use criterion::{criterion_group, criterion_main, Criterion}; use common_crypto::{ Crypto, PrivateKey, Secp256k1Recoverable, Secp256k1RecoverablePrivateKey, Signature, }; -use core_storage::ImplStorage; use protocol::rand::{random, rngs::OsRng}; use protocol::traits::{Context, Storage}; use protocol::types::{ @@ -13,7 +12,8 @@ use protocol::types::{ SignedTransaction, TransactionAction, UnsignedTransaction, UnverifiedTransaction, }; -use core_storage::adapter::memory::MemoryAdapter; +use core_db::MemoryAdapter; +use core_storage::ImplStorage; macro_rules! exec { ($func: expr) => { diff --git a/core/storage/src/adapter/mod.rs b/core/storage/src/adapter/mod.rs deleted file mode 100644 index 3a0dc7d9d..000000000 --- a/core/storage/src/adapter/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod memory; -pub mod rocks; diff --git a/core/storage/src/lib.rs b/core/storage/src/lib.rs index 51b423475..43975a2e6 100644 --- a/core/storage/src/lib.rs +++ b/core/storage/src/lib.rs @@ -1,6 +1,5 @@ #![allow(clippy::mutable_key_type, clippy::uninlined_format_args)] -pub mod adapter; mod cache; mod hash_key; mod schema; diff --git a/core/storage/src/tests/adapter.rs b/core/storage/src/tests/adapter.rs index dddddb88c..cf0e6802d 100644 --- a/core/storage/src/tests/adapter.rs +++ b/core/storage/src/tests/adapter.rs @@ -1,9 +1,7 @@ +use core_db::{MemoryAdapter, RocksAdapter}; use protocol::traits::{StorageAdapter, StorageBatchModify}; -use crate::adapter::memory::MemoryAdapter; -use crate::adapter::rocks::RocksAdapter; -use crate::tests::mock_signed_tx; -use crate::{CommonHashKey, TransactionSchema}; +use crate::{tests::mock_signed_tx, CommonHashKey, TransactionSchema}; #[test] fn test_adapter_insert() { diff --git a/core/storage/src/tests/storage.rs b/core/storage/src/tests/storage.rs index d04d0ccab..c96fd3b83 100644 --- a/core/storage/src/tests/storage.rs +++ b/core/storage/src/tests/storage.rs @@ -3,7 +3,8 @@ use std::sync::Arc; use protocol::traits::{Context, ReadOnlyStorage, Storage}; use protocol::types::Hasher; -use crate::adapter::memory::MemoryAdapter; +use core_db::MemoryAdapter; + use crate::tests::{get_random_bytes, mock_block, mock_proof, mock_receipt, mock_signed_tx}; use crate::ImplStorage; diff --git a/protocol/Cargo.toml b/protocol/Cargo.toml index ad83758da..9b0796022 100644 --- a/protocol/Cargo.toml +++ b/protocol/Cargo.toml @@ -24,13 +24,13 @@ lazy_static = "1.4" ophelia = "0.3" ophelia-secp256k1 = "0.3" overlord = "0.4" -prost = "0.11" rand = "0.7" rlp = "0.5" rlp-derive = "0.1" serde = { version = "1.0", features = ["derive"] } -tokio = { version = "1.31", features = ["full"] } -trie = { package = "cita_trie", version = "4.0" } +thiserror = "1.0" +tokio = { version = "1.32", features = ["full"] } +trie = { package = "cita_trie", git = "https://github.com/KaoImin/cita-trie.git", rev = "eea569c", version = "4.1" } common-crypto = { path = "../common/crypto" } common-hasher = { path = "../common/hasher" } diff --git a/protocol/src/lib.rs b/protocol/src/lib.rs index 9c248924f..c12e4f2f0 100644 --- a/protocol/src/lib.rs +++ b/protocol/src/lib.rs @@ -13,9 +13,12 @@ pub mod types; use std::error::Error; pub use derive_more::{Constructor, Display, From}; -pub use {async_trait::async_trait, ckb_hash::blake2b_256 as ckb_blake2b_256, rand, tokio, trie}; +pub use { + async_trait::async_trait, ckb_hash::blake2b_256 as ckb_blake2b_256, rand, thiserror, tokio, + trie, +}; -#[derive(Debug, Clone)] +#[derive(Copy, Clone, Debug)] pub enum ProtocolErrorKind { // traits API, @@ -24,6 +27,7 @@ pub enum ProtocolErrorKind { Consensus, Contract, CrossChain, + DB, Executor, Interoperation, Mempool, diff --git a/protocol/src/traits/mod.rs b/protocol/src/traits/mod.rs index e3cdd5abd..9251d43f6 100644 --- a/protocol/src/traits/mod.rs +++ b/protocol/src/traits/mod.rs @@ -21,6 +21,6 @@ pub use network::{ Gossip, MessageCodec, MessageHandler, Network, PeerTag, PeerTrust, Priority, Rpc, TrustFeedback, }; pub use storage::{ - IntoIteratorByRef, ReadOnlyStorage, Storage, StorageAdapter, StorageBatchModify, - StorageCategory, StorageIterator, StorageSchema, + IntoIteratorByRef, ReadOnlyStorage, StateStorageCategory, Storage, StorageAdapter, + StorageBatchModify, StorageCategory, StorageIterator, StorageSchema, }; diff --git a/protocol/src/traits/storage.rs b/protocol/src/traits/storage.rs index bf9debd11..7b17b3628 100644 --- a/protocol/src/traits/storage.rs +++ b/protocol/src/traits/storage.rs @@ -1,7 +1,7 @@ use crate::types::{Block, Bytes, Hash, Header, Proof, Receipt, SignedTransaction, H256}; use crate::{async_trait, codec::ProtocolCodec, traits::Context, Display, ProtocolResult}; -#[derive(Debug, Copy, Clone, Display)] +#[derive(Copy, Clone, Display, Debug)] pub enum StorageCategory { Block, BlockHeader, @@ -10,6 +10,26 @@ pub enum StorageCategory { Wal, HashHeight, Code, + EvmState, + MetadataState, + CkbLightClientState, +} + +#[derive(Copy, Clone, Debug)] +pub enum StateStorageCategory { + EvmState, + MetadataState, + CkbLightClientState, +} + +impl From for StorageCategory { + fn from(value: StateStorageCategory) -> Self { + match value { + StateStorageCategory::EvmState => StorageCategory::EvmState, + StateStorageCategory::MetadataState => StorageCategory::MetadataState, + StateStorageCategory::CkbLightClientState => StorageCategory::CkbLightClientState, + } + } } pub type StorageIterator<'a, S> = Box< @@ -28,6 +48,10 @@ pub trait IntoIteratorByRef { fn ref_to_iter<'a, 'b: 'a>(&'b self) -> StorageIterator<'a, S>; } +pub trait TrieStorage: trie::DB { + fn category(&self) -> StateStorageCategory; +} + #[async_trait] pub trait ReadOnlyStorage: Sync + Send { async fn get_block(&self, ctx: Context, height: u64) -> ProtocolResult>; @@ -127,7 +151,7 @@ pub enum StorageBatchModify { Insert(::Value), } -pub trait StorageAdapter: Send + Sync { +pub trait StorageAdapter: Send + Sync + 'static { fn insert( &self, key: ::Key, diff --git a/protocol/src/types/primitive.rs b/protocol/src/types/primitive.rs index 949fb91ce..446222fbd 100644 --- a/protocol/src/types/primitive.rs +++ b/protocol/src/types/primitive.rs @@ -52,6 +52,12 @@ impl AsRef<[u8]> for DBBytes { } } +impl From> for DBBytes { + fn from(value: Vec) -> Self { + DBBytes(value.into()) + } +} + pub struct Hasher; impl Hasher { From 990a5ea6b9cc48f57a55e993a204508928f524c0 Mon Sep 17 00:00:00 2001 From: Eason Date: Tue, 22 Aug 2023 15:17:22 +0800 Subject: [PATCH 4/4] rebase main --- core/executor/src/adapter/backend/read_only.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/executor/src/adapter/backend/read_only.rs b/core/executor/src/adapter/backend/read_only.rs index 5159fd8da..361f3e78f 100644 --- a/core/executor/src/adapter/backend/read_only.rs +++ b/core/executor/src/adapter/backend/read_only.rs @@ -91,7 +91,7 @@ where } fn block_difficulty(&self) -> U256 { - self.exec_ctx.difficulty + U256::one() } fn block_gas_limit(&self) -> U256 {