From 28434d1dd47d0a862b61a227f12835f5a702037e Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 29 Aug 2019 12:09:42 +0200 Subject: [PATCH 01/21] Move snapshot to own crate Sort out imports --- Cargo.lock | 36 +++++++++++ Cargo.toml | 2 + ethcore/res/ethereum/foundation.json | 16 ++++- .../{src/snapshot => snapshot/src}/account.rs | 18 +++--- .../{src/snapshot => snapshot/src}/block.rs | 28 ++++----- .../src}/consensus/authority.rs | 18 +++--- .../src}/consensus/mod.rs | 4 +- .../src}/consensus/work.rs | 32 +++++----- ethcore/{src/snapshot => snapshot/src}/io.rs | 11 ++-- .../snapshot/mod.rs => snapshot/src/lib.rs} | 42 ++++++------- .../{src/snapshot => snapshot/src}/service.rs | 62 +++++++++---------- .../src}/tests/helpers.rs | 8 +-- .../snapshot => snapshot/src}/tests/mod.rs | 0 .../src}/tests/proof_of_authority.rs | 2 +- .../src}/tests/proof_of_work.rs | 4 +- .../src}/tests/service.rs | 4 +- .../snapshot => snapshot/src}/tests/state.rs | 8 +-- .../src}/tests/test_validator_contract.json | 0 .../{src/snapshot => snapshot/src}/traits.rs | 7 +-- .../{src/snapshot => snapshot/src}/watcher.rs | 14 ++--- ethcore/state-db/src/lib.rs | 6 +- 21 files changed, 187 insertions(+), 135 deletions(-) rename ethcore/{src/snapshot => snapshot/src}/account.rs (98%) rename ethcore/{src/snapshot => snapshot/src}/block.rs (94%) rename ethcore/{src/snapshot => snapshot/src}/consensus/authority.rs (99%) rename ethcore/{src/snapshot => snapshot/src}/consensus/mod.rs (94%) rename ethcore/{src/snapshot => snapshot/src}/consensus/work.rs (95%) rename ethcore/{src/snapshot => snapshot/src}/io.rs (99%) rename ethcore/{src/snapshot/mod.rs => snapshot/src/lib.rs} (98%) rename ethcore/{src/snapshot => snapshot/src}/service.rs (98%) rename ethcore/{src/snapshot => snapshot/src}/tests/helpers.rs (97%) rename ethcore/{src/snapshot => snapshot/src}/tests/mod.rs (100%) rename ethcore/{src/snapshot => snapshot/src}/tests/proof_of_authority.rs (99%) rename ethcore/{src/snapshot => snapshot/src}/tests/proof_of_work.rs (99%) rename ethcore/{src/snapshot => snapshot/src}/tests/service.rs (99%) rename ethcore/{src/snapshot => snapshot/src}/tests/state.rs (97%) rename ethcore/{src/snapshot => snapshot/src}/tests/test_validator_contract.json (100%) rename ethcore/{src/snapshot => snapshot/src}/traits.rs (99%) rename ethcore/{src/snapshot => snapshot/src}/watcher.rs (97%) diff --git a/Cargo.lock b/Cargo.lock index 98940990fae..a997b517de6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4202,6 +4202,42 @@ name = "smallvec" version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "snapshot" +version = "0.1.0" +dependencies = [ + "account-db 0.1.0", + "account-state 0.1.0", + "client-traits 0.1.0", + "common-types 0.1.0", + "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "engine 0.1.0", + "ethcore-blockchain 0.1.0", + "ethcore-bloom-journal 0.1.0", + "ethcore-db 0.1.0", + "ethcore-io 1.12.0", + "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", + "journaldb 0.2.0", + "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hasher 0.1.1", + "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "patricia-trie-ethereum 0.1.0", + "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rlp_derive 0.1.0", + "state-db 0.1.0", + "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "triehash-ethereum 0.2.0", +] + [[package]] name = "socket2" version = "0.3.8" diff --git a/Cargo.toml b/Cargo.toml index 298c1542e8f..acd8fb42782 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,4 +134,6 @@ members = [ "ethcore/wasm/run", "evmbin", "parity-clib", + + "ethcore/snapshot" ] diff --git a/ethcore/res/ethereum/foundation.json b/ethcore/res/ethereum/foundation.json index 5a7fd3be318..12a456a453e 100644 --- a/ethcore/res/ethereum/foundation.json +++ b/ethcore/res/ethereum/foundation.json @@ -3942,7 +3942,8 @@ } }, "0x0000000000000000000000000000000000000008": { - "builtin": { + "builtin": [ + { "name": "alt_bn128_pairing", "activate_at": "0x42ae50", "pricing": { @@ -3951,7 +3952,18 @@ "pair": 80000 } } - } + }, + { + "name": "alt_bn128_pairing", + "activate_at": "0xFFFFFF", + "pricing": { + "alt_bn128_pairing": { + "base": 123, + "pair": 456 + } + } + }, + ], }, "0x3282791d6fd713f1e94f4bfd565eaa78b3a0599d": { "balance": "0x487a9a304539440000" diff --git a/ethcore/src/snapshot/account.rs b/ethcore/snapshot/src/account.rs similarity index 98% rename from ethcore/src/snapshot/account.rs rename to ethcore/snapshot/src/account.rs index 62d9ac71d0d..5b277e3f11d 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/snapshot/src/account.rs @@ -16,21 +16,23 @@ //! Account state encoding and decoding +use std::collections::HashSet; +use std::sync::atomic::Ordering; + use account_db::{AccountDB, AccountDBMut}; -use types::{ +use bytes::Bytes; +use common_types::{ basic_account::BasicAccount, snapshot::Progress, errors::SnapshotError as Error, }; -use bytes::Bytes; use ethereum_types::{H256, U256}; use ethtrie::{TrieDB, TrieDBMut}; -use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP}; use hash_db::HashDB; +use keccak_hash::{KECCAK_EMPTY, KECCAK_NULL_RLP}; +use log::{trace, warn}; use rlp::{RlpStream, Rlp}; -use std::collections::HashSet; -use trie::{Trie, TrieMut}; -use std::sync::atomic::Ordering; +use trie_db::{Trie, TrieMut}; // An empty account -- these were replaced with RLP null data for a space optimization in v1. const ACC_EMPTY: BasicAccount = BasicAccount { @@ -246,12 +248,12 @@ pub fn from_fat_rlp( #[cfg(test)] mod tests { use account_db::{AccountDB, AccountDBMut}; - use types::basic_account::BasicAccount; + use common_types::basic_account::BasicAccount; use test_helpers::get_temp_state_db; use snapshot::tests::helpers::fill_storage; use snapshot::Progress; - use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; + use keccak_hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; use ethereum_types::{H256, Address}; use hash_db::{HashDB, EMPTY_PREFIX}; use kvdb::DBValue; diff --git a/ethcore/src/snapshot/block.rs b/ethcore/snapshot/src/block.rs similarity index 94% rename from ethcore/src/snapshot/block.rs rename to ethcore/snapshot/src/block.rs index f19b2611be4..fbc5d320742 100644 --- a/ethcore/src/snapshot/block.rs +++ b/ethcore/snapshot/src/block.rs @@ -17,18 +17,20 @@ //! Block RLP compression. use bytes::Bytes; +use common_types::{ + block::Block, + header::Header, + views::BlockView, +}; use ethereum_types::H256; -use hash::keccak; +use keccak_hash::keccak; use rlp::{DecoderError, RlpStream, Rlp}; use triehash::ordered_trie_root; -use types::block::Block; -use types::header::Header; -use types::views::BlockView; const HEADER_FIELDS: usize = 8; const BLOCK_FIELDS: usize = 2; -pub struct AbridgedBlock { +pub(crate) struct AbridgedBlock { rlp: Bytes, } @@ -122,11 +124,7 @@ impl AbridgedBlock { header.set_seal(seal_fields); - Ok(Block { - header: header, - transactions: transactions, - uncles: uncles, - }) + Ok(Block { header, transactions, uncles }) } } @@ -136,10 +134,12 @@ mod tests { use bytes::Bytes; use ethereum_types::{H256, U256, Address}; - use types::transaction::{Action, Transaction}; - use types::block::Block; - use types::view; - use types::views::BlockView; + use common_types::{ + transaction::{Action, Transaction}, + block::Block, + view, + views::BlockView, + }; fn encode_block(b: &Block) -> Bytes { b.rlp_bytes() diff --git a/ethcore/src/snapshot/consensus/authority.rs b/ethcore/snapshot/src/consensus/authority.rs similarity index 99% rename from ethcore/src/snapshot/consensus/authority.rs rename to ethcore/snapshot/src/consensus/authority.rs index 426ca1e54d2..6135ba8820f 100644 --- a/ethcore/src/snapshot/consensus/authority.rs +++ b/ethcore/snapshot/src/consensus/authority.rs @@ -22,14 +22,9 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use engine::{Engine, EpochVerifier}; use blockchain::{BlockChain, BlockChainDB, BlockProvider}; use bytes::Bytes; -use ethereum_types::{H256, U256}; -use itertools::{Position, Itertools}; -use kvdb::KeyValueDB; -use rlp::{RlpStream, Rlp}; -use types::{ +use common_types::{ encoded, engines::epoch::Transition as EpochTransition, header::Header, @@ -38,8 +33,15 @@ use types::{ receipt::Receipt, snapshot::{ChunkSink, Progress, ManifestData} }; +use engine::{Engine, EpochVerifier}; +use ethereum_types::{H256, U256}; +use itertools::{Position, Itertools}; +use kvdb::KeyValueDB; +use log::trace; +use rlp::{RlpStream, Rlp}; + +use crate::{SnapshotComponents, Rebuilder}; -use snapshot::{SnapshotComponents, Rebuilder}; /// Snapshot creation and restoration for PoA chains. /// Chunk format: @@ -320,7 +322,7 @@ impl Rebuilder for ChunkRebuilder { } if is_last_chunk { - use types::block::Block; + use common_types::block::Block; let last_rlp = rlp.at(num_items - 1)?; let block = Block { diff --git a/ethcore/src/snapshot/consensus/mod.rs b/ethcore/snapshot/src/consensus/mod.rs similarity index 94% rename from ethcore/src/snapshot/consensus/mod.rs rename to ethcore/snapshot/src/consensus/mod.rs index aa323b6afd6..280ebd8a759 100644 --- a/ethcore/src/snapshot/consensus/mod.rs +++ b/ethcore/snapshot/src/consensus/mod.rs @@ -23,8 +23,8 @@ mod work; pub use self::authority::*; pub use self::work::*; -use snapshot::SnapshotComponents; -use types::snapshot::Snapshotting::{self, *}; +use crate::SnapshotComponents; +use common_types::snapshot::Snapshotting::{self, *}; /// Create a factory for building snapshot chunks and restoring from them. /// `None` indicates that the engine doesn't support snapshot creation. diff --git a/ethcore/src/snapshot/consensus/work.rs b/ethcore/snapshot/src/consensus/work.rs similarity index 95% rename from ethcore/src/snapshot/consensus/work.rs rename to ethcore/snapshot/src/consensus/work.rs index ee52f85ba4a..eb035fb7734 100644 --- a/ethcore/src/snapshot/consensus/work.rs +++ b/ethcore/snapshot/src/consensus/work.rs @@ -25,21 +25,27 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use blockchain::{BlockChain, BlockChainDB, BlockProvider}; -use engine::Engine; -use snapshot::block::AbridgedBlock; -use ethereum_types::H256; -use kvdb::KeyValueDB; use bytes::Bytes; -use rlp::{RlpStream, Rlp}; -use rand::rngs::OsRng; -use types::{ +use common_types::{ encoded, engines::epoch::Transition as EpochTransition, errors::{SnapshotError, EthcoreError}, snapshot::{ChunkSink, ManifestData, Progress}, + receipt::Receipt, }; +use engine::Engine; +use ethereum_types::{H256, U256}; +use kvdb::KeyValueDB; +use log::trace; +use rand::rngs::OsRng; +use rlp::{RlpStream, Rlp}; +use triehash::ordered_trie_root; -use snapshot::{SnapshotComponents, Rebuilder}; +use crate::{ + SnapshotComponents, Rebuilder, + block::AbridgedBlock, + verify_old_block +}; /// Snapshot creation and restoration for PoW chains. /// This includes blocks from the head of the chain as a @@ -93,8 +99,8 @@ impl SnapshotComponents for PowSnapshot { ).map(|r| Box::new(r) as Box<_>) } - fn min_supported_version(&self) -> u64 { ::snapshot::MIN_SUPPORTED_STATE_CHUNK_VERSION } - fn current_version(&self) -> u64 { ::snapshot::STATE_CHUNK_VERSION } + fn min_supported_version(&self) -> u64 { crate::MIN_SUPPORTED_STATE_CHUNK_VERSION } + fn current_version(&self) -> u64 { crate::STATE_CHUNK_VERSION } } /// Used to build block chunks. @@ -231,10 +237,6 @@ impl Rebuilder for PowRebuilder { /// Feed the rebuilder an uncompressed block chunk. /// Returns the number of blocks fed or any errors. fn feed(&mut self, chunk: &[u8], engine: &dyn Engine, abort_flag: &AtomicBool) -> Result<(), EthcoreError> { - use snapshot::verify_old_block; - use ethereum_types::U256; - use triehash::ordered_trie_root; - let rlp = Rlp::new(chunk); let item_count = rlp.item_count()?; let num_blocks = (item_count - 3) as u64; @@ -256,7 +258,7 @@ impl Rebuilder for PowRebuilder { let pair = rlp.at(idx)?; let abridged_rlp = pair.at(0)?.as_raw().to_owned(); let abridged_block = AbridgedBlock::from_raw(abridged_rlp); - let receipts: Vec<::types::receipt::Receipt> = pair.list_at(1)?; + let receipts: Vec = pair.list_at(1)?; let receipts_root = ordered_trie_root(pair.at(1)?.iter().map(|r| r.as_raw())); let block = abridged_block.to_block(parent_hash, cur_number, receipts_root)?; diff --git a/ethcore/src/snapshot/io.rs b/ethcore/snapshot/src/io.rs similarity index 99% rename from ethcore/src/snapshot/io.rs rename to ethcore/snapshot/src/io.rs index b996651bcf5..ef22e04ff17 100644 --- a/ethcore/src/snapshot/io.rs +++ b/ethcore/snapshot/src/io.rs @@ -27,16 +27,17 @@ use std::path::{Path, PathBuf}; use bytes::Bytes; use client_traits::SnapshotWriter; -use ethereum_types::H256; -use rlp::{RlpStream, Rlp}; -use types::{ +use common_types::{ errors::{SnapshotError, EthcoreError}, snapshot::ManifestData, }; +use ethereum_types::H256; +use log::trace; +use rlp::{RlpStream, Rlp}; +use rlp_derive::*; const SNAPSHOT_VERSION: u64 = 2; - // (hash, len, offset) #[derive(RlpEncodable, RlpDecodable)] struct ChunkInfo(H256, u64, u64); @@ -320,7 +321,7 @@ impl SnapshotReader for LooseReader { #[cfg(test)] mod tests { use tempdir::TempDir; - use hash::keccak; + use keccak_hash::keccak; use snapshot::ManifestData; use super::{SnapshotWriter, SnapshotReader, PackedWriter, PackedReader, LooseWriter, LooseReader, SNAPSHOT_VERSION}; diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/snapshot/src/lib.rs similarity index 98% rename from ethcore/src/snapshot/mod.rs rename to ethcore/snapshot/src/lib.rs index 66a97719648..76d48ba569e 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/snapshot/src/lib.rs @@ -23,46 +23,44 @@ use std::collections::{HashMap, HashSet}; use std::cmp; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use hash::{keccak, KECCAK_NULL_RLP, KECCAK_EMPTY}; + +use keccak_hash::{keccak, KECCAK_NULL_RLP, KECCAK_EMPTY}; use account_db::{AccountDB, AccountDBMut}; +use account_state::Account as StateAccount; use blockchain::{BlockChain, BlockProvider}; -use types::{ +use bloom_journal::Bloom; +use bytes::Bytes; +// todo[dvdplm] put back in snapshots once it's extracted +use client_traits::SnapshotWriter; +use common_types::{ ids::BlockId, header::Header, errors::{SnapshotError as Error, EthcoreError}, - snapshot::Progress, + snapshot::{Progress, ManifestData}, }; +use crossbeam_utils::thread; +use engine::Engine; use ethereum_types::{H256, U256}; +use ethtrie::{TrieDB, TrieDBMut}; use hash_db::HashDB; +use journaldb::{self, Algorithm, JournalDB}; use keccak_hasher::KeccakHasher; -use snappy; -use bytes::Bytes; use parking_lot::Mutex; -use journaldb::{self, Algorithm, JournalDB}; use kvdb::{KeyValueDB, DBValue}; -use trie::{Trie, TrieMut}; -use ethtrie::{TrieDB, TrieDBMut}; -use rlp::{RlpStream, Rlp}; -use bloom_journal::Bloom; +use log::{debug, info, trace}; use num_cpus; -use types::snapshot::ManifestData; - -// todo[dvdplm] put back in snapshots once it's extracted -use client_traits::SnapshotWriter; - -use super::state_db::StateDB; -use account_state::Account as StateAccount; -use engine::Engine; - -use crossbeam_utils::thread; use rand::{Rng, rngs::OsRng}; +use rlp::{RlpStream, Rlp}; +use snappy; +use state_db::StateDB; +use trie_db::{Trie, TrieMut}; pub use self::consensus::*; pub use self::service::Service; pub use self::traits::{SnapshotService, SnapshotComponents, Rebuilder}; pub use self::watcher::Watcher; -pub use types::basic_account::BasicAccount; +pub use common_types::basic_account::BasicAccount; pub mod io; pub mod service; @@ -377,7 +375,7 @@ impl StateRebuilder { /// Create a new state rebuilder to write into the given backing DB. pub fn new(db: Arc, pruning: Algorithm) -> Self { StateRebuilder { - db: journaldb::new(db.clone(), pruning, ::db::COL_STATE), + db: journaldb::new(db.clone(), pruning, ethcore_db::COL_STATE), state_root: KECCAK_NULL_RLP, known_code: HashMap::new(), missing_code: HashMap::new(), diff --git a/ethcore/src/snapshot/service.rs b/ethcore/snapshot/src/service.rs similarity index 98% rename from ethcore/src/snapshot/service.rs rename to ethcore/snapshot/src/service.rs index 6ca50d70636..4299d90d08f 100644 --- a/ethcore/src/snapshot/service.rs +++ b/ethcore/snapshot/src/service.rs @@ -24,39 +24,39 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::cmp; -use super::{ - StateRebuilder, - SnapshotService, - Rebuilder, - MAX_CHUNK_SIZE, - io::{SnapshotReader, LooseReader, LooseWriter}, - chunker, -}; - use blockchain::{BlockChain, BlockChainDB, BlockChainDBHandler}; -use client::Client; +use bytes::Bytes; +use client::Client; // todo[dvdplm] +use common_types::{ + io_message::ClientIoMessage, + errors::{EthcoreError as Error, SnapshotError, SnapshotError::UnlinkedAncientBlockChain}, + ids::BlockId, + snapshot::{ManifestData, Progress, RestorationStatus}, +}; // todo[dvdplm] put SnapshotWriter back in snapshots once extracted use client_traits::{ BlockInfo, BlockChainClient, ChainInfo, SnapshotClient, SnapshotWriter, DatabaseRestore, }; use engine::Engine; -use hash::keccak; -use types::{ - io_message::ClientIoMessage, - errors::{EthcoreError as Error, SnapshotError, SnapshotError::UnlinkedAncientBlockChain}, - ids::BlockId, - snapshot::{ManifestData, Progress, RestorationStatus}, -}; - -use io::IoChannel; - use ethereum_types::H256; -use parking_lot::{Mutex, RwLock, RwLockReadGuard}; -use bytes::Bytes; +use ethcore_io::IoChannel; use journaldb::Algorithm; +use keccak_hash::keccak; use kvdb::DBTransaction; +use log::{error, info, trace, warn}; +use parking_lot::{Mutex, RwLock, RwLockReadGuard}; use snappy; +use trie_db::TrieError; + +use super::{ + StateRebuilder, + SnapshotService, + Rebuilder, + MAX_CHUNK_SIZE, + io::{SnapshotReader, LooseReader, LooseWriter}, + chunker, +}; /// Helper for removing directories in case of error. struct Guard(bool, PathBuf); @@ -140,7 +140,7 @@ impl Restoration { let expected_len = snappy::decompressed_len(chunk)?; if expected_len > MAX_CHUNK_SIZE { trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE); - return Err(::snapshot::Error::ChunkTooLarge.into()); + return Err(SnapshotError::ChunkTooLarge.into()); } let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?; @@ -162,7 +162,7 @@ impl Restoration { let expected_len = snappy::decompressed_len(chunk)?; if expected_len > MAX_CHUNK_SIZE { trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE); - return Err(::snapshot::Error::ChunkTooLarge.into()); + return Err(SnapshotError::ChunkTooLarge.into()); } let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?; @@ -179,8 +179,6 @@ impl Restoration { // finish up restoration. fn finalize(mut self) -> Result<(), Error> { - use trie::TrieError; - if !self.is_done() { return Ok(()) } // verify final state root. @@ -807,10 +805,6 @@ impl SnapshotService for Service { .map(|c| (c.min_supported_version(), c.current_version())) } - fn chunk(&self, hash: H256) -> Option { - self.reader.read().as_ref().and_then(|r| r.chunk(hash).ok()) - } - fn completed_chunks(&self) -> Option> { let restoration = self.restoration.lock(); @@ -833,6 +827,10 @@ impl SnapshotService for Service { } } + fn chunk(&self, hash: H256) -> Option { + self.reader.read().as_ref().and_then(|r| r.chunk(hash).ok()) + } + fn status(&self) -> RestorationStatus { let mut cur_status = self.status.lock(); @@ -905,12 +903,12 @@ impl Drop for Service { #[cfg(test)] mod tests { use client::Client; - use io::{IoService}; + use ethcore_io::IoService; use spec; use journaldb::Algorithm; use snapshot::SnapshotService; use super::*; - use types::{ + use common_types::{ io_message::ClientIoMessage, snapshot::{ManifestData, RestorationStatus} }; diff --git a/ethcore/src/snapshot/tests/helpers.rs b/ethcore/snapshot/src/tests/helpers.rs similarity index 97% rename from ethcore/src/snapshot/tests/helpers.rs rename to ethcore/snapshot/src/tests/helpers.rs index 3f23377a5f4..e79a538fa8a 100644 --- a/ethcore/src/snapshot/tests/helpers.rs +++ b/ethcore/snapshot/src/tests/helpers.rs @@ -20,10 +20,10 @@ extern crate trie_standardmap; use std::sync::Arc; -use hash::{KECCAK_NULL_RLP}; +use keccak_hash::{KECCAK_NULL_RLP}; use account_db::AccountDBMut; -use types::basic_account::BasicAccount; +use common_types::basic_account::BasicAccount; use blockchain::{BlockChain, BlockChainDB}; use client::Client; use client_traits::{ChainInfo, SnapshotClient}; @@ -42,7 +42,7 @@ use journaldb; use trie::{TrieMut, Trie}; use ethtrie::{SecTrieDBMut, TrieDB, TrieDBMut}; use self::trie_standardmap::{Alphabet, StandardMap, ValueMode}; -use types::errors::EthcoreError; +use common_types::errors::EthcoreError; // the proportion of accounts we will alter each tick. const ACCOUNT_CHURN: f32 = 0.01; @@ -135,7 +135,7 @@ pub fn fill_storage(mut db: AccountDBMut, root: &mut H256, seed: &mut H256) { /// Take a snapshot from the given client into a temporary file. /// Return a snapshot reader for it. pub fn snap(client: &Client) -> (Box, TempDir) { - use types::ids::BlockId; + use common_types::ids::BlockId; let tempdir = TempDir::new("").unwrap(); let path = tempdir.path().join("file"); diff --git a/ethcore/src/snapshot/tests/mod.rs b/ethcore/snapshot/src/tests/mod.rs similarity index 100% rename from ethcore/src/snapshot/tests/mod.rs rename to ethcore/snapshot/src/tests/mod.rs diff --git a/ethcore/src/snapshot/tests/proof_of_authority.rs b/ethcore/snapshot/src/tests/proof_of_authority.rs similarity index 99% rename from ethcore/src/snapshot/tests/proof_of_authority.rs rename to ethcore/snapshot/src/tests/proof_of_authority.rs index a0161acdf98..7a916caaf28 100644 --- a/ethcore/src/snapshot/tests/proof_of_authority.rs +++ b/ethcore/snapshot/src/tests/proof_of_authority.rs @@ -27,7 +27,7 @@ use ethkey::Secret; use snapshot::tests::helpers as snapshot_helpers; use spec::Spec; use test_helpers::generate_dummy_client_with_spec; -use types::transaction::{Transaction, Action, SignedTransaction}; +use common_types::transaction::{Transaction, Action, SignedTransaction}; use tempdir::TempDir; use ethereum_types::Address; diff --git a/ethcore/src/snapshot/tests/proof_of_work.rs b/ethcore/snapshot/src/tests/proof_of_work.rs similarity index 99% rename from ethcore/src/snapshot/tests/proof_of_work.rs rename to ethcore/snapshot/src/tests/proof_of_work.rs index 3b6106313c7..9fd1ca95020 100644 --- a/ethcore/src/snapshot/tests/proof_of_work.rs +++ b/ethcore/snapshot/src/tests/proof_of_work.rs @@ -18,7 +18,7 @@ use std::sync::atomic::AtomicBool; use tempdir::TempDir; -use types::{ +use common_types::{ errors::EthcoreError as Error, engines::ForkChoice, snapshot::Progress, @@ -31,7 +31,7 @@ use snapshot::{chunk_secondary, Error as SnapshotError, SnapshotComponents}; use snapshot::io::{PackedReader, PackedWriter, SnapshotReader}; use parking_lot::Mutex; -use snappy; +use parity_snappy; use kvdb::DBTransaction; use test_helpers; use spec; diff --git a/ethcore/src/snapshot/tests/service.rs b/ethcore/snapshot/src/tests/service.rs similarity index 99% rename from ethcore/src/snapshot/tests/service.rs rename to ethcore/snapshot/src/tests/service.rs index aed14e9b456..6081e1d9b04 100644 --- a/ethcore/src/snapshot/tests/service.rs +++ b/ethcore/snapshot/src/tests/service.rs @@ -23,7 +23,7 @@ use tempdir::TempDir; use blockchain::BlockProvider; use client::{Client, ClientConfig}; use client_traits::{BlockInfo, ImportBlock, SnapshotWriter}; -use types::{ +use common_types::{ ids::BlockId, snapshot::Progress, verification::Unverified, @@ -36,7 +36,7 @@ use spec; use test_helpers::{new_db, new_temp_db, generate_dummy_client_with_spec_and_data, restoration_db_handler}; use parking_lot::Mutex; -use io::IoChannel; +use ethcore_io::IoChannel; use kvdb_rocksdb::DatabaseConfig; #[test] diff --git a/ethcore/src/snapshot/tests/state.rs b/ethcore/snapshot/src/tests/state.rs similarity index 97% rename from ethcore/src/snapshot/tests/state.rs rename to ethcore/snapshot/src/tests/state.rs index 50a505e3e71..1b3c0617534 100644 --- a/ethcore/src/snapshot/tests/state.rs +++ b/ethcore/snapshot/src/tests/state.rs @@ -18,9 +18,9 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; -use hash::{KECCAK_NULL_RLP, keccak}; +use keccak_hash::{KECCAK_NULL_RLP, keccak}; -use types::{ +use common_types::{ basic_account::BasicAccount, errors::EthcoreError as Error, }; @@ -82,7 +82,7 @@ fn snap_and_restore() { for chunk_hash in &reader.manifest().state_hashes { let raw = reader.chunk(*chunk_hash).unwrap(); - let chunk = ::snappy::decompress(&raw).unwrap(); + let chunk = snappy::decompress(&raw).unwrap(); rebuilder.feed(&chunk, &flag).unwrap(); } @@ -197,7 +197,7 @@ fn checks_flag() { for chunk_hash in &reader.manifest().state_hashes { let raw = reader.chunk(*chunk_hash).unwrap(); - let chunk = ::snappy::decompress(&raw).unwrap(); + let chunk = snappy::decompress(&raw).unwrap(); match rebuilder.feed(&chunk, &flag) { Err(Error::Snapshot(SnapshotError::RestorationAborted)) => {}, diff --git a/ethcore/src/snapshot/tests/test_validator_contract.json b/ethcore/snapshot/src/tests/test_validator_contract.json similarity index 100% rename from ethcore/src/snapshot/tests/test_validator_contract.json rename to ethcore/snapshot/src/tests/test_validator_contract.json diff --git a/ethcore/src/snapshot/traits.rs b/ethcore/snapshot/src/traits.rs similarity index 99% rename from ethcore/src/snapshot/traits.rs rename to ethcore/snapshot/src/traits.rs index 3eb2542bcba..7425ada6375 100644 --- a/ethcore/src/snapshot/traits.rs +++ b/ethcore/snapshot/src/traits.rs @@ -18,11 +18,12 @@ use std::sync::{Arc, atomic::AtomicBool}; use blockchain::{BlockChain, BlockChainDB}; use bytes::Bytes; -use ethereum_types::H256; -use types::{ +use common_types::{ errors::{EthcoreError as Error, SnapshotError}, snapshot::{ManifestData, ChunkSink, Progress, RestorationStatus}, }; +use engine::Engine; +use ethereum_types::H256; /// The interface for a snapshot network service. @@ -69,8 +70,6 @@ pub trait SnapshotService : Sync + Send { fn shutdown(&self); } -use crate::engine::Engine; - /// Restore from secondary snapshot chunks. pub trait Rebuilder: Send { /// Feed a chunk, potentially out of order. diff --git a/ethcore/src/snapshot/watcher.rs b/ethcore/snapshot/src/watcher.rs similarity index 97% rename from ethcore/src/snapshot/watcher.rs rename to ethcore/snapshot/src/watcher.rs index 828fb8d4d45..8fd05cbcb92 100644 --- a/ethcore/src/snapshot/watcher.rs +++ b/ethcore/snapshot/src/watcher.rs @@ -16,18 +16,18 @@ //! Watcher for snapshot-related chain events. -use parking_lot::Mutex; -use client::{Client, ChainNotify, NewBlocks}; +use std::sync::Arc; + +use client::{Client, ChainNotify, NewBlocks}; // todo[dvdplm] use client_traits::BlockInfo; -use types::{ +use common_types::{ ids::BlockId, io_message::ClientIoMessage, }; - -use io::IoChannel; use ethereum_types::H256; - -use std::sync::Arc; +use ethcore_io::IoChannel; +use log::{trace, warn}; +use parking_lot::Mutex; // helper trait for transforming hashes to numbers and checking if syncing. trait Oracle: Send + Sync { diff --git a/ethcore/state-db/src/lib.rs b/ethcore/state-db/src/lib.rs index 67d70343aad..7b121dfb7b1 100644 --- a/ethcore/state-db/src/lib.rs +++ b/ethcore/state-db/src/lib.rs @@ -207,9 +207,9 @@ impl StateDB { /// Journal all recent operations under the given era and ID. pub fn journal_under(&mut self, batch: &mut DBTransaction, now: u64, id: &H256) -> io::Result { { - let mut bloom_lock = self.account_bloom.lock(); - Self::commit_bloom(batch, bloom_lock.drain_journal())?; - } + let mut bloom_lock = self.account_bloom.lock(); + Self::commit_bloom(batch, bloom_lock.drain_journal())?; + } let records = self.db.journal_under(batch, now, id)?; self.commit_hash = Some(id.clone()); self.commit_number = Some(now); From b83d0cb0afc0d1ac1225775b393996554e0c8f70 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 29 Aug 2019 12:11:55 +0200 Subject: [PATCH 02/21] WIP cargo toml --- ethcore/snapshot/Cargo.toml | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 ethcore/snapshot/Cargo.toml diff --git a/ethcore/snapshot/Cargo.toml b/ethcore/snapshot/Cargo.toml new file mode 100644 index 00000000000..ecc78baf48f --- /dev/null +++ b/ethcore/snapshot/Cargo.toml @@ -0,0 +1,56 @@ +[package] +description = "Snapshot TODO" +name = "snapshot" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" + +[dependencies] +account-db = { path = "../account-db" } +account-state = { path = "../account-state" } +bloom-journal = { package = "ethcore-bloom-journal", path = "../../util/bloom" } +bytes = { package = "parity-bytes", version = "0.1.0" } +client-traits = { path = "../client-traits" } +common-types = { path = "../types" } +crossbeam-utils = "0.6" +blockchain = { package = "ethcore-blockchain", path = "../blockchain" } +engine = { path = "../engine" } +ethcore-db = { path = "../db" } +ethcore-io = { path = "../../util/io" } +ethereum-types = "0.6.0" +ethtrie = { package = "patricia-trie-ethereum", path = "../../util/patricia-trie-ethereum" } +hash-db = "0.15.0" +itertools = "0.5" +journaldb = { path = "../../util/journaldb" } +keccak-hash = "0.2.0" +keccak-hasher = { path = "../../util/keccak-hasher" } +kvdb = "0.1.0" +log = "0.4.8" +#machine = { path = "../machine" } +num_cpus = "1.10.1" +parking_lot = "0.8.0" +snappy = { package = "parity-snappy", version ="0.1.0" } +rand = "0.6" +rand_xorshift = "0.1.1" +rlp = "0.4.2" +rlp_derive = { path = "../../util/rlp-derive" } +state-db = { path = "../state-db" } +#trace = { path = "../trace" } +triehash = { package = "triehash-ethereum", version = "0.2", path = "../../util/triehash-ethereum" } +trie-db = "0.15.0" + +#trie-vm-factories = { path = "../trie-vm-factories" } +#vm = { path = "../vm" } + +[dev-dependencies] +#env_logger = "0.5" +#ethcore = { path = "..", features = ["test-helpers"] } +#ethkey = { path = "../../accounts/ethkey" } +#evm = { path = "../evm" } +#keccak-hash = "0.2.0" +#pod = { path = "../pod" } +#rustc-hex = "1.0" +#spec = { path = "../spec" } +#trie-db = "0.15.0" +#ethtrie = { package = "patricia-trie-ethereum", path = "../../util/patricia-trie-ethereum" } From 11dada9f24dc436f6eff16555677ab1f5864e800 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 29 Aug 2019 15:32:57 +0200 Subject: [PATCH 03/21] Make snapshotting generic over the client Sort out tests --- ethcore/snapshot/Cargo.toml | 34 ++++++++--------- ethcore/snapshot/src/account.rs | 6 +-- ethcore/snapshot/src/io.rs | 6 +-- ethcore/snapshot/src/service.rs | 37 +++++++++---------- ethcore/snapshot/src/tests/helpers.rs | 23 ++++++------ ethcore/snapshot/src/tests/mod.rs | 2 +- .../snapshot/src/tests/proof_of_authority.rs | 22 ++++++----- ethcore/snapshot/src/tests/proof_of_work.rs | 28 ++++++++------ ethcore/snapshot/src/tests/service.rs | 36 ++++++++++-------- ethcore/snapshot/src/tests/state.rs | 29 ++++++++------- ethcore/snapshot/src/watcher.rs | 37 ++++++++++--------- 11 files changed, 136 insertions(+), 124 deletions(-) diff --git a/ethcore/snapshot/Cargo.toml b/ethcore/snapshot/Cargo.toml index ecc78baf48f..87ad36c4185 100644 --- a/ethcore/snapshot/Cargo.toml +++ b/ethcore/snapshot/Cargo.toml @@ -9,12 +9,12 @@ license = "GPL-3.0" [dependencies] account-db = { path = "../account-db" } account-state = { path = "../account-state" } +blockchain = { package = "ethcore-blockchain", path = "../blockchain" } bloom-journal = { package = "ethcore-bloom-journal", path = "../../util/bloom" } bytes = { package = "parity-bytes", version = "0.1.0" } client-traits = { path = "../client-traits" } common-types = { path = "../types" } crossbeam-utils = "0.6" -blockchain = { package = "ethcore-blockchain", path = "../blockchain" } engine = { path = "../engine" } ethcore-db = { path = "../db" } ethcore-io = { path = "../../util/io" } @@ -27,30 +27,28 @@ keccak-hash = "0.2.0" keccak-hasher = { path = "../../util/keccak-hasher" } kvdb = "0.1.0" log = "0.4.8" -#machine = { path = "../machine" } num_cpus = "1.10.1" parking_lot = "0.8.0" -snappy = { package = "parity-snappy", version ="0.1.0" } rand = "0.6" rand_xorshift = "0.1.1" rlp = "0.4.2" rlp_derive = { path = "../../util/rlp-derive" } +snappy = { package = "parity-snappy", version ="0.1.0" } state-db = { path = "../state-db" } -#trace = { path = "../trace" } -triehash = { package = "triehash-ethereum", version = "0.2", path = "../../util/triehash-ethereum" } trie-db = "0.15.0" - -#trie-vm-factories = { path = "../trie-vm-factories" } -#vm = { path = "../vm" } +triehash = { package = "triehash-ethereum", version = "0.2", path = "../../util/triehash-ethereum" } [dev-dependencies] -#env_logger = "0.5" -#ethcore = { path = "..", features = ["test-helpers"] } -#ethkey = { path = "../../accounts/ethkey" } -#evm = { path = "../evm" } -#keccak-hash = "0.2.0" -#pod = { path = "../pod" } -#rustc-hex = "1.0" -#spec = { path = "../spec" } -#trie-db = "0.15.0" -#ethtrie = { package = "patricia-trie-ethereum", path = "../../util/patricia-trie-ethereum" } +accounts = { package = "ethcore-accounts", path = "../../accounts" } +engine = { path = "../engine", features = ["test-helpers"] } +env_logger = "0.5" +ethabi = "8.0" +ethabi-contract = "8.0" +ethabi-derive = "8.0" +ethcore = { path = "..", features = ["test-helpers"] } +ethkey = { path = "../../accounts/ethkey" } +kvdb-rocksdb = { version = "0.1.3" } +lazy_static = { version = "1.3" } +spec = { path = "../spec" } +tempdir = "0.3" +trie-standardmap = "0.15.0" diff --git a/ethcore/snapshot/src/account.rs b/ethcore/snapshot/src/account.rs index 5b277e3f11d..760576e370b 100644 --- a/ethcore/snapshot/src/account.rs +++ b/ethcore/snapshot/src/account.rs @@ -249,9 +249,9 @@ pub fn from_fat_rlp( mod tests { use account_db::{AccountDB, AccountDBMut}; use common_types::basic_account::BasicAccount; - use test_helpers::get_temp_state_db; - use snapshot::tests::helpers::fill_storage; - use snapshot::Progress; + use ethcore::test_helpers::get_temp_state_db; + use crate::tests::helpers::fill_storage; + use common_types::snapshot::Progress; use keccak_hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; use ethereum_types::{H256, Address}; diff --git a/ethcore/snapshot/src/io.rs b/ethcore/snapshot/src/io.rs index ef22e04ff17..1afba21fb37 100644 --- a/ethcore/snapshot/src/io.rs +++ b/ethcore/snapshot/src/io.rs @@ -323,7 +323,7 @@ mod tests { use tempdir::TempDir; use keccak_hash::keccak; - use snapshot::ManifestData; + use common_types::snapshot::ManifestData; use super::{SnapshotWriter, SnapshotReader, PackedWriter, PackedReader, LooseWriter, LooseReader, SNAPSHOT_VERSION}; const STATE_CHUNKS: &'static [&'static [u8]] = &[b"dog", b"cat", b"hello world", b"hi", b"notarealchunk"]; @@ -352,8 +352,8 @@ mod tests { let manifest = ManifestData { version: SNAPSHOT_VERSION, - state_hashes: state_hashes, - block_hashes: block_hashes, + state_hashes, + block_hashes, state_root: keccak(b"notarealroot"), block_number: 12345678987654321, block_hash: keccak(b"notarealblock"), diff --git a/ethcore/snapshot/src/service.rs b/ethcore/snapshot/src/service.rs index 4299d90d08f..491b215db60 100644 --- a/ethcore/snapshot/src/service.rs +++ b/ethcore/snapshot/src/service.rs @@ -26,7 +26,7 @@ use std::cmp; use blockchain::{BlockChain, BlockChainDB, BlockChainDBHandler}; use bytes::Bytes; -use client::Client; // todo[dvdplm] +//use client::Client; // todo[dvdplm] use common_types::{ io_message::ClientIoMessage, errors::{EthcoreError as Error, SnapshotError, SnapshotError::UnlinkedAncientBlockChain}, @@ -210,10 +210,10 @@ impl Restoration { } /// Type alias for client io channel. -pub type Channel = IoChannel>; +pub type Channel = IoChannel>; /// Snapshot service parameters. -pub struct ServiceParams { +pub struct ServiceParams { /// The consensus engine this is built on. pub engine: Arc, /// The chain's genesis block. @@ -223,21 +223,21 @@ pub struct ServiceParams { /// Handler for opening a restoration DB. pub restoration_db_handler: Box, /// Async IO channel for sending messages. - pub channel: Channel, + pub channel: Channel, /// The directory to put snapshots in. /// Usually "/snapshot" pub snapshot_root: PathBuf, /// A handle for database restoration. - pub client: Arc, + pub client: Arc, } /// `SnapshotService` implementation. /// This controls taking snapshots and restoring from them. -pub struct Service { +pub struct Service { restoration: Mutex>, restoration_db_handler: Box, snapshot_root: PathBuf, - io_channel: Mutex, + io_channel: Mutex>, pruning: Algorithm, status: Mutex, reader: RwLock>, @@ -245,15 +245,15 @@ pub struct Service { genesis_block: Bytes, state_chunks: AtomicUsize, block_chunks: AtomicUsize, - client: Arc, + client: Arc, progress: Progress, taking_snapshot: AtomicBool, restoring_snapshot: AtomicBool, } -impl Service { +impl Service where C: SnapshotClient + ChainInfo { /// Create a new snapshot service from the given parameters. - pub fn new(params: ServiceParams) -> Result { + pub fn new(params: ServiceParams) -> Result { let mut service = Service { restoration: Mutex::new(None), restoration_db_handler: params.restoration_db_handler, @@ -477,10 +477,7 @@ impl Service { /// calling this while a restoration is in progress or vice versa /// will lead to a race condition where the first one to finish will /// have their produced snapshot overwritten. - pub fn take_snapshot(&self, client: &C, num: u64) -> Result<(), Error> - where - C: ChainInfo + SnapshotClient - { + pub fn take_snapshot(&self, client: &C, num: u64) -> Result<(), Error> { if self.taking_snapshot.compare_and_swap(false, true, Ordering::SeqCst) { info!("Skipping snapshot at #{} as another one is currently in-progress.", num); return Ok(()); @@ -795,7 +792,7 @@ impl Service { } } -impl SnapshotService for Service { +impl SnapshotService for Service { fn manifest(&self) -> Option { self.reader.read().as_ref().map(|r| r.manifest().clone()) } @@ -890,7 +887,7 @@ impl SnapshotService for Service { } } -impl Drop for Service { +impl Drop for Service { fn drop(&mut self) { trace!(target: "shutdown", "Dropping Service"); self.abort_restore(); @@ -902,18 +899,18 @@ impl Drop for Service { #[cfg(test)] mod tests { - use client::Client; + use ethcore::client::Client; use ethcore_io::IoService; use spec; use journaldb::Algorithm; - use snapshot::SnapshotService; + use crate::SnapshotService; use super::*; use common_types::{ io_message::ClientIoMessage, snapshot::{ManifestData, RestorationStatus} }; use tempdir::TempDir; - use test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler}; + use ethcore::test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler}; #[test] fn sends_async_messages() { @@ -966,7 +963,7 @@ mod tests { let state_hashes: Vec<_> = (0..5).map(|_| H256::random()).collect(); let block_hashes: Vec<_> = (0..5).map(|_| H256::random()).collect(); - let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_config = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); let gb = spec.genesis_block(); let flag = ::std::sync::atomic::AtomicBool::new(true); diff --git a/ethcore/snapshot/src/tests/helpers.rs b/ethcore/snapshot/src/tests/helpers.rs index e79a538fa8a..98b206ed4e2 100644 --- a/ethcore/snapshot/src/tests/helpers.rs +++ b/ethcore/snapshot/src/tests/helpers.rs @@ -17,31 +17,32 @@ //! Snapshot test helpers. These are used to build blockchains and state tries //! which can be queried before and after a full snapshot/restore cycle. -extern crate trie_standardmap; - use std::sync::Arc; -use keccak_hash::{KECCAK_NULL_RLP}; +use std::sync::atomic::AtomicBool; +use keccak_hash::{KECCAK_NULL_RLP}; use account_db::AccountDBMut; use common_types::basic_account::BasicAccount; use blockchain::{BlockChain, BlockChainDB}; -use client::Client; +use ethcore::client::Client; use client_traits::{ChainInfo, SnapshotClient}; use engine::Engine; -use snapshot::{StateRebuilder}; -use snapshot::io::{SnapshotReader, PackedWriter, PackedReader}; +use crate::{ + StateRebuilder, + io::{SnapshotReader, PackedWriter, PackedReader}, +}; use tempdir::TempDir; use rand::Rng; - +use log::trace; use kvdb::DBValue; use ethereum_types::H256; use hash_db::HashDB; use keccak_hasher::KeccakHasher; use journaldb; -use trie::{TrieMut, Trie}; +use trie_db::{TrieMut, Trie}; use ethtrie::{SecTrieDBMut, TrieDB, TrieDBMut}; -use self::trie_standardmap::{Alphabet, StandardMap, ValueMode}; +use trie_standardmap::{Alphabet, StandardMap, ValueMode}; use common_types::errors::EthcoreError; // the proportion of accounts we will alter each tick. @@ -158,10 +159,8 @@ pub fn restore( reader: &dyn SnapshotReader, genesis: &[u8], ) -> Result<(), EthcoreError> { - use std::sync::atomic::AtomicBool; - let flag = AtomicBool::new(true); - let chunker = crate::snapshot::chunker(engine.snapshot_mode()).expect("the engine used here supports snapshots"); + let chunker = crate::chunker(engine.snapshot_mode()).expect("the engine used here supports snapshots"); let manifest = reader.manifest(); let mut state = StateRebuilder::new(db.key_value().clone(), journaldb::Algorithm::Archive); diff --git a/ethcore/snapshot/src/tests/mod.rs b/ethcore/snapshot/src/tests/mod.rs index f25fd03b258..c1139e7b5fa 100644 --- a/ethcore/snapshot/src/tests/mod.rs +++ b/ethcore/snapshot/src/tests/mod.rs @@ -23,7 +23,7 @@ mod service; pub mod helpers; -use super::ManifestData; +use common_types::snapshot::ManifestData; #[test] fn manifest_rlp() { diff --git a/ethcore/snapshot/src/tests/proof_of_authority.rs b/ethcore/snapshot/src/tests/proof_of_authority.rs index 7a916caaf28..7b9437625a2 100644 --- a/ethcore/snapshot/src/tests/proof_of_authority.rs +++ b/ethcore/snapshot/src/tests/proof_of_authority.rs @@ -21,26 +21,32 @@ use std::sync::Arc; use std::str::FromStr; use accounts::AccountProvider; -use client::Client; +use ethcore::client::Client; use client_traits::{BlockChainClient, ChainInfo}; use ethkey::Secret; -use snapshot::tests::helpers as snapshot_helpers; +use crate::tests::helpers as snapshot_helpers; use spec::Spec; -use test_helpers::generate_dummy_client_with_spec; +use ethcore::test_helpers::generate_dummy_client_with_spec; use common_types::transaction::{Transaction, Action, SignedTransaction}; use tempdir::TempDir; - +use log::trace; use ethereum_types::Address; -use test_helpers; +use ethcore::{ + test_helpers, + miner::{self, MinerService}, +}; +use keccak_hash::keccak; +use ethabi_contract::use_contract; +use lazy_static::lazy_static; -use_contract!(test_validator_set, "res/contracts/test_validator_set.json"); +use_contract!(test_validator_set, "../res/contracts/test_validator_set.json"); const PASS: &'static str = ""; const TRANSITION_BLOCK_1: usize = 2; // block at which the contract becomes activated. const TRANSITION_BLOCK_2: usize = 10; // block at which the second contract activates. macro_rules! secret { - ($e: expr) => { Secret::from($crate::hash::keccak($e).0) } + ($e: expr) => { Secret::from(keccak($e).0) } } lazy_static! { @@ -100,8 +106,6 @@ fn make_chain(accounts: Arc, blocks_beyond: usize, transitions: { // push a block with given number, signed by one of the signers, with given transactions. let push_block = |signers: &[Address], n, txs: Vec| { - use miner::{self, MinerService}; - let idx = n as usize % signers.len(); trace!(target: "snapshot", "Pushing block #{}, {} txs, author={}", n, txs.len(), signers[idx]); diff --git a/ethcore/snapshot/src/tests/proof_of_work.rs b/ethcore/snapshot/src/tests/proof_of_work.rs index 9fd1ca95020..53dc071423a 100644 --- a/ethcore/snapshot/src/tests/proof_of_work.rs +++ b/ethcore/snapshot/src/tests/proof_of_work.rs @@ -21,22 +21,26 @@ use tempdir::TempDir; use common_types::{ errors::EthcoreError as Error, engines::ForkChoice, - snapshot::Progress, + snapshot::{Progress, ManifestData}, }; use blockchain::generator::{BlockGenerator, BlockBuilder}; use blockchain::{BlockChain, ExtrasInsert}; use client_traits::SnapshotWriter; -use snapshot::{chunk_secondary, Error as SnapshotError, SnapshotComponents}; -use snapshot::io::{PackedReader, PackedWriter, SnapshotReader}; - +use crate::{ + chunk_secondary, + Error as SnapshotError, SnapshotComponents, + io::{PackedReader, PackedWriter, SnapshotReader}, + PowSnapshot, +}; use parking_lot::Mutex; -use parity_snappy; +use snappy; +use keccak_hash::KECCAK_NULL_RLP; use kvdb::DBTransaction; -use test_helpers; +use ethcore::test_helpers; use spec; -const SNAPSHOT_MODE: ::snapshot::PowSnapshot = ::snapshot::PowSnapshot { blocks: 30000, max_restore_blocks: 30000 }; +const SNAPSHOT_MODE: PowSnapshot = PowSnapshot { blocks: 30000, max_restore_blocks: 30000 }; fn chunk_and_restore(amount: u64) { let genesis = BlockBuilder::genesis(); @@ -75,11 +79,11 @@ fn chunk_and_restore(amount: u64) { &Progress::default() ).unwrap(); - let manifest = ::snapshot::ManifestData { + let manifest = ManifestData { version: 2, state_hashes: Vec::new(), - block_hashes: block_hashes, - state_root: ::hash::KECCAK_NULL_RLP, + block_hashes, + state_root: KECCAK_NULL_RLP, block_number: amount, block_hash: best_hash, }; @@ -137,11 +141,11 @@ fn checks_flag() { let engine = spec::new_test().engine; let chain = BlockChain::new(Default::default(), genesis.last().encoded().raw(), db.clone()); - let manifest = ::snapshot::ManifestData { + let manifest = ManifestData { version: 2, state_hashes: Vec::new(), block_hashes: Vec::new(), - state_root: ::hash::KECCAK_NULL_RLP, + state_root: KECCAK_NULL_RLP, block_number: 102, block_hash: H256::zero(), }; diff --git a/ethcore/snapshot/src/tests/service.rs b/ethcore/snapshot/src/tests/service.rs index 6081e1d9b04..5e696bcfab3 100644 --- a/ethcore/snapshot/src/tests/service.rs +++ b/ethcore/snapshot/src/tests/service.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use tempdir::TempDir; use blockchain::BlockProvider; -use client::{Client, ClientConfig}; +use ethcore::client::{Client, ClientConfig}; use client_traits::{BlockInfo, ImportBlock, SnapshotWriter}; use common_types::{ ids::BlockId, @@ -29,11 +29,17 @@ use common_types::{ verification::Unverified, snapshot::{ManifestData, RestorationStatus}, }; -use snapshot::io::{PackedReader, PackedWriter, SnapshotReader}; -use snapshot::service::{Service, ServiceParams}; -use snapshot::{chunk_state, chunk_secondary, SnapshotService}; +use crate::{ + chunk_state, chunk_secondary, SnapshotService, + io::{PackedReader, PackedWriter, SnapshotReader}, + service::{Service, ServiceParams}, + PowSnapshot, +}; use spec; -use test_helpers::{new_db, new_temp_db, generate_dummy_client_with_spec_and_data, restoration_db_handler}; +use ethcore::{ + miner, + test_helpers::{new_db, new_temp_db, generate_dummy_client_with_spec_and_data, restoration_db_handler} +}; use parking_lot::Mutex; use ethcore_io::IoChannel; @@ -53,7 +59,7 @@ fn restored_is_equivalent() { let client_db = tempdir.path().join("client_db"); let path = tempdir.path().join("snapshot"); - let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_config = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); let restoration = restoration_db_handler(db_config); let blockchain_db = restoration.open(&client_db).unwrap(); @@ -62,7 +68,7 @@ fn restored_is_equivalent() { Default::default(), &spec, blockchain_db, - Arc::new(::miner::Miner::new_for_tests(&spec, None)), + Arc::new(miner::Miner::new_for_tests(&spec, None)), IoChannel::disconnected(), ).unwrap(); @@ -118,7 +124,7 @@ fn guards_delete_folders() { let service_params = ServiceParams { engine: spec.engine.clone(), genesis_block: spec.genesis_block(), - restoration_db_handler: restoration_db_handler(DatabaseConfig::with_columns(::db::NUM_COLUMNS)), + restoration_db_handler: restoration_db_handler(DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS)), pruning: ::journaldb::Algorithm::Archive, channel: IoChannel::disconnected(), snapshot_root: tempdir.path().to_owned(), @@ -157,11 +163,11 @@ fn guards_delete_folders() { #[test] fn keep_ancient_blocks() { let _ = ::env_logger::try_init(); - + use client_traits::BlockChainClient; // Test variables const NUM_BLOCKS: u64 = 500; const NUM_SNAPSHOT_BLOCKS: u64 = 300; - const SNAPSHOT_MODE: ::snapshot::PowSnapshot = ::snapshot::PowSnapshot { blocks: NUM_SNAPSHOT_BLOCKS, max_restore_blocks: NUM_SNAPSHOT_BLOCKS }; + const SNAPSHOT_MODE: PowSnapshot = PowSnapshot { blocks: NUM_SNAPSHOT_BLOCKS, max_restore_blocks: NUM_SNAPSHOT_BLOCKS }; // Temporary folders let tempdir = TempDir::new("").unwrap(); @@ -197,7 +203,7 @@ fn keep_ancient_blocks() { 0 ).unwrap(); - let manifest = ::snapshot::ManifestData { + let manifest = ManifestData { version: 2, state_hashes, state_root, @@ -209,13 +215,13 @@ fn keep_ancient_blocks() { writer.into_inner().finish(manifest.clone()).unwrap(); // Initialize the Client - let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_config = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); let client_db = new_temp_db(&tempdir.path()); let client2 = Client::new( ClientConfig::default(), &spec, client_db, - Arc::new(::miner::Miner::new_for_tests(&spec, None)), + Arc::new(miner::Miner::new_for_tests(&spec, None)), IoChannel::disconnected(), ).unwrap(); @@ -283,13 +289,13 @@ fn recover_aborted_recovery() { let spec = spec::new_null(); let tempdir = TempDir::new("").unwrap(); - let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_config = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); let client_db = new_db(); let client2 = Client::new( Default::default(), &spec, client_db, - Arc::new(::miner::Miner::new_for_tests(&spec, None)), + Arc::new(miner::Miner::new_for_tests(&spec, None)), IoChannel::disconnected(), ).unwrap(); let service_params = ServiceParams { diff --git a/ethcore/snapshot/src/tests/state.rs b/ethcore/snapshot/src/tests/state.rs index 1b3c0617534..c56cfe31da6 100644 --- a/ethcore/snapshot/src/tests/state.rs +++ b/ethcore/snapshot/src/tests/state.rs @@ -23,12 +23,15 @@ use keccak_hash::{KECCAK_NULL_RLP, keccak}; use common_types::{ basic_account::BasicAccount, errors::EthcoreError as Error, + snapshot::ManifestData, }; use client_traits::SnapshotWriter; -use snapshot::account; -use snapshot::{chunk_state, Error as SnapshotError, Progress, StateRebuilder, SNAPSHOT_SUBPARTS}; -use snapshot::io::{PackedReader, PackedWriter, SnapshotReader}; -use super::helpers::StateProducer; +use crate::{ + account, + {chunk_state, Error as SnapshotError, Progress, StateRebuilder, SNAPSHOT_SUBPARTS}, + io::{PackedReader, PackedWriter, SnapshotReader}, + tests::helpers::StateProducer, +}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use ethereum_types::H256; @@ -45,7 +48,7 @@ fn snap_and_restore() { let mut producer = StateProducer::new(); let mut rng = XorShiftRng::from_seed(RNG_SEED); let mut old_db = journaldb::new_memory_db(); - let db_cfg = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_cfg = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); for _ in 0..150 { producer.tick(&mut rng, &mut old_db); @@ -63,11 +66,11 @@ fn snap_and_restore() { state_hashes.append(&mut hashes); } - writer.into_inner().finish(::snapshot::ManifestData { + writer.into_inner().finish(ManifestData { version: 2, - state_hashes: state_hashes, + state_hashes, block_hashes: Vec::new(), - state_root: state_root, + state_root, block_number: 1000, block_hash: H256::zero(), }).unwrap(); @@ -93,7 +96,7 @@ fn snap_and_restore() { new_db }; - let new_db = journaldb::new(db, Algorithm::OverlayRecent, ::db::COL_STATE); + let new_db = journaldb::new(db, Algorithm::OverlayRecent, ethcore_db::COL_STATE); assert_eq!(new_db.earliest_era(), Some(1000)); let keys = old_db.keys(); @@ -141,7 +144,7 @@ fn get_code_from_prev_chunk() { let chunk2 = make_chunk(acc, h2); let tempdir = TempDir::new("").unwrap(); - let db_cfg = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_cfg = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); let new_db = Arc::new(Database::open(&db_cfg, tempdir.path().to_str().unwrap()).unwrap()); { @@ -154,7 +157,7 @@ fn get_code_from_prev_chunk() { rebuilder.finalize(1000, H256::random()).unwrap(); } - let state_db = journaldb::new(new_db, Algorithm::OverlayRecent, ::db::COL_STATE); + let state_db = journaldb::new(new_db, Algorithm::OverlayRecent, ethcore_db::COL_STATE); assert_eq!(state_db.earliest_era(), Some(1000)); } @@ -163,7 +166,7 @@ fn checks_flag() { let mut producer = StateProducer::new(); let mut rng = XorShiftRng::from_seed(RNG_SEED); let mut old_db = journaldb::new_memory_db(); - let db_cfg = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_cfg = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); for _ in 0..10 { producer.tick(&mut rng, &mut old_db); @@ -177,7 +180,7 @@ fn checks_flag() { let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default(), None, 0).unwrap(); - writer.into_inner().finish(::snapshot::ManifestData { + writer.into_inner().finish(ManifestData { version: 2, state_hashes, block_hashes: Vec::new(), diff --git a/ethcore/snapshot/src/watcher.rs b/ethcore/snapshot/src/watcher.rs index 8fd05cbcb92..db07ac0435d 100644 --- a/ethcore/snapshot/src/watcher.rs +++ b/ethcore/snapshot/src/watcher.rs @@ -18,11 +18,12 @@ use std::sync::Arc; -use client::{Client, ChainNotify, NewBlocks}; // todo[dvdplm] -use client_traits::BlockInfo; +//use client::{Client, ChainNotify, NewBlocks}; // todo[dvdplm] +use client_traits::{BlockInfo, ChainNotify}; use common_types::{ ids::BlockId, io_message::ClientIoMessage, + client_types::NewBlocks, }; use ethereum_types::H256; use ethcore_io::IoChannel; @@ -37,7 +38,7 @@ trait Oracle: Send + Sync { } struct StandardOracle where F: 'static + Send + Sync + Fn() -> bool { - client: Arc, + client: Arc, sync_status: F, } @@ -58,7 +59,7 @@ trait Broadcast: Send + Sync { fn take_at(&self, num: Option); } -impl Broadcast for Mutex>> { +impl Broadcast for Mutex>> { fn take_at(&self, num: Option) { let num = match num { Some(n) => n, @@ -86,17 +87,16 @@ impl Watcher { /// Create a new `Watcher` which will trigger a snapshot event /// once every `period` blocks, but only after that block is /// `history` blocks old. - pub fn new(client: Arc, sync_status: F, channel: IoChannel>, period: u64, history: u64) -> Self - where F: 'static + Send + Sync + Fn() -> bool + pub fn new(client: Arc, sync_status: F, channel: IoChannel>, period: u64, history: u64) -> Self + where + F: 'static + Send + Sync + Fn() -> bool, + C: 'static + Send + Sync, { Watcher { - oracle: Box::new(StandardOracle { - client: client, - sync_status: sync_status, - }), + oracle: Box::new(StandardOracle { client, sync_status }), broadcast: Box::new(Mutex::new(channel)), - period: period, - history: history, + period, + history, } } } @@ -123,14 +123,15 @@ impl ChainNotify for Watcher { #[cfg(test)] mod tests { - use super::{Broadcast, Oracle, Watcher}; + use std::collections::HashMap; + use std::time::Duration; - use client::{ChainNotify, NewBlocks, ChainRoute}; + use client_traits::ChainNotify; + use common_types::client_types::{NewBlocks, ChainRouteType, ChainRoute}; use ethereum_types::{H256, U256, BigEndianHash}; - use std::collections::HashMap; - use std::time::Duration; + use super::{Broadcast, Oracle, Watcher}; struct TestOracle(HashMap); @@ -161,8 +162,8 @@ mod tests { let watcher = Watcher { oracle: Box::new(TestOracle(map)), broadcast: Box::new(TestBroadcast(expected)), - period: period, - history: history, + period, + history, }; watcher.new_blocks(NewBlocks::new( From a4fc0f2dc4067c009d0df1fd6f696bd67e3abd2b Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 29 Aug 2019 16:09:53 +0200 Subject: [PATCH 04/21] Sort out types from blockchain and client --- Cargo.lock | 13 ++++ ethcore/Cargo.toml | 1 + ethcore/blockchain/src/block_info.rs | 70 ++++++++++--------- ethcore/blockchain/src/blockchain.rs | 31 ++++---- ethcore/blockchain/src/lib.rs | 3 - ethcore/blockchain/src/update.rs | 11 +-- ethcore/client-traits/src/lib.rs | 43 +++++++++++- ethcore/snapshot/src/service.rs | 5 +- ethcore/snapshot/src/tests/service.rs | 1 - ethcore/snapshot/src/watcher.rs | 4 +- ethcore/src/client/client.rs | 16 +++-- ethcore/src/client/mod.rs | 4 +- ethcore/src/lib.rs | 2 +- ethcore/src/test_helpers.rs | 5 +- ethcore/types/src/block.rs | 38 ++++++++++ .../{src/client => types/src}/chain_notify.rs | 45 ++---------- ethcore/types/src/client_types.rs | 8 ++- .../{blockchain => types}/src/import_route.rs | 6 +- ethcore/types/src/lib.rs | 2 + 19 files changed, 188 insertions(+), 120 deletions(-) rename ethcore/{src/client => types/src}/chain_notify.rs (81%) rename ethcore/{blockchain => types}/src/import_route.rs (95%) diff --git a/Cargo.lock b/Cargo.lock index a997b517de6..8801c4e918a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1074,6 +1074,7 @@ dependencies = [ "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", "spec 0.1.0", "state-db 0.1.0", "stats 0.1.0", @@ -4212,17 +4213,26 @@ dependencies = [ "common-types 0.1.0", "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "engine 0.1.0", + "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi-contract 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi-derive 8.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethcore 1.12.0", + "ethcore-accounts 0.1.0", "ethcore-blockchain 0.1.0", "ethcore-bloom-journal 0.1.0", "ethcore-db 0.1.0", "ethcore-io 1.12.0", "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethkey 0.3.0", "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4233,8 +4243,11 @@ dependencies = [ "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp_derive 0.1.0", + "spec 0.1.0", "state-db 0.1.0", + "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "triehash-ethereum 0.2.0", ] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index cf27c1a6585..5535012ba91 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -62,6 +62,7 @@ rlp_derive = { path = "../util/rlp-derive" } rustc-hex = { version = "1", optional = true } serde = "1.0" serde_derive = "1.0" +snapshot = { path = "snapshot" } spec = { path = "spec" } state-db = { path = "state-db" } tempdir = { version = "0.3", optional = true } diff --git a/ethcore/blockchain/src/block_info.rs b/ethcore/blockchain/src/block_info.rs index 15f71ecb855..43c1e3c530f 100644 --- a/ethcore/blockchain/src/block_info.rs +++ b/ethcore/blockchain/src/block_info.rs @@ -17,38 +17,40 @@ use ethereum_types::{H256, U256}; use common_types::BlockNumber; -/// Brief info about inserted block. -#[derive(Clone)] -pub struct BlockInfo { - /// Block hash. - pub hash: H256, - /// Block number. - pub number: BlockNumber, - /// Total block difficulty. - pub total_difficulty: U256, - /// Block location in blockchain. - pub location: BlockLocation -} +// todo[dvdplm]: moved to common-types -/// Describes location of newly inserted block. -#[derive(Debug, Clone, PartialEq)] -pub enum BlockLocation { - /// It's part of the canon chain. - CanonChain, - /// It's not a part of the canon chain. - Branch, - /// It's part of the fork which should become canon chain, - /// because its total difficulty is higher than current - /// canon chain difficulty. - BranchBecomingCanonChain(BranchBecomingCanonChainData), -} - -#[derive(Debug, Clone, PartialEq)] -pub struct BranchBecomingCanonChainData { - /// Hash of the newest common ancestor with old canon chain. - pub ancestor: H256, - /// Hashes of the blocks between ancestor and this block. - pub enacted: Vec, - /// Hashes of the blocks which were invalidated. - pub retracted: Vec, -} +///// Brief info about inserted block. +//#[derive(Clone)] +//pub struct BlockInfo { +// /// Block hash. +// pub hash: H256, +// /// Block number. +// pub number: BlockNumber, +// /// Total block difficulty. +// pub total_difficulty: U256, +// /// Block location in blockchain. +// pub location: BlockLocation +//} +// +///// Describes location of newly inserted block. +//#[derive(Debug, Clone, PartialEq)] +//pub enum BlockLocation { +// /// It's part of the canon chain. +// CanonChain, +// /// It's not a part of the canon chain. +// Branch, +// /// It's part of the fork which should become canon chain, +// /// because its total difficulty is higher than current +// /// canon chain difficulty. +// BranchBecomingCanonChain(BranchBecomingCanonChainData), +//} +// +//#[derive(Debug, Clone, PartialEq)] +//pub struct BranchBecomingCanonChainData { +// /// Hash of the newest common ancestor with old canon chain. +// pub ancestor: H256, +// /// Hashes of the blocks between ancestor and this block. +// pub enacted: Vec, +// /// Hashes of the blocks which were invalidated. +// pub retracted: Vec, +//} diff --git a/ethcore/blockchain/src/blockchain.rs b/ethcore/blockchain/src/blockchain.rs index a85de546dfd..05aae018a52 100644 --- a/ethcore/blockchain/src/blockchain.rs +++ b/ethcore/blockchain/src/blockchain.rs @@ -23,18 +23,22 @@ use std::sync::Arc; use ansi_term::Colour; use blooms_db; -use common_types::BlockNumber; -use common_types::blockchain_info::BlockChainInfo; -use common_types::encoded; -use common_types::engines::ForkChoice; -use common_types::engines::epoch::{Transition as EpochTransition, PendingTransition as PendingEpochTransition}; -use common_types::header::{Header, ExtendedHeader}; -use common_types::log_entry::{LogEntry, LocalizedLogEntry}; -use common_types::receipt::Receipt; -use common_types::transaction::LocalizedTransaction; -use common_types::tree_route::TreeRoute; -use common_types::view; -use common_types::views::{BlockView, HeaderView}; +use common_types::{ + BlockNumber, + blockchain_info::BlockChainInfo, + block::{BlockInfo, BlockLocation, BranchBecomingCanonChainData}, + encoded, + engines::ForkChoice, + engines::epoch::{Transition as EpochTransition, PendingTransition as PendingEpochTransition}, + header::{Header, ExtendedHeader}, + import_route::ImportRoute, + log_entry::{LogEntry, LocalizedLogEntry}, + receipt::Receipt, + transaction::LocalizedTransaction, + tree_route::TreeRoute, + view, + views::{BlockView, HeaderView}, +}; use ethcore_db::cache_manager::CacheManager; use ethcore_db::keys::{BlockReceipts, BlockDetails, TransactionAddress, EPOCH_KEY_PREFIX, EpochTransitions}; use ethcore_db::{self as db, Writable, Readable, CacheUpdatePolicy}; @@ -50,9 +54,8 @@ use rlp::RlpStream; use rlp_compress::{compress, decompress, blocks_swapper}; use crate::best_block::{BestBlock, BestAncientBlock}; -use crate::block_info::{BlockInfo, BlockLocation, BranchBecomingCanonChainData}; use crate::update::{ExtrasUpdate, ExtrasInsert}; -use crate::{CacheSize, ImportRoute, Config}; +use crate::{CacheSize, Config}; /// Database backing `BlockChain`. pub trait BlockChainDB: Send + Sync { diff --git a/ethcore/blockchain/src/lib.rs b/ethcore/blockchain/src/lib.rs index 77933922eaf..ceada247c7a 100644 --- a/ethcore/blockchain/src/lib.rs +++ b/ethcore/blockchain/src/lib.rs @@ -22,11 +22,9 @@ extern crate parity_util_mem as util_mem; extern crate parity_util_mem as malloc_size_of; mod best_block; -mod block_info; mod blockchain; mod cache; mod config; -mod import_route; mod update; pub mod generator; @@ -35,7 +33,6 @@ pub use crate::{ blockchain::{BlockProvider, BlockChain, BlockChainDB, BlockChainDBHandler}, cache::CacheSize, config::Config, - import_route::ImportRoute, update::ExtrasInsert, }; pub use ethcore_db::keys::{BlockReceipts, BlockDetails, TransactionAddress, BlockNumberKey}; diff --git a/ethcore/blockchain/src/update.rs b/ethcore/blockchain/src/update.rs index 959f55fdff5..24d04cab8e9 100644 --- a/ethcore/blockchain/src/update.rs +++ b/ethcore/blockchain/src/update.rs @@ -16,14 +16,15 @@ use std::collections::HashMap; -use common_types::BlockNumber; -use common_types::encoded::Block; -use common_types::engines::ForkChoice; +use common_types::{ + BlockNumber, + encoded::Block, + engines::ForkChoice, + block::BlockInfo, +}; use ethcore_db::keys::{BlockDetails, BlockReceipts, TransactionAddress}; use ethereum_types::{H256, Bloom}; -use crate::block_info::BlockInfo; - /// Block extras update info. pub struct ExtrasUpdate { /// Block info. diff --git a/ethcore/client-traits/src/lib.rs b/ethcore/client-traits/src/lib.rs index 896f9515b8b..7654c005920 100644 --- a/ethcore/client-traits/src/lib.rs +++ b/ethcore/client-traits/src/lib.rs @@ -29,6 +29,7 @@ use common_types::{ blockchain_info::BlockChainInfo, BlockNumber, call_analytics::CallAnalytics, + chain_notify::{NewBlocks, ChainMessageType}, client_types::Mode, encoded, engines::{epoch::Transition as EpochTransition, machine::Executed}, @@ -40,7 +41,7 @@ use common_types::{ pruning_info::PruningInfo, receipt::LocalizedReceipt, trace_filter::Filter as TraceFilter, - transaction::{self, LocalizedTransaction, CallError, SignedTransaction}, + transaction::{self, LocalizedTransaction, CallError, SignedTransaction, UnverifiedTransaction}, tree_route::TreeRoute, verification::{VerificationQueueInfo, Unverified}, }; @@ -119,7 +120,7 @@ pub trait ChainInfo { } /// Provides various information on a block by it's ID -pub trait BlockInfo { +pub trait BlockInfo: Send + Sync { /// Get raw block header data by block id. fn block_header(&self, id: BlockId) -> Option; @@ -474,3 +475,41 @@ pub trait SnapshotWriter { /// with the chunks written. fn finish(self, manifest: common_types::snapshot::ManifestData) -> std::io::Result<()> where Self: Sized; } + + +/// Represents what has to be handled by actor listening to chain events +pub trait ChainNotify : Send + Sync { + /// fires when chain has new blocks. + fn new_blocks(&self, _new_blocks: NewBlocks) { + // does nothing by default + } + + /// fires when chain achieves active mode + fn start(&self) { + // does nothing by default + } + + /// fires when chain achieves passive mode + fn stop(&self) { + // does nothing by default + } + + /// fires when chain broadcasts a message + fn broadcast(&self, _message_type: ChainMessageType) { + // does nothing by default + } + + /// fires when new block is about to be imported + /// implementations should be light + fn block_pre_import(&self, _bytes: &Bytes, _hash: &H256, _difficulty: &U256) { + // does nothing by default + } + + /// fires when new transactions are received from a peer + fn transactions_received(&self, + _txs: &[UnverifiedTransaction], + _peer_id: usize, + ) { + // does nothing by default + } +} diff --git a/ethcore/snapshot/src/service.rs b/ethcore/snapshot/src/service.rs index 491b215db60..38d6d4e56d0 100644 --- a/ethcore/snapshot/src/service.rs +++ b/ethcore/snapshot/src/service.rs @@ -34,10 +34,7 @@ use common_types::{ snapshot::{ManifestData, Progress, RestorationStatus}, }; // todo[dvdplm] put SnapshotWriter back in snapshots once extracted -use client_traits::{ - BlockInfo, BlockChainClient, ChainInfo, - SnapshotClient, SnapshotWriter, DatabaseRestore, -}; +use client_traits::{ChainInfo, SnapshotClient, SnapshotWriter}; use engine::Engine; use ethereum_types::H256; use ethcore_io::IoChannel; diff --git a/ethcore/snapshot/src/tests/service.rs b/ethcore/snapshot/src/tests/service.rs index 5e696bcfab3..05a2322b8cf 100644 --- a/ethcore/snapshot/src/tests/service.rs +++ b/ethcore/snapshot/src/tests/service.rs @@ -163,7 +163,6 @@ fn guards_delete_folders() { #[test] fn keep_ancient_blocks() { let _ = ::env_logger::try_init(); - use client_traits::BlockChainClient; // Test variables const NUM_BLOCKS: u64 = 500; const NUM_SNAPSHOT_BLOCKS: u64 = 300; diff --git a/ethcore/snapshot/src/watcher.rs b/ethcore/snapshot/src/watcher.rs index db07ac0435d..88d5cf250b1 100644 --- a/ethcore/snapshot/src/watcher.rs +++ b/ethcore/snapshot/src/watcher.rs @@ -23,7 +23,7 @@ use client_traits::{BlockInfo, ChainNotify}; use common_types::{ ids::BlockId, io_message::ClientIoMessage, - client_types::NewBlocks, + chain_notify::NewBlocks, }; use ethereum_types::H256; use ethcore_io::IoChannel; @@ -127,7 +127,7 @@ mod tests { use std::time::Duration; use client_traits::ChainNotify; - use common_types::client_types::{NewBlocks, ChainRouteType, ChainRoute}; + use common_types::chain_notify::{NewBlocks, ChainRoute}; use ethereum_types::{H256, U256, BigEndianHash}; diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index bb4855e189e..000a0cd7583 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -21,7 +21,7 @@ use std::sync::{Arc, Weak}; use std::time::{Instant, Duration}; use account_state::state::StateInfo; -use blockchain::{BlockReceipts, BlockChain, BlockChainDB, BlockProvider, TreeRoute, ImportRoute, TransactionAddress, ExtrasInsert, BlockNumberKey}; +use blockchain::{BlockReceipts, BlockChain, BlockChainDB, BlockProvider, TreeRoute, TransactionAddress, ExtrasInsert, BlockNumberKey}; use bytes::Bytes; use call_contract::{CallContract, RegistryInfo}; use ethcore_miner::pool::VerifiedTransaction; @@ -41,15 +41,15 @@ use block::{LockedBlock, Drain, ClosedBlock, OpenBlock, enact_verified, SealedBl use client::ancient_import::AncientVerifier; use client::{ ReopenBlock, PrepareOpenBlock, ImportSealedBlock, BroadcastProposalBlock, - Call, BlockProducer, SealedBlockImporter, ChainNotify, EngineInfo, - ClientConfig, NewBlocks, ChainRoute, ChainMessageType, bad_blocks, + Call, BlockProducer, SealedBlockImporter, EngineInfo, + ClientConfig, bad_blocks, }; use client_traits::{ BlockInfo, ScheduleInfo, StateClient, BlockChainReset, Nonce, Balance, ChainInfo, TransactionInfo, ImportBlock, AccountData, BlockChain as BlockChainTrait, BlockChainClient, IoClient, BadBlocks, ProvingBlockChainClient, SnapshotClient, - DatabaseRestore, SnapshotWriter, Tick, + DatabaseRestore, SnapshotWriter, Tick, ChainNotify, StateOrBlock, }; use engine::Engine; @@ -72,7 +72,9 @@ use types::{ block::PreverifiedBlock, block_status::BlockStatus, blockchain_info::BlockChainInfo, + chain_notify::{NewBlocks, ChainRoute, ChainMessageType}, client_types::ClientReport, + import_route::ImportRoute, io_message::ClientIoMessage, encoded, engines::{ @@ -986,12 +988,14 @@ impl Client { self.importer.miner.clone() } - #[cfg(test)] + /// Access state from tests + #[cfg(any(test, feature = "test-helpers"))] pub fn state_db(&self) -> ::parking_lot::RwLockReadGuard { self.state_db.read() } - #[cfg(test)] + /// Access the BlockChain from tests + #[cfg(any(test, feature = "test-helpers"))] pub fn chain(&self) -> Arc { self.chain.read().clone() } diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs index 58b8b7ffa65..56cd4273dea 100644 --- a/ethcore/src/client/mod.rs +++ b/ethcore/src/client/mod.rs @@ -31,7 +31,7 @@ pub use self::config::{ClientConfig, DatabaseCompactionProfile, BlockChainConfig pub use self::evm_test_client::{EvmTestClient, EvmTestError, TransactErr, TransactSuccess}; #[cfg(any(test, feature = "test-helpers"))] pub use self::test_client::{TestBlockChainClient, EachBlockWith, TestState}; -pub use self::chain_notify::{ChainNotify, NewBlocks, ChainRoute, ChainRouteType, ChainMessageType}; +//pub use self::chain_notify::{ChainNotify, NewBlocks, ChainRoute, ChainRouteType, ChainMessageType}; pub use self::traits::{ ReopenBlock, PrepareOpenBlock, ImportSealedBlock, BroadcastProposalBlock, Call, EngineInfo, BlockProducer, SealedBlockImporter, @@ -40,4 +40,4 @@ pub use self::traits::{ pub use verification::VerifierType; mod traits; -mod chain_notify; +//mod chain_notify; diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 8beaf991f53..26c8c46bf1c 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -88,6 +88,7 @@ extern crate rand; extern crate rayon; extern crate rlp; extern crate serde; +extern crate snapshot; extern crate spec; extern crate state_db; extern crate trace; @@ -157,7 +158,6 @@ extern crate parity_runtime; pub mod block; pub mod client; pub mod miner; -pub mod snapshot; #[cfg(test)] mod tests; diff --git a/ethcore/src/test_helpers.rs b/ethcore/src/test_helpers.rs index a89b342def5..30484bf0d45 100644 --- a/ethcore/src/test_helpers.rs +++ b/ethcore/src/test_helpers.rs @@ -34,6 +34,7 @@ use parking_lot::RwLock; use rlp::{self, RlpStream}; use tempdir::TempDir; use types::{ + chain_notify::ChainMessageType, transaction::{Action, Transaction, SignedTransaction}, encoded, engines::ForkChoice, @@ -44,8 +45,8 @@ use types::{ }; use block::{OpenBlock, Drain}; -use client::{Client, ClientConfig, ChainNotify, ChainMessageType, PrepareOpenBlock}; -use client_traits::{ChainInfo, ImportBlock}; +use client::{Client, ClientConfig, PrepareOpenBlock}; +use client_traits::{ChainInfo, ChainNotify, ImportBlock}; use trie_vm_factories::Factories; use miner::Miner; use spec::{Spec, self}; diff --git a/ethcore/types/src/block.rs b/ethcore/types/src/block.rs index e110fc7dc99..76ab66d2f3f 100644 --- a/ethcore/types/src/block.rs +++ b/ethcore/types/src/block.rs @@ -21,8 +21,10 @@ //! Other block types are found in `ethcore` use bytes::Bytes; +use ethereum_types::{H256, U256}; use parity_util_mem::MallocSizeOf; +use BlockNumber; use header::Header; use rlp::{Rlp, RlpStream, Decodable, DecoderError}; use transaction::{UnverifiedTransaction, SignedTransaction}; @@ -77,3 +79,39 @@ pub struct PreverifiedBlock { /// Block bytes pub bytes: Bytes, } + +/// Brief info about inserted block. +#[derive(Clone)] +pub struct BlockInfo { + /// Block hash. + pub hash: H256, + /// Block number. + pub number: BlockNumber, + /// Total block difficulty. + pub total_difficulty: U256, + /// Block location in blockchain. + pub location: BlockLocation +} + +/// Describes location of newly inserted block. +#[derive(Debug, Clone, PartialEq)] +pub enum BlockLocation { + /// It's part of the canon chain. + CanonChain, + /// It's not a part of the canon chain. + Branch, + /// It's part of the fork which should become canon chain, + /// because its total difficulty is higher than current + /// canon chain difficulty. + BranchBecomingCanonChain(BranchBecomingCanonChainData), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct BranchBecomingCanonChainData { + /// Hash of the newest common ancestor with old canon chain. + pub ancestor: H256, + /// Hashes of the blocks between ancestor and this block. + pub enacted: Vec, + /// Hashes of the blocks which were invalidated. + pub retracted: Vec, +} diff --git a/ethcore/src/client/chain_notify.rs b/ethcore/types/src/chain_notify.rs similarity index 81% rename from ethcore/src/client/chain_notify.rs rename to ethcore/types/src/chain_notify.rs index 1567f6c5c9e..6230e2eab31 100644 --- a/ethcore/src/client/chain_notify.rs +++ b/ethcore/types/src/chain_notify.rs @@ -14,10 +14,14 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . +// todo[dvdplm] module docs + use bytes::Bytes; use ethereum_types::{H256, U256}; -use types::transaction::UnverifiedTransaction; -use blockchain::ImportRoute; +use crate::{ + import_route::ImportRoute, + transaction::UnverifiedTransaction +}; use std::time::Duration; use std::collections::HashMap; @@ -156,40 +160,3 @@ impl NewBlocks { } } } - -/// Represents what has to be handled by actor listening to chain events -pub trait ChainNotify : Send + Sync { - /// fires when chain has new blocks. - fn new_blocks(&self, _new_blocks: NewBlocks) { - // does nothing by default - } - - /// fires when chain achieves active mode - fn start(&self) { - // does nothing by default - } - - /// fires when chain achieves passive mode - fn stop(&self) { - // does nothing by default - } - - /// fires when chain broadcasts a message - fn broadcast(&self, _message_type: ChainMessageType) { - // does nothing by default - } - - /// fires when new block is about to be imported - /// implementations should be light - fn block_pre_import(&self, _bytes: &Bytes, _hash: &H256, _difficulty: &U256) { - // does nothing by default - } - - /// fires when new transactions are received from a peer - fn transactions_received(&self, - _txs: &[UnverifiedTransaction], - _peer_id: usize, - ) { - // does nothing by default - } -} diff --git a/ethcore/types/src/client_types.rs b/ethcore/types/src/client_types.rs index a67eebcab1f..6e6cff7d9a4 100644 --- a/ethcore/types/src/client_types.rs +++ b/ethcore/types/src/client_types.rs @@ -17,12 +17,15 @@ //! Client related types. use std::{ + cmp, + collections::HashMap, fmt::{Display, Formatter, Error as FmtError}, ops, - cmp, time::Duration, }; -use ethereum_types::U256; + +use bytes::Bytes; +use ethereum_types::{H256, U256}; use crate::header::Header; /// Operating mode for the client. @@ -88,4 +91,3 @@ impl<'a> ops::Sub<&'a ClientReport> for ClientReport { self } } - diff --git a/ethcore/blockchain/src/import_route.rs b/ethcore/types/src/import_route.rs similarity index 95% rename from ethcore/blockchain/src/import_route.rs rename to ethcore/types/src/import_route.rs index 46ebc254b4e..22deee8f0d6 100644 --- a/ethcore/blockchain/src/import_route.rs +++ b/ethcore/types/src/import_route.rs @@ -17,7 +17,9 @@ //! Import route. use ethereum_types::H256; -use crate::block_info::{BlockInfo, BlockLocation}; +use crate::block::{BlockInfo, BlockLocation}; + +// todo[dvdplm]: what to do about the tests? /// Import route for newly inserted block. #[derive(Debug, PartialEq, Clone)] @@ -69,7 +71,7 @@ impl From for ImportRoute { #[cfg(test)] mod tests { use ethereum_types::{U256, BigEndianHash}; - use crate::block_info::{BlockInfo, BlockLocation, BranchBecomingCanonChainData}; + use crate::block::{BlockInfo, BlockLocation, BranchBecomingCanonChainData}; use super::ImportRoute; #[test] diff --git a/ethcore/types/src/lib.rs b/ethcore/types/src/lib.rs index fb39ad6eb7c..3b5f1f68ad6 100644 --- a/ethcore/types/src/lib.rs +++ b/ethcore/types/src/lib.rs @@ -64,6 +64,7 @@ pub mod block; pub mod block_status; pub mod blockchain_info; pub mod call_analytics; +pub mod chain_notify; pub mod client_types; pub mod encoded; pub mod engines; @@ -72,6 +73,7 @@ pub mod filter; pub mod header; pub mod ids; pub mod io_message; +pub mod import_route; pub mod log_entry; pub mod pruning_info; pub mod receipt; From 8acf2d18fd6bd3752156ddb50edc013ec679b9d3 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 29 Aug 2019 17:59:24 +0200 Subject: [PATCH 05/21] Sort out sync --- ethcore/sync/Cargo.toml | 11 ++++++----- ethcore/sync/src/api.rs | 13 +++++++------ ethcore/sync/src/chain/handler.rs | 10 ++++++---- ethcore/sync/src/chain/mod.rs | 6 ++++-- ethcore/sync/src/lib.rs | 7 ++++--- ethcore/sync/src/{snapshot.rs => snapshot_sync.rs} | 2 +- ethcore/sync/src/sync_io.rs | 2 +- ethcore/sync/src/tests/helpers.rs | 2 +- ethcore/sync/src/tests/snapshot.rs | 2 +- 9 files changed, 31 insertions(+), 24 deletions(-) rename ethcore/sync/src/{snapshot.rs => snapshot_sync.rs} (99%) diff --git a/ethcore/sync/Cargo.toml b/ethcore/sync/Cargo.toml index 7216148876e..c60b2cb030c 100644 --- a/ethcore/sync/Cargo.toml +++ b/ethcore/sync/Cargo.toml @@ -17,11 +17,12 @@ ethcore-io = { path = "../../util/io" } ethcore-light = { path = "../light" } ethcore-network = { path = "../../util/network" } ethcore-network-devp2p = { path = "../../util/network-devp2p" } -ethereum-types = "0.6.0" ethcore-private-tx = { path = "../private-tx" } +ethereum-types = "0.6.0" ethkey = { path = "../../accounts/ethkey" } ethstore = { path = "../../accounts/ethstore" } fastmap = { path = "../../util/fastmap" } +futures = "0.1" hash-db = "0.15.0" keccak-hash = "0.2.0" keccak-hasher = { path = "../../util/keccak-hasher" } @@ -30,20 +31,20 @@ log = "0.4" machine = { path = "../machine" } macros = { path = "../../util/macros" } parity-bytes = "0.1" -parking_lot = "0.8" +parity-runtime = { path = "../../util/runtime" } parity-util-mem = "0.2.0" +parking_lot = "0.8" rand = "0.6" rlp = "0.4.0" +snapshot = { path = "../snapshot" } trace-time = "0.1" triehash-ethereum = {version = "0.2", path = "../../util/triehash-ethereum" } -futures = "0.1" -parity-runtime = { path = "../../util/runtime" } [dev-dependencies] env_logger = "0.5" ethcore = { path = "..", features = ["test-helpers"] } ethcore-io = { path = "../../util/io", features = ["mio"] } kvdb-memorydb = "0.1" -rustc-hex = "1.0" rand_xorshift = "0.1.1" +rustc-hex = "1.0" spec = { path = "../spec" } diff --git a/ethcore/sync/src/api.rs b/ethcore/sync/src/api.rs index eeb6cb41d42..75f9634ad69 100644 --- a/ethcore/sync/src/api.rs +++ b/ethcore/sync/src/api.rs @@ -25,16 +25,13 @@ use network::{NetworkProtocolHandler, NetworkContext, PeerId, ProtocolId, NetworkConfiguration as BasicNetworkConfiguration, NonReservedPeerMode, Error, ConnectionFilter}; use network::client_version::ClientVersion; - -use types::pruning_info::PruningInfo; use ethereum_types::{H256, H512, U256}; use futures::sync::mpsc as futures_mpsc; use futures::Stream; use io::{TimerToken}; use ethkey::Secret; -use ethcore::client::{ChainNotify, NewBlocks, ChainMessageType}; -use client_traits::BlockChainClient; -use ethcore::snapshot::SnapshotService; +use client_traits::{BlockChainClient, ChainNotify}; +use snapshot::SnapshotService; use ethcore_private_tx::PrivateStateDB; use types::BlockNumber; use sync_io::NetSyncIo; @@ -55,7 +52,11 @@ use parity_runtime::Executor; use std::sync::atomic::{AtomicBool, Ordering}; use network::IpFilter; use private_tx::PrivateTxHandler; -use types::transaction::UnverifiedTransaction; +use types::{ + chain_notify::{NewBlocks, ChainMessageType}, + pruning_info::PruningInfo, + transaction::UnverifiedTransaction, +}; use super::light_sync::SyncInfo; diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index e7a54fa95be..8c0ab8741d0 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -23,10 +23,12 @@ use hash::keccak; use network::PeerId; use network::client_version::ClientVersion; use rlp::Rlp; -use snapshot::ChunkType; +use crate::{ + snapshot_sync::ChunkType, + sync_io::SyncIo, +}; use std::time::Instant; use std::{mem, cmp}; -use sync_io::SyncIo; use types::{ BlockNumber, block_status::BlockStatus, @@ -711,7 +713,7 @@ impl SyncHandler { Err(e) => { trace!(target: "privatetx", "Ignoring the message, error queueing: {}", e); } - } + } Ok(()) } @@ -739,7 +741,7 @@ impl SyncHandler { Err(e) => { trace!(target: "privatetx", "Ignoring the message, error queueing: {}", e); } - } + } Ok(()) } diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 07be14f7fe0..ed0f867169b 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -109,11 +109,13 @@ use rlp::{RlpStream, DecoderError}; use network::{self, PeerId, PacketId}; use network::client_version::ClientVersion; use client_traits::BlockChainClient; -use sync_io::SyncIo; +use crate::{ + sync_io::SyncIo, + snapshot_sync::Snapshot, +}; use super::{WarpSync, SyncConfig}; use block_sync::{BlockDownloader, DownloadAction}; use rand::{Rng, seq::SliceRandom}; -use snapshot::{Snapshot}; use api::{EthProtocolInfo as PeerInfoDigest, WARP_SYNC_PROTOCOL_ID, PriorityTask}; use private_tx::PrivateTxHandler; use transactions_stats::{TransactionsStats, Stats as TransactionStats}; diff --git a/ethcore/sync/src/lib.rs b/ethcore/sync/src/lib.rs index 32baddec0d7..54e013fd453 100644 --- a/ethcore/sync/src/lib.rs +++ b/ethcore/sync/src/lib.rs @@ -27,19 +27,20 @@ extern crate ethcore; extern crate ethcore_io as io; extern crate ethcore_network as network; extern crate ethcore_network_devp2p as devp2p; +extern crate ethcore_private_tx; extern crate ethereum_types; extern crate ethkey; extern crate ethstore; extern crate fastmap; +extern crate futures; extern crate keccak_hash as hash; extern crate parity_bytes as bytes; extern crate parity_runtime; extern crate parking_lot; -extern crate ethcore_private_tx; extern crate rand; extern crate rlp; +extern crate snapshot; extern crate triehash_ethereum; -extern crate futures; extern crate ethcore_light as light; @@ -69,7 +70,7 @@ mod blocks; mod block_sync; mod sync_io; mod private_tx; -mod snapshot; +mod snapshot_sync; mod transactions_stats; pub mod light_sync; diff --git a/ethcore/sync/src/snapshot.rs b/ethcore/sync/src/snapshot_sync.rs similarity index 99% rename from ethcore/sync/src/snapshot.rs rename to ethcore/sync/src/snapshot_sync.rs index 882ee188091..6aa00738031 100644 --- a/ethcore/sync/src/snapshot.rs +++ b/ethcore/sync/src/snapshot_sync.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use ethereum_types::H256; use hash::keccak; use types::snapshot::ManifestData; diff --git a/ethcore/sync/src/sync_io.rs b/ethcore/sync/src/sync_io.rs index 5c974e712fe..0f92220af6e 100644 --- a/ethcore/sync/src/sync_io.rs +++ b/ethcore/sync/src/sync_io.rs @@ -23,7 +23,7 @@ use bytes::Bytes; use client_traits::BlockChainClient; use ethcore_private_tx::PrivateStateDB; use types::BlockNumber; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use parking_lot::RwLock; /// IO interface for the syncing handler. diff --git a/ethcore/sync/src/tests/helpers.rs b/ethcore/sync/src/tests/helpers.rs index 666d362ebb5..c13e7c239a5 100644 --- a/ethcore/sync/src/tests/helpers.rs +++ b/ethcore/sync/src/tests/helpers.rs @@ -26,7 +26,7 @@ use types::io_message::ClientIoMessage; use client_traits::BlockChainClient; use ethcore::client::{TestBlockChainClient, Client as EthcoreClient, ClientConfig, ChainNotify, NewBlocks, ChainMessageType}; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use spec::{self, Spec}; use ethcore_private_tx::PrivateStateDB; use ethcore::miner::Miner; diff --git a/ethcore/sync/src/tests/snapshot.rs b/ethcore/sync/src/tests/snapshot.rs index 8136dc50c41..8152bc69ca2 100644 --- a/ethcore/sync/src/tests/snapshot.rs +++ b/ethcore/sync/src/tests/snapshot.rs @@ -20,7 +20,7 @@ use hash::keccak; use ethereum_types::H256; use parking_lot::Mutex; use bytes::Bytes; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use ethcore::client::EachBlockWith; use types::{ BlockNumber, From fd67d2faeb015cea1fab01364b047c009968ca16 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 29 Aug 2019 18:00:02 +0200 Subject: [PATCH 06/21] Sort out imports and generics --- ethcore/node-filter/src/lib.rs | 8 +++++--- ethcore/private-tx/src/lib.rs | 7 +++---- ethcore/service/Cargo.toml | 7 ++++--- ethcore/service/src/lib.rs | 1 + ethcore/service/src/service.rs | 18 +++++++++--------- ethcore/snapshot/src/watcher.rs | 2 +- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/ethcore/node-filter/src/lib.rs b/ethcore/node-filter/src/lib.rs index f34b6571b58..2c80cb2d2ab 100644 --- a/ethcore/node-filter/src/lib.rs +++ b/ethcore/node-filter/src/lib.rs @@ -42,9 +42,11 @@ extern crate log; use std::collections::{HashMap, VecDeque}; use std::sync::Weak; -use common_types::ids::BlockId; -use ethcore::client::{ChainNotify, NewBlocks}; -use client_traits::BlockChainClient; +use common_types::{ + ids::BlockId, + chain_notify::NewBlocks, +}; +use client_traits::{BlockChainClient, ChainNotify}; use ethereum_types::{H256, Address}; use ethabi::FunctionOutputDecoder; use network::{ConnectionFilter, ConnectionDirection}; diff --git a/ethcore/private-tx/src/lib.rs b/ethcore/private-tx/src/lib.rs index 8673d4be7c3..fc0686541a8 100644 --- a/ethcore/private-tx/src/lib.rs +++ b/ethcore/private-tx/src/lib.rs @@ -103,15 +103,14 @@ use machine::{ executed::Executed as FlatExecuted, }; use types::{ + chain_notify::{NewBlocks, ChainMessageType}, ids::BlockId, io_message::ClientIoMessage, transaction::{SignedTransaction, Transaction, Action, UnverifiedTransaction}, engines::machine::Executed, }; -use ethcore::client::{ - Client, ChainNotify, NewBlocks, ChainMessageType, Call -}; -use client_traits::BlockInfo; +use ethcore::client::{Client, Call}; +use client_traits::{BlockInfo, ChainNotify}; use ethcore::miner::{self, Miner, MinerService, pool_client::NonceCache}; use state_db::StateDB; use account_state::State; diff --git a/ethcore/service/Cargo.toml b/ethcore/service/Cargo.toml index f333e979d06..4956d9c7535 100644 --- a/ethcore/service/Cargo.toml +++ b/ethcore/service/Cargo.toml @@ -6,8 +6,8 @@ authors = ["Parity Technologies "] [dependencies] ansi_term = "0.11" -common-types = { path = "../types" } client-traits = { path = "../client-traits" } +common-types = { path = "../types" } ethcore = { path = ".." } ethcore-blockchain = { path = "../blockchain" } ethcore-io = { path = "../../util/io" } @@ -16,11 +16,12 @@ ethcore-sync = { path = "../sync" } ethereum-types = "0.6.0" kvdb = "0.1" log = "0.4" +snapshot = { path = "../snapshot" } spec = { path = "../spec" } trace-time = "0.1" [dev-dependencies] -ethcore-db = { path = "../db" } ethcore = { path = "..", features = ["test-helpers"] } -tempdir = "0.3" +ethcore-db = { path = "../db" } kvdb-rocksdb = "0.1.3" +tempdir = "0.3" diff --git a/ethcore/service/src/lib.rs b/ethcore/service/src/lib.rs index 662a2567669..ed82c7dfad0 100644 --- a/ethcore/service/src/lib.rs +++ b/ethcore/service/src/lib.rs @@ -25,6 +25,7 @@ extern crate ethcore_sync as sync; extern crate ethereum_types; extern crate kvdb; extern crate spec; +extern crate snapshot; #[macro_use] extern crate log; diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs index add9eb1d0d6..786b8b8db06 100644 --- a/ethcore/service/src/service.rs +++ b/ethcore/service/src/service.rs @@ -23,13 +23,13 @@ use std::time::Duration; use ansi_term::Colour; use ethereum_types::H256; use io::{IoContext, TimerToken, IoHandler, IoService, IoError}; - +use client_traits::ChainNotify; use sync::PrivateTxHandler; use blockchain::{BlockChainDB, BlockChainDBHandler}; -use ethcore::client::{Client, ClientConfig, ChainNotify}; +use ethcore::client::{Client, ClientConfig}; use ethcore::miner::Miner; -use ethcore::snapshot::service::{Service as SnapshotService, ServiceParams as SnapServiceParams}; -use ethcore::snapshot::{SnapshotService as _SnapshotService}; +use snapshot::service::{Service as SnapshotService, ServiceParams as SnapServiceParams}; +use snapshot::{SnapshotService as _SnapshotService}; use spec::Spec; use common_types::{ io_message::ClientIoMessage, @@ -94,7 +94,7 @@ impl PrivateTxHandler for PrivateTxService { pub struct ClientService { io_service: Arc>>, client: Arc, - snapshot: Arc, + snapshot: Arc>, private_tx: Arc, database: Arc, } @@ -185,7 +185,7 @@ impl ClientService { } /// Get snapshot interface. - pub fn snapshot_service(&self) -> Arc { + pub fn snapshot_service(&self) -> Arc> { self.snapshot.clone() } @@ -215,9 +215,9 @@ impl ClientService { } /// IO interface for the Client handler -struct ClientIoHandler { +struct ClientIoHandler { client: Arc, - snapshot: Arc, + snapshot: Arc>, } const CLIENT_TICK_TIMER: TimerToken = 0; @@ -239,7 +239,7 @@ where trace_time!("service::read"); match timer { CLIENT_TICK_TIMER => { - use ethcore::snapshot::SnapshotService; + use snapshot::SnapshotService; let snapshot_restoration = if let RestorationStatus::Ongoing{..} = self.snapshot.status() { true } else { false }; self.client.tick(snapshot_restoration) }, diff --git a/ethcore/snapshot/src/watcher.rs b/ethcore/snapshot/src/watcher.rs index 88d5cf250b1..9d70f032952 100644 --- a/ethcore/snapshot/src/watcher.rs +++ b/ethcore/snapshot/src/watcher.rs @@ -59,7 +59,7 @@ trait Broadcast: Send + Sync { fn take_at(&self, num: Option); } -impl Broadcast for Mutex>> { +impl Broadcast for Mutex>> { fn take_at(&self, num: Option) { let num = match num { Some(n) => n, From 67820aeccc8eb5440d8d311fb97f983798036a67 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 29 Aug 2019 18:02:33 +0200 Subject: [PATCH 07/21] Sort out main binary --- Cargo.lock | 4 ++++ Cargo.toml | 3 ++- ethcore/res/ethereum/foundation.json | 16 ++------------- parity/configuration.rs | 8 ++++---- parity/informant.rs | 21 ++++++++++---------- parity/lib.rs | 5 +++-- parity/modules.rs | 5 ++--- parity/rpc_apis.rs | 2 +- parity/run.rs | 2 +- parity/{snapshot.rs => snapshot_cmd.rs} | 10 +++++----- rpc/Cargo.toml | 1 + rpc/src/lib.rs | 1 + rpc/src/v1/impls/eth.rs | 2 +- rpc/src/v1/impls/eth_pubsub.rs | 4 ++-- rpc/src/v1/impls/parity.rs | 2 +- rpc/src/v1/tests/helpers/snapshot_service.rs | 2 +- updater/src/updater.rs | 4 ++-- 17 files changed, 44 insertions(+), 48 deletions(-) rename parity/{snapshot.rs => snapshot_cmd.rs} (96%) diff --git a/Cargo.lock b/Cargo.lock index 8801c4e918a..9f57bf3b335 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1463,6 +1463,7 @@ dependencies = [ "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb-rocksdb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", "spec 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1519,6 +1520,7 @@ dependencies = [ "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", "spec 0.1.0", "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "triehash-ethereum 0.2.0", @@ -3035,6 +3037,7 @@ dependencies = [ "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", "spec 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3183,6 +3186,7 @@ dependencies = [ "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", "spec 0.1.0", "stats 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index acd8fb42782..1c06a734dbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,7 @@ semver = "0.9" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" +snapshot = { path = "ethcore/snapshot" } spec = { path = "ethcore/spec" } term_size = "0.3" textwrap = "0.9" @@ -121,7 +122,7 @@ name = "parity" [profile.release] debug = false -lto = true +lto = false [workspace] # This should only list projects that are not diff --git a/ethcore/res/ethereum/foundation.json b/ethcore/res/ethereum/foundation.json index 12a456a453e..5a7fd3be318 100644 --- a/ethcore/res/ethereum/foundation.json +++ b/ethcore/res/ethereum/foundation.json @@ -3942,8 +3942,7 @@ } }, "0x0000000000000000000000000000000000000008": { - "builtin": [ - { + "builtin": { "name": "alt_bn128_pairing", "activate_at": "0x42ae50", "pricing": { @@ -3952,18 +3951,7 @@ "pair": 80000 } } - }, - { - "name": "alt_bn128_pairing", - "activate_at": "0xFFFFFF", - "pricing": { - "alt_bn128_pairing": { - "base": 123, - "pair": 456 - } - } - }, - ], + } }, "0x3282791d6fd713f1e94f4bfd565eaa78b3a0599d": { "balance": "0x487a9a304539440000" diff --git a/parity/configuration.rs b/parity/configuration.rs index 2066d590854..d19e39fa433 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -31,7 +31,7 @@ use sync::{NetworkConfiguration, validate_node_url, self}; use ethkey::{Secret, Public}; use ethcore::client::{VMType}; use ethcore::miner::{stratum, MinerOptions}; -use ethcore::snapshot::SnapshotConfiguration; +use snapshot::SnapshotConfiguration; use miner::pool; use verification::queue::VerifierSettings; @@ -52,7 +52,7 @@ use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, KillBlockcha use export_hardcoded_sync::ExportHsyncCmd; use presale::ImportWallet; use account::{AccountCmd, NewAccount, ListAccounts, ImportAccounts, ImportFromGethAccounts}; -use snapshot::{self, SnapshotCommand}; +use snapshot_cmd::{self, SnapshotCommand}; use network::{IpFilter}; const DEFAULT_MAX_PEERS: u16 = 50; @@ -321,7 +321,7 @@ impl Configuration { fat_db: fat_db, compaction: compaction, file_path: self.args.arg_snapshot_file.clone(), - kind: snapshot::Kind::Take, + kind: snapshot_cmd::Kind::Take, block_at: to_block_id(&self.args.arg_snapshot_at)?, max_round_blocks_to_import: self.args.arg_max_round_blocks_to_import, snapshot_conf: snapshot_conf, @@ -339,7 +339,7 @@ impl Configuration { fat_db: fat_db, compaction: compaction, file_path: self.args.arg_restore_file.clone(), - kind: snapshot::Kind::Restore, + kind: snapshot_cmd::Kind::Restore, block_at: to_block_id("latest")?, // unimportant. max_round_blocks_to_import: self.args.arg_max_round_blocks_to_import, snapshot_conf: snapshot_conf, diff --git a/parity/informant.rs b/parity/informant.rs index e4511aff483..38f02d45859 100644 --- a/parity/informant.rs +++ b/parity/informant.rs @@ -23,10 +23,11 @@ use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering}; use std::time::{Instant, Duration}; use atty; -use ethcore::client::{ChainNotify, NewBlocks, Client}; -use client_traits::{BlockInfo, ChainInfo, BlockChainClient}; +use ethcore::client::Client; +use client_traits::{BlockInfo, ChainInfo, BlockChainClient, ChainNotify}; use types::{ BlockNumber, + chain_notify::NewBlocks, client_types::ClientReport, ids::BlockId, io_message::ClientIoMessage, @@ -34,8 +35,8 @@ use types::{ verification::VerificationQueueInfo as BlockQueueInfo, snapshot::RestorationStatus, }; -use ethcore::snapshot::SnapshotService as SS; -use ethcore::snapshot::service::Service as SnapshotService; +use snapshot::SnapshotService as SS; +use snapshot::service::Service as SnapshotService; use sync::{LightSyncProvider, LightSync, SyncProvider, ManageNetwork}; use io::{TimerToken, IoContext, IoHandler}; use light::Cache as LightDataCache; @@ -224,7 +225,7 @@ pub struct Informant { last_tick: RwLock, with_color: bool, target: T, - snapshot: Option>, + snapshot: Option>>, rpc_stats: Option>, last_import: Mutex, skipped: AtomicUsize, @@ -237,16 +238,16 @@ impl Informant { /// Make a new instance potentially `with_color` output. pub fn new( target: T, - snapshot: Option>, + snapshot: Option>>, rpc_stats: Option>, with_color: bool, ) -> Self { Informant { last_tick: RwLock::new(Instant::now()), - with_color: with_color, - target: target, - snapshot: snapshot, - rpc_stats: rpc_stats, + with_color, + target, + snapshot, + rpc_stats, last_import: Mutex::new(Instant::now()), skipped: AtomicUsize::new(0), skipped_txs: AtomicUsize::new(0), diff --git a/parity/lib.rs b/parity/lib.rs index 2ea24c0bfc8..3ecf1e84c5a 100644 --- a/parity/lib.rs +++ b/parity/lib.rs @@ -73,6 +73,7 @@ extern crate parity_runtime; extern crate parity_updater as updater; extern crate parity_version; extern crate registrar; +extern crate snapshot; extern crate spec; extern crate verification; @@ -112,7 +113,7 @@ mod rpc_apis; mod run; mod secretstore; mod signer; -mod snapshot; +mod snapshot_cmd; mod upgrade; mod user_defaults; mod db; @@ -213,7 +214,7 @@ fn execute( Cmd::SignerSign { id, pwfile, port, authfile } => cli_signer::signer_sign(id, pwfile, port, authfile).map(|s| ExecutionAction::Instant(Some(s))), Cmd::SignerList { port, authfile } => cli_signer::signer_list(port, authfile).map(|s| ExecutionAction::Instant(Some(s))), Cmd::SignerReject { id, port, authfile } => cli_signer::signer_reject(id, port, authfile).map(|s| ExecutionAction::Instant(Some(s))), - Cmd::Snapshot(snapshot_cmd) => snapshot::execute(snapshot_cmd).map(|s| ExecutionAction::Instant(Some(s))), + Cmd::Snapshot(snapshot_cmd) => snapshot_cmd::execute(snapshot_cmd).map(|s| ExecutionAction::Instant(Some(s))), Cmd::ExportHardcodedSync(export_hs_cmd) => export_hardcoded_sync::execute(export_hs_cmd).map(|s| ExecutionAction::Instant(Some(s))), } } diff --git a/parity/modules.rs b/parity/modules.rs index 783d6fcb177..28ba663162d 100644 --- a/parity/modules.rs +++ b/parity/modules.rs @@ -16,15 +16,14 @@ use std::sync::{Arc, mpsc}; -use client_traits::BlockChainClient; +use client_traits::{BlockChainClient, ChainNotify}; use sync::{self, SyncConfig, NetworkConfiguration, Params, ConnectionFilter}; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use ethcore_private_tx::PrivateStateDB; use light::Provider; use parity_runtime::Executor; pub use sync::{EthSync, SyncProvider, ManageNetwork, PrivateTxHandler}; -pub use ethcore::client::ChainNotify; use ethcore_logger::Config as LogConfig; pub type SyncModules = ( diff --git a/parity/rpc_apis.rs b/parity/rpc_apis.rs index 35e65c8ada9..a3f1836f114 100644 --- a/parity/rpc_apis.rs +++ b/parity/rpc_apis.rs @@ -24,7 +24,7 @@ pub use parity_rpc::signer::SignerService; use account_utils::{self, AccountProvider}; use ethcore::client::Client; use ethcore::miner::Miner; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use client_traits::BlockChainClient; use sync::SyncState; use ethcore_logger::RotatingLogger; diff --git a/parity/run.rs b/parity/run.rs index 7b88747374c..400541336ce 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -25,7 +25,7 @@ use call_contract::CallContract; use client_traits::{BlockInfo, BlockChainClient}; use ethcore::client::{Client, DatabaseCompactionProfile, VMType}; use ethcore::miner::{self, stratum, Miner, MinerService, MinerOptions}; -use ethcore::snapshot::{self, SnapshotConfiguration}; +use snapshot::{self, SnapshotConfiguration}; use spec::SpecParams; use verification::queue::VerifierSettings; use ethcore_logger::{Config as LogConfig, RotatingLogger}; diff --git a/parity/snapshot.rs b/parity/snapshot_cmd.rs similarity index 96% rename from parity/snapshot.rs rename to parity/snapshot_cmd.rs index c9657b2d6ac..8c4a1cb10c9 100644 --- a/parity/snapshot.rs +++ b/parity/snapshot_cmd.rs @@ -22,10 +22,10 @@ use std::sync::Arc; use client_traits::SnapshotClient; use hash::keccak; -use ethcore::snapshot::{SnapshotConfiguration, SnapshotService as SS}; -use ethcore::snapshot::io::{SnapshotReader, PackedReader, PackedWriter}; -use ethcore::snapshot::service::Service as SnapshotService; -use ethcore::client::{DatabaseCompactionProfile, VMType}; +use snapshot::{SnapshotConfiguration, SnapshotService as SS}; +use snapshot::io::{SnapshotReader, PackedReader, PackedWriter}; +use snapshot::service::Service as SnapshotService; +use ethcore::client::{Client, DatabaseCompactionProfile, VMType}; use ethcore::miner::Miner; use ethcore_service::ClientService; use types::{ @@ -73,7 +73,7 @@ pub struct SnapshotCommand { // helper for reading chunks from arbitrary reader and feeding them into the // service. -fn restore_using(snapshot: Arc, reader: &R, recover: bool) -> Result<(), String> { +fn restore_using(snapshot: Arc>, reader: &R, recover: bool) -> Result<(), String> { let manifest = reader.manifest(); info!("Restoring to block #{} (0x{:?})", manifest.block_number, manifest.block_hash); diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 27b2b6fda47..aa555d75fdf 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -64,6 +64,7 @@ parity-updater = { path = "../updater" } parity-version = { path = "../util/version" } rlp = "0.4.0" account-state = { path = "../ethcore/account-state" } +snapshot = { path = "../ethcore/snapshot" } stats = { path = "../util/stats" } trace = { path = "../ethcore/trace" } vm = { path = "../ethcore/vm" } diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 5cb96fa1f14..955947339c8 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -87,6 +87,7 @@ extern crate rlp; extern crate account_state; extern crate stats; +extern crate snapshot; extern crate tempdir; extern crate trace; extern crate vm; diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 5c572b8ee48..1e6b088472f 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -29,7 +29,7 @@ use client_traits::{BlockChainClient, StateClient, ProvingBlockChainClient, Stat use ethash::{self, SeedHashCompute}; use ethcore::client::{Call, EngineInfo}; use ethcore::miner::{self, MinerService}; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use hash::keccak; use miner::external::ExternalMinerService; use sync::SyncProvider; diff --git a/rpc/src/v1/impls/eth_pubsub.rs b/rpc/src/v1/impls/eth_pubsub.rs index 7841827e934..57f7d7e1aa0 100644 --- a/rpc/src/v1/impls/eth_pubsub.rs +++ b/rpc/src/v1/impls/eth_pubsub.rs @@ -31,8 +31,7 @@ use v1::traits::EthPubSub; use v1::types::{pubsub, RichHeader, Log}; use sync::{SyncState, Notification}; -use ethcore::client::{ChainNotify, NewBlocks, ChainRouteType}; -use client_traits::BlockChainClient; +use client_traits::{BlockChainClient, ChainNotify}; use ethereum_types::H256; use light::cache::Cache; use light::client::{LightChainClient, LightChainNotify}; @@ -43,6 +42,7 @@ use parking_lot::{RwLock, Mutex}; use sync::{LightSyncProvider, LightNetworkDispatcher, ManageNetwork}; use types::{ + chain_notify::{NewBlocks, ChainRouteType}, ids::BlockId, encoded, filter::Filter as EthFilter, diff --git a/rpc/src/v1/impls/parity.rs b/rpc/src/v1/impls/parity.rs index 3503af586b1..2905324d65d 100644 --- a/rpc/src/v1/impls/parity.rs +++ b/rpc/src/v1/impls/parity.rs @@ -23,7 +23,7 @@ use ethereum_types::{H64, H160, H256, H512, U64, U256}; use ethcore::client::Call; use client_traits::{BlockChainClient, StateClient}; use ethcore::miner::{self, MinerService, FilterOptions}; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use account_state::state::StateInfo; use ethcore_logger::RotatingLogger; use ethkey::{crypto::ecies, Brain, Generator}; diff --git a/rpc/src/v1/tests/helpers/snapshot_service.rs b/rpc/src/v1/tests/helpers/snapshot_service.rs index aee9a6681c1..7f049ab2094 100644 --- a/rpc/src/v1/tests/helpers/snapshot_service.rs +++ b/rpc/src/v1/tests/helpers/snapshot_service.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use bytes::Bytes; use ethereum_types::H256; diff --git a/updater/src/updater.rs b/updater/src/updater.rs index c675fef9417..561f2c3356a 100644 --- a/updater/src/updater.rs +++ b/updater/src/updater.rs @@ -29,9 +29,9 @@ use common_types::{ BlockNumber, ids::BlockId, filter::Filter, + chain_notify::NewBlocks, }; -use ethcore::client::{ChainNotify, NewBlocks}; -use client_traits::BlockChainClient; +use client_traits::{BlockChainClient, ChainNotify}; use ethereum_types::{H256, H160}; use hash_fetch::{self as fetch, HashFetch}; use parity_path::restrict_permissions_owner; From d5a9a8142f929777e4a8d4fc4d4f12967317c108 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 29 Aug 2019 18:52:12 +0200 Subject: [PATCH 08/21] Fix sync test-helpers --- ethcore/sync/src/tests/helpers.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ethcore/sync/src/tests/helpers.rs b/ethcore/sync/src/tests/helpers.rs index c13e7c239a5..938569f5603 100644 --- a/ethcore/sync/src/tests/helpers.rs +++ b/ethcore/sync/src/tests/helpers.rs @@ -22,10 +22,16 @@ use bytes::Bytes; use network::{self, PeerId, ProtocolId, PacketId, SessionInfo}; use network::client_version::ClientVersion; use tests::snapshot::*; -use types::io_message::ClientIoMessage; -use client_traits::BlockChainClient; -use ethcore::client::{TestBlockChainClient, Client as EthcoreClient, - ClientConfig, ChainNotify, NewBlocks, ChainMessageType}; +use types::{ + chain_notify::{NewBlocks, ChainMessageType}, + io_message::ClientIoMessage, +}; +use client_traits::{BlockChainClient, ChainNotify}; +use ethcore::client::{ + TestBlockChainClient, + Client as EthcoreClient, + ClientConfig, +}; use snapshot::SnapshotService; use spec::{self, Spec}; use ethcore_private_tx::PrivateStateDB; From 23add8aae560641e426dc88b795977fb1690fb57 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 29 Aug 2019 21:20:30 +0200 Subject: [PATCH 09/21] Sort out import for secret-store --- secret-store/src/acl_storage.rs | 7 +++++-- secret-store/src/key_server_set.rs | 9 ++++++--- secret-store/src/listener/service_contract_listener.rs | 7 ++++--- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/secret-store/src/acl_storage.rs b/secret-store/src/acl_storage.rs index 110ce3d5ce6..f8f295ecc66 100644 --- a/secret-store/src/acl_storage.rs +++ b/secret-store/src/acl_storage.rs @@ -16,10 +16,13 @@ use std::sync::Arc; use std::collections::{HashMap, HashSet}; -use common_types::ids::BlockId; +use common_types::{ + chain_notify::NewBlocks, + ids::BlockId +}; use parking_lot::{Mutex, RwLock}; use call_contract::CallContract; -use ethcore::client::{ChainNotify, NewBlocks}; +use client_traits::ChainNotify; use ethereum_types::Address; use ethabi::FunctionOutputDecoder; use trusted_client::TrustedClient; diff --git a/secret-store/src/key_server_set.rs b/secret-store/src/key_server_set.rs index 5ffaa7fb767..c69be8ce7ea 100644 --- a/secret-store/src/key_server_set.rs +++ b/secret-store/src/key_server_set.rs @@ -20,9 +20,12 @@ use std::collections::{BTreeMap, HashSet}; use parking_lot::Mutex; use call_contract::CallContract; use ethabi::FunctionOutputDecoder; -use ethcore::client::{Client, ChainNotify, NewBlocks}; -use client_traits::BlockChainClient; -use common_types::ids::BlockId; +use ethcore::client::Client; +use client_traits::{BlockChainClient, ChainNotify}; +use common_types::{ + chain_notify::NewBlocks, + ids::BlockId, +}; use ethereum_types::{H256, Address}; use ethkey::public_to_address; use bytes::Bytes; diff --git a/secret-store/src/listener/service_contract_listener.rs b/secret-store/src/listener/service_contract_listener.rs index 1b6a6afdfef..c0b30605071 100644 --- a/secret-store/src/listener/service_contract_listener.rs +++ b/secret-store/src/listener/service_contract_listener.rs @@ -18,10 +18,10 @@ use std::collections::HashSet; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::thread; -use parking_lot::Mutex; -use ethcore::client::{ChainNotify, NewBlocks}; -use ethkey::{Public, public_to_address}; +use client_traits::ChainNotify; +use common_types::chain_notify::NewBlocks; use bytes::Bytes; +use ethkey::{Public, public_to_address}; use ethereum_types::{H256, U256, Address, BigEndianHash as _}; use key_server_set::KeyServerSet; use key_server_cluster::{NodeId, ClusterClient, ClusterSessionsListener, ClusterSession}; @@ -32,6 +32,7 @@ use key_server_cluster::decryption_session::SessionImpl as DecryptionSession; use key_server_cluster::key_version_negotiation_session::{SessionImpl as KeyVersionNegotiationSession, IsolatedSessionTransport as KeyVersionNegotiationTransport, FailedContinueAction}; use key_storage::KeyStorage; +use parking_lot::Mutex; use acl_storage::AclStorage; use listener::service_contract::ServiceContract; use listener::tasks_queue::TasksQueue; From 3cb1e90d91e7be52b1852efe2650b12b36da7588 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 29 Aug 2019 21:38:26 +0200 Subject: [PATCH 10/21] Sort out more imports --- ethcore/node-filter/src/lib.rs | 2 +- rpc/src/v1/tests/mocked/eth_pubsub.rs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ethcore/node-filter/src/lib.rs b/ethcore/node-filter/src/lib.rs index 2c80cb2d2ab..332b93fc163 100644 --- a/ethcore/node-filter/src/lib.rs +++ b/ethcore/node-filter/src/lib.rs @@ -161,7 +161,7 @@ mod test { Arc::new(Miner::new_for_tests(&spec, None)), IoChannel::disconnected(), ).unwrap(); - let filter = NodeFilter::new(Arc::downgrade(&client) as Weak, contract_addr); + let filter = NodeFilter::new(Arc::downgrade(&client) as Weak, contract_addr); let self1 = NodeId::from_str("00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002").unwrap(); let self2 = NodeId::from_str("00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003").unwrap(); let node1 = NodeId::from_str("00000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000012").unwrap(); diff --git a/rpc/src/v1/tests/mocked/eth_pubsub.rs b/rpc/src/v1/tests/mocked/eth_pubsub.rs index a2bfa118de2..c3563475aea 100644 --- a/rpc/src/v1/tests/mocked/eth_pubsub.rs +++ b/rpc/src/v1/tests/mocked/eth_pubsub.rs @@ -23,11 +23,12 @@ use jsonrpc_pubsub::Session; use std::time::Duration; use v1::{EthPubSub, EthPubSubClient, Metadata}; -use ethcore::client::{TestBlockChainClient, EachBlockWith, ChainNotify, NewBlocks, ChainRoute, ChainRouteType}; +use ethcore::client::{TestBlockChainClient, EachBlockWith}; use parity_runtime::Runtime; use ethereum_types::{Address, H256}; -use client_traits::BlockInfo; +use client_traits::{BlockInfo, ChainNotify}; use types::{ + chain_notify::{NewBlocks, ChainRoute, ChainRouteType}, log_entry::{LocalizedLogEntry, LogEntry}, ids::BlockId, }; From 4288cd303184791b47322fc524ed91a8ec0069c4 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 29 Aug 2019 21:49:54 +0200 Subject: [PATCH 11/21] Fix easy todos --- ethcore/blockchain/src/block_info.rs | 56 ---------------------------- ethcore/client-traits/src/lib.rs | 5 +-- ethcore/snapshot/src/service.rs | 1 - ethcore/snapshot/src/watcher.rs | 1 - ethcore/types/src/chain_notify.rs | 6 ++- 5 files changed, 5 insertions(+), 64 deletions(-) delete mode 100644 ethcore/blockchain/src/block_info.rs diff --git a/ethcore/blockchain/src/block_info.rs b/ethcore/blockchain/src/block_info.rs deleted file mode 100644 index 43c1e3c530f..00000000000 --- a/ethcore/blockchain/src/block_info.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . - -use ethereum_types::{H256, U256}; -use common_types::BlockNumber; - -// todo[dvdplm]: moved to common-types - -///// Brief info about inserted block. -//#[derive(Clone)] -//pub struct BlockInfo { -// /// Block hash. -// pub hash: H256, -// /// Block number. -// pub number: BlockNumber, -// /// Total block difficulty. -// pub total_difficulty: U256, -// /// Block location in blockchain. -// pub location: BlockLocation -//} -// -///// Describes location of newly inserted block. -//#[derive(Debug, Clone, PartialEq)] -//pub enum BlockLocation { -// /// It's part of the canon chain. -// CanonChain, -// /// It's not a part of the canon chain. -// Branch, -// /// It's part of the fork which should become canon chain, -// /// because its total difficulty is higher than current -// /// canon chain difficulty. -// BranchBecomingCanonChain(BranchBecomingCanonChainData), -//} -// -//#[derive(Debug, Clone, PartialEq)] -//pub struct BranchBecomingCanonChainData { -// /// Hash of the newest common ancestor with old canon chain. -// pub ancestor: H256, -// /// Hashes of the blocks between ancestor and this block. -// pub enacted: Vec, -// /// Hashes of the blocks which were invalidated. -// pub retracted: Vec, -//} diff --git a/ethcore/client-traits/src/lib.rs b/ethcore/client-traits/src/lib.rs index 7654c005920..9db24469a0b 100644 --- a/ethcore/client-traits/src/lib.rs +++ b/ethcore/client-traits/src/lib.rs @@ -506,10 +506,7 @@ pub trait ChainNotify : Send + Sync { } /// fires when new transactions are received from a peer - fn transactions_received(&self, - _txs: &[UnverifiedTransaction], - _peer_id: usize, - ) { + fn transactions_received(&self, _txs: &[UnverifiedTransaction], _peer_id: usize) { // does nothing by default } } diff --git a/ethcore/snapshot/src/service.rs b/ethcore/snapshot/src/service.rs index 38d6d4e56d0..747c987dbd0 100644 --- a/ethcore/snapshot/src/service.rs +++ b/ethcore/snapshot/src/service.rs @@ -26,7 +26,6 @@ use std::cmp; use blockchain::{BlockChain, BlockChainDB, BlockChainDBHandler}; use bytes::Bytes; -//use client::Client; // todo[dvdplm] use common_types::{ io_message::ClientIoMessage, errors::{EthcoreError as Error, SnapshotError, SnapshotError::UnlinkedAncientBlockChain}, diff --git a/ethcore/snapshot/src/watcher.rs b/ethcore/snapshot/src/watcher.rs index 9d70f032952..864efd356ea 100644 --- a/ethcore/snapshot/src/watcher.rs +++ b/ethcore/snapshot/src/watcher.rs @@ -18,7 +18,6 @@ use std::sync::Arc; -//use client::{Client, ChainNotify, NewBlocks}; // todo[dvdplm] use client_traits::{BlockInfo, ChainNotify}; use common_types::{ ids::BlockId, diff --git a/ethcore/types/src/chain_notify.rs b/ethcore/types/src/chain_notify.rs index 6230e2eab31..8d74e0490c5 100644 --- a/ethcore/types/src/chain_notify.rs +++ b/ethcore/types/src/chain_notify.rs @@ -14,7 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . -// todo[dvdplm] module docs +//! Types pertaining to sending messages and finding routes through the chain. Used mostly by the +//! ChainNotify trait. use bytes::Bytes; use ethereum_types::{H256, U256}; @@ -120,7 +121,8 @@ impl ChainRoute { } } -/// Used by `ChainNotify` `new_blocks()` +/// Used by `ChainNotify` `new_blocks()` and contains information about new blocks imported to the +/// chain. pub struct NewBlocks { /// Imported blocks pub imported: Vec, From 4487e84c5bc47dbc259d6ce474de2276b20363d4 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 30 Aug 2019 08:57:08 +0200 Subject: [PATCH 12/21] cleanup --- ethcore/types/src/block.rs | 1 + ethcore/types/src/chain_notify.rs | 3 +-- ethcore/types/src/client_types.rs | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ethcore/types/src/block.rs b/ethcore/types/src/block.rs index 76ab66d2f3f..334ab691c0e 100644 --- a/ethcore/types/src/block.rs +++ b/ethcore/types/src/block.rs @@ -106,6 +106,7 @@ pub enum BlockLocation { BranchBecomingCanonChain(BranchBecomingCanonChainData), } +/// Info about heaviest fork #[derive(Debug, Clone, PartialEq)] pub struct BranchBecomingCanonChainData { /// Hash of the newest common ancestor with old canon chain. diff --git a/ethcore/types/src/chain_notify.rs b/ethcore/types/src/chain_notify.rs index 8d74e0490c5..fe41644efa4 100644 --- a/ethcore/types/src/chain_notify.rs +++ b/ethcore/types/src/chain_notify.rs @@ -18,10 +18,9 @@ //! ChainNotify trait. use bytes::Bytes; -use ethereum_types::{H256, U256}; +use ethereum_types::H256; use crate::{ import_route::ImportRoute, - transaction::UnverifiedTransaction }; use std::time::Duration; use std::collections::HashMap; diff --git a/ethcore/types/src/client_types.rs b/ethcore/types/src/client_types.rs index 6e6cff7d9a4..00fcd91be40 100644 --- a/ethcore/types/src/client_types.rs +++ b/ethcore/types/src/client_types.rs @@ -18,14 +18,12 @@ use std::{ cmp, - collections::HashMap, fmt::{Display, Formatter, Error as FmtError}, ops, time::Duration, }; -use bytes::Bytes; -use ethereum_types::{H256, U256}; +use ethereum_types::U256; use crate::header::Header; /// Operating mode for the client. From 8995858230c1cc9a7370f9c96fdb092a4874e621 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 30 Aug 2019 15:47:56 +0200 Subject: [PATCH 13/21] Move SnapshotClient and SnapshotWriter to their proper places Sort out the circular dependency between snapshot and ethcore by moving all snapshot tests to own crate, snapshot-tests --- Cargo.lock | 48 ++++- Cargo.toml | 3 +- ethcore/Cargo.toml | 7 +- ethcore/client-traits/src/lib.rs | 32 --- ethcore/service/src/service.rs | 4 +- ethcore/snapshot/Cargo.toml | 3 + ethcore/snapshot/snapshot-tests/Cargo.toml | 44 ++++ .../snapshot-tests/src/abridged_block.rs | 89 ++++++++ .../snapshot/snapshot-tests/src/account.rs | 195 ++++++++++++++++++ .../tests => snapshot-tests/src}/helpers.rs | 44 ++-- ethcore/snapshot/snapshot-tests/src/io.rs | 109 ++++++++++ .../mod.rs => snapshot-tests/src/lib.rs} | 38 ++-- .../src}/proof_of_authority.rs | 22 +- .../src}/proof_of_work.rs | 10 +- .../tests => snapshot-tests/src}/service.rs | 102 ++++++++- .../tests => snapshot-tests/src}/state.rs | 16 +- .../src}/test_validator_contract.json | 0 .../snapshot/snapshot-tests/src/watcher.rs | 94 +++++++++ ethcore/snapshot/src/account.rs | 184 +---------------- ethcore/snapshot/src/block.rs | 150 +++++++------- ethcore/snapshot/src/io.rs | 111 ++-------- ethcore/snapshot/src/lib.rs | 24 ++- ethcore/snapshot/src/service.rs | 140 +++---------- ethcore/snapshot/src/traits.rs | 15 ++ ethcore/snapshot/src/watcher.rs | 165 ++++++++------- ethcore/src/client/client.rs | 5 +- ethcore/src/client/mod.rs | 2 - ethcore/src/lib.rs | 8 +- parity/snapshot_cmd.rs | 3 +- 29 files changed, 995 insertions(+), 672 deletions(-) create mode 100644 ethcore/snapshot/snapshot-tests/Cargo.toml create mode 100644 ethcore/snapshot/snapshot-tests/src/abridged_block.rs create mode 100644 ethcore/snapshot/snapshot-tests/src/account.rs rename ethcore/snapshot/{src/tests => snapshot-tests/src}/helpers.rs (92%) create mode 100644 ethcore/snapshot/snapshot-tests/src/io.rs rename ethcore/snapshot/{src/tests/mod.rs => snapshot-tests/src/lib.rs} (60%) rename ethcore/snapshot/{src/tests => snapshot-tests/src}/proof_of_authority.rs (97%) rename ethcore/snapshot/{src/tests => snapshot-tests/src}/proof_of_work.rs (96%) rename ethcore/snapshot/{src/tests => snapshot-tests/src}/service.rs (78%) rename ethcore/snapshot/{src/tests => snapshot-tests/src}/state.rs (95%) rename ethcore/snapshot/{src/tests => snapshot-tests/src}/test_validator_contract.json (100%) create mode 100644 ethcore/snapshot/snapshot-tests/src/watcher.rs diff --git a/Cargo.lock b/Cargo.lock index 9f57bf3b335..630de859cd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,7 +1020,6 @@ dependencies = [ "client-traits 0.1.0", "common-types 0.1.0", "criterion 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "engine 0.1.0", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1029,7 +1028,6 @@ dependencies = [ "ethash 1.12.0", "ethcore-accounts 0.1.0", "ethcore-blockchain 0.1.0", - "ethcore-bloom-journal 0.1.0", "ethcore-builtin 0.1.0", "ethcore-call-contract 0.1.0", "ethcore-db 0.1.0", @@ -1047,7 +1045,6 @@ dependencies = [ "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak-hasher 0.1.1", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb-memorydb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb-rocksdb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1057,10 +1054,8 @@ dependencies = [ "macros 0.1.0", "memory-cache 0.1.0", "null-engine 0.1.0", - "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-runtime 0.1.0", - "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie-ethereum 0.1.0", "pod 0.1.0", @@ -4255,6 +4250,49 @@ dependencies = [ "triehash-ethereum 0.2.0", ] +[[package]] +name = "snapshot-tests" +version = "0.1.0" +dependencies = [ + "account-db 0.1.0", + "account-state 0.1.0", + "client-traits 0.1.0", + "common-types 0.1.0", + "engine 0.1.0", + "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi-contract 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi-derive 8.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethcore 1.12.0", + "ethcore-accounts 0.1.0", + "ethcore-blockchain 0.1.0", + "ethcore-db 0.1.0", + "ethcore-io 1.12.0", + "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethkey 0.3.0", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "journaldb 0.2.0", + "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hasher 0.1.1", + "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "patricia-trie-ethereum 0.1.0", + "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", + "spec 0.1.0", + "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "triehash-ethereum 0.2.0", +] + [[package]] name = "socket2" version = "0.3.8" diff --git a/Cargo.toml b/Cargo.toml index 1c06a734dbd..302e97d1bb2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,6 +122,7 @@ name = "parity" [profile.release] debug = false +# todo[dvdplm] re-enable lto = false [workspace] @@ -136,5 +137,5 @@ members = [ "evmbin", "parity-clib", - "ethcore/snapshot" + "ethcore/snapshot/snapshot-tests" ] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 5535012ba91..4b94e4addec 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -7,14 +7,12 @@ version = "1.12.0" authors = ["Parity Technologies "] [dependencies] -account-db = { path = "account-db" } account-state = { path = "account-state" } ansi_term = "0.11" basic-authority = { path = "./engines/basic-authority", optional = true} # used by test-helpers feature blooms-db = { path = "../util/blooms-db", optional = true } client-traits = { path = "./client-traits" } common-types = { path = "types" } -crossbeam-utils = "0.6" engine = { path = "./engine" } env_logger = { version = "0.5", optional = true } ethabi = "8.0" @@ -24,7 +22,6 @@ ethash = { path = "../ethash", optional = true } ethjson = { path = "../json", optional = true } ethkey = { path = "../accounts/ethkey", optional = true } ethcore-blockchain = { path = "./blockchain" } -ethcore-bloom-journal = { path = "../util/bloom" } ethcore-call-contract = { path = "./call-contract" } ethcore-db = { path = "./db" } ethcore-io = { path = "../util/io" } @@ -38,7 +35,6 @@ hash-db = "0.15.0" itertools = "0.5" journaldb = { path = "../util/journaldb" } keccak-hash = "0.2.0" -keccak-hasher = { path = "../util/keccak-hasher" } kvdb = "0.1" kvdb-memorydb = { version = "0.1", optional = true } kvdb-rocksdb = { version = "0.1.3", optional = true } @@ -47,9 +43,7 @@ log = "0.4" macros = { path = "../util/macros", optional = true } machine = { path = "./machine" } memory-cache = { path = "../util/memory-cache" } -num_cpus = "1.2" parity-bytes = "0.1" -parity-snappy = "0.1" parking_lot = "0.8" pod = { path = "pod", optional = true } trie-db = "0.15.0" @@ -76,6 +70,7 @@ verification = { path = "./verification" } vm = { path = "vm" } [dev-dependencies] +account-db = { path = "account-db" } blooms-db = { path = "../util/blooms-db" } ethcore-builtin = { path = "./builtin" } criterion = "0.2" diff --git a/ethcore/client-traits/src/lib.rs b/ethcore/client-traits/src/lib.rs index 9db24469a0b..ace2d1a0361 100644 --- a/ethcore/client-traits/src/lib.rs +++ b/ethcore/client-traits/src/lib.rs @@ -57,8 +57,6 @@ use trace::{ }; use vm::{LastHashes, Schedule}; -use common_types::snapshot::Progress; - /// State information to be used during client query pub enum StateOrBlock { /// State to be used, may be pending @@ -447,36 +445,6 @@ pub trait DatabaseRestore: Send + Sync { fn restore_db(&self, new_db: &str) -> Result<(), EthcoreError>; } -/// Snapshot related functionality -pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + BlockChainReset { - /// Take a snapshot at the given block. - /// If the ID given is "latest", this will default to 1000 blocks behind. - fn take_snapshot( - &self, - writer: W, - at: BlockId, - p: &Progress, - ) -> Result<(), EthcoreError>; -} - - -// todo[dvdplm] move this back to snapshot once extracted from ethcore -/// Something which can write snapshots. -/// Writing the same chunk multiple times will lead to implementation-defined -/// behavior, and is not advised. -pub trait SnapshotWriter { - /// Write a compressed state chunk. - fn write_state_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>; - - /// Write a compressed block chunk. - fn write_block_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>; - - /// Complete writing. The manifest's chunk lists must be consistent - /// with the chunks written. - fn finish(self, manifest: common_types::snapshot::ManifestData) -> std::io::Result<()> where Self: Sized; -} - - /// Represents what has to be handled by actor listening to chain events pub trait ChainNotify : Send + Sync { /// fires when chain has new blocks. diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs index 786b8b8db06..06f1efe7658 100644 --- a/ethcore/service/src/service.rs +++ b/ethcore/service/src/service.rs @@ -29,14 +29,14 @@ use blockchain::{BlockChainDB, BlockChainDBHandler}; use ethcore::client::{Client, ClientConfig}; use ethcore::miner::Miner; use snapshot::service::{Service as SnapshotService, ServiceParams as SnapServiceParams}; -use snapshot::{SnapshotService as _SnapshotService}; +use snapshot::{SnapshotService as _SnapshotService, SnapshotClient}; use spec::Spec; use common_types::{ io_message::ClientIoMessage, errors::{EthcoreError, SnapshotError}, snapshot::RestorationStatus, }; -use client_traits::{ImportBlock, SnapshotClient, Tick}; +use client_traits::{ImportBlock, Tick}; use ethcore_private_tx::{self, Importer, Signer}; diff --git a/ethcore/snapshot/Cargo.toml b/ethcore/snapshot/Cargo.toml index 87ad36c4185..fcfb97e2130 100644 --- a/ethcore/snapshot/Cargo.toml +++ b/ethcore/snapshot/Cargo.toml @@ -52,3 +52,6 @@ lazy_static = { version = "1.3" } spec = { path = "../spec" } tempdir = "0.3" trie-standardmap = "0.15.0" + +[features] +test-helpers = [] diff --git a/ethcore/snapshot/snapshot-tests/Cargo.toml b/ethcore/snapshot/snapshot-tests/Cargo.toml new file mode 100644 index 00000000000..e64491797f2 --- /dev/null +++ b/ethcore/snapshot/snapshot-tests/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "snapshot-tests" +version = "0.1.0" +authors = ["David Palm "] +edition = "2018" + +[dependencies] +accounts = { package = "ethcore-accounts", path = "../../../accounts" } +account-db = { path = "../../account-db" } +account-state = { path = "../../account-state" } +blockchain = { package = "ethcore-blockchain", path = "../../blockchain" } +bytes = { package = "parity-bytes", version = "0.1.0" } +client-traits = { path = "../../client-traits" } +common-types = { path = "../../types" } +engine = { path = "../../engine", features = ["test-helpers"] } +env_logger = "0.5" +ethcore = { path = "../..", features = ["test-helpers"] } +ethcore-db = { path = "../../db" } +ethcore-io = { path = "../../../util/io" } +ethereum-types = "0.6.0" +ethkey = { path = "../../../accounts/ethkey" } +ethtrie = { package = "patricia-trie-ethereum", path = "../../../util/patricia-trie-ethereum" } +hash-db = "0.15.0" +journaldb = { path = "../../../util/journaldb" } +keccak-hash = "0.2.0" +keccak-hasher = { path = "../../../util/keccak-hasher" } +kvdb = "0.1.0" +kvdb-rocksdb = { version = "0.1.3" } +log = "0.4.8" +parking_lot = "0.8.0" +rand = "0.6" +rand_xorshift = "0.1.1" +rlp = "0.4.2" +snappy = { package = "parity-snappy", version ="0.1.0" } +snapshot = { path = "../../snapshot", features = ["test-helpers"] } +spec = { path = "../../spec" } +tempdir = "0.3" +trie-db = "0.15.0" +trie-standardmap = "0.15.0" +ethabi = "8.0" +ethabi-contract = "8.0" +ethabi-derive = "8.0" +lazy_static = { version = "1.3" } +triehash = { package = "triehash-ethereum", version = "0.2", path = "../../../util/triehash-ethereum" } diff --git a/ethcore/snapshot/snapshot-tests/src/abridged_block.rs b/ethcore/snapshot/snapshot-tests/src/abridged_block.rs new file mode 100644 index 00000000000..7d17b727bc3 --- /dev/null +++ b/ethcore/snapshot/snapshot-tests/src/abridged_block.rs @@ -0,0 +1,89 @@ +// Copyright 2015-2019 Parity Technologies (UK) Ltd. +// This file is part of Parity Ethereum. + +// Parity Ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity Ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity Ethereum. If not, see . + +//! Tests for block RLP encoding + +use snapshot::block::AbridgedBlock; + +use bytes::Bytes; +use ethereum_types::{H256, U256, Address}; +use common_types::{ + transaction::{Action, Transaction}, + block::Block, + view, + views::BlockView, +}; + +fn encode_block(b: &Block) -> Bytes { + b.rlp_bytes() +} + +#[test] +fn empty_block_abridging() { + let b = Block::default(); + let receipts_root = b.header.receipts_root().clone(); + let encoded = encode_block(&b); + + let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded)); + assert_eq!(abridged.to_block(H256::zero(), 0, receipts_root).unwrap(), b); +} + +#[test] +#[should_panic] +fn wrong_number() { + let b = Block::default(); + let receipts_root = b.header.receipts_root().clone(); + let encoded = encode_block(&b); + + let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded)); + assert_eq!(abridged.to_block(H256::zero(), 2, receipts_root).unwrap(), b); +} + +#[test] +fn with_transactions() { + let mut b = Block::default(); + + let t1 = Transaction { + action: Action::Create, + nonce: U256::from(42), + gas_price: U256::from(3000), + gas: U256::from(50_000), + value: U256::from(1), + data: b"Hello!".to_vec() + }.fake_sign(Address::from_low_u64_be(0x69)); + + let t2 = Transaction { + action: Action::Create, + nonce: U256::from(88), + gas_price: U256::from(12345), + gas: U256::from(300000), + value: U256::from(1000000000), + data: "Eep!".into(), + }.fake_sign(Address::from_low_u64_be(0x55)); + + b.transactions.push(t1.into()); + b.transactions.push(t2.into()); + + let receipts_root = b.header.receipts_root().clone(); + b.header.set_transactions_root(triehash::ordered_trie_root( + b.transactions.iter().map(::rlp::encode) + )); + + let encoded = encode_block(&b); + + let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded[..])); + assert_eq!(abridged.to_block(H256::zero(), 0, receipts_root).unwrap(), b); +} diff --git a/ethcore/snapshot/snapshot-tests/src/account.rs b/ethcore/snapshot/snapshot-tests/src/account.rs new file mode 100644 index 00000000000..62c199fbd85 --- /dev/null +++ b/ethcore/snapshot/snapshot-tests/src/account.rs @@ -0,0 +1,195 @@ +// Copyright 2015-2019 Parity Technologies (UK) Ltd. +// This file is part of Parity Ethereum. + +// Parity Ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity Ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity Ethereum. If not, see . + +//! Tests for account state encoding and decoding + +use std::collections::HashSet; + +use account_db::{AccountDB, AccountDBMut}; +use common_types::{ + basic_account::BasicAccount, + snapshot::Progress +}; +use ethcore::test_helpers::get_temp_state_db; +use ethereum_types::{H256, Address}; +use hash_db::{HashDB, EMPTY_PREFIX}; +use keccak_hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; +use kvdb::DBValue; +use rlp::Rlp; +use snapshot::account::{ACC_EMPTY, to_fat_rlps, from_fat_rlp}; + +use crate::helpers::fill_storage; + +#[test] +fn encoding_basic() { + let mut db = get_temp_state_db(); + let addr = Address::random(); + + let account = BasicAccount { + nonce: 50.into(), + balance: 123456789.into(), + storage_root: KECCAK_NULL_RLP, + code_hash: KECCAK_EMPTY, + code_version: 0.into(), + }; + + let thin_rlp = ::rlp::encode(&account); + assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); + let p = Progress::default(); + let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); + let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap(); + assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account); +} + +#[test] +fn encoding_version() { + let mut db = get_temp_state_db(); + let addr = Address::random(); + + let account = BasicAccount { + nonce: 50.into(), + balance: 123456789.into(), + storage_root: KECCAK_NULL_RLP, + code_hash: KECCAK_EMPTY, + code_version: 1.into(), + }; + + let thin_rlp = ::rlp::encode(&account); + assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); + let p = Progress::default(); + let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); + let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap(); + assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account); +} + +#[test] +fn encoding_storage() { + let mut db = get_temp_state_db(); + let addr = Address::random(); + + let account = { + let acct_db = AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)); + let mut root = KECCAK_NULL_RLP; + fill_storage(acct_db, &mut root, &mut H256::zero()); + BasicAccount { + nonce: 25.into(), + balance: 987654321.into(), + storage_root: root, + code_hash: KECCAK_EMPTY, + code_version: 0.into(), + } + }; + + let thin_rlp = ::rlp::encode(&account); + assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); + + let p = Progress::default(); + + let fat_rlp = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); + let fat_rlp = Rlp::new(&fat_rlp[0]).at(1).unwrap(); + assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account); +} + +#[test] +fn encoding_storage_split() { + let mut db = get_temp_state_db(); + let addr = Address::random(); + + let account = { + let acct_db = AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)); + let mut root = KECCAK_NULL_RLP; + fill_storage(acct_db, &mut root, &mut H256::zero()); + BasicAccount { + nonce: 25.into(), + balance: 987654321.into(), + storage_root: root, + code_hash: KECCAK_EMPTY, + code_version: 0.into(), + } + }; + + let thin_rlp = ::rlp::encode(&account); + assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); + + let p = Progress::default(); + let fat_rlps = to_fat_rlps(&keccak(addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), 500, 1000, &p).unwrap(); + let mut root = KECCAK_NULL_RLP; + let mut restored_account = None; + for rlp in fat_rlps { + let fat_rlp = Rlp::new(&rlp).at(1).unwrap(); + restored_account = Some(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, root).unwrap().0); + root = restored_account.as_ref().unwrap().storage_root.clone(); + } + assert_eq!(restored_account, Some(account)); +} + +#[test] +fn encoding_code() { + let mut db = get_temp_state_db(); + + let addr1 = Address::random(); + let addr2 = Address::random(); + + let code_hash = { + let mut acct_db = AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr1)); + acct_db.insert(EMPTY_PREFIX, b"this is definitely code") + }; + + { + let mut acct_db = AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr2)); + acct_db.emplace(code_hash.clone(), EMPTY_PREFIX, DBValue::from_slice(b"this is definitely code")); + } + + let account1 = BasicAccount { + nonce: 50.into(), + balance: 123456789.into(), + storage_root: KECCAK_NULL_RLP, + code_hash, + code_version: 0.into(), + }; + + let account2 = BasicAccount { + nonce: 400.into(), + balance: 98765432123456789usize.into(), + storage_root: KECCAK_NULL_RLP, + code_hash, + code_version: 0.into(), + }; + + let mut used_code = HashSet::new(); + let p1 = Progress::default(); + let p2 = Progress::default(); + let fat_rlp1 = to_fat_rlps(&keccak(&addr1), &account1, &AccountDB::from_hash(db.as_hash_db(), keccak(addr1)), &mut used_code, usize::max_value(), usize::max_value(), &p1).unwrap(); + let fat_rlp2 = to_fat_rlps(&keccak(&addr2), &account2, &AccountDB::from_hash(db.as_hash_db(), keccak(addr2)), &mut used_code, usize::max_value(), usize::max_value(), &p2).unwrap(); + assert_eq!(used_code.len(), 1); + + let fat_rlp1 = Rlp::new(&fat_rlp1[0]).at(1).unwrap(); + let fat_rlp2 = Rlp::new(&fat_rlp2[0]).at(1).unwrap(); + + let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr2)), fat_rlp2, H256::zero()).unwrap(); + assert!(maybe_code.is_none()); + assert_eq!(acc, account2); + + let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr1)), fat_rlp1, H256::zero()).unwrap(); + assert_eq!(maybe_code, Some(b"this is definitely code".to_vec())); + assert_eq!(acc, account1); +} + +#[test] +fn encoding_empty_acc() { + let mut db = get_temp_state_db(); + assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(Address::zero())), Rlp::new(&::rlp::NULL_RLP), H256::zero()).unwrap(), (ACC_EMPTY, None)); +} diff --git a/ethcore/snapshot/src/tests/helpers.rs b/ethcore/snapshot/snapshot-tests/src/helpers.rs similarity index 92% rename from ethcore/snapshot/src/tests/helpers.rs rename to ethcore/snapshot/snapshot-tests/src/helpers.rs index 98b206ed4e2..afc4979068d 100644 --- a/ethcore/snapshot/src/tests/helpers.rs +++ b/ethcore/snapshot/snapshot-tests/src/helpers.rs @@ -20,30 +20,36 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; -use keccak_hash::{KECCAK_NULL_RLP}; use account_db::AccountDBMut; -use common_types::basic_account::BasicAccount; +use account_state; use blockchain::{BlockChain, BlockChainDB}; -use ethcore::client::Client; -use client_traits::{ChainInfo, SnapshotClient}; +use client_traits::ChainInfo; +use common_types::{ + ids::BlockId, + basic_account::BasicAccount, + errors::EthcoreError +}; use engine::Engine; -use crate::{ +use ethcore::client::Client; +use ethereum_types::H256; +use ethtrie::{SecTrieDBMut, TrieDB, TrieDBMut}; +use hash_db::HashDB; +use journaldb; +use keccak_hash::{KECCAK_NULL_RLP}; +use keccak_hasher::KeccakHasher; +use kvdb::DBValue; +use log::trace; +use rand::Rng; +use rlp; +use snapshot::{ + SnapshotClient, StateRebuilder, io::{SnapshotReader, PackedWriter, PackedReader}, + chunker, }; - use tempdir::TempDir; -use rand::Rng; -use log::trace; -use kvdb::DBValue; -use ethereum_types::H256; -use hash_db::HashDB; -use keccak_hasher::KeccakHasher; -use journaldb; use trie_db::{TrieMut, Trie}; -use ethtrie::{SecTrieDBMut, TrieDB, TrieDBMut}; use trie_standardmap::{Alphabet, StandardMap, ValueMode}; -use common_types::errors::EthcoreError; // the proportion of accounts we will alter each tick. const ACCOUNT_CHURN: f32 = 0.01; @@ -80,10 +86,10 @@ impl StateProducer { // sweep once to alter storage tries. for &mut (ref mut address_hash, ref mut account_data) in &mut accounts_to_modify { - let mut account: BasicAccount = ::rlp::decode(&*account_data).expect("error decoding basic account"); + let mut account: BasicAccount = rlp::decode(&*account_data).expect("error decoding basic account"); let acct_db = AccountDBMut::from_hash(db, *address_hash); fill_storage(acct_db, &mut account.storage_root, &mut self.storage_seed); - *account_data = DBValue::from_vec(::rlp::encode(&account)); + *account_data = DBValue::from_vec(rlp::encode(&account)); } // sweep again to alter account trie. @@ -136,8 +142,6 @@ pub fn fill_storage(mut db: AccountDBMut, root: &mut H256, seed: &mut H256) { /// Take a snapshot from the given client into a temporary file. /// Return a snapshot reader for it. pub fn snap(client: &Client) -> (Box, TempDir) { - use common_types::ids::BlockId; - let tempdir = TempDir::new("").unwrap(); let path = tempdir.path().join("file"); let writer = PackedWriter::new(&path).unwrap(); @@ -160,7 +164,7 @@ pub fn restore( genesis: &[u8], ) -> Result<(), EthcoreError> { let flag = AtomicBool::new(true); - let chunker = crate::chunker(engine.snapshot_mode()).expect("the engine used here supports snapshots"); + let chunker = chunker(engine.snapshot_mode()).expect("the engine used here supports snapshots"); let manifest = reader.manifest(); let mut state = StateRebuilder::new(db.key_value().clone(), journaldb::Algorithm::Archive); diff --git a/ethcore/snapshot/snapshot-tests/src/io.rs b/ethcore/snapshot/snapshot-tests/src/io.rs new file mode 100644 index 00000000000..8618e6862d9 --- /dev/null +++ b/ethcore/snapshot/snapshot-tests/src/io.rs @@ -0,0 +1,109 @@ +// Copyright 2015-2019 Parity Technologies (UK) Ltd. +// This file is part of Parity Ethereum. + +// Parity Ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity Ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity Ethereum. If not, see . + +//! Tests for snapshot i/o. + +use tempdir::TempDir; +use keccak_hash::keccak; + +use common_types::snapshot::ManifestData; +use snapshot::io::{ + SnapshotWriter,SnapshotReader, + PackedWriter, PackedReader, LooseWriter, LooseReader, + SNAPSHOT_VERSION, +}; + +const STATE_CHUNKS: &'static [&'static [u8]] = &[b"dog", b"cat", b"hello world", b"hi", b"notarealchunk"]; +const BLOCK_CHUNKS: &'static [&'static [u8]] = &[b"hello!", b"goodbye!", b"abcdefg", b"hijklmnop", b"qrstuvwxy", b"and", b"z"]; + +#[test] +fn packed_write_and_read() { + let tempdir = TempDir::new("").unwrap(); + let path = tempdir.path().join("packed"); + let mut writer = PackedWriter::new(&path).unwrap(); + + let mut state_hashes = Vec::new(); + let mut block_hashes = Vec::new(); + + for chunk in STATE_CHUNKS { + let hash = keccak(&chunk); + state_hashes.push(hash.clone()); + writer.write_state_chunk(hash, chunk).unwrap(); + } + + for chunk in BLOCK_CHUNKS { + let hash = keccak(&chunk); + block_hashes.push(hash.clone()); + writer.write_block_chunk(keccak(&chunk), chunk).unwrap(); + } + + let manifest = ManifestData { + version: SNAPSHOT_VERSION, + state_hashes, + block_hashes, + state_root: keccak(b"notarealroot"), + block_number: 12345678987654321, + block_hash: keccak(b"notarealblock"), + }; + + writer.finish(manifest.clone()).unwrap(); + + let reader = PackedReader::new(&path).unwrap().unwrap(); + assert_eq!(reader.manifest(), &manifest); + + for hash in manifest.state_hashes.iter().chain(&manifest.block_hashes) { + reader.chunk(hash.clone()).unwrap(); + } +} + +#[test] +fn loose_write_and_read() { + let tempdir = TempDir::new("").unwrap(); + let mut writer = LooseWriter::new(tempdir.path().into()).unwrap(); + + let mut state_hashes = Vec::new(); + let mut block_hashes = Vec::new(); + + for chunk in STATE_CHUNKS { + let hash = keccak(&chunk); + state_hashes.push(hash.clone()); + writer.write_state_chunk(hash, chunk).unwrap(); + } + + for chunk in BLOCK_CHUNKS { + let hash = keccak(&chunk); + block_hashes.push(hash.clone()); + writer.write_block_chunk(keccak(&chunk), chunk).unwrap(); + } + + let manifest = ManifestData { + version: SNAPSHOT_VERSION, + state_hashes, + block_hashes, + state_root: keccak(b"notarealroot"), + block_number: 12345678987654321, + block_hash: keccak(b"notarealblock)"), + }; + + writer.finish(manifest.clone()).unwrap(); + + let reader = LooseReader::new(tempdir.path().into()).unwrap(); + assert_eq!(reader.manifest(), &manifest); + + for hash in manifest.state_hashes.iter().chain(&manifest.block_hashes) { + reader.chunk(hash.clone()).unwrap(); + } +} diff --git a/ethcore/snapshot/src/tests/mod.rs b/ethcore/snapshot/snapshot-tests/src/lib.rs similarity index 60% rename from ethcore/snapshot/src/tests/mod.rs rename to ethcore/snapshot/snapshot-tests/src/lib.rs index c1139e7b5fa..c4fcc940188 100644 --- a/ethcore/snapshot/src/tests/mod.rs +++ b/ethcore/snapshot/snapshot-tests/src/lib.rs @@ -16,25 +16,37 @@ //! Snapshot tests. +#[cfg(test)] +mod abridged_block; +#[cfg(test)] +mod account; +#[cfg(test)] +mod io; +#[cfg(test)] mod proof_of_work; +#[cfg(test)] mod proof_of_authority; +#[cfg(test)] mod state; +#[cfg(test)] mod service; +#[cfg(test)] +mod watcher; -pub mod helpers; - -use common_types::snapshot::ManifestData; +#[cfg(test)] +mod helpers; #[test] fn manifest_rlp() { - let manifest = ManifestData { - version: 2, - block_hashes: Vec::new(), - state_hashes: Vec::new(), - block_number: 1234567, - state_root: Default::default(), - block_hash: Default::default(), - }; - let raw = manifest.clone().into_rlp(); - assert_eq!(ManifestData::from_rlp(&raw).unwrap(), manifest); + use common_types::snapshot::ManifestData; + let manifest = ManifestData { + version: 2, + block_hashes: Vec::new(), + state_hashes: Vec::new(), + block_number: 1234567, + state_root: Default::default(), + block_hash: Default::default(), + }; + let raw = manifest.clone().into_rlp(); + assert_eq!(ManifestData::from_rlp(&raw).unwrap(), manifest); } diff --git a/ethcore/snapshot/src/tests/proof_of_authority.rs b/ethcore/snapshot/snapshot-tests/src/proof_of_authority.rs similarity index 97% rename from ethcore/snapshot/src/tests/proof_of_authority.rs rename to ethcore/snapshot/snapshot-tests/src/proof_of_authority.rs index 7b9437625a2..b36ab4674dd 100644 --- a/ethcore/snapshot/src/tests/proof_of_authority.rs +++ b/ethcore/snapshot/snapshot-tests/src/proof_of_authority.rs @@ -21,25 +21,25 @@ use std::sync::Arc; use std::str::FromStr; use accounts::AccountProvider; -use ethcore::client::Client; use client_traits::{BlockChainClient, ChainInfo}; -use ethkey::Secret; -use crate::tests::helpers as snapshot_helpers; -use spec::Spec; -use ethcore::test_helpers::generate_dummy_client_with_spec; use common_types::transaction::{Transaction, Action, SignedTransaction}; -use tempdir::TempDir; -use log::trace; -use ethereum_types::Address; +use ethabi_contract::use_contract; use ethcore::{ - test_helpers, + client::Client, + test_helpers::{self, generate_dummy_client_with_spec}, miner::{self, MinerService}, }; +use ethereum_types::Address; +use ethkey::Secret; use keccak_hash::keccak; -use ethabi_contract::use_contract; use lazy_static::lazy_static; +use log::trace; +use spec::Spec; +use tempdir::TempDir; + +use crate::helpers as snapshot_helpers; -use_contract!(test_validator_set, "../res/contracts/test_validator_set.json"); +use_contract!(test_validator_set, "../../res/contracts/test_validator_set.json"); const PASS: &'static str = ""; const TRANSITION_BLOCK_1: usize = 2; // block at which the contract becomes activated. diff --git a/ethcore/snapshot/src/tests/proof_of_work.rs b/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs similarity index 96% rename from ethcore/snapshot/src/tests/proof_of_work.rs rename to ethcore/snapshot/snapshot-tests/src/proof_of_work.rs index 53dc071423a..81dcfe2cd5b 100644 --- a/ethcore/snapshot/src/tests/proof_of_work.rs +++ b/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs @@ -19,18 +19,16 @@ use std::sync::atomic::AtomicBool; use tempdir::TempDir; use common_types::{ - errors::EthcoreError as Error, + errors::{EthcoreError as Error, SnapshotError}, engines::ForkChoice, snapshot::{Progress, ManifestData}, }; - use blockchain::generator::{BlockGenerator, BlockBuilder}; use blockchain::{BlockChain, ExtrasInsert}; -use client_traits::SnapshotWriter; -use crate::{ +use snapshot::{ chunk_secondary, - Error as SnapshotError, SnapshotComponents, - io::{PackedReader, PackedWriter, SnapshotReader}, + SnapshotComponents, + io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}, PowSnapshot, }; use parking_lot::Mutex; diff --git a/ethcore/snapshot/src/tests/service.rs b/ethcore/snapshot/snapshot-tests/src/service.rs similarity index 78% rename from ethcore/snapshot/src/tests/service.rs rename to ethcore/snapshot/snapshot-tests/src/service.rs index 05a2322b8cf..b2e7a98256f 100644 --- a/ethcore/snapshot/src/tests/service.rs +++ b/ethcore/snapshot/snapshot-tests/src/service.rs @@ -22,17 +22,18 @@ use std::sync::Arc; use tempdir::TempDir; use blockchain::BlockProvider; use ethcore::client::{Client, ClientConfig}; -use client_traits::{BlockInfo, ImportBlock, SnapshotWriter}; +use client_traits::{BlockInfo, ImportBlock}; use common_types::{ + io_message::ClientIoMessage, ids::BlockId, snapshot::Progress, verification::Unverified, snapshot::{ManifestData, RestorationStatus}, }; -use crate::{ +use snapshot::{ chunk_state, chunk_secondary, SnapshotService, - io::{PackedReader, PackedWriter, SnapshotReader}, - service::{Service, ServiceParams}, + io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}, + service::{Service, ServiceParams, Guard, Restoration, RestorationParams}, PowSnapshot, }; use spec; @@ -42,8 +43,97 @@ use ethcore::{ }; use parking_lot::Mutex; -use ethcore_io::IoChannel; +use ethcore_io::{IoChannel, IoService}; use kvdb_rocksdb::DatabaseConfig; +use journaldb::Algorithm; + +#[test] +fn sends_async_messages() { + let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; + let client = generate_dummy_client_with_spec_and_data(spec::new_null, 400, 5, &gas_prices); + let service = IoService::>::start().unwrap(); + let spec = spec::new_test(); + + let tempdir = TempDir::new("").unwrap(); + let dir = tempdir.path().join("snapshot"); + + let snapshot_params = ServiceParams { + engine: spec.engine.clone(), + genesis_block: spec.genesis_block(), + restoration_db_handler: restoration_db_handler(Default::default()), + pruning: Algorithm::Archive, + channel: service.channel(), + snapshot_root: dir, + client, + }; + + let service = Service::new(snapshot_params).unwrap(); + + assert!(service.manifest().is_none()); + assert!(service.chunk(Default::default()).is_none()); + assert_eq!(service.status(), RestorationStatus::Inactive); + + let manifest = ManifestData { + version: 2, + state_hashes: vec![], + block_hashes: vec![], + state_root: Default::default(), + block_number: 0, + block_hash: Default::default(), + }; + + service.begin_restore(manifest); + service.abort_restore(); + service.restore_state_chunk(Default::default(), vec![]); + service.restore_block_chunk(Default::default(), vec![]); +} + +#[test] +fn cannot_finish_with_invalid_chunks() { + use ethereum_types::H256; + use kvdb_rocksdb::DatabaseConfig; + + let spec = spec::new_test(); + let tempdir = TempDir::new("").unwrap(); + + let state_hashes: Vec<_> = (0..5).map(|_| H256::random()).collect(); + let block_hashes: Vec<_> = (0..5).map(|_| H256::random()).collect(); + let db_config = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); + let gb = spec.genesis_block(); + let flag = ::std::sync::atomic::AtomicBool::new(true); + + let engine = &*spec.engine.clone(); + let params = RestorationParams::new( + ManifestData { + version: 2, + state_hashes: state_hashes.clone(), + block_hashes: block_hashes.clone(), + state_root: H256::zero(), + block_number: 100000, + block_hash: H256::zero(), + }, + Algorithm::Archive, + restoration_db_handler(db_config).open(&tempdir.path().to_owned()).unwrap(), + None, + &gb, + Guard::benign(), + engine, + ); + + let mut restoration = Restoration::new(params).unwrap(); + let definitely_bad_chunk = [1, 2, 3, 4, 5]; + + for hash in state_hashes { + assert!(restoration.feed_state(hash, &definitely_bad_chunk, &flag).is_err()); + assert!(!restoration.is_done()); + } + + for hash in block_hashes { + assert!(restoration.feed_blocks(hash, &definitely_bad_chunk, &*spec.engine, &flag).is_err()); + assert!(!restoration.is_done()); + } +} + #[test] fn restored_is_equivalent() { @@ -280,7 +370,7 @@ fn keep_ancient_blocks() { #[test] fn recover_aborted_recovery() { - let _ = ::env_logger::try_init(); + let _ = env_logger::try_init(); const NUM_BLOCKS: u32 = 400; let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; diff --git a/ethcore/snapshot/src/tests/state.rs b/ethcore/snapshot/snapshot-tests/src/state.rs similarity index 95% rename from ethcore/snapshot/src/tests/state.rs rename to ethcore/snapshot/snapshot-tests/src/state.rs index c56cfe31da6..071370bc41e 100644 --- a/ethcore/snapshot/src/tests/state.rs +++ b/ethcore/snapshot/snapshot-tests/src/state.rs @@ -18,19 +18,17 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; -use keccak_hash::{KECCAK_NULL_RLP, keccak}; +use keccak_hash::{KECCAK_NULL_RLP, keccak}; use common_types::{ basic_account::BasicAccount, - errors::EthcoreError as Error, - snapshot::ManifestData, + errors::{EthcoreError as Error, SnapshotError}, + snapshot::{ManifestData, Progress}, }; -use client_traits::SnapshotWriter; -use crate::{ +use snapshot::{ account, - {chunk_state, Error as SnapshotError, Progress, StateRebuilder, SNAPSHOT_SUBPARTS}, - io::{PackedReader, PackedWriter, SnapshotReader}, - tests::helpers::StateProducer, + chunk_state, StateRebuilder, SNAPSHOT_SUBPARTS, + io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}, }; use rand::SeedableRng; use rand_xorshift::XorShiftRng; @@ -40,6 +38,8 @@ use kvdb_rocksdb::{Database, DatabaseConfig}; use parking_lot::Mutex; use tempdir::TempDir; +use crate::helpers::StateProducer; + const RNG_SEED: [u8; 16] = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]; #[test] diff --git a/ethcore/snapshot/src/tests/test_validator_contract.json b/ethcore/snapshot/snapshot-tests/src/test_validator_contract.json similarity index 100% rename from ethcore/snapshot/src/tests/test_validator_contract.json rename to ethcore/snapshot/snapshot-tests/src/test_validator_contract.json diff --git a/ethcore/snapshot/snapshot-tests/src/watcher.rs b/ethcore/snapshot/snapshot-tests/src/watcher.rs new file mode 100644 index 00000000000..b0ccd7dedfc --- /dev/null +++ b/ethcore/snapshot/snapshot-tests/src/watcher.rs @@ -0,0 +1,94 @@ +// Copyright 2015-2019 Parity Technologies (UK) Ltd. +// This file is part of Parity Ethereum. + +// Parity Ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity Ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity Ethereum. If not, see . + +//! Tests for block RLP encoding + +use std::collections::HashMap; +use std::time::Duration; + +use client_traits::ChainNotify; +use common_types::chain_notify::{NewBlocks, ChainRoute}; +use ethereum_types::{H256, U256, BigEndianHash}; + +use snapshot::{ + Watcher, + watcher::{Broadcast, Oracle}, +}; + +struct TestOracle(HashMap); + +impl Oracle for TestOracle { + fn to_number(&self, hash: H256) -> Option { + self.0.get(&hash).cloned() + } + + fn is_major_importing(&self) -> bool { false } +} + +struct TestBroadcast(Option); +impl Broadcast for TestBroadcast { + fn take_at(&self, num: Option) { + if num != self.0 { + panic!("Watcher broadcast wrong number. Expected {:?}, found {:?}", self.0, num); + } + } +} + +// helper harness for tests which expect a notification. +fn harness(numbers: Vec, period: u64, history: u64, expected: Option) { + const DURATION_ZERO: Duration = Duration::from_millis(0); + + let hashes: Vec<_> = numbers.clone().into_iter().map(|x| BigEndianHash::from_uint(&U256::from(x))).collect(); + let map = hashes.clone().into_iter().zip(numbers).collect(); + let watcher = Watcher::new_test( + Box::new(TestOracle(map)), + Box::new(TestBroadcast(expected)), + period, + history, + ); + + watcher.new_blocks(NewBlocks::new( + hashes, + vec![], + ChainRoute::default(), + vec![], + vec![], + DURATION_ZERO, + false + )); +} + +// helper + +#[test] +fn should_not_fire() { + harness(vec![0], 5, 0, None); +} + +#[test] +fn fires_once_for_two() { + harness(vec![14, 15], 10, 5, Some(10)); +} + +#[test] +fn finds_highest() { + harness(vec![15, 25], 10, 5, Some(20)); +} + +#[test] +fn doesnt_fire_before_history() { + harness(vec![10, 11], 10, 5, None); +} diff --git a/ethcore/snapshot/src/account.rs b/ethcore/snapshot/src/account.rs index 760576e370b..1e1341301a0 100644 --- a/ethcore/snapshot/src/account.rs +++ b/ethcore/snapshot/src/account.rs @@ -35,7 +35,7 @@ use rlp::{RlpStream, Rlp}; use trie_db::{Trie, TrieMut}; // An empty account -- these were replaced with RLP null data for a space optimization in v1. -const ACC_EMPTY: BasicAccount = BasicAccount { +pub const ACC_EMPTY: BasicAccount = BasicAccount { nonce: U256([0, 0, 0, 0]), balance: U256([0, 0, 0, 0]), storage_root: KECCAK_NULL_RLP, @@ -69,6 +69,7 @@ impl CodeState { } } +// todo[dvdplm] move to test helpers? // walk the account's storage trie, returning a vector of RLP items containing the // account address hash, account properties and the storage. Each item contains at most `max_storage_items` // storage records split according to snapshot format definition. @@ -170,6 +171,7 @@ pub fn to_fat_rlps( } } +// todo[dvdplm] move to test helpers? // decode a fat rlp, and rebuild the storage trie as we go. // returns the account structure along with its newly recovered code, // if it exists. @@ -244,183 +246,3 @@ pub fn from_fat_rlp( Ok((acc, new_code)) } - -#[cfg(test)] -mod tests { - use account_db::{AccountDB, AccountDBMut}; - use common_types::basic_account::BasicAccount; - use ethcore::test_helpers::get_temp_state_db; - use crate::tests::helpers::fill_storage; - use common_types::snapshot::Progress; - - use keccak_hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; - use ethereum_types::{H256, Address}; - use hash_db::{HashDB, EMPTY_PREFIX}; - use kvdb::DBValue; - use rlp::Rlp; - - use std::collections::HashSet; - - use super::{ACC_EMPTY, to_fat_rlps, from_fat_rlp}; - - #[test] - fn encoding_basic() { - let mut db = get_temp_state_db(); - let addr = Address::random(); - - let account = BasicAccount { - nonce: 50.into(), - balance: 123456789.into(), - storage_root: KECCAK_NULL_RLP, - code_hash: KECCAK_EMPTY, - code_version: 0.into(), - }; - - let thin_rlp = ::rlp::encode(&account); - assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); - let p = Progress::default(); - let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); - let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap(); - assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account); - } - - #[test] - fn encoding_version() { - let mut db = get_temp_state_db(); - let addr = Address::random(); - - let account = BasicAccount { - nonce: 50.into(), - balance: 123456789.into(), - storage_root: KECCAK_NULL_RLP, - code_hash: KECCAK_EMPTY, - code_version: 1.into(), - }; - - let thin_rlp = ::rlp::encode(&account); - assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); - let p = Progress::default(); - let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); - let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap(); - assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account); - } - - #[test] - fn encoding_storage() { - let mut db = get_temp_state_db(); - let addr = Address::random(); - - let account = { - let acct_db = AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)); - let mut root = KECCAK_NULL_RLP; - fill_storage(acct_db, &mut root, &mut H256::zero()); - BasicAccount { - nonce: 25.into(), - balance: 987654321.into(), - storage_root: root, - code_hash: KECCAK_EMPTY, - code_version: 0.into(), - } - }; - - let thin_rlp = ::rlp::encode(&account); - assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); - - let p = Progress::default(); - - let fat_rlp = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); - let fat_rlp = Rlp::new(&fat_rlp[0]).at(1).unwrap(); - assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account); - } - - #[test] - fn encoding_storage_split() { - let mut db = get_temp_state_db(); - let addr = Address::random(); - - let account = { - let acct_db = AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)); - let mut root = KECCAK_NULL_RLP; - fill_storage(acct_db, &mut root, &mut H256::zero()); - BasicAccount { - nonce: 25.into(), - balance: 987654321.into(), - storage_root: root, - code_hash: KECCAK_EMPTY, - code_version: 0.into(), - } - }; - - let thin_rlp = ::rlp::encode(&account); - assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); - - let p = Progress::default(); - let fat_rlps = to_fat_rlps(&keccak(addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), 500, 1000, &p).unwrap(); - let mut root = KECCAK_NULL_RLP; - let mut restored_account = None; - for rlp in fat_rlps { - let fat_rlp = Rlp::new(&rlp).at(1).unwrap(); - restored_account = Some(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, root).unwrap().0); - root = restored_account.as_ref().unwrap().storage_root.clone(); - } - assert_eq!(restored_account, Some(account)); - } - - #[test] - fn encoding_code() { - let mut db = get_temp_state_db(); - - let addr1 = Address::random(); - let addr2 = Address::random(); - - let code_hash = { - let mut acct_db = AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr1)); - acct_db.insert(EMPTY_PREFIX, b"this is definitely code") - }; - - { - let mut acct_db = AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr2)); - acct_db.emplace(code_hash.clone(), EMPTY_PREFIX, DBValue::from_slice(b"this is definitely code")); - } - - let account1 = BasicAccount { - nonce: 50.into(), - balance: 123456789.into(), - storage_root: KECCAK_NULL_RLP, - code_hash, - code_version: 0.into(), - }; - - let account2 = BasicAccount { - nonce: 400.into(), - balance: 98765432123456789usize.into(), - storage_root: KECCAK_NULL_RLP, - code_hash, - code_version: 0.into(), - }; - - let mut used_code = HashSet::new(); - let p1 = Progress::default(); - let p2 = Progress::default(); - let fat_rlp1 = to_fat_rlps(&keccak(&addr1), &account1, &AccountDB::from_hash(db.as_hash_db(), keccak(addr1)), &mut used_code, usize::max_value(), usize::max_value(), &p1).unwrap(); - let fat_rlp2 = to_fat_rlps(&keccak(&addr2), &account2, &AccountDB::from_hash(db.as_hash_db(), keccak(addr2)), &mut used_code, usize::max_value(), usize::max_value(), &p2).unwrap(); - assert_eq!(used_code.len(), 1); - - let fat_rlp1 = Rlp::new(&fat_rlp1[0]).at(1).unwrap(); - let fat_rlp2 = Rlp::new(&fat_rlp2[0]).at(1).unwrap(); - - let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr2)), fat_rlp2, H256::zero()).unwrap(); - assert!(maybe_code.is_none()); - assert_eq!(acc, account2); - - let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr1)), fat_rlp1, H256::zero()).unwrap(); - assert_eq!(maybe_code, Some(b"this is definitely code".to_vec())); - assert_eq!(acc, account1); - } - - #[test] - fn encoding_empty_acc() { - let mut db = get_temp_state_db(); - assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(Address::zero())), Rlp::new(&::rlp::NULL_RLP), H256::zero()).unwrap(), (ACC_EMPTY, None)); - } -} diff --git a/ethcore/snapshot/src/block.rs b/ethcore/snapshot/src/block.rs index fbc5d320742..e7253855148 100644 --- a/ethcore/snapshot/src/block.rs +++ b/ethcore/snapshot/src/block.rs @@ -30,7 +30,7 @@ use triehash::ordered_trie_root; const HEADER_FIELDS: usize = 8; const BLOCK_FIELDS: usize = 2; -pub(crate) struct AbridgedBlock { +pub struct AbridgedBlock { rlp: Bytes, } @@ -128,77 +128,77 @@ impl AbridgedBlock { } } -#[cfg(test)] -mod tests { - use super::AbridgedBlock; - - use bytes::Bytes; - use ethereum_types::{H256, U256, Address}; - use common_types::{ - transaction::{Action, Transaction}, - block::Block, - view, - views::BlockView, - }; - - fn encode_block(b: &Block) -> Bytes { - b.rlp_bytes() - } - - #[test] - fn empty_block_abridging() { - let b = Block::default(); - let receipts_root = b.header.receipts_root().clone(); - let encoded = encode_block(&b); - - let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded)); - assert_eq!(abridged.to_block(H256::zero(), 0, receipts_root).unwrap(), b); - } - - #[test] - #[should_panic] - fn wrong_number() { - let b = Block::default(); - let receipts_root = b.header.receipts_root().clone(); - let encoded = encode_block(&b); - - let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded)); - assert_eq!(abridged.to_block(H256::zero(), 2, receipts_root).unwrap(), b); - } - - #[test] - fn with_transactions() { - let mut b = Block::default(); - - let t1 = Transaction { - action: Action::Create, - nonce: U256::from(42), - gas_price: U256::from(3000), - gas: U256::from(50_000), - value: U256::from(1), - data: b"Hello!".to_vec() - }.fake_sign(Address::from_low_u64_be(0x69)); - - let t2 = Transaction { - action: Action::Create, - nonce: U256::from(88), - gas_price: U256::from(12345), - gas: U256::from(300000), - value: U256::from(1000000000), - data: "Eep!".into(), - }.fake_sign(Address::from_low_u64_be(0x55)); - - b.transactions.push(t1.into()); - b.transactions.push(t2.into()); - - let receipts_root = b.header.receipts_root().clone(); - b.header.set_transactions_root(::triehash::ordered_trie_root( - b.transactions.iter().map(::rlp::encode) - )); - - let encoded = encode_block(&b); - - let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded[..])); - assert_eq!(abridged.to_block(H256::zero(), 0, receipts_root).unwrap(), b); - } -} +//#[cfg(test)] +//mod tests { +// use super::AbridgedBlock; +// +// use bytes::Bytes; +// use ethereum_types::{H256, U256, Address}; +// use common_types::{ +// transaction::{Action, Transaction}, +// block::Block, +// view, +// views::BlockView, +// }; +// +// fn encode_block(b: &Block) -> Bytes { +// b.rlp_bytes() +// } +// +// #[test] +// fn empty_block_abridging() { +// let b = Block::default(); +// let receipts_root = b.header.receipts_root().clone(); +// let encoded = encode_block(&b); +// +// let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded)); +// assert_eq!(abridged.to_block(H256::zero(), 0, receipts_root).unwrap(), b); +// } +// +// #[test] +// #[should_panic] +// fn wrong_number() { +// let b = Block::default(); +// let receipts_root = b.header.receipts_root().clone(); +// let encoded = encode_block(&b); +// +// let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded)); +// assert_eq!(abridged.to_block(H256::zero(), 2, receipts_root).unwrap(), b); +// } +// +// #[test] +// fn with_transactions() { +// let mut b = Block::default(); +// +// let t1 = Transaction { +// action: Action::Create, +// nonce: U256::from(42), +// gas_price: U256::from(3000), +// gas: U256::from(50_000), +// value: U256::from(1), +// data: b"Hello!".to_vec() +// }.fake_sign(Address::from_low_u64_be(0x69)); +// +// let t2 = Transaction { +// action: Action::Create, +// nonce: U256::from(88), +// gas_price: U256::from(12345), +// gas: U256::from(300000), +// value: U256::from(1000000000), +// data: "Eep!".into(), +// }.fake_sign(Address::from_low_u64_be(0x55)); +// +// b.transactions.push(t1.into()); +// b.transactions.push(t2.into()); +// +// let receipts_root = b.header.receipts_root().clone(); +// b.header.set_transactions_root(::triehash::ordered_trie_root( +// b.transactions.iter().map(::rlp::encode) +// )); +// +// let encoded = encode_block(&b); +// +// let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded[..])); +// assert_eq!(abridged.to_block(H256::zero(), 0, receipts_root).unwrap(), b); +// } +//} diff --git a/ethcore/snapshot/src/io.rs b/ethcore/snapshot/src/io.rs index 1afba21fb37..5e67d1c1bf8 100644 --- a/ethcore/snapshot/src/io.rs +++ b/ethcore/snapshot/src/io.rs @@ -26,7 +26,6 @@ use std::fs::{self, File}; use std::path::{Path, PathBuf}; use bytes::Bytes; -use client_traits::SnapshotWriter; use common_types::{ errors::{SnapshotError, EthcoreError}, snapshot::ManifestData, @@ -36,7 +35,7 @@ use log::trace; use rlp::{RlpStream, Rlp}; use rlp_derive::*; -const SNAPSHOT_VERSION: u64 = 2; +pub const SNAPSHOT_VERSION: u64 = 2; // (hash, len, offset) #[derive(RlpEncodable, RlpDecodable)] @@ -184,6 +183,23 @@ pub trait SnapshotReader { fn chunk(&self, hash: H256) -> io::Result; } + +/// Something which can write snapshots. +/// Writing the same chunk multiple times will lead to implementation-defined +/// behavior, and is not advised. +pub trait SnapshotWriter { + /// Write a compressed state chunk. + fn write_state_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>; + + /// Write a compressed block chunk. + fn write_block_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>; + + /// Complete writing. The manifest's chunk lists must be consistent + /// with the chunks written. + fn finish(self, manifest: ManifestData) -> std::io::Result<()> where Self: Sized; +} + + /// Packed snapshot reader. pub struct PackedReader { file: File, @@ -317,94 +333,3 @@ impl SnapshotReader for LooseReader { Ok(buf) } } - -#[cfg(test)] -mod tests { - use tempdir::TempDir; - use keccak_hash::keccak; - - use common_types::snapshot::ManifestData; - use super::{SnapshotWriter, SnapshotReader, PackedWriter, PackedReader, LooseWriter, LooseReader, SNAPSHOT_VERSION}; - - const STATE_CHUNKS: &'static [&'static [u8]] = &[b"dog", b"cat", b"hello world", b"hi", b"notarealchunk"]; - const BLOCK_CHUNKS: &'static [&'static [u8]] = &[b"hello!", b"goodbye!", b"abcdefg", b"hijklmnop", b"qrstuvwxy", b"and", b"z"]; - - #[test] - fn packed_write_and_read() { - let tempdir = TempDir::new("").unwrap(); - let path = tempdir.path().join("packed"); - let mut writer = PackedWriter::new(&path).unwrap(); - - let mut state_hashes = Vec::new(); - let mut block_hashes = Vec::new(); - - for chunk in STATE_CHUNKS { - let hash = keccak(&chunk); - state_hashes.push(hash.clone()); - writer.write_state_chunk(hash, chunk).unwrap(); - } - - for chunk in BLOCK_CHUNKS { - let hash = keccak(&chunk); - block_hashes.push(hash.clone()); - writer.write_block_chunk(keccak(&chunk), chunk).unwrap(); - } - - let manifest = ManifestData { - version: SNAPSHOT_VERSION, - state_hashes, - block_hashes, - state_root: keccak(b"notarealroot"), - block_number: 12345678987654321, - block_hash: keccak(b"notarealblock"), - }; - - writer.finish(manifest.clone()).unwrap(); - - let reader = PackedReader::new(&path).unwrap().unwrap(); - assert_eq!(reader.manifest(), &manifest); - - for hash in manifest.state_hashes.iter().chain(&manifest.block_hashes) { - reader.chunk(hash.clone()).unwrap(); - } - } - - #[test] - fn loose_write_and_read() { - let tempdir = TempDir::new("").unwrap(); - let mut writer = LooseWriter::new(tempdir.path().into()).unwrap(); - - let mut state_hashes = Vec::new(); - let mut block_hashes = Vec::new(); - - for chunk in STATE_CHUNKS { - let hash = keccak(&chunk); - state_hashes.push(hash.clone()); - writer.write_state_chunk(hash, chunk).unwrap(); - } - - for chunk in BLOCK_CHUNKS { - let hash = keccak(&chunk); - block_hashes.push(hash.clone()); - writer.write_block_chunk(keccak(&chunk), chunk).unwrap(); - } - - let manifest = ManifestData { - version: SNAPSHOT_VERSION, - state_hashes: state_hashes, - block_hashes: block_hashes, - state_root: keccak(b"notarealroot"), - block_number: 12345678987654321, - block_hash: keccak(b"notarealblock)"), - }; - - writer.finish(manifest.clone()).unwrap(); - - let reader = LooseReader::new(tempdir.path().into()).unwrap(); - assert_eq!(reader.manifest(), &manifest); - - for hash in manifest.state_hashes.iter().chain(&manifest.block_hashes) { - reader.chunk(hash.clone()).unwrap(); - } - } -} diff --git a/ethcore/snapshot/src/lib.rs b/ethcore/snapshot/src/lib.rs index 76d48ba569e..06a21754854 100644 --- a/ethcore/snapshot/src/lib.rs +++ b/ethcore/snapshot/src/lib.rs @@ -31,8 +31,6 @@ use account_state::Account as StateAccount; use blockchain::{BlockChain, BlockProvider}; use bloom_journal::Bloom; use bytes::Bytes; -// todo[dvdplm] put back in snapshots once it's extracted -use client_traits::SnapshotWriter; use common_types::{ ids::BlockId, header::Header, @@ -57,21 +55,31 @@ use state_db::StateDB; use trie_db::{Trie, TrieMut}; pub use self::consensus::*; -pub use self::service::Service; -pub use self::traits::{SnapshotService, SnapshotComponents, Rebuilder}; +pub use self::service::{Service, Guard, Restoration, RestorationParams}; +pub use self::traits::{SnapshotService, SnapshotClient, SnapshotComponents, Rebuilder}; +pub use self::io::SnapshotWriter; pub use self::watcher::Watcher; -pub use common_types::basic_account::BasicAccount; +use common_types::basic_account::BasicAccount; pub mod io; pub mod service; +#[cfg(any(test, feature = "test-helpers"))] +pub mod account; +#[cfg(not(any(test, feature = "test-helpers")))] mod account; +#[cfg(any(test, feature = "test-helpers"))] +pub mod block; +#[cfg(not(any(test, feature = "test-helpers")))] mod block; mod consensus; +#[cfg(any(test, feature = "test-helpers"))] +pub mod watcher; +#[cfg(not(any(test, feature = "test-helpers")))] mod watcher; -#[cfg(test)] -mod tests; +//#[cfg(test)] +//mod tests; mod traits; @@ -88,7 +96,7 @@ const MIN_SUPPORTED_STATE_CHUNK_VERSION: u64 = 1; // current state chunk version. const STATE_CHUNK_VERSION: u64 = 2; /// number of snapshot subparts, must be a power of 2 in [1; 256] -const SNAPSHOT_SUBPARTS: usize = 16; +pub const SNAPSHOT_SUBPARTS: usize = 16; /// Maximum number of snapshot subparts (must be a multiple of `SNAPSHOT_SUBPARTS`) const MAX_SNAPSHOT_SUBPARTS: usize = 256; diff --git a/ethcore/snapshot/src/service.rs b/ethcore/snapshot/src/service.rs index 747c987dbd0..4fc3132e692 100644 --- a/ethcore/snapshot/src/service.rs +++ b/ethcore/snapshot/src/service.rs @@ -32,8 +32,7 @@ use common_types::{ ids::BlockId, snapshot::{ManifestData, Progress, RestorationStatus}, }; -// todo[dvdplm] put SnapshotWriter back in snapshots once extracted -use client_traits::{ChainInfo, SnapshotClient, SnapshotWriter}; +use client_traits::ChainInfo; use engine::Engine; use ethereum_types::H256; use ethcore_io::IoChannel; @@ -45,6 +44,8 @@ use parking_lot::{Mutex, RwLock, RwLockReadGuard}; use snappy; use trie_db::TrieError; +use crate::{SnapshotClient, SnapshotWriter}; + use super::{ StateRebuilder, SnapshotService, @@ -55,13 +56,13 @@ use super::{ }; /// Helper for removing directories in case of error. -struct Guard(bool, PathBuf); +pub struct Guard(bool, PathBuf); impl Guard { fn new(path: PathBuf) -> Self { Guard(true, path) } - #[cfg(test)] - fn benign() -> Self { Guard(false, PathBuf::default()) } + #[cfg(any(test, feature = "test-helpers"))] + pub fn benign() -> Self { Guard(false, PathBuf::default()) } fn disarm(mut self) { self.0 = false } } @@ -75,7 +76,7 @@ impl Drop for Guard { } /// State restoration manager. -struct Restoration { +pub struct Restoration { manifest: ManifestData, state_chunks_left: HashSet, block_chunks_left: HashSet, @@ -88,7 +89,7 @@ struct Restoration { db: Arc, } -struct RestorationParams<'a> { +pub struct RestorationParams<'a> { manifest: ManifestData, // manifest to base restoration on. pruning: Algorithm, // pruning algorithm for the database. db: Arc, // database @@ -98,9 +99,24 @@ struct RestorationParams<'a> { engine: &'a dyn Engine, } +#[cfg(any(test, feature = "test-helpers"))] +impl<'a> RestorationParams<'a> { + pub fn new( + manifest: ManifestData, + pruning: Algorithm, + db: Arc, + writer: Option, + genesis: &'a [u8], + guard: Guard, + engine: &'a dyn Engine, + ) -> Self { + Self { manifest, pruning, db, writer, genesis, guard, engine } + } +} + impl Restoration { // make a new restoration using the given parameters. - fn new(params: RestorationParams) -> Result { + pub fn new(params: RestorationParams) -> Result { let manifest = params.manifest; let state_chunks = manifest.state_hashes.iter().cloned().collect(); @@ -131,7 +147,7 @@ impl Restoration { } // feeds a state chunk, aborts early if `flag` becomes false. - fn feed_state(&mut self, hash: H256, chunk: &[u8], flag: &AtomicBool) -> Result<(), Error> { + pub fn feed_state(&mut self, hash: H256, chunk: &[u8], flag: &AtomicBool) -> Result<(), Error> { if self.state_chunks_left.contains(&hash) { let expected_len = snappy::decompressed_len(chunk)?; if expected_len > MAX_CHUNK_SIZE { @@ -153,7 +169,7 @@ impl Restoration { } // feeds a block chunk - fn feed_blocks(&mut self, hash: H256, chunk: &[u8], engine: &dyn Engine, flag: &AtomicBool) -> Result<(), Error> { + pub fn feed_blocks(&mut self, hash: H256, chunk: &[u8], engine: &dyn Engine, flag: &AtomicBool) -> Result<(), Error> { if self.block_chunks_left.contains(&hash) { let expected_len = snappy::decompressed_len(chunk)?; if expected_len > MAX_CHUNK_SIZE { @@ -200,7 +216,7 @@ impl Restoration { } // is everything done? - fn is_done(&self) -> bool { + pub fn is_done(&self) -> bool { self.block_chunks_left.is_empty() && self.state_chunks_left.is_empty() } } @@ -892,105 +908,3 @@ impl Drop for Service { trace!(target: "shutdown", "Dropping Service - snapshot aborted"); } } - -#[cfg(test)] -mod tests { - use ethcore::client::Client; - use ethcore_io::IoService; - use spec; - use journaldb::Algorithm; - use crate::SnapshotService; - use super::*; - use common_types::{ - io_message::ClientIoMessage, - snapshot::{ManifestData, RestorationStatus} - }; - use tempdir::TempDir; - use ethcore::test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler}; - - #[test] - fn sends_async_messages() { - let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; - let client = generate_dummy_client_with_spec_and_data(spec::new_null, 400, 5, &gas_prices); - let service = IoService::>::start().unwrap(); - let spec = spec::new_test(); - - let tempdir = TempDir::new("").unwrap(); - let dir = tempdir.path().join("snapshot"); - - let snapshot_params = ServiceParams { - engine: spec.engine.clone(), - genesis_block: spec.genesis_block(), - restoration_db_handler: restoration_db_handler(Default::default()), - pruning: Algorithm::Archive, - channel: service.channel(), - snapshot_root: dir, - client, - }; - - let service = Service::new(snapshot_params).unwrap(); - - assert!(service.manifest().is_none()); - assert!(service.chunk(Default::default()).is_none()); - assert_eq!(service.status(), RestorationStatus::Inactive); - - let manifest = ManifestData { - version: 2, - state_hashes: vec![], - block_hashes: vec![], - state_root: Default::default(), - block_number: 0, - block_hash: Default::default(), - }; - - service.begin_restore(manifest); - service.abort_restore(); - service.restore_state_chunk(Default::default(), vec![]); - service.restore_block_chunk(Default::default(), vec![]); - } - - #[test] - fn cannot_finish_with_invalid_chunks() { - use ethereum_types::H256; - use kvdb_rocksdb::DatabaseConfig; - - let spec = spec::new_test(); - let tempdir = TempDir::new("").unwrap(); - - let state_hashes: Vec<_> = (0..5).map(|_| H256::random()).collect(); - let block_hashes: Vec<_> = (0..5).map(|_| H256::random()).collect(); - let db_config = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); - let gb = spec.genesis_block(); - let flag = ::std::sync::atomic::AtomicBool::new(true); - - let params = RestorationParams { - manifest: ManifestData { - version: 2, - state_hashes: state_hashes.clone(), - block_hashes: block_hashes.clone(), - state_root: H256::zero(), - block_number: 100000, - block_hash: H256::zero(), - }, - pruning: Algorithm::Archive, - db: restoration_db_handler(db_config).open(&tempdir.path().to_owned()).unwrap(), - writer: None, - genesis: &gb, - guard: Guard::benign(), - engine: &*spec.engine.clone(), - }; - - let mut restoration = Restoration::new(params).unwrap(); - let definitely_bad_chunk = [1, 2, 3, 4, 5]; - - for hash in state_hashes { - assert!(restoration.feed_state(hash, &definitely_bad_chunk, &flag).is_err()); - assert!(!restoration.is_done()); - } - - for hash in block_hashes { - assert!(restoration.feed_blocks(hash, &definitely_bad_chunk, &*spec.engine, &flag).is_err()); - assert!(!restoration.is_done()); - } - } -} diff --git a/ethcore/snapshot/src/traits.rs b/ethcore/snapshot/src/traits.rs index 7425ada6375..a1b40d57001 100644 --- a/ethcore/snapshot/src/traits.rs +++ b/ethcore/snapshot/src/traits.rs @@ -18,13 +18,16 @@ use std::sync::{Arc, atomic::AtomicBool}; use blockchain::{BlockChain, BlockChainDB}; use bytes::Bytes; +use client_traits::{BlockChainClient, BlockInfo, DatabaseRestore, BlockChainReset}; use common_types::{ + ids::BlockId, errors::{EthcoreError as Error, SnapshotError}, snapshot::{ManifestData, ChunkSink, Progress, RestorationStatus}, }; use engine::Engine; use ethereum_types::H256; +use crate::io::SnapshotWriter; /// The interface for a snapshot network service. /// This handles: @@ -129,3 +132,15 @@ pub trait SnapshotComponents: Send { /// Current version number fn current_version(&self) -> u64; } + +/// Snapshot related functionality +pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + BlockChainReset { + /// Take a snapshot at the given block. + /// If the ID given is "latest", this will default to 1000 blocks behind. + fn take_snapshot( + &self, + writer: W, + at: BlockId, + p: &Progress, + ) -> Result<(), Error>; +} diff --git a/ethcore/snapshot/src/watcher.rs b/ethcore/snapshot/src/watcher.rs index 864efd356ea..a8aa9ebc7a2 100644 --- a/ethcore/snapshot/src/watcher.rs +++ b/ethcore/snapshot/src/watcher.rs @@ -30,7 +30,7 @@ use log::{trace, warn}; use parking_lot::Mutex; // helper trait for transforming hashes to numbers and checking if syncing. -trait Oracle: Send + Sync { +pub trait Oracle: Send + Sync { fn to_number(&self, hash: H256) -> Option; fn is_major_importing(&self) -> bool; @@ -54,7 +54,7 @@ impl Oracle for StandardOracle } // helper trait for broadcasting a block to take a snapshot at. -trait Broadcast: Send + Sync { +pub trait Broadcast: Send + Sync { fn take_at(&self, num: Option); } @@ -98,6 +98,11 @@ impl Watcher { history, } } + + #[cfg(any(test, feature = "test-helpers"))] + pub fn new_test(oracle: Box, broadcast: Box, period: u64, history: u64) -> Self { + Watcher { oracle, broadcast, period, history } + } } impl ChainNotify for Watcher { @@ -120,81 +125,81 @@ impl ChainNotify for Watcher { } } -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::time::Duration; - - use client_traits::ChainNotify; - use common_types::chain_notify::{NewBlocks, ChainRoute}; - - use ethereum_types::{H256, U256, BigEndianHash}; - - use super::{Broadcast, Oracle, Watcher}; - - struct TestOracle(HashMap); - - impl Oracle for TestOracle { - fn to_number(&self, hash: H256) -> Option { - self.0.get(&hash).cloned() - } - - fn is_major_importing(&self) -> bool { false } - } - - struct TestBroadcast(Option); - impl Broadcast for TestBroadcast { - fn take_at(&self, num: Option) { - if num != self.0 { - panic!("Watcher broadcast wrong number. Expected {:?}, found {:?}", self.0, num); - } - } - } - - // helper harness for tests which expect a notification. - fn harness(numbers: Vec, period: u64, history: u64, expected: Option) { - const DURATION_ZERO: Duration = Duration::from_millis(0); - - let hashes: Vec<_> = numbers.clone().into_iter().map(|x| BigEndianHash::from_uint(&U256::from(x))).collect(); - let map = hashes.clone().into_iter().zip(numbers).collect(); - - let watcher = Watcher { - oracle: Box::new(TestOracle(map)), - broadcast: Box::new(TestBroadcast(expected)), - period, - history, - }; - - watcher.new_blocks(NewBlocks::new( - hashes, - vec![], - ChainRoute::default(), - vec![], - vec![], - DURATION_ZERO, - false - )); - } - - // helper - - #[test] - fn should_not_fire() { - harness(vec![0], 5, 0, None); - } - - #[test] - fn fires_once_for_two() { - harness(vec![14, 15], 10, 5, Some(10)); - } - - #[test] - fn finds_highest() { - harness(vec![15, 25], 10, 5, Some(20)); - } - - #[test] - fn doesnt_fire_before_history() { - harness(vec![10, 11], 10, 5, None); - } -} +//#[cfg(test)] +//mod tests { +// use std::collections::HashMap; +// use std::time::Duration; +// +// use client_traits::ChainNotify; +// use common_types::chain_notify::{NewBlocks, ChainRoute}; +// +// use ethereum_types::{H256, U256, BigEndianHash}; +// +// use super::{Broadcast, Oracle, Watcher}; +// +// struct TestOracle(HashMap); +// +// impl Oracle for TestOracle { +// fn to_number(&self, hash: H256) -> Option { +// self.0.get(&hash).cloned() +// } +// +// fn is_major_importing(&self) -> bool { false } +// } +// +// struct TestBroadcast(Option); +// impl Broadcast for TestBroadcast { +// fn take_at(&self, num: Option) { +// if num != self.0 { +// panic!("Watcher broadcast wrong number. Expected {:?}, found {:?}", self.0, num); +// } +// } +// } +// +// // helper harness for tests which expect a notification. +// fn harness(numbers: Vec, period: u64, history: u64, expected: Option) { +// const DURATION_ZERO: Duration = Duration::from_millis(0); +// +// let hashes: Vec<_> = numbers.clone().into_iter().map(|x| BigEndianHash::from_uint(&U256::from(x))).collect(); +// let map = hashes.clone().into_iter().zip(numbers).collect(); +// +// let watcher = Watcher { +// oracle: Box::new(TestOracle(map)), +// broadcast: Box::new(TestBroadcast(expected)), +// period, +// history, +// }; +// +// watcher.new_blocks(NewBlocks::new( +// hashes, +// vec![], +// ChainRoute::default(), +// vec![], +// vec![], +// DURATION_ZERO, +// false +// )); +// } +// +// // helper +// +// #[test] +// fn should_not_fire() { +// harness(vec![0], 5, 0, None); +// } +// +// #[test] +// fn fires_once_for_two() { +// harness(vec![14, 15], 10, 5, Some(10)); +// } +// +// #[test] +// fn finds_highest() { +// harness(vec![15, 25], 10, 5, Some(20)); +// } +// +// #[test] +// fn doesnt_fire_before_history() { +// harness(vec![10, 11], 10, 5, None); +// } +//} diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 000a0cd7583..5cd6030ff36 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -48,8 +48,8 @@ use client_traits::{ BlockInfo, ScheduleInfo, StateClient, BlockChainReset, Nonce, Balance, ChainInfo, TransactionInfo, ImportBlock, AccountData, BlockChain as BlockChainTrait, BlockChainClient, - IoClient, BadBlocks, ProvingBlockChainClient, SnapshotClient, - DatabaseRestore, SnapshotWriter, Tick, ChainNotify, + IoClient, BadBlocks, ProvingBlockChainClient, + DatabaseRestore, Tick, ChainNotify, StateOrBlock, }; use engine::Engine; @@ -64,6 +64,7 @@ use snapshot; use spec::Spec; use account_state::State; use executive_state; +use snapshot::{SnapshotClient, SnapshotWriter}; use state_db::StateDB; use trace::{self, TraceDB, ImportRequest as TraceImportRequest, LocalizedTrace, Database as TraceDatabase}; use types::{ diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs index 56cd4273dea..c6b338444ef 100644 --- a/ethcore/src/client/mod.rs +++ b/ethcore/src/client/mod.rs @@ -31,7 +31,6 @@ pub use self::config::{ClientConfig, DatabaseCompactionProfile, BlockChainConfig pub use self::evm_test_client::{EvmTestClient, EvmTestError, TransactErr, TransactSuccess}; #[cfg(any(test, feature = "test-helpers"))] pub use self::test_client::{TestBlockChainClient, EachBlockWith, TestState}; -//pub use self::chain_notify::{ChainNotify, NewBlocks, ChainRoute, ChainRouteType, ChainMessageType}; pub use self::traits::{ ReopenBlock, PrepareOpenBlock, ImportSealedBlock, BroadcastProposalBlock, Call, EngineInfo, BlockProducer, SealedBlockImporter, @@ -40,4 +39,3 @@ pub use self::traits::{ pub use verification::VerifierType; mod traits; -//mod chain_notify; diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 26c8c46bf1c..7071b34994d 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -53,16 +53,13 @@ //! cargo build --release //! ``` -extern crate account_db; extern crate account_state; extern crate ansi_term; extern crate client_traits; extern crate common_types as types; -extern crate crossbeam_utils; extern crate engine; extern crate ethabi; extern crate ethcore_blockchain as blockchain; -extern crate ethcore_bloom_journal as bloom_journal; extern crate ethcore_call_contract as call_contract; extern crate ethcore_db as db; extern crate ethcore_io as io; @@ -74,13 +71,10 @@ extern crate hash_db; extern crate itertools; extern crate journaldb; extern crate keccak_hash as hash; -extern crate keccak_hasher; extern crate kvdb; extern crate machine; extern crate memory_cache; -extern crate num_cpus; extern crate parity_bytes as bytes; -extern crate parity_snappy as snappy; extern crate parking_lot; extern crate trie_db as trie; extern crate patricia_trie_ethereum as ethtrie; @@ -102,6 +96,8 @@ extern crate vm; #[cfg(test)] extern crate rand_xorshift; #[cfg(test)] +extern crate account_db; +#[cfg(test)] extern crate ethcore_accounts as accounts; #[cfg(feature = "stratum")] extern crate ethcore_stratum; diff --git a/parity/snapshot_cmd.rs b/parity/snapshot_cmd.rs index 8c4a1cb10c9..bd41400a3cd 100644 --- a/parity/snapshot_cmd.rs +++ b/parity/snapshot_cmd.rs @@ -20,9 +20,8 @@ use std::time::Duration; use std::path::{Path, PathBuf}; use std::sync::Arc; -use client_traits::SnapshotClient; use hash::keccak; -use snapshot::{SnapshotConfiguration, SnapshotService as SS}; +use snapshot::{SnapshotConfiguration, SnapshotService as SS, SnapshotClient}; use snapshot::io::{SnapshotReader, PackedReader, PackedWriter}; use snapshot::service::Service as SnapshotService; use ethcore::client::{Client, DatabaseCompactionProfile, VMType}; From caa2d2d5d3fefd7d8ec6e3638626b2a4cb71a58f Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 30 Aug 2019 17:50:11 +0200 Subject: [PATCH 14/21] cleanup --- Cargo.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 302e97d1bb2..3a18c6c9e69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,8 +122,7 @@ name = "parity" [profile.release] debug = false -# todo[dvdplm] re-enable -lto = false +lto = true [workspace] # This should only list projects that are not @@ -136,6 +135,4 @@ members = [ "ethcore/wasm/run", "evmbin", "parity-clib", - - "ethcore/snapshot/snapshot-tests" ] From 145d77cd3779528bdf9ed5fd09e35e91c3f71ebe Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 30 Aug 2019 18:03:41 +0200 Subject: [PATCH 15/21] Cleanup --- Cargo.lock | 43 ---------- .../snapshot/snapshot-tests/src/watcher.rs | 2 - ethcore/snapshot/src/account.rs | 2 - ethcore/snapshot/src/block.rs | 75 ------------------ ethcore/snapshot/src/io.rs | 2 - ethcore/snapshot/src/lib.rs | 3 - ethcore/snapshot/src/watcher.rs | 79 ------------------- 7 files changed, 206 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 630de859cd4..59ef08be579 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4250,49 +4250,6 @@ dependencies = [ "triehash-ethereum 0.2.0", ] -[[package]] -name = "snapshot-tests" -version = "0.1.0" -dependencies = [ - "account-db 0.1.0", - "account-state 0.1.0", - "client-traits 0.1.0", - "common-types 0.1.0", - "engine 0.1.0", - "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", - "ethabi 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "ethabi-contract 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "ethabi-derive 8.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ethcore 1.12.0", - "ethcore-accounts 0.1.0", - "ethcore-blockchain 0.1.0", - "ethcore-db 0.1.0", - "ethcore-io 1.12.0", - "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ethkey 0.3.0", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", - "journaldb 0.2.0", - "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak-hasher 0.1.1", - "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-rocksdb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "patricia-trie-ethereum 0.1.0", - "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "snapshot 0.1.0", - "spec 0.1.0", - "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", - "triehash-ethereum 0.2.0", -] - [[package]] name = "socket2" version = "0.3.8" diff --git a/ethcore/snapshot/snapshot-tests/src/watcher.rs b/ethcore/snapshot/snapshot-tests/src/watcher.rs index b0ccd7dedfc..1f04a71164a 100644 --- a/ethcore/snapshot/snapshot-tests/src/watcher.rs +++ b/ethcore/snapshot/snapshot-tests/src/watcher.rs @@ -71,8 +71,6 @@ fn harness(numbers: Vec, period: u64, history: u64, expected: Option) )); } -// helper - #[test] fn should_not_fire() { harness(vec![0], 5, 0, None); diff --git a/ethcore/snapshot/src/account.rs b/ethcore/snapshot/src/account.rs index 1e1341301a0..da227f17657 100644 --- a/ethcore/snapshot/src/account.rs +++ b/ethcore/snapshot/src/account.rs @@ -69,7 +69,6 @@ impl CodeState { } } -// todo[dvdplm] move to test helpers? // walk the account's storage trie, returning a vector of RLP items containing the // account address hash, account properties and the storage. Each item contains at most `max_storage_items` // storage records split according to snapshot format definition. @@ -171,7 +170,6 @@ pub fn to_fat_rlps( } } -// todo[dvdplm] move to test helpers? // decode a fat rlp, and rebuild the storage trie as we go. // returns the account structure along with its newly recovered code, // if it exists. diff --git a/ethcore/snapshot/src/block.rs b/ethcore/snapshot/src/block.rs index e7253855148..2d6716d674d 100644 --- a/ethcore/snapshot/src/block.rs +++ b/ethcore/snapshot/src/block.rs @@ -127,78 +127,3 @@ impl AbridgedBlock { Ok(Block { header, transactions, uncles }) } } - -//#[cfg(test)] -//mod tests { -// use super::AbridgedBlock; -// -// use bytes::Bytes; -// use ethereum_types::{H256, U256, Address}; -// use common_types::{ -// transaction::{Action, Transaction}, -// block::Block, -// view, -// views::BlockView, -// }; -// -// fn encode_block(b: &Block) -> Bytes { -// b.rlp_bytes() -// } -// -// #[test] -// fn empty_block_abridging() { -// let b = Block::default(); -// let receipts_root = b.header.receipts_root().clone(); -// let encoded = encode_block(&b); -// -// let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded)); -// assert_eq!(abridged.to_block(H256::zero(), 0, receipts_root).unwrap(), b); -// } -// -// #[test] -// #[should_panic] -// fn wrong_number() { -// let b = Block::default(); -// let receipts_root = b.header.receipts_root().clone(); -// let encoded = encode_block(&b); -// -// let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded)); -// assert_eq!(abridged.to_block(H256::zero(), 2, receipts_root).unwrap(), b); -// } -// -// #[test] -// fn with_transactions() { -// let mut b = Block::default(); -// -// let t1 = Transaction { -// action: Action::Create, -// nonce: U256::from(42), -// gas_price: U256::from(3000), -// gas: U256::from(50_000), -// value: U256::from(1), -// data: b"Hello!".to_vec() -// }.fake_sign(Address::from_low_u64_be(0x69)); -// -// let t2 = Transaction { -// action: Action::Create, -// nonce: U256::from(88), -// gas_price: U256::from(12345), -// gas: U256::from(300000), -// value: U256::from(1000000000), -// data: "Eep!".into(), -// }.fake_sign(Address::from_low_u64_be(0x55)); -// -// b.transactions.push(t1.into()); -// b.transactions.push(t2.into()); -// -// let receipts_root = b.header.receipts_root().clone(); -// b.header.set_transactions_root(::triehash::ordered_trie_root( -// b.transactions.iter().map(::rlp::encode) -// )); -// -// let encoded = encode_block(&b); -// -// let abridged = AbridgedBlock::from_block_view(&view!(BlockView, &encoded[..])); -// assert_eq!(abridged.to_block(H256::zero(), 0, receipts_root).unwrap(), b); -// } -//} diff --git a/ethcore/snapshot/src/io.rs b/ethcore/snapshot/src/io.rs index 5e67d1c1bf8..116d7e6991b 100644 --- a/ethcore/snapshot/src/io.rs +++ b/ethcore/snapshot/src/io.rs @@ -183,7 +183,6 @@ pub trait SnapshotReader { fn chunk(&self, hash: H256) -> io::Result; } - /// Something which can write snapshots. /// Writing the same chunk multiple times will lead to implementation-defined /// behavior, and is not advised. @@ -199,7 +198,6 @@ pub trait SnapshotWriter { fn finish(self, manifest: ManifestData) -> std::io::Result<()> where Self: Sized; } - /// Packed snapshot reader. pub struct PackedReader { file: File, diff --git a/ethcore/snapshot/src/lib.rs b/ethcore/snapshot/src/lib.rs index 06a21754854..3ce7e71eb94 100644 --- a/ethcore/snapshot/src/lib.rs +++ b/ethcore/snapshot/src/lib.rs @@ -78,9 +78,6 @@ pub mod watcher; #[cfg(not(any(test, feature = "test-helpers")))] mod watcher; -//#[cfg(test)] -//mod tests; - mod traits; // Try to have chunks be around 4MB (before compression) diff --git a/ethcore/snapshot/src/watcher.rs b/ethcore/snapshot/src/watcher.rs index a8aa9ebc7a2..b0e2ada339e 100644 --- a/ethcore/snapshot/src/watcher.rs +++ b/ethcore/snapshot/src/watcher.rs @@ -124,82 +124,3 @@ impl ChainNotify for Watcher { } } } - -//#[cfg(test)] -//mod tests { -// use std::collections::HashMap; -// use std::time::Duration; -// -// use client_traits::ChainNotify; -// use common_types::chain_notify::{NewBlocks, ChainRoute}; -// -// use ethereum_types::{H256, U256, BigEndianHash}; -// -// use super::{Broadcast, Oracle, Watcher}; -// -// struct TestOracle(HashMap); -// -// impl Oracle for TestOracle { -// fn to_number(&self, hash: H256) -> Option { -// self.0.get(&hash).cloned() -// } -// -// fn is_major_importing(&self) -> bool { false } -// } -// -// struct TestBroadcast(Option); -// impl Broadcast for TestBroadcast { -// fn take_at(&self, num: Option) { -// if num != self.0 { -// panic!("Watcher broadcast wrong number. Expected {:?}, found {:?}", self.0, num); -// } -// } -// } -// -// // helper harness for tests which expect a notification. -// fn harness(numbers: Vec, period: u64, history: u64, expected: Option) { -// const DURATION_ZERO: Duration = Duration::from_millis(0); -// -// let hashes: Vec<_> = numbers.clone().into_iter().map(|x| BigEndianHash::from_uint(&U256::from(x))).collect(); -// let map = hashes.clone().into_iter().zip(numbers).collect(); -// -// let watcher = Watcher { -// oracle: Box::new(TestOracle(map)), -// broadcast: Box::new(TestBroadcast(expected)), -// period, -// history, -// }; -// -// watcher.new_blocks(NewBlocks::new( -// hashes, -// vec![], -// ChainRoute::default(), -// vec![], -// vec![], -// DURATION_ZERO, -// false -// )); -// } -// -// // helper -// -// #[test] -// fn should_not_fire() { -// harness(vec![0], 5, 0, None); -// } -// -// #[test] -// fn fires_once_for_two() { -// harness(vec![14, 15], 10, 5, Some(10)); -// } -// -// #[test] -// fn finds_highest() { -// harness(vec![15, 25], 10, 5, Some(20)); -// } -// -// #[test] -// fn doesnt_fire_before_history() { -// harness(vec![10, 11], 10, 5, None); -// } -//} From d6e05749212e6477bb7a29543e13fc042b2897cc Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 3 Sep 2019 11:49:39 +0200 Subject: [PATCH 16/21] fix merge issues --- ethcore/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 6c165aa1f21..727665a6c00 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -96,8 +96,6 @@ extern crate vm; #[cfg(test)] extern crate rand_xorshift; #[cfg(test)] -extern crate account_db; -#[cfg(test)] extern crate ethcore_accounts as accounts; #[cfg(feature = "stratum")] extern crate ethcore_stratum; From e86c991ed9cb576c3ae221510b91b7ee7ebf106b Mon Sep 17 00:00:00 2001 From: David Date: Thu, 5 Sep 2019 16:18:20 +0200 Subject: [PATCH 17/21] Update ethcore/snapshot/snapshot-tests/Cargo.toml Co-Authored-By: Andronik Ordian --- ethcore/snapshot/snapshot-tests/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/snapshot/snapshot-tests/Cargo.toml b/ethcore/snapshot/snapshot-tests/Cargo.toml index e64491797f2..0c186f43109 100644 --- a/ethcore/snapshot/snapshot-tests/Cargo.toml +++ b/ethcore/snapshot/snapshot-tests/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snapshot-tests" version = "0.1.0" -authors = ["David Palm "] +authors = ["Parity Technologies "] edition = "2018" [dependencies] From 3717ecf46d4614356ec64f5e975ee2f325c8b735 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 5 Sep 2019 16:38:07 +0200 Subject: [PATCH 18/21] Sort out botched merge --- .../snapshot/snapshot-tests/src/helpers.rs | 34 ------------------- ethcore/snapshot/snapshot-tests/src/lib.rs | 6 ---- .../snapshot-tests/src/proof_of_authority.rs | 23 ------------- .../snapshot-tests/src/proof_of_work.rs | 12 ------- .../snapshot/snapshot-tests/src/service.rs | 16 --------- ethcore/snapshot/snapshot-tests/src/state.rs | 21 ------------ 6 files changed, 112 deletions(-) diff --git a/ethcore/snapshot/snapshot-tests/src/helpers.rs b/ethcore/snapshot/snapshot-tests/src/helpers.rs index ea3f5ac3527..afc4979068d 100644 --- a/ethcore/snapshot/snapshot-tests/src/helpers.rs +++ b/ethcore/snapshot/snapshot-tests/src/helpers.rs @@ -20,9 +20,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; -use keccak_hash::{KECCAK_NULL_RLP}; use account_db::AccountDBMut; -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/helpers.rs use account_state; use blockchain::{BlockChain, BlockChainDB}; use client_traits::ChainInfo; @@ -33,27 +31,10 @@ use common_types::{ }; use engine::Engine; use ethcore::client::Client; -======= -use common_types::basic_account::BasicAccount; -use blockchain::{BlockChain, BlockChainDB}; -use ethcore::client::Client; -use client_traits::{ChainInfo, SnapshotClient}; -use engine::Engine; -use crate::{ - StateRebuilder, - io::{SnapshotReader, PackedWriter, PackedReader}, -}; - -use tempdir::TempDir; -use rand::Rng; -use log::trace; -use kvdb::DBValue; ->>>>>>> master:ethcore/snapshot/src/tests/helpers.rs use ethereum_types::H256; use ethtrie::{SecTrieDBMut, TrieDB, TrieDBMut}; use hash_db::HashDB; use journaldb; -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/helpers.rs use keccak_hash::{KECCAK_NULL_RLP}; use keccak_hasher::KeccakHasher; use kvdb::DBValue; @@ -69,12 +50,6 @@ use snapshot::{ use tempdir::TempDir; use trie_db::{TrieMut, Trie}; use trie_standardmap::{Alphabet, StandardMap, ValueMode}; -======= -use trie_db::{TrieMut, Trie}; -use ethtrie::{SecTrieDBMut, TrieDB, TrieDBMut}; -use trie_standardmap::{Alphabet, StandardMap, ValueMode}; -use common_types::errors::EthcoreError; ->>>>>>> master:ethcore/snapshot/src/tests/helpers.rs // the proportion of accounts we will alter each tick. const ACCOUNT_CHURN: f32 = 0.01; @@ -167,11 +142,6 @@ pub fn fill_storage(mut db: AccountDBMut, root: &mut H256, seed: &mut H256) { /// Take a snapshot from the given client into a temporary file. /// Return a snapshot reader for it. pub fn snap(client: &Client) -> (Box, TempDir) { -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/helpers.rs -======= - use common_types::ids::BlockId; - ->>>>>>> master:ethcore/snapshot/src/tests/helpers.rs let tempdir = TempDir::new("").unwrap(); let path = tempdir.path().join("file"); let writer = PackedWriter::new(&path).unwrap(); @@ -194,11 +164,7 @@ pub fn restore( genesis: &[u8], ) -> Result<(), EthcoreError> { let flag = AtomicBool::new(true); -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/helpers.rs let chunker = chunker(engine.snapshot_mode()).expect("the engine used here supports snapshots"); -======= - let chunker = crate::chunker(engine.snapshot_mode()).expect("the engine used here supports snapshots"); ->>>>>>> master:ethcore/snapshot/src/tests/helpers.rs let manifest = reader.manifest(); let mut state = StateRebuilder::new(db.key_value().clone(), journaldb::Algorithm::Archive); diff --git a/ethcore/snapshot/snapshot-tests/src/lib.rs b/ethcore/snapshot/snapshot-tests/src/lib.rs index 98da54fc713..c4fcc940188 100644 --- a/ethcore/snapshot/snapshot-tests/src/lib.rs +++ b/ethcore/snapshot/snapshot-tests/src/lib.rs @@ -33,14 +33,8 @@ mod service; #[cfg(test)] mod watcher; -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/lib.rs #[cfg(test)] mod helpers; -======= -pub mod helpers; - -use common_types::snapshot::ManifestData; ->>>>>>> master:ethcore/snapshot/src/tests/mod.rs #[test] fn manifest_rlp() { diff --git a/ethcore/snapshot/snapshot-tests/src/proof_of_authority.rs b/ethcore/snapshot/snapshot-tests/src/proof_of_authority.rs index 01c6f87f79c..b36ab4674dd 100644 --- a/ethcore/snapshot/snapshot-tests/src/proof_of_authority.rs +++ b/ethcore/snapshot/snapshot-tests/src/proof_of_authority.rs @@ -21,10 +21,6 @@ use std::sync::Arc; use std::str::FromStr; use accounts::AccountProvider; -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/proof_of_authority.rs -======= -use ethcore::client::Client; ->>>>>>> master:ethcore/snapshot/src/tests/proof_of_authority.rs use client_traits::{BlockChainClient, ChainInfo}; use common_types::transaction::{Transaction, Action, SignedTransaction}; use ethabi_contract::use_contract; @@ -35,7 +31,6 @@ use ethcore::{ }; use ethereum_types::Address; use ethkey::Secret; -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/proof_of_authority.rs use keccak_hash::keccak; use lazy_static::lazy_static; use log::trace; @@ -45,24 +40,6 @@ use tempdir::TempDir; use crate::helpers as snapshot_helpers; use_contract!(test_validator_set, "../../res/contracts/test_validator_set.json"); -======= -use crate::tests::helpers as snapshot_helpers; -use spec::Spec; -use ethcore::test_helpers::generate_dummy_client_with_spec; -use common_types::transaction::{Transaction, Action, SignedTransaction}; -use tempdir::TempDir; -use log::trace; -use ethereum_types::Address; -use ethcore::{ - test_helpers, - miner::{self, MinerService}, -}; -use keccak_hash::keccak; -use ethabi_contract::use_contract; -use lazy_static::lazy_static; - -use_contract!(test_validator_set, "../res/contracts/test_validator_set.json"); ->>>>>>> master:ethcore/snapshot/src/tests/proof_of_authority.rs const PASS: &'static str = ""; const TRANSITION_BLOCK_1: usize = 2; // block at which the contract becomes activated. diff --git a/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs b/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs index 3fd125b2182..81dcfe2cd5b 100644 --- a/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs +++ b/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs @@ -19,28 +19,16 @@ use std::sync::atomic::AtomicBool; use tempdir::TempDir; use common_types::{ -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/proof_of_work.rs errors::{EthcoreError as Error, SnapshotError}, -======= - errors::EthcoreError as Error, ->>>>>>> master:ethcore/snapshot/src/tests/proof_of_work.rs engines::ForkChoice, snapshot::{Progress, ManifestData}, }; use blockchain::generator::{BlockGenerator, BlockBuilder}; use blockchain::{BlockChain, ExtrasInsert}; -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/proof_of_work.rs use snapshot::{ chunk_secondary, SnapshotComponents, io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}, -======= -use client_traits::SnapshotWriter; -use crate::{ - chunk_secondary, - Error as SnapshotError, SnapshotComponents, - io::{PackedReader, PackedWriter, SnapshotReader}, ->>>>>>> master:ethcore/snapshot/src/tests/proof_of_work.rs PowSnapshot, }; use parking_lot::Mutex; diff --git a/ethcore/snapshot/snapshot-tests/src/service.rs b/ethcore/snapshot/snapshot-tests/src/service.rs index 129fa8653ff..b2e7a98256f 100644 --- a/ethcore/snapshot/snapshot-tests/src/service.rs +++ b/ethcore/snapshot/snapshot-tests/src/service.rs @@ -22,30 +22,18 @@ use std::sync::Arc; use tempdir::TempDir; use blockchain::BlockProvider; use ethcore::client::{Client, ClientConfig}; -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/service.rs use client_traits::{BlockInfo, ImportBlock}; use common_types::{ io_message::ClientIoMessage, -======= -use client_traits::{BlockInfo, ImportBlock, SnapshotWriter}; -use common_types::{ ->>>>>>> master:ethcore/snapshot/src/tests/service.rs ids::BlockId, snapshot::Progress, verification::Unverified, snapshot::{ManifestData, RestorationStatus}, }; -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/service.rs use snapshot::{ chunk_state, chunk_secondary, SnapshotService, io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}, service::{Service, ServiceParams, Guard, Restoration, RestorationParams}, -======= -use crate::{ - chunk_state, chunk_secondary, SnapshotService, - io::{PackedReader, PackedWriter, SnapshotReader}, - service::{Service, ServiceParams}, ->>>>>>> master:ethcore/snapshot/src/tests/service.rs PowSnapshot, }; use spec; @@ -55,11 +43,7 @@ use ethcore::{ }; use parking_lot::Mutex; -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/service.rs use ethcore_io::{IoChannel, IoService}; -======= -use ethcore_io::IoChannel; ->>>>>>> master:ethcore/snapshot/src/tests/service.rs use kvdb_rocksdb::DatabaseConfig; use journaldb::Algorithm; diff --git a/ethcore/snapshot/snapshot-tests/src/state.rs b/ethcore/snapshot/snapshot-tests/src/state.rs index c4490f257ed..071370bc41e 100644 --- a/ethcore/snapshot/snapshot-tests/src/state.rs +++ b/ethcore/snapshot/snapshot-tests/src/state.rs @@ -18,7 +18,6 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/state.rs use keccak_hash::{KECCAK_NULL_RLP, keccak}; use common_types::{ @@ -31,22 +30,6 @@ use snapshot::{ chunk_state, StateRebuilder, SNAPSHOT_SUBPARTS, io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}, }; -======= -use keccak_hash::{KECCAK_NULL_RLP, keccak}; - -use common_types::{ - basic_account::BasicAccount, - errors::EthcoreError as Error, - snapshot::ManifestData, -}; -use client_traits::SnapshotWriter; -use crate::{ - account, - {chunk_state, Error as SnapshotError, Progress, StateRebuilder, SNAPSHOT_SUBPARTS}, - io::{PackedReader, PackedWriter, SnapshotReader}, - tests::helpers::StateProducer, -}; ->>>>>>> master:ethcore/snapshot/src/tests/state.rs use rand::SeedableRng; use rand_xorshift::XorShiftRng; use ethereum_types::H256; @@ -113,11 +96,7 @@ fn snap_and_restore() { new_db }; -<<<<<<< HEAD:ethcore/snapshot/snapshot-tests/src/state.rs let new_db = journaldb::new(db, Algorithm::OverlayRecent, ethcore_db::COL_STATE); -======= - let new_db = journaldb::new(db, Algorithm::OverlayRecent, ethcore_db::COL_STATE); ->>>>>>> master:ethcore/snapshot/src/tests/state.rs assert_eq!(new_db.earliest_era(), Some(1000)); let keys = old_db.keys(); From dff7adc69ff6e303d4fb0087a93e8d055f326fbf Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 6 Sep 2019 10:41:53 +0200 Subject: [PATCH 19/21] Ensure snapshot-tests run --- Cargo.lock | 44 +++++++++++++++++++++++++++++++++++++ ethcore/snapshot/Cargo.toml | 3 +++ 2 files changed, 47 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 59ef08be579..e3d93d76f0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4242,6 +4242,7 @@ dependencies = [ "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp_derive 0.1.0", + "snapshot-tests 0.1.0", "spec 0.1.0", "state-db 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4250,6 +4251,49 @@ dependencies = [ "triehash-ethereum 0.2.0", ] +[[package]] +name = "snapshot-tests" +version = "0.1.0" +dependencies = [ + "account-db 0.1.0", + "account-state 0.1.0", + "client-traits 0.1.0", + "common-types 0.1.0", + "engine 0.1.0", + "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi-contract 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi-derive 8.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethcore 1.12.0", + "ethcore-accounts 0.1.0", + "ethcore-blockchain 0.1.0", + "ethcore-db 0.1.0", + "ethcore-io 1.12.0", + "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethkey 0.3.0", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "journaldb 0.2.0", + "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hasher 0.1.1", + "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "patricia-trie-ethereum 0.1.0", + "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", + "spec 0.1.0", + "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "triehash-ethereum 0.2.0", +] + [[package]] name = "socket2" version = "0.3.8" diff --git a/ethcore/snapshot/Cargo.toml b/ethcore/snapshot/Cargo.toml index 9abff45548c..3b5d7da921e 100644 --- a/ethcore/snapshot/Cargo.toml +++ b/ethcore/snapshot/Cargo.toml @@ -52,6 +52,9 @@ lazy_static = { version = "1.3" } spec = { path = "../spec" } tempdir = "0.3" trie-standardmap = "0.15.0" +# Note[dvdplm]: Ensure the snapshot tests are included in the dependency tree, which in turn means that +# `cargo test --all` runs the tests. +snapshot-tests = { path = "./snapshot-tests" } [features] test-helpers = [] From 3c2d64993eeb0d6ed49fb65909200d9b628754b2 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 6 Sep 2019 11:04:08 +0200 Subject: [PATCH 20/21] Docs --- ethcore/snapshot/src/service.rs | 9 +++++---- ethcore/snapshot/src/watcher.rs | 18 ++++++++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/ethcore/snapshot/src/service.rs b/ethcore/snapshot/src/service.rs index 4fc3132e692..141a32360c9 100644 --- a/ethcore/snapshot/src/service.rs +++ b/ethcore/snapshot/src/service.rs @@ -89,6 +89,7 @@ pub struct Restoration { db: Arc, } +/// Params to initialise restoration pub struct RestorationParams<'a> { manifest: ManifestData, // manifest to base restoration on. pruning: Algorithm, // pruning algorithm for the database. @@ -115,7 +116,7 @@ impl<'a> RestorationParams<'a> { } impl Restoration { - // make a new restoration using the given parameters. + /// Build a Restoration using the given parameters. pub fn new(params: RestorationParams) -> Result { let manifest = params.manifest; @@ -146,7 +147,7 @@ impl Restoration { }) } - // feeds a state chunk, aborts early if `flag` becomes false. + /// Feeds a chunk of state data to the Restoration. Aborts early if `flag` becomes false. pub fn feed_state(&mut self, hash: H256, chunk: &[u8], flag: &AtomicBool) -> Result<(), Error> { if self.state_chunks_left.contains(&hash) { let expected_len = snappy::decompressed_len(chunk)?; @@ -168,7 +169,7 @@ impl Restoration { Ok(()) } - // feeds a block chunk + /// Feeds a chunk of block data to the `Restoration`. Aborts early if `flag` becomes false. pub fn feed_blocks(&mut self, hash: H256, chunk: &[u8], engine: &dyn Engine, flag: &AtomicBool) -> Result<(), Error> { if self.block_chunks_left.contains(&hash) { let expected_len = snappy::decompressed_len(chunk)?; @@ -215,7 +216,7 @@ impl Restoration { Ok(()) } - // is everything done? + /// Check if we're done restoring: no more block chunks and no more state chunks to process. pub fn is_done(&self) -> bool { self.block_chunks_left.is_empty() && self.state_chunks_left.is_empty() } diff --git a/ethcore/snapshot/src/watcher.rs b/ethcore/snapshot/src/watcher.rs index b0e2ada339e..6ac95bac1dc 100644 --- a/ethcore/snapshot/src/watcher.rs +++ b/ethcore/snapshot/src/watcher.rs @@ -29,10 +29,12 @@ use ethcore_io::IoChannel; use log::{trace, warn}; use parking_lot::Mutex; -// helper trait for transforming hashes to numbers and checking if syncing. +/// Helper trait for transforming hashes to block numbers and checking if syncing. pub trait Oracle: Send + Sync { + /// Maps a block hash to a block number fn to_number(&self, hash: H256) -> Option; + /// Are we currently syncing? fn is_major_importing(&self) -> bool; } @@ -53,7 +55,7 @@ impl Oracle for StandardOracle } } -// helper trait for broadcasting a block to take a snapshot at. +/// Helper trait for broadcasting a block to take a snapshot at. pub trait Broadcast: Send + Sync { fn take_at(&self, num: Option); } @@ -65,7 +67,7 @@ impl Broadcast for Mutex>> { None => return, }; - trace!(target: "snapshot_watcher", "broadcast: {}", num); + trace!(target: "snapshot_watcher", "Snapshot requested at block #{}", num); if let Err(e) = self.lock().send(ClientIoMessage::TakeSnapshot(num)) { warn!("Snapshot watcher disconnected from IoService: {}", e); @@ -86,7 +88,14 @@ impl Watcher { /// Create a new `Watcher` which will trigger a snapshot event /// once every `period` blocks, but only after that block is /// `history` blocks old. - pub fn new(client: Arc, sync_status: F, channel: IoChannel>, period: u64, history: u64) -> Self + pub fn new( + client: Arc, + sync_status: F, + channel: + IoChannel>, + period: u64, + history: u64 + ) -> Self where F: 'static + Send + Sync + Fn() -> bool, C: 'static + Send + Sync, @@ -100,6 +109,7 @@ impl Watcher { } #[cfg(any(test, feature = "test-helpers"))] + /// Instantiate a `Watcher` using anything that impls `Oracle` and `Broadcast`. Test only. pub fn new_test(oracle: Box, broadcast: Box, period: u64, history: u64) -> Self { Watcher { oracle, broadcast, period, history } } From 78dac6a0619675060d47bdc929ed5e3df3e872aa Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 10 Sep 2019 21:55:59 +0200 Subject: [PATCH 21/21] Fix grumbles --- .../snapshot-tests/src/abridged_block.rs | 2 +- .../snapshot/snapshot-tests/src/account.rs | 2 +- ethcore/snapshot/snapshot-tests/src/lib.rs | 11 ++------- ethcore/snapshot/snapshot-tests/src/state.rs | 4 ++-- .../snapshot/snapshot-tests/src/watcher.rs | 5 ++-- ethcore/snapshot/src/block.rs | 1 + ethcore/snapshot/src/lib.rs | 23 +++++++++---------- ethcore/snapshot/src/traits.rs | 16 +++++++++++++ ethcore/snapshot/src/watcher.rs | 17 ++------------ 9 files changed, 39 insertions(+), 42 deletions(-) diff --git a/ethcore/snapshot/snapshot-tests/src/abridged_block.rs b/ethcore/snapshot/snapshot-tests/src/abridged_block.rs index 7d17b727bc3..ec41267043b 100644 --- a/ethcore/snapshot/snapshot-tests/src/abridged_block.rs +++ b/ethcore/snapshot/snapshot-tests/src/abridged_block.rs @@ -16,7 +16,7 @@ //! Tests for block RLP encoding -use snapshot::block::AbridgedBlock; +use snapshot::test_helpers::AbridgedBlock; use bytes::Bytes; use ethereum_types::{H256, U256, Address}; diff --git a/ethcore/snapshot/snapshot-tests/src/account.rs b/ethcore/snapshot/snapshot-tests/src/account.rs index 62c199fbd85..09707c31156 100644 --- a/ethcore/snapshot/snapshot-tests/src/account.rs +++ b/ethcore/snapshot/snapshot-tests/src/account.rs @@ -29,7 +29,7 @@ use hash_db::{HashDB, EMPTY_PREFIX}; use keccak_hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; use kvdb::DBValue; use rlp::Rlp; -use snapshot::account::{ACC_EMPTY, to_fat_rlps, from_fat_rlp}; +use snapshot::test_helpers::{ACC_EMPTY, to_fat_rlps, from_fat_rlp}; use crate::helpers::fill_storage; diff --git a/ethcore/snapshot/snapshot-tests/src/lib.rs b/ethcore/snapshot/snapshot-tests/src/lib.rs index c4fcc940188..8bb5569bae5 100644 --- a/ethcore/snapshot/snapshot-tests/src/lib.rs +++ b/ethcore/snapshot/snapshot-tests/src/lib.rs @@ -16,24 +16,17 @@ //! Snapshot tests. -#[cfg(test)] +#![cfg(test)] + mod abridged_block; -#[cfg(test)] mod account; -#[cfg(test)] mod io; -#[cfg(test)] mod proof_of_work; -#[cfg(test)] mod proof_of_authority; -#[cfg(test)] mod state; -#[cfg(test)] mod service; -#[cfg(test)] mod watcher; -#[cfg(test)] mod helpers; #[test] diff --git a/ethcore/snapshot/snapshot-tests/src/state.rs b/ethcore/snapshot/snapshot-tests/src/state.rs index 071370bc41e..4cccd0d82e9 100644 --- a/ethcore/snapshot/snapshot-tests/src/state.rs +++ b/ethcore/snapshot/snapshot-tests/src/state.rs @@ -26,7 +26,7 @@ use common_types::{ snapshot::{ManifestData, Progress}, }; use snapshot::{ - account, + test_helpers::to_fat_rlps, chunk_state, StateRebuilder, SNAPSHOT_SUBPARTS, io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}, }; @@ -134,7 +134,7 @@ fn get_code_from_prev_chunk() { let mut db = journaldb::new_memory_db(); AccountDBMut::from_hash(&mut db, hash).insert(EMPTY_PREFIX, &code[..]); let p = Progress::default(); - let fat_rlp = account::to_fat_rlps(&hash, &acc, &AccountDB::from_hash(&db, hash), &mut used_code, usize::max_value(), usize::max_value(), &p).unwrap(); + let fat_rlp = to_fat_rlps(&hash, &acc, &AccountDB::from_hash(&db, hash), &mut used_code, usize::max_value(), usize::max_value(), &p).unwrap(); let mut stream = RlpStream::new_list(1); stream.append_raw(&fat_rlp[0], 1); stream.out() diff --git a/ethcore/snapshot/snapshot-tests/src/watcher.rs b/ethcore/snapshot/snapshot-tests/src/watcher.rs index 1f04a71164a..d7cf35ced3e 100644 --- a/ethcore/snapshot/snapshot-tests/src/watcher.rs +++ b/ethcore/snapshot/snapshot-tests/src/watcher.rs @@ -24,8 +24,9 @@ use common_types::chain_notify::{NewBlocks, ChainRoute}; use ethereum_types::{H256, U256, BigEndianHash}; use snapshot::{ - Watcher, - watcher::{Broadcast, Oracle}, + Broadcast, + Oracle, + test_helpers::Watcher, }; struct TestOracle(HashMap); diff --git a/ethcore/snapshot/src/block.rs b/ethcore/snapshot/src/block.rs index 2d6716d674d..998b1392d96 100644 --- a/ethcore/snapshot/src/block.rs +++ b/ethcore/snapshot/src/block.rs @@ -30,6 +30,7 @@ use triehash::ordered_trie_root; const HEADER_FIELDS: usize = 8; const BLOCK_FIELDS: usize = 2; +/// Convenience type to convert raw RLP to and from blocks. pub struct AbridgedBlock { rlp: Bytes, } diff --git a/ethcore/snapshot/src/lib.rs b/ethcore/snapshot/src/lib.rs index 3ce7e71eb94..e6797a661a9 100644 --- a/ethcore/snapshot/src/lib.rs +++ b/ethcore/snapshot/src/lib.rs @@ -56,7 +56,7 @@ use trie_db::{Trie, TrieMut}; pub use self::consensus::*; pub use self::service::{Service, Guard, Restoration, RestorationParams}; -pub use self::traits::{SnapshotService, SnapshotClient, SnapshotComponents, Rebuilder}; +pub use self::traits::{Broadcast, Oracle, SnapshotService, SnapshotClient, SnapshotComponents, Rebuilder}; pub use self::io::SnapshotWriter; pub use self::watcher::Watcher; use common_types::basic_account::BasicAccount; @@ -64,21 +64,20 @@ use common_types::basic_account::BasicAccount; pub mod io; pub mod service; -#[cfg(any(test, feature = "test-helpers"))] -pub mod account; -#[cfg(not(any(test, feature = "test-helpers")))] +#[cfg(feature = "test-helpers" )] +pub mod test_helpers { + pub use super::{ + account::{ACC_EMPTY, to_fat_rlps, from_fat_rlp}, + block::AbridgedBlock, + watcher::Watcher, + }; +} + mod account; -#[cfg(any(test, feature = "test-helpers"))] -pub mod block; -#[cfg(not(any(test, feature = "test-helpers")))] mod block; mod consensus; -#[cfg(any(test, feature = "test-helpers"))] -pub mod watcher; -#[cfg(not(any(test, feature = "test-helpers")))] -mod watcher; - mod traits; +mod watcher; // Try to have chunks be around 4MB (before compression) const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024; diff --git a/ethcore/snapshot/src/traits.rs b/ethcore/snapshot/src/traits.rs index a1b40d57001..b72d6b7afe0 100644 --- a/ethcore/snapshot/src/traits.rs +++ b/ethcore/snapshot/src/traits.rs @@ -144,3 +144,19 @@ pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + Block p: &Progress, ) -> Result<(), Error>; } + +/// Helper trait for broadcasting a block to take a snapshot at. +pub trait Broadcast: Send + Sync { + /// Start a snapshot from the given block number. + fn take_at(&self, num: Option); +} + + +/// Helper trait for transforming hashes to block numbers and checking if syncing. +pub trait Oracle: Send + Sync { + /// Maps a block hash to a block number + fn to_number(&self, hash: H256) -> Option; + + /// Are we currently syncing? + fn is_major_importing(&self) -> bool; +} diff --git a/ethcore/snapshot/src/watcher.rs b/ethcore/snapshot/src/watcher.rs index 6ac95bac1dc..d6d9bcef8c0 100644 --- a/ethcore/snapshot/src/watcher.rs +++ b/ethcore/snapshot/src/watcher.rs @@ -29,14 +29,7 @@ use ethcore_io::IoChannel; use log::{trace, warn}; use parking_lot::Mutex; -/// Helper trait for transforming hashes to block numbers and checking if syncing. -pub trait Oracle: Send + Sync { - /// Maps a block hash to a block number - fn to_number(&self, hash: H256) -> Option; - - /// Are we currently syncing? - fn is_major_importing(&self) -> bool; -} +use crate::traits::{Broadcast, Oracle}; struct StandardOracle where F: 'static + Send + Sync + Fn() -> bool { client: Arc, @@ -55,11 +48,6 @@ impl Oracle for StandardOracle } } -/// Helper trait for broadcasting a block to take a snapshot at. -pub trait Broadcast: Send + Sync { - fn take_at(&self, num: Option); -} - impl Broadcast for Mutex>> { fn take_at(&self, num: Option) { let num = match num { @@ -91,8 +79,7 @@ impl Watcher { pub fn new( client: Arc, sync_status: F, - channel: - IoChannel>, + channel: IoChannel>, period: u64, history: u64 ) -> Self