diff --git a/Cargo.lock b/Cargo.lock index d4867ad3362..1d8e330b43b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1052,7 +1052,6 @@ dependencies = [ "kvdb-memorydb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb-rocksdb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "len-caching-lock 0.1.1", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "machine 0.1.0", "macros 0.1.0", @@ -1062,7 +1061,6 @@ dependencies = [ "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-runtime 0.1.0", "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie-ethereum 0.1.0", "pod 0.1.0", @@ -1080,7 +1078,6 @@ dependencies = [ "state-db 0.1.0", "stats 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "time-utils 0.1.0", "trace 0.1.0", "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1089,6 +1086,7 @@ dependencies = [ "triehash-ethereum 0.2.0", "unexpected 0.1.0", "using_queue 0.1.0", + "verification 0.1.0", "vm 0.1.0", ] @@ -1245,6 +1243,7 @@ dependencies = [ "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "triehash-ethereum 0.2.0", + "verification 0.1.0", "vm 0.1.0", ] @@ -1451,6 +1450,7 @@ name = "ethcore-service" version = "0.1.0" dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "client-traits 0.1.0", "common-types 0.1.0", "ethcore 1.12.0", "ethcore-blockchain 0.1.0", @@ -3021,6 +3021,7 @@ dependencies = [ "term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "verification 0.1.0", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3171,6 +3172,7 @@ dependencies = [ "trace 0.1.0", "transaction-pool 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "transient-hashmap 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "verification 0.1.0", "vm 0.1.0", ] @@ -4947,6 +4949,35 @@ dependencies = [ "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "verification" +version = "0.1.0" +dependencies = [ + "client-traits 0.1.0", + "common-types 0.1.0", + "engine 0.1.0", + "ethcore 1.12.0", + "ethcore-blockchain 0.1.0", + "ethcore-call-contract 0.1.0", + "ethcore-io 1.12.0", + "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethkey 0.3.0", + "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "len-caching-lock 0.1.1", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "machine 0.1.0", + "null-engine 0.1.0", + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "spec 0.1.0", + "time-utils 0.1.0", + "triehash-ethereum 0.2.0", + "unexpected 0.1.0", +] + [[package]] name = "version_check" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index d8eb2179793..298c1542e8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ spec = { path = "ethcore/spec" } term_size = "0.3" textwrap = "0.9" toml = "0.4" +verification = { path = "ethcore/verification" } [build-dependencies] rustc_version = "0.2" diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index d87133a502b..cf27c1a6585 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -35,16 +35,14 @@ evm = { path = "evm" } executive-state = { path = "executive-state" } futures = "0.1" hash-db = "0.15.0" -parity-util-mem = "0.2.0" itertools = "0.5" journaldb = { path = "../util/journaldb" } keccak-hash = "0.2.0" keccak-hasher = { path = "../util/keccak-hasher" } kvdb = "0.1" -kvdb-memorydb = "0.1" +kvdb-memorydb = { version = "0.1", optional = true } kvdb-rocksdb = { version = "0.1.3", optional = true } -lazy_static = "1.3.0" -len-caching-lock = { path = "../util/len-caching-lock" } +lazy_static = { version = "1.3", optional = true } log = "0.4" macros = { path = "../util/macros", optional = true } machine = { path = "./machine" } @@ -61,20 +59,19 @@ rand_xorshift = "0.1.1" rayon = "1.1" rlp = "0.4.0" rlp_derive = { path = "../util/rlp-derive" } -rustc-hex = "1.0" +rustc-hex = { version = "1", optional = true } serde = "1.0" serde_derive = "1.0" spec = { path = "spec" } state-db = { path = "state-db" } -stats = { path = "../util/stats" } tempdir = { version = "0.3", optional = true } -time-utils = { path = "../util/time-utils" } trace = { path = "trace" } trace-time = "0.1" trie-vm-factories = { path = "trie-vm-factories" } triehash-ethereum = { version = "0.2", path = "../util/triehash-ethereum" } unexpected = { path = "../util/unexpected" } using_queue = { path = "../miner/using-queue" } +verification = { path = "./verification" } vm = { path = "vm" } [dev-dependencies] @@ -88,14 +85,18 @@ ethcore-accounts = { path = "../accounts" } ethjson = { path = "../json" } ethkey = { path = "../accounts/ethkey" } fetch = { path = "../util/fetch" } +kvdb-memorydb = "0.1" kvdb-rocksdb = "0.1.3" +lazy_static = { version = "1.3" } machine = { path = "./machine", features = ["test-helpers"] } macros = { path = "../util/macros" } null-engine = { path = "./engines/null-engine" } parity-runtime = { path = "../util/runtime" } pod = { path = "pod" } rlp_compress = { path = "../util/rlp-compress" } +rustc-hex = "1" serde_json = "1.0" +stats = { path = "../util/stats" } tempdir = "0.3" trie-standardmap = "0.15.0" @@ -121,14 +122,14 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"] # EVM debug traces are printed. slow-blocks = [] # Run JSON consensus tests. -json-tests = ["env_logger", "test-helpers", "machine/test-helpers"] +json-tests = ["env_logger", "test-helpers", "lazy_static", "machine/test-helpers"] # Skip JSON consensus tests with pending issues. ci-skip-tests = [] # Run memory/cpu heavy tests. test-heavy = [] # Compile test helpers # note[dvdplm]: "basic-authority/test-helpers" is needed so that `generate_dummy_client_with_spec` works -test-helpers = ["tempdir", "kvdb-rocksdb", "blooms-db", "ethash", "ethjson", "ethkey", "macros", "pod", "basic-authority/test-helpers"] +test-helpers = ["tempdir", "kvdb-memorydb", "kvdb-rocksdb", "blooms-db", "ethash", "ethjson", "ethkey", "macros", "pod", "rustc-hex", "basic-authority/test-helpers"] [[bench]] name = "builtin" diff --git a/ethcore/client-traits/src/lib.rs b/ethcore/client-traits/src/lib.rs index af68b9b3f6c..896f9515b8b 100644 --- a/ethcore/client-traits/src/lib.rs +++ b/ethcore/client-traits/src/lib.rs @@ -32,7 +32,7 @@ use common_types::{ client_types::Mode, encoded, engines::{epoch::Transition as EpochTransition, machine::Executed}, - errors::EthcoreResult, + errors::{EthcoreError, EthcoreResult}, filter::Filter, header::Header, ids::{BlockId, TransactionId, TraceId, UncleId}, @@ -56,6 +56,8 @@ use trace::{ }; use vm::{LastHashes, Schedule}; +use common_types::snapshot::Progress; + /// State information to be used during client query pub enum StateOrBlock { /// State to be used, may be pending @@ -168,11 +170,14 @@ pub trait EngineClient: Sync + Send + ChainInfo { fn block_header(&self, id: BlockId) -> Option; } -// FIXME Why these methods belong to BlockChainClient and not MiningBlockChainClient? /// Provides methods to import block into blockchain pub trait ImportBlock { /// Import a block into the blockchain. fn import_block(&self, block: Unverified) -> EthcoreResult; + + /// Triggered by a message from a block queue when the block is ready for insertion. + /// Returns the number of blocks imported. + fn import_verified_blocks(&self) -> usize; } /// IO operations that should off-load heavy work to another thread. @@ -187,6 +192,14 @@ pub trait IoClient: Sync + Send { fn queue_consensus_message(&self, message: Bytes); } +/// Implement this for clients that need logic to decide when/how to advance. +pub trait Tick { + /// Tick the client + fn tick(&self, _prevent_sleep: bool) {} +} + +impl Tick for () {} + /// Provides recently seen bad blocks. pub trait BadBlocks { /// Returns a list of blocks that were recently not imported because they were invalid. @@ -377,6 +390,9 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra pub trait BlockChainReset { /// reset to best_block - n fn reset(&self, num: u32) -> Result<(), String>; + + /// Number of eras kept in a journal before they are pruned + fn pruning_history(&self) -> u64; } @@ -423,3 +439,38 @@ pub trait ProvingBlockChainClient: BlockChainClient { /// Get an epoch change signal by block hash. fn epoch_signal(&self, hash: H256) -> Option>; } + +/// External database restoration handler +pub trait DatabaseRestore: Send + Sync { + /// Restart with a new backend. Takes ownership of passed database and moves it to a new location. + fn restore_db(&self, new_db: &str) -> Result<(), EthcoreError>; +} + +/// Snapshot related functionality +pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + BlockChainReset { + /// Take a snapshot at the given block. + /// If the ID given is "latest", this will default to 1000 blocks behind. + fn take_snapshot( + &self, + writer: W, + at: BlockId, + p: &Progress, + ) -> Result<(), EthcoreError>; +} + + +// todo[dvdplm] move this back to snapshot once extracted from ethcore +/// Something which can write snapshots. +/// Writing the same chunk multiple times will lead to implementation-defined +/// behavior, and is not advised. +pub trait SnapshotWriter { + /// Write a compressed state chunk. + fn write_state_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>; + + /// Write a compressed block chunk. + fn write_block_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>; + + /// Complete writing. The manifest's chunk lists must be consistent + /// with the chunks written. + fn finish(self, manifest: common_types::snapshot::ManifestData) -> std::io::Result<()> where Self: Sized; +} diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml index abc233744d1..32e0c5e00d0 100644 --- a/ethcore/light/Cargo.toml +++ b/ethcore/light/Cargo.toml @@ -13,7 +13,6 @@ client-traits = { path = "../client-traits" } common-types = { path = "../types" } derive_more = "0.14.0" engine = { path = "../engine" } -ethcore = { path = ".."} ethcore-db = { path = "../db" } ethcore-blockchain = { path = "../blockchain" } ethereum-types = "0.6.0" @@ -49,6 +48,7 @@ kvdb = "0.1" memory-cache = { path = "../../util/memory-cache" } error-chain = { version = "0.12", default-features = false } journaldb = { path = "../../util/journaldb" } +verification = { path = "../verification" } [dev-dependencies] ethcore = { path = "..", features = ["test-helpers"] } diff --git a/ethcore/light/src/client/mod.rs b/ethcore/light/src/client/mod.rs index 98149ea2f8b..0339cfe1b10 100644 --- a/ethcore/light/src/client/mod.rs +++ b/ethcore/light/src/client/mod.rs @@ -19,8 +19,7 @@ use std::sync::{Weak, Arc}; use engine::{Engine, EpochChange, Proof}; -use ethcore::client::{ClientReport, ClientIoMessage}; -use ethcore::verification::queue::{self, HeaderQueue}; +use verification::queue::{self, HeaderQueue}; use spec::{Spec, SpecHardcodedSync}; use io::IoChannel; use parking_lot::{Mutex, RwLock}; @@ -30,12 +29,14 @@ use common_types::{ BlockNumber, block_status::BlockStatus, blockchain_info::BlockChainInfo, + client_types::ClientReport, encoded, engines::epoch::{Transition as EpochTransition, PendingTransition}, errors::EthcoreError as Error, errors::EthcoreResult, header::Header, ids::BlockId, + io_message::ClientIoMessage, verification::VerificationQueueInfo as BlockQueueInfo, }; use kvdb::KeyValueDB; @@ -162,7 +163,7 @@ impl AsLightClient for T { /// Light client implementation. pub struct Client { - queue: HeaderQueue, + queue: HeaderQueue<()>, engine: Arc, chain: HeaderChain, report: RwLock, @@ -183,7 +184,7 @@ impl Client { chain_col: Option, spec: &Spec, fetcher: T, - io_channel: IoChannel, + io_channel: IoChannel>, cache: Arc> ) -> Result { Ok(Self { @@ -649,3 +650,5 @@ impl client_traits::EngineClient for Client { Client::block_header(self, id) } } + +impl client_traits::Tick for Client {} diff --git a/ethcore/light/src/client/service.rs b/ethcore/light/src/client/service.rs index 2a55322b464..2b22a7c15bf 100644 --- a/ethcore/light/src/client/service.rs +++ b/ethcore/light/src/client/service.rs @@ -20,10 +20,12 @@ use std::fmt; use std::sync::Arc; -use common_types::errors::EthcoreError as CoreError; +use common_types::{ + errors::EthcoreError as CoreError, + io_message::ClientIoMessage, +}; use ethcore_db as db; use ethcore_blockchain::BlockChainDB; -use ethcore::client::ClientIoMessage; use spec::Spec; use io::{IoContext, IoError, IoHandler, IoService}; @@ -58,15 +60,15 @@ impl fmt::Display for Error { } /// Light client service. -pub struct Service { +pub struct Service { client: Arc>, - io_service: IoService, + io_service: IoService>, } impl Service { /// Start the service: initialize I/O workers and client itself. pub fn start(config: ClientConfig, spec: &Spec, fetcher: T, db: Arc, cache: Arc>) -> Result { - let io_service = IoService::::start().map_err(Error::Io)?; + let io_service = IoService::>::start().map_err(Error::Io)?; let client = Arc::new(Client::new(config, db.key_value().clone(), db::COL_LIGHT_CHAIN, @@ -90,7 +92,7 @@ impl Service { } /// Register an I/O handler on the service. - pub fn register_handler(&self, handler: Arc + Send>) -> Result<(), IoError> { + pub fn register_handler(&self, handler: Arc> + Send>) -> Result<(), IoError> { self.io_service.register_handler(handler) } @@ -102,8 +104,8 @@ impl Service { struct ImportBlocks(Arc>); -impl IoHandler for ImportBlocks { - fn message(&self, _io: &IoContext, message: &ClientIoMessage) { +impl IoHandler> for ImportBlocks { + fn message(&self, _io: &IoContext>, message: &ClientIoMessage<()>) { if let ClientIoMessage::BlockVerified = *message { self.0.import_verified(); } diff --git a/ethcore/light/src/lib.rs b/ethcore/light/src/lib.rs index 3d2bd3752d7..61ce17e5632 100644 --- a/ethcore/light/src/lib.rs +++ b/ethcore/light/src/lib.rs @@ -65,7 +65,6 @@ extern crate executive_state; extern crate parity_bytes as bytes; extern crate ethereum_types; extern crate ethcore_miner as miner; -extern crate ethcore; extern crate hash_db; extern crate parity_util_mem; extern crate parity_util_mem as mem; @@ -94,7 +93,10 @@ extern crate triehash_ethereum as triehash; extern crate kvdb; extern crate memory_cache; extern crate derive_more; +extern crate verification; +#[cfg(test)] +extern crate ethcore; #[cfg(test)] extern crate kvdb_memorydb; #[cfg(test)] diff --git a/ethcore/private-tx/src/lib.rs b/ethcore/private-tx/src/lib.rs index 2368205baf8..8673d4be7c3 100644 --- a/ethcore/private-tx/src/lib.rs +++ b/ethcore/private-tx/src/lib.rs @@ -104,11 +104,12 @@ use machine::{ }; use types::{ ids::BlockId, + io_message::ClientIoMessage, transaction::{SignedTransaction, Transaction, Action, UnverifiedTransaction}, engines::machine::Executed, }; use ethcore::client::{ - Client, ChainNotify, NewBlocks, ChainMessageType, ClientIoMessage, Call + Client, ChainNotify, NewBlocks, ChainMessageType, Call }; use client_traits::BlockInfo; use ethcore::miner::{self, Miner, MinerService, pool_client::NonceCache}; @@ -213,7 +214,7 @@ pub struct Provider { client: Arc, miner: Arc, accounts: Arc, - channel: IoChannel, + channel: IoChannel>, keys_provider: Arc, logging: Option, use_offchain_storage: bool, @@ -236,7 +237,7 @@ impl Provider { accounts: Arc, encryptor: Box, config: ProviderConfig, - channel: IoChannel, + channel: IoChannel>, keys_provider: Arc, db: Arc, ) -> Self { @@ -490,7 +491,7 @@ impl Provider { } } Ok(()) - } + } fn contract_address_from_transaction(transaction: &SignedTransaction) -> Result { match transaction.action { @@ -876,14 +877,14 @@ impl Provider { } } -impl IoHandler for Provider { - fn initialize(&self, io: &IoContext) { +impl IoHandler> for Provider { + fn initialize(&self, io: &IoContext>) { if self.use_offchain_storage { io.register_timer(STATE_RETRIEVAL_TIMER, STATE_RETRIEVAL_TICK).expect("Error registering state retrieval timer"); } } - fn timeout(&self, _io: &IoContext, timer: TimerToken) { + fn timeout(&self, _io: &IoContext>, timer: TimerToken) { match timer { STATE_RETRIEVAL_TIMER => self.state_storage.tick(&self.logging), _ => warn!("IO service triggered unregistered timer '{}'", timer), diff --git a/ethcore/service/Cargo.toml b/ethcore/service/Cargo.toml index 6ab4afecd7a..f333e979d06 100644 --- a/ethcore/service/Cargo.toml +++ b/ethcore/service/Cargo.toml @@ -7,6 +7,7 @@ authors = ["Parity Technologies "] [dependencies] ansi_term = "0.11" common-types = { path = "../types" } +client-traits = { path = "../client-traits" } ethcore = { path = ".." } ethcore-blockchain = { path = "../blockchain" } ethcore-io = { path = "../../util/io" } diff --git a/ethcore/service/src/lib.rs b/ethcore/service/src/lib.rs index d49a3d30d39..662a2567669 100644 --- a/ethcore/service/src/lib.rs +++ b/ethcore/service/src/lib.rs @@ -16,6 +16,7 @@ extern crate ansi_term; extern crate common_types; +extern crate client_traits; extern crate ethcore; extern crate ethcore_blockchain as blockchain; extern crate ethcore_io as io; diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs index a1bda778b80..add9eb1d0d6 100644 --- a/ethcore/service/src/service.rs +++ b/ethcore/service/src/service.rs @@ -26,15 +26,17 @@ use io::{IoContext, TimerToken, IoHandler, IoService, IoError}; use sync::PrivateTxHandler; use blockchain::{BlockChainDB, BlockChainDBHandler}; -use ethcore::client::{Client, ClientConfig, ChainNotify, ClientIoMessage}; +use ethcore::client::{Client, ClientConfig, ChainNotify}; use ethcore::miner::Miner; use ethcore::snapshot::service::{Service as SnapshotService, ServiceParams as SnapServiceParams}; use ethcore::snapshot::{SnapshotService as _SnapshotService}; use spec::Spec; use common_types::{ + io_message::ClientIoMessage, errors::{EthcoreError, SnapshotError}, snapshot::RestorationStatus, }; +use client_traits::{ImportBlock, SnapshotClient, Tick}; use ethcore_private_tx::{self, Importer, Signer}; @@ -90,7 +92,7 @@ impl PrivateTxHandler for PrivateTxService { /// Client service setup. Creates and registers client and network services with the IO subsystem. pub struct ClientService { - io_service: Arc>, + io_service: Arc>>, client: Arc, snapshot: Arc, private_tx: Arc, @@ -113,7 +115,7 @@ impl ClientService { private_encryptor_conf: ethcore_private_tx::EncryptorConfig, ) -> Result { - let io_service = IoService::::start()?; + let io_service = IoService::>::start()?; info!("Configured for {} using {} engine", Colour::White.bold().paint(spec.name.clone()), Colour::Yellow.bold().paint(spec.engine.name().to_string())); @@ -165,15 +167,15 @@ impl ClientService { Ok(ClientService { io_service: Arc::new(io_service), - client: client, - snapshot: snapshot, + client, + snapshot, private_tx, database: blockchain_db, }) } /// Get general IO interface - pub fn register_io_handler(&self, handler: Arc + Send>) -> Result<(), IoError> { + pub fn register_io_handler(&self, handler: Arc> + Send>) -> Result<(), IoError> { self.io_service.register_handler(handler) } @@ -193,7 +195,7 @@ impl ClientService { } /// Get network service component - pub fn io(&self) -> Arc> { + pub fn io(&self) -> Arc>> { self.io_service.clone() } @@ -213,8 +215,8 @@ impl ClientService { } /// IO interface for the Client handler -struct ClientIoHandler { - client: Arc, +struct ClientIoHandler { + client: Arc, snapshot: Arc, } @@ -224,13 +226,16 @@ const SNAPSHOT_TICK_TIMER: TimerToken = 1; const CLIENT_TICK: Duration = Duration::from_secs(5); const SNAPSHOT_TICK: Duration = Duration::from_secs(10); -impl IoHandler for ClientIoHandler { - fn initialize(&self, io: &IoContext) { +impl IoHandler> for ClientIoHandler +where + C: ImportBlock + SnapshotClient + Tick + 'static, +{ + fn initialize(&self, io: &IoContext>) { io.register_timer(CLIENT_TICK_TIMER, CLIENT_TICK).expect("Error registering client timer"); io.register_timer(SNAPSHOT_TICK_TIMER, SNAPSHOT_TICK).expect("Error registering snapshot timer"); } - fn timeout(&self, _io: &IoContext, timer: TimerToken) { + fn timeout(&self, _io: &IoContext>, timer: TimerToken) { trace_time!("service::read"); match timer { CLIENT_TICK_TIMER => { @@ -243,7 +248,7 @@ impl IoHandler for ClientIoHandler { } } - fn message(&self, _io: &IoContext, net_message: &ClientIoMessage) { + fn message(&self, _io: &IoContext>, net_message: &ClientIoMessage) { trace_time!("service::message"); use std::thread; diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 60e4263aefc..bb4855e189e 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . -use std::{cmp, ops}; +use std::cmp; use std::collections::{HashSet, BTreeMap, VecDeque}; use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering}; use std::sync::{Arc, Weak}; @@ -42,14 +42,15 @@ use client::ancient_import::AncientVerifier; use client::{ ReopenBlock, PrepareOpenBlock, ImportSealedBlock, BroadcastProposalBlock, Call, BlockProducer, SealedBlockImporter, ChainNotify, EngineInfo, - ClientConfig, NewBlocks, ChainRoute, ChainMessageType, bad_blocks, ClientIoMessage, + ClientConfig, NewBlocks, ChainRoute, ChainMessageType, bad_blocks, }; use client_traits::{ BlockInfo, ScheduleInfo, StateClient, BlockChainReset, Nonce, Balance, ChainInfo, TransactionInfo, ImportBlock, AccountData, BlockChain as BlockChainTrait, BlockChainClient, - IoClient, BadBlocks, ProvingBlockChainClient, - StateOrBlock + IoClient, BadBlocks, ProvingBlockChainClient, SnapshotClient, + DatabaseRestore, SnapshotWriter, Tick, + StateOrBlock, }; use engine::Engine; use machine::{ @@ -59,7 +60,7 @@ use machine::{ }; use trie_vm_factories::{Factories, VmFactory}; use miner::{Miner, MinerService}; -use snapshot::{self, io as snapshot_io, SnapshotClient}; +use snapshot; use spec::Spec; use account_state::State; use executive_state; @@ -71,6 +72,8 @@ use types::{ block::PreverifiedBlock, block_status::BlockStatus, blockchain_info::BlockChainInfo, + client_types::ClientReport, + io_message::ClientIoMessage, encoded, engines::{ ForkChoice, @@ -111,44 +114,6 @@ const MAX_ANCIENT_BLOCKS_TO_IMPORT: usize = 4; const MAX_QUEUE_SIZE_TO_SLEEP_ON: usize = 2; const MIN_HISTORY_SIZE: u64 = 8; -/// Report on the status of a client. -#[derive(Default, Clone, Debug, Eq, PartialEq)] -pub struct ClientReport { - /// How many blocks have been imported so far. - pub blocks_imported: usize, - /// How many transactions have been applied so far. - pub transactions_applied: usize, - /// How much gas has been processed so far. - pub gas_processed: U256, - /// Memory used by state DB - pub state_db_mem: usize, -} - -impl ClientReport { - /// Alter internal reporting to reflect the additional `block` has been processed. - pub fn accrue_block(&mut self, header: &Header, transactions: usize) { - self.blocks_imported += 1; - self.transactions_applied += transactions; - self.gas_processed = self.gas_processed + *header.gas_used(); - } -} - -impl<'a> ops::Sub<&'a ClientReport> for ClientReport { - type Output = Self; - - fn sub(mut self, other: &'a ClientReport) -> Self { - let higher_mem = cmp::max(self.state_db_mem, other.state_db_mem); - let lower_mem = cmp::min(self.state_db_mem, other.state_db_mem); - - self.blocks_imported -= other.blocks_imported; - self.transactions_applied -= other.transactions_applied; - self.gas_processed = self.gas_processed - other.gas_processed; - self.state_db_mem = higher_mem - lower_mem; - - self - } -} - struct SleepState { last_activity: Option, last_autosleep: Option, @@ -171,7 +136,7 @@ struct Importer { pub verifier: Box>, /// Queue containing pending blocks - pub block_queue: BlockQueue, + pub block_queue: BlockQueue, /// Handles block sealing pub miner: Arc, @@ -222,7 +187,7 @@ pub struct Client { /// Flag changed by `sleep` and `wake_up` methods. Not to be confused with `enabled`. liveness: AtomicBool, - io_channel: RwLock>, + io_channel: RwLock>>, /// List of actors to be notified on certain chain events notify: RwLock>>, @@ -261,7 +226,7 @@ impl Importer { pub fn new( config: &ClientConfig, engine: Arc, - message_channel: IoChannel, + message_channel: IoChannel>, miner: Arc, ) -> Result { let block_queue = BlockQueue::new( @@ -723,7 +688,7 @@ impl Client { spec: &Spec, db: Arc, miner: Arc, - message_channel: IoChannel, + message_channel: IoChannel>, ) -> Result, EthcoreError> { let trie_spec = match config.fat_db { true => TrieSpec::Fat, @@ -948,11 +913,6 @@ impl Client { Arc::new(last_hashes) } - /// This is triggered by a message coming from a block queue when the block is ready for insertion - pub fn import_verified_blocks(&self) -> usize { - self.importer.import_verified_blocks(self) - } - // use a state-proving closure for the given block. fn with_proving_caller(&self, id: BlockId, with_call: F) -> T where F: FnOnce(&MachineCall) -> T @@ -1037,7 +997,7 @@ impl Client { } /// Replace io channel. Useful for testing. - pub fn set_io_channel(&self, io_channel: IoChannel) { + pub fn set_io_channel(&self, io_channel: IoChannel>) { *self.io_channel.write() = io_channel; } @@ -1112,15 +1072,6 @@ impl Client { report } - /// Tick the client. - // TODO: manage by real events. - pub fn tick(&self, prevent_sleep: bool) { - self.check_garbage(); - if !prevent_sleep { - self.check_snooze(); - } - } - fn check_garbage(&self) { self.chain.read().collect_garbage(); self.importer.block_queue.collect_garbage(); @@ -1161,64 +1112,6 @@ impl Client { } } - /// Take a snapshot at the given block. - /// If the ID given is "latest", this will default to 1000 blocks behind. - pub fn take_snapshot( - &self, - writer: W, - at: BlockId, - p: &Progress, - ) -> Result<(), EthcoreError> { - if let Snapshotting::Unsupported = self.engine.snapshot_mode() { - return Err(EthcoreError::Snapshot(SnapshotError::SnapshotsUnsupported)); - } - let db = self.state_db.read().journal_db().boxed_clone(); - let best_block_number = self.chain_info().best_block_number; - let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?; - - if db.is_prunable() && self.pruning_info().earliest_state > block_number { - return Err(SnapshotError::OldBlockPrunedDB.into()); - } - - let history = cmp::min(self.history, 1000); - - let start_hash = match at { - BlockId::Latest => { - let start_num = match db.earliest_era() { - Some(era) => cmp::max(era, best_block_number.saturating_sub(history)), - None => best_block_number.saturating_sub(history), - }; - - match self.block_hash(BlockId::Number(start_num)) { - Some(h) => h, - None => return Err(SnapshotError::InvalidStartingBlock(at).into()), - } - } - _ => match self.block_hash(at) { - Some(hash) => hash, - None => return Err(SnapshotError::InvalidStartingBlock(at).into()), - }, - }; - - let processing_threads = self.config.snapshot.processing_threads; - let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?; - snapshot::take_snapshot( - chunker, - &self.chain.read(), - start_hash, - db.as_hash_db(), - writer, - p, - processing_threads, - )?; - Ok(()) - } - - /// Ask the client what the history parameter is. - pub fn pruning_history(&self) -> u64 { - self.history - } - fn block_hash(chain: &BlockChain, id: BlockId) -> Option { match id { BlockId::Hash(hash) => Some(hash), @@ -1342,7 +1235,7 @@ impl Client { } } -impl snapshot::DatabaseRestore for Client { +impl DatabaseRestore for Client { /// Restart the client with a new backend fn restore_db(&self, new_db: &str) -> Result<(), EthcoreError> { trace!(target: "snapshot", "Replacing client database with {:?}", new_db); @@ -1426,6 +1319,11 @@ impl BlockChainReset for Client { Ok(()) } + + /// Ask the client what the history parameter is. + fn pruning_history(&self) -> u64 { + self.history + } } impl Nonce for Client { @@ -1547,6 +1445,11 @@ impl ImportBlock for Client { Err((_, e)) => Err(e), } } + + /// Triggered by a message from a block queue when the block is ready for insertion + fn import_verified_blocks(&self) -> usize { + self.importer.import_verified_blocks(self) + } } impl StateClient for Client { @@ -2341,6 +2244,18 @@ impl IoClient for Client { } } } + +} + +impl Tick for Client { + /// Tick the client. + // TODO: manage by real events. + fn tick(&self, prevent_sleep: bool) { + self.check_garbage(); + if !prevent_sleep { + self.check_snooze(); + } + } } impl ReopenBlock for Client { @@ -2581,7 +2496,60 @@ impl ProvingBlockChainClient for Client { } } -impl SnapshotClient for Client {} +impl SnapshotClient for Client { + fn take_snapshot( + &self, + writer: W, + at: BlockId, + p: &Progress, + ) -> Result<(), EthcoreError> { + if let Snapshotting::Unsupported = self.engine.snapshot_mode() { + return Err(EthcoreError::Snapshot(SnapshotError::SnapshotsUnsupported)); + } + let db = self.state_db.read().journal_db().boxed_clone(); + let best_block_number = self.chain_info().best_block_number; + let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?; + + if db.is_prunable() && self.pruning_info().earliest_state > block_number { + return Err(SnapshotError::OldBlockPrunedDB.into()); + } + + let history = cmp::min(self.history, 1000); + + let start_hash = match at { + BlockId::Latest => { + let start_num = match db.earliest_era() { + Some(era) => cmp::max(era, best_block_number.saturating_sub(history)), + None => best_block_number.saturating_sub(history), + }; + + match self.block_hash(BlockId::Number(start_num)) { + Some(h) => h, + None => return Err(SnapshotError::InvalidStartingBlock(at).into()), + } + } + _ => match self.block_hash(at) { + Some(hash) => hash, + None => return Err(SnapshotError::InvalidStartingBlock(at).into()), + }, + }; + + let processing_threads = self.config.snapshot.processing_threads; + let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?; + snapshot::take_snapshot( + chunker, + &self.chain.read(), + start_hash, + db.as_hash_db(), + writer, + p, + processing_threads, + )?; + Ok(()) + } + + +} /// Returns `LocalizedReceipt` given `LocalizedTransaction` /// and a vector of receipts from given block up to transaction index. @@ -2641,7 +2609,7 @@ impl IoChannelQueue { } } - pub fn queue(&self, channel: &IoChannel, count: usize, fun: F) -> EthcoreResult<()> where + pub fn queue(&self, channel: &IoChannel>, count: usize, fun: F) -> EthcoreResult<()> where F: Fn(&Client) + Send + Sync + 'static, { let queue_size = self.currently_queued.load(AtomicOrdering::Relaxed); diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs index 60e36d172a0..58b8b7ffa65 100644 --- a/ethcore/src/client/mod.rs +++ b/ethcore/src/client/mod.rs @@ -22,15 +22,13 @@ mod client; mod config; #[cfg(any(test, feature = "test-helpers"))] mod evm_test_client; -mod io_message; #[cfg(any(test, feature = "test-helpers"))] mod test_client; -pub use self::client::{Client, ClientReport}; +pub use self::client::Client; pub use self::config::{ClientConfig, DatabaseCompactionProfile, BlockChainConfig, VMType}; #[cfg(any(test, feature = "test-helpers"))] pub use self::evm_test_client::{EvmTestClient, EvmTestError, TransactErr, TransactSuccess}; -pub use self::io_message::ClientIoMessage; #[cfg(any(test, feature = "test-helpers"))] pub use self::test_client::{TestBlockChainClient, EachBlockWith, TestState}; pub use self::chain_notify::{ChainNotify, NewBlocks, ChainRoute, ChainRouteType, ChainMessageType}; diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 6f243cb4c45..51cbed75e2a 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -592,6 +592,10 @@ impl ImportBlock for TestBlockChainClient { } Ok(h) } + + fn import_verified_blocks(&self) -> usize { + unimplemented!("TestClient does not implement import_verified_blocks()") + } } impl Call for TestBlockChainClient { diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index 4bfe3c7c962..28e1f567319 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -100,7 +100,6 @@ pub fn json_chain_test(json_data: &[u8], start_stop_ho if let Ok(block) = Unverified::from_rlp(b) { let _ = client.import_block(block); client.flush_queue(); - client.import_verified_blocks(); } } fail_unless(client.chain_info().best_block_hash == blockchain.best_block.into()); diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index a34db6c6522..8beaf991f53 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -69,7 +69,6 @@ extern crate ethcore_io as io; extern crate ethcore_miner; extern crate ethereum_types; extern crate executive_state; -extern crate trie_vm_factories; extern crate futures; extern crate hash_db; extern crate itertools; @@ -77,10 +76,6 @@ extern crate journaldb; extern crate keccak_hash as hash; extern crate keccak_hasher; extern crate kvdb; -#[cfg(any(test, feature = "test-helpers"))] -extern crate kvdb_memorydb; - -extern crate len_caching_lock; extern crate machine; extern crate memory_cache; extern crate num_cpus; @@ -92,18 +87,15 @@ extern crate patricia_trie_ethereum as ethtrie; extern crate rand; extern crate rayon; extern crate rlp; -extern crate parity_util_mem; -extern crate parity_util_mem as malloc_size_of; -#[cfg(any(test, feature = "test-helpers"))] -extern crate rustc_hex; extern crate serde; extern crate spec; extern crate state_db; -extern crate time_utils; extern crate trace; +extern crate trie_vm_factories; extern crate triehash_ethereum as triehash; extern crate unexpected; extern crate using_queue; +extern crate verification; extern crate vm; #[cfg(test)] @@ -119,8 +111,8 @@ extern crate ethash; extern crate ethkey; #[cfg(any(test, feature = "test-helpers"))] extern crate ethjson; -#[cfg(any(test, feature = "tempdir"))] -extern crate tempdir; +#[cfg(any(test, feature = "test-helpers"))] +extern crate kvdb_memorydb; #[cfg(any(test, feature = "kvdb-rocksdb"))] extern crate kvdb_rocksdb; #[cfg(any(test, feature = "json-tests"))] @@ -137,8 +129,12 @@ extern crate pod; extern crate blooms_db; #[cfg(any(test, feature = "env_logger"))] extern crate env_logger; +#[cfg(any(test, feature = "test-helpers"))] +extern crate rustc_hex; #[cfg(test)] extern crate serde_json; +#[cfg(any(test, feature = "tempdir"))] +extern crate tempdir; #[macro_use] extern crate ethabi_contract; @@ -162,7 +158,6 @@ pub mod block; pub mod client; pub mod miner; pub mod snapshot; -pub mod verification; #[cfg(test)] mod tests; diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index bd12ede1965..47559f1707f 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -36,36 +36,30 @@ use miner::pool_client::{PoolClient, CachedNonceClient, NonceCache}; use miner; use parking_lot::{Mutex, RwLock}; use rayon::prelude::*; -use types::transaction::{ - self, - Action, - UnverifiedTransaction, - SignedTransaction, - PendingTransaction, -}; use types::{ BlockNumber, ids::TransactionId, block::Block, header::Header, ids::BlockId, + io_message::ClientIoMessage, engines::{Seal, SealingState}, errors::{EthcoreError as Error, ExecutionError}, receipt::RichReceipt, + transaction::{ + self, + Action, + UnverifiedTransaction, + SignedTransaction, + PendingTransaction, + }, }; use using_queue::{UsingQueue, GetAction}; use block::{ClosedBlock, SealedBlock}; -use client::{ - BlockProducer, SealedBlockImporter, ClientIoMessage, -}; -use client_traits::{ - BlockChain, ChainInfo, Nonce, TransactionInfo, -}; -use engine::{ - Engine, - signer::EngineSigner -}; +use client::{BlockProducer, SealedBlockImporter, Client}; +use client_traits::{BlockChain, ChainInfo, Nonce, TransactionInfo}; +use engine::{Engine, signer::EngineSigner}; use machine::executive::contract_address; use spec::Spec; use account_state::State; @@ -262,7 +256,7 @@ pub struct Miner { transaction_queue: Arc, engine: Arc, accounts: Arc, - io_channel: RwLock>>, + io_channel: RwLock>>>, service_transaction_checker: Option, } @@ -346,7 +340,7 @@ impl Miner { } /// Sets `IoChannel` - pub fn set_io_channel(&self, io_channel: IoChannel) { + pub fn set_io_channel(&self, io_channel: IoChannel>) { *self.io_channel.write() = Some(io_channel); } @@ -1426,7 +1420,7 @@ impl miner::MinerService for Miner { let accounts = self.accounts.clone(); let service_transaction_checker = self.service_transaction_checker.clone(); - let cull = move |chain: &::client::Client| { + let cull = move |chain: &Client| { let client = PoolClient::new( chain, &nonce_cache, @@ -1437,7 +1431,7 @@ impl miner::MinerService for Miner { queue.cull(client); }; - if let Err(e) = channel.send(ClientIoMessage::execute(cull)) { + if let Err(e) = channel.send(ClientIoMessage::::execute(cull)) { warn!(target: "miner", "Error queueing cull: {:?}", e); } } else { diff --git a/ethcore/src/miner/pool_client.rs b/ethcore/src/miner/pool_client.rs index 97a4d004d84..8b8740dff5d 100644 --- a/ethcore/src/miner/pool_client.rs +++ b/ethcore/src/miner/pool_client.rs @@ -221,30 +221,30 @@ impl<'a, C: 'a> CachedNonceClient<'a, C> { impl<'a, C: 'a> NonceClient for CachedNonceClient<'a, C> where C: Nonce + Sync, { - fn account_nonce(&self, address: &Address) -> U256 { - if let Some(nonce) = self.cache.nonces.read().get(address) { - return *nonce; - } - - // We don't check again if cache has been populated. - // It's not THAT expensive to fetch the nonce from state. - let mut cache = self.cache.nonces.write(); - let nonce = self.client.latest_nonce(address); - cache.insert(*address, nonce); - - if cache.len() < self.cache.limit { - return nonce - } - - debug!(target: "txpool", "NonceCache: reached limit."); - trace_time!("nonce_cache:clear"); - - // Remove excessive amount of entries from the cache - let to_remove: Vec<_> = cache.keys().take(self.cache.limit / 2).cloned().collect(); - for x in to_remove { - cache.remove(&x); - } - - nonce - } + fn account_nonce(&self, address: &Address) -> U256 { + if let Some(nonce) = self.cache.nonces.read().get(address) { + return *nonce; + } + + // We don't check again if cache has been populated. + // It's not THAT expensive to fetch the nonce from state. + let mut cache = self.cache.nonces.write(); + let nonce = self.client.latest_nonce(address); + cache.insert(*address, nonce); + + if cache.len() < self.cache.limit { + return nonce + } + + debug!(target: "txpool", "NonceCache: reached limit."); + trace_time!("nonce_cache:clear"); + + // Remove excessive amount of entries from the cache + let to_remove: Vec<_> = cache.keys().take(self.cache.limit / 2).cloned().collect(); + for x in to_remove { + cache.remove(&x); + } + + nonce + } } diff --git a/ethcore/src/snapshot/io.rs b/ethcore/src/snapshot/io.rs index 5a959932b36..b996651bcf5 100644 --- a/ethcore/src/snapshot/io.rs +++ b/ethcore/src/snapshot/io.rs @@ -26,6 +26,7 @@ use std::fs::{self, File}; use std::path::{Path, PathBuf}; use bytes::Bytes; +use client_traits::SnapshotWriter; use ethereum_types::H256; use rlp::{RlpStream, Rlp}; use types::{ @@ -35,20 +36,6 @@ use types::{ const SNAPSHOT_VERSION: u64 = 2; -/// Something which can write snapshots. -/// Writing the same chunk multiple times will lead to implementation-defined -/// behavior, and is not advised. -pub trait SnapshotWriter { - /// Write a compressed state chunk. - fn write_state_chunk(&mut self, hash: H256, chunk: &[u8]) -> io::Result<()>; - - /// Write a compressed block chunk. - fn write_block_chunk(&mut self, hash: H256, chunk: &[u8]) -> io::Result<()>; - - /// Complete writing. The manifest's chunk lists must be consistent - /// with the chunks written. - fn finish(self, manifest: ManifestData) -> io::Result<()> where Self: Sized; -} // (hash, len, offset) #[derive(RlpEncodable, RlpDecodable)] diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index ff3faa2c539..66a97719648 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -48,7 +48,8 @@ use bloom_journal::Bloom; use num_cpus; use types::snapshot::ManifestData; -use self::io::SnapshotWriter; +// todo[dvdplm] put back in snapshots once it's extracted +use client_traits::SnapshotWriter; use super::state_db::StateDB; use account_state::Account as StateAccount; @@ -58,7 +59,7 @@ use crossbeam_utils::thread; use rand::{Rng, rngs::OsRng}; pub use self::consensus::*; -pub use self::service::{SnapshotClient, Service, DatabaseRestore}; +pub use self::service::Service; pub use self::traits::{SnapshotService, SnapshotComponents, Rebuilder}; pub use self::watcher::Watcher; pub use types::basic_account::BasicAccount; diff --git a/ethcore/src/snapshot/service.rs b/ethcore/src/snapshot/service.rs index cdc0d6a1888..6ca50d70636 100644 --- a/ethcore/src/snapshot/service.rs +++ b/ethcore/src/snapshot/service.rs @@ -29,16 +29,21 @@ use super::{ SnapshotService, Rebuilder, MAX_CHUNK_SIZE, - io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter}, + io::{SnapshotReader, LooseReader, LooseWriter}, chunker, }; use blockchain::{BlockChain, BlockChainDB, BlockChainDBHandler}; -use client::{Client, ClientIoMessage}; -use client_traits::{BlockInfo, BlockChainClient, ChainInfo}; +use client::Client; +// todo[dvdplm] put SnapshotWriter back in snapshots once extracted +use client_traits::{ + BlockInfo, BlockChainClient, ChainInfo, + SnapshotClient, SnapshotWriter, DatabaseRestore, +}; use engine::Engine; use hash::keccak; use types::{ + io_message::ClientIoMessage, errors::{EthcoreError as Error, SnapshotError, SnapshotError::UnlinkedAncientBlockChain}, ids::BlockId, snapshot::{ManifestData, Progress, RestorationStatus}, @@ -73,12 +78,6 @@ impl Drop for Guard { } } -/// External database restoration handler -pub trait DatabaseRestore: Send + Sync { - /// Restart with a new backend. Takes ownership of passed database and moves it to a new location. - fn restore_db(&self, new_db: &str) -> Result<(), Error>; -} - /// State restoration manager. struct Restoration { manifest: ManifestData, @@ -213,10 +212,7 @@ impl Restoration { } /// Type alias for client io channel. -pub type Channel = IoChannel; - -/// Trait alias for the Client Service used -pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore {} +pub type Channel = IoChannel>; /// Snapshot service parameters. pub struct ServiceParams { @@ -234,7 +230,7 @@ pub struct ServiceParams { /// Usually "/snapshot" pub snapshot_root: PathBuf, /// A handle for database restoration. - pub client: Arc, + pub client: Arc, } /// `SnapshotService` implementation. @@ -251,7 +247,7 @@ pub struct Service { genesis_block: Bytes, state_chunks: AtomicUsize, block_chunks: AtomicUsize, - client: Arc, + client: Arc, progress: Progress, taking_snapshot: AtomicBool, restoring_snapshot: AtomicBool, @@ -483,7 +479,10 @@ impl Service { /// calling this while a restoration is in progress or vice versa /// will lead to a race condition where the first one to finish will /// have their produced snapshot overwritten. - pub fn take_snapshot(&self, client: &Client, num: u64) -> Result<(), Error> { + pub fn take_snapshot(&self, client: &C, num: u64) -> Result<(), Error> + where + C: ChainInfo + SnapshotClient + { if self.taking_snapshot.compare_and_swap(false, true, Ordering::SeqCst) { info!("Skipping snapshot at #{} as another one is currently in-progress.", num); return Ok(()); @@ -905,13 +904,16 @@ impl Drop for Service { #[cfg(test)] mod tests { - use client::ClientIoMessage; + use client::Client; use io::{IoService}; use spec; use journaldb::Algorithm; use snapshot::SnapshotService; use super::*; - use types::snapshot::{ManifestData, RestorationStatus}; + use types::{ + io_message::ClientIoMessage, + snapshot::{ManifestData, RestorationStatus} + }; use tempdir::TempDir; use test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler}; @@ -919,7 +921,7 @@ mod tests { fn sends_async_messages() { let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; let client = generate_dummy_client_with_spec_and_data(spec::new_null, 400, 5, &gas_prices); - let service = IoService::::start().unwrap(); + let service = IoService::>::start().unwrap(); let spec = spec::new_test(); let tempdir = TempDir::new("").unwrap(); @@ -932,7 +934,7 @@ mod tests { pruning: Algorithm::Archive, channel: service.channel(), snapshot_root: dir, - client: client, + client, }; let service = Service::new(snapshot_params).unwrap(); diff --git a/ethcore/src/snapshot/tests/helpers.rs b/ethcore/src/snapshot/tests/helpers.rs index c9420fcfcc1..3f23377a5f4 100644 --- a/ethcore/src/snapshot/tests/helpers.rs +++ b/ethcore/src/snapshot/tests/helpers.rs @@ -26,7 +26,7 @@ use account_db::AccountDBMut; use types::basic_account::BasicAccount; use blockchain::{BlockChain, BlockChainDB}; use client::Client; -use client_traits::ChainInfo; +use client_traits::{ChainInfo, SnapshotClient}; use engine::Engine; use snapshot::{StateRebuilder}; use snapshot::io::{SnapshotReader, PackedWriter, PackedReader}; diff --git a/ethcore/src/snapshot/tests/proof_of_work.rs b/ethcore/src/snapshot/tests/proof_of_work.rs index 85982a6df3c..3b6106313c7 100644 --- a/ethcore/src/snapshot/tests/proof_of_work.rs +++ b/ethcore/src/snapshot/tests/proof_of_work.rs @@ -26,8 +26,9 @@ use types::{ use blockchain::generator::{BlockGenerator, BlockBuilder}; use blockchain::{BlockChain, ExtrasInsert}; +use client_traits::SnapshotWriter; use snapshot::{chunk_secondary, Error as SnapshotError, SnapshotComponents}; -use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}; +use snapshot::io::{PackedReader, PackedWriter, SnapshotReader}; use parking_lot::Mutex; use snappy; diff --git a/ethcore/src/snapshot/tests/service.rs b/ethcore/src/snapshot/tests/service.rs index 5702ddf9703..aed14e9b456 100644 --- a/ethcore/src/snapshot/tests/service.rs +++ b/ethcore/src/snapshot/tests/service.rs @@ -22,14 +22,14 @@ use std::sync::Arc; use tempdir::TempDir; use blockchain::BlockProvider; use client::{Client, ClientConfig}; -use client_traits::{BlockInfo, ImportBlock}; +use client_traits::{BlockInfo, ImportBlock, SnapshotWriter}; use types::{ ids::BlockId, snapshot::Progress, verification::Unverified, snapshot::{ManifestData, RestorationStatus}, }; -use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}; +use snapshot::io::{PackedReader, PackedWriter, SnapshotReader}; use snapshot::service::{Service, ServiceParams}; use snapshot::{chunk_state, chunk_secondary, SnapshotService}; use spec; @@ -77,7 +77,7 @@ fn restored_is_equivalent() { }; let service = Service::new(service_params).unwrap(); - service.take_snapshot(&client, NUM_BLOCKS as u64).unwrap(); + service.take_snapshot(&*client, NUM_BLOCKS as u64).unwrap(); let manifest = service.manifest().unwrap(); @@ -226,7 +226,6 @@ fn keep_ancient_blocks() { client2.import_block(Unverified::from_rlp(block.into_inner()).unwrap()).unwrap(); } - client2.import_verified_blocks(); client2.flush_queue(); // Restore the Snapshot @@ -304,7 +303,7 @@ fn recover_aborted_recovery() { }; let service = Service::new(service_params).unwrap(); - service.take_snapshot(&client, NUM_BLOCKS as u64).unwrap(); + service.take_snapshot(&*client, NUM_BLOCKS as u64).unwrap(); let manifest = service.manifest().unwrap(); service.init_restore(manifest.clone(), true).unwrap(); diff --git a/ethcore/src/snapshot/tests/state.rs b/ethcore/src/snapshot/tests/state.rs index d78f33d8d52..50a505e3e71 100644 --- a/ethcore/src/snapshot/tests/state.rs +++ b/ethcore/src/snapshot/tests/state.rs @@ -24,9 +24,10 @@ use types::{ basic_account::BasicAccount, errors::EthcoreError as Error, }; +use client_traits::SnapshotWriter; use snapshot::account; use snapshot::{chunk_state, Error as SnapshotError, Progress, StateRebuilder, SNAPSHOT_SUBPARTS}; -use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}; +use snapshot::io::{PackedReader, PackedWriter, SnapshotReader}; use super::helpers::StateProducer; use rand::SeedableRng; use rand_xorshift::XorShiftRng; diff --git a/ethcore/src/snapshot/watcher.rs b/ethcore/src/snapshot/watcher.rs index 4fc9d881f63..828fb8d4d45 100644 --- a/ethcore/src/snapshot/watcher.rs +++ b/ethcore/src/snapshot/watcher.rs @@ -17,9 +17,12 @@ //! Watcher for snapshot-related chain events. use parking_lot::Mutex; -use client::{Client, ChainNotify, NewBlocks, ClientIoMessage}; +use client::{Client, ChainNotify, NewBlocks}; use client_traits::BlockInfo; -use types::ids::BlockId; +use types::{ + ids::BlockId, + io_message::ClientIoMessage, +}; use io::IoChannel; use ethereum_types::H256; @@ -55,7 +58,7 @@ trait Broadcast: Send + Sync { fn take_at(&self, num: Option); } -impl Broadcast for Mutex> { +impl Broadcast for Mutex>> { fn take_at(&self, num: Option) { let num = match num { Some(n) => n, @@ -83,7 +86,7 @@ impl Watcher { /// Create a new `Watcher` which will trigger a snapshot event /// once every `period` blocks, but only after that block is /// `history` blocks old. - pub fn new(client: Arc, sync_status: F, channel: IoChannel, period: u64, history: u64) -> Self + pub fn new(client: Arc, sync_status: F, channel: IoChannel>, period: u64, history: u64) -> Self where F: 'static + Send + Sync + Fn() -> bool { Watcher { diff --git a/ethcore/src/test_helpers.rs b/ethcore/src/test_helpers.rs index 4ab63846a0a..a89b342def5 100644 --- a/ethcore/src/test_helpers.rs +++ b/ethcore/src/test_helpers.rs @@ -186,7 +186,6 @@ pub fn generate_dummy_client_with_spec_and_data(test_spec: F, block_number: u db = b.drain().state.drop().1; } client.flush_queue(); - client.import_verified_blocks(); client } @@ -239,7 +238,6 @@ pub fn push_block_with_transactions(client: &Arc, transactions: &[Signed } client.flush_queue(); - client.import_verified_blocks(); } /// Creates dummy client (not test client) with corresponding blocks @@ -261,7 +259,6 @@ pub fn get_test_client_with_blocks(blocks: Vec) -> Arc { } } client.flush_queue(); - client.import_verified_blocks(); client } diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index b376b3c0f62..5ffa201b7a8 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -32,7 +32,10 @@ use types::{ }; use client::{Client, ClientConfig, PrepareOpenBlock, ImportSealedBlock}; -use client_traits::{BlockInfo, BlockChainClient, BlockChainReset, ChainInfo, ImportBlock}; +use client_traits::{ + BlockInfo, BlockChainClient, BlockChainReset, ChainInfo, + ImportBlock, Tick, +}; use spec; use machine::executive::{Executive, TransactOptions}; use miner::{Miner, PendingOrdering, MinerService}; @@ -55,7 +58,6 @@ fn imports_from_empty() { Arc::new(Miner::new_for_tests(&spec, None)), IoChannel::disconnected(), ).unwrap(); - client.import_verified_blocks(); client.flush_queue(); } @@ -102,7 +104,6 @@ fn imports_good_block() { panic!("error importing block being good by definition"); } client.flush_queue(); - client.import_verified_blocks(); let block = client.block_header(BlockId::Number(1)).unwrap(); assert!(!block.into_inner().is_empty()); diff --git a/ethcore/src/tests/trace.rs b/ethcore/src/tests/trace.rs index 26cb7e2bece..76d7234d176 100644 --- a/ethcore/src/tests/trace.rs +++ b/ethcore/src/tests/trace.rs @@ -181,7 +181,6 @@ fn can_trace_block_and_uncle_reward() { block.drain(); client.flush_queue(); - client.import_verified_blocks(); // Test0. Check overall filter let filter = TraceFilter { diff --git a/ethcore/sync/src/tests/consensus.rs b/ethcore/sync/src/tests/consensus.rs index 8879897f0ea..1223678046b 100644 --- a/ethcore/sync/src/tests/consensus.rs +++ b/ethcore/sync/src/tests/consensus.rs @@ -20,11 +20,14 @@ use ethereum_types::{U256, Address}; use io::{IoHandler, IoChannel}; use client_traits::ChainInfo; use engine::signer; -use ethcore::client::{ClientIoMessage}; use spec; +use ethcore::client::Client; use ethcore::miner::{self, MinerService}; use ethkey::{KeyPair, Secret}; -use types::transaction::{Action, PendingTransaction, Transaction}; +use types::{ + io_message::ClientIoMessage, + transaction::{Action, PendingTransaction, Transaction} +}; use super::helpers::*; use SyncConfig; @@ -47,8 +50,8 @@ fn authority_round() { let chain_id = spec::new_test_round().chain_id(); let mut net = TestNet::with_spec(2, SyncConfig::default(), spec::new_test_round); - let io_handler0: Arc> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone())); - let io_handler1: Arc> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone())); + let io_handler0: Arc>> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone())); + let io_handler1: Arc>> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone())); // Push transaction to both clients. Only one of them gets lucky to produce a block. net.peer(0).miner.set_author(miner::Author::Sealer(signer::from_keypair(s0.clone()))); net.peer(1).miner.set_author(miner::Author::Sealer(signer::from_keypair(s1.clone()))); @@ -115,7 +118,7 @@ fn authority_round() { net.peer(1).chain.engine().step(); net.peer(1).chain.engine().step(); assert_eq!(net.peer(1).chain.chain_info().best_block_number, 5); - // Reorg to the longest chain one not ealier view one. + // Reorg to the longest chain one not earlier view one. net.sync(); let ci0 = net.peer(0).chain.chain_info(); let ci1 = net.peer(1).chain.chain_info(); diff --git a/ethcore/sync/src/tests/helpers.rs b/ethcore/sync/src/tests/helpers.rs index b991480788f..666d362ebb5 100644 --- a/ethcore/sync/src/tests/helpers.rs +++ b/ethcore/sync/src/tests/helpers.rs @@ -22,9 +22,10 @@ use bytes::Bytes; use network::{self, PeerId, ProtocolId, PacketId, SessionInfo}; use network::client_version::ClientVersion; use tests::snapshot::*; +use types::io_message::ClientIoMessage; use client_traits::BlockChainClient; use ethcore::client::{TestBlockChainClient, Client as EthcoreClient, - ClientConfig, ChainNotify, NewBlocks, ChainMessageType, ClientIoMessage}; + ClientConfig, ChainNotify, NewBlocks, ChainMessageType}; use ethcore::snapshot::SnapshotService; use spec::{self, Spec}; use ethcore_private_tx::PrivateStateDB; @@ -554,8 +555,8 @@ impl TestIoHandler { } } -impl IoHandler for TestIoHandler { - fn message(&self, _io: &IoContext, net_message: &ClientIoMessage) { +impl IoHandler> for TestIoHandler { + fn message(&self, _io: &IoContext>, net_message: &ClientIoMessage) { match *net_message { ClientIoMessage::Execute(ref exec) => { *self.private_tx_queued.lock() += 1; diff --git a/ethcore/sync/src/tests/private.rs b/ethcore/sync/src/tests/private.rs index aabcaa82566..50c361c9f56 100644 --- a/ethcore/sync/src/tests/private.rs +++ b/ethcore/sync/src/tests/private.rs @@ -18,12 +18,15 @@ use std::sync::Arc; use hash::keccak; use io::{IoHandler, IoChannel}; use types::transaction::{Transaction, Action}; -use types::ids::BlockId; +use types::{ + ids::BlockId, + io_message::ClientIoMessage, +}; use client_traits::BlockChainClient; use engine::signer; use ethcore::{ + client::Client, CreateContractAddress, - client::ClientIoMessage, miner::{self, MinerService}, test_helpers::{push_block_with_transactions, new_db}, }; @@ -52,8 +55,8 @@ fn send_private_transaction() { let mut net = TestNet::with_spec(2, SyncConfig::default(), seal_spec); let client0 = net.peer(0).chain.clone(); let client1 = net.peer(1).chain.clone(); - let io_handler0: Arc> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone())); - let io_handler1: Arc> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone())); + let io_handler0: Arc>> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone())); + let io_handler1: Arc>> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone())); net.peer(0).miner.set_author(miner::Author::Sealer(signer::from_keypair(s0.clone()))); net.peer(1).miner.set_author(miner::Author::Sealer(signer::from_keypair(s1.clone()))); @@ -65,7 +68,7 @@ fn send_private_transaction() { let (address, _) = contract_address(CreateContractAddress::FromSenderAndNonce, &s0.address(), &0.into(), &[]); let chain_id = client0.signing_chain_id(); - // Exhange statuses + // Exchange statuses net.sync(); // Setup private providers @@ -173,8 +176,8 @@ fn sync_private_state() { let mut net = TestNet::with_spec(2, SyncConfig::default(), seal_spec); let client0 = net.peer(0).chain.clone(); let client1 = net.peer(1).chain.clone(); - let io_handler0: Arc> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone())); - let io_handler1: Arc> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone())); + let io_handler0: Arc>> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone())); + let io_handler1: Arc>> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone())); net.peer(0).miner.set_author(miner::Author::Sealer(signer::from_keypair(s0.clone()))); net.peer(1).miner.set_author(miner::Author::Sealer(signer::from_keypair(s1.clone()))); diff --git a/ethcore/types/src/client_types.rs b/ethcore/types/src/client_types.rs index ad90e0db0d6..a67eebcab1f 100644 --- a/ethcore/types/src/client_types.rs +++ b/ethcore/types/src/client_types.rs @@ -18,8 +18,12 @@ use std::{ fmt::{Display, Formatter, Error as FmtError}, + ops, + cmp, time::Duration, }; +use ethereum_types::U256; +use crate::header::Header; /// Operating mode for the client. #[derive(Debug, Eq, PartialEq, Clone)] @@ -47,3 +51,41 @@ impl Display for Mode { } } +/// Report on the status of a client. +#[derive(Default, Clone, Debug, Eq, PartialEq)] +pub struct ClientReport { + /// How many blocks have been imported so far. + pub blocks_imported: usize, + /// How many transactions have been applied so far. + pub transactions_applied: usize, + /// How much gas has been processed so far. + pub gas_processed: U256, + /// Memory used by state DB + pub state_db_mem: usize, +} + +impl ClientReport { + /// Alter internal reporting to reflect the additional `block` has been processed. + pub fn accrue_block(&mut self, header: &Header, transactions: usize) { + self.blocks_imported += 1; + self.transactions_applied += transactions; + self.gas_processed = self.gas_processed + *header.gas_used(); + } +} + +impl<'a> ops::Sub<&'a ClientReport> for ClientReport { + type Output = Self; + + fn sub(mut self, other: &'a ClientReport) -> Self { + let higher_mem = cmp::max(self.state_db_mem, other.state_db_mem); + let lower_mem = cmp::min(self.state_db_mem, other.state_db_mem); + + self.blocks_imported -= other.blocks_imported; + self.transactions_applied -= other.transactions_applied; + self.gas_processed = self.gas_processed - other.gas_processed; + self.state_db_mem = higher_mem - lower_mem; + + self + } +} + diff --git a/ethcore/src/client/io_message.rs b/ethcore/types/src/io_message.rs similarity index 78% rename from ethcore/src/client/io_message.rs rename to ethcore/types/src/io_message.rs index 1b4725da51c..d475cb1b377 100644 --- a/ethcore/src/client/io_message.rs +++ b/ethcore/types/src/io_message.rs @@ -14,15 +14,17 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . +//! Defines the `ClientIoMessage` type, used pervasively throughout various parts of the project to +//! communicate between each other. + use std::fmt; use bytes::Bytes; -use client::Client; use ethereum_types::H256; -use types::snapshot::ManifestData; +use crate::snapshot::ManifestData; /// Message type for external and internal events #[derive(Debug)] -pub enum ClientIoMessage { +pub enum ClientIoMessage { /// Best Block Hash in chain has been changed NewChainHead, /// A block is ready @@ -36,20 +38,20 @@ pub enum ClientIoMessage { /// Take a snapshot for the block with given number. TakeSnapshot(u64), /// Execute wrapped closure - Execute(Callback), + Execute(Callback), } -impl ClientIoMessage { +impl ClientIoMessage { /// Create new `ClientIoMessage` that executes given procedure. - pub fn execute(fun: F) -> Self { + pub fn execute(fun: F) -> Self { ClientIoMessage::Execute(Callback(Box::new(fun))) } } /// A function to invoke in the client thread. -pub struct Callback(pub Box); +pub struct Callback(pub Box); -impl fmt::Debug for Callback { +impl fmt::Debug for Callback { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { write!(fmt, "") } diff --git a/ethcore/types/src/lib.rs b/ethcore/types/src/lib.rs index 42a3c6f8166..fb39ad6eb7c 100644 --- a/ethcore/types/src/lib.rs +++ b/ethcore/types/src/lib.rs @@ -71,6 +71,7 @@ pub mod errors; pub mod filter; pub mod header; pub mod ids; +pub mod io_message; pub mod log_entry; pub mod pruning_info; pub mod receipt; diff --git a/ethcore/verification/Cargo.toml b/ethcore/verification/Cargo.toml new file mode 100644 index 00000000000..0a19861a457 --- /dev/null +++ b/ethcore/verification/Cargo.toml @@ -0,0 +1,34 @@ +[package] +description = "Block verification utilities." +name = "verification" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" + +[dependencies] +blockchain = { package = "ethcore-blockchain", path = "../blockchain" } +call-contract = { package = "ethcore-call-contract", path = "../call-contract" } +client-traits = { path = "../client-traits" } +common-types = { path = "../types" } +engine = { path = "../engine" } +ethcore-io = { path = "../../util/io" } +ethereum-types = "0.6.0" +keccak-hash = "0.2.0" +len-caching-lock = { path = "../../util/len-caching-lock" } +log = "0.4" +num_cpus = "1.2" +parity-bytes = "0.1.0" +parity-util-mem = "0.2.0" +parking_lot = "0.8.0" +rlp = "0.4.2" +time-utils = { path = "../../util/time-utils" } +triehash = { package = "triehash-ethereum", version = "0.2", path = "../../util/triehash-ethereum" } +unexpected = { path = "../../util/unexpected" } + +[dev-dependencies] +ethcore = { path = "../", features = ["test-helpers"] } +ethkey = { path = "../../accounts/ethkey" } +machine = { path = "../machine" } +null-engine = { path = "../engines/null-engine" } +spec = { path = "../spec" } diff --git a/ethcore/src/verification/canon_verifier.rs b/ethcore/verification/src/canon_verifier.rs similarity index 98% rename from ethcore/src/verification/canon_verifier.rs rename to ethcore/verification/src/canon_verifier.rs index c0c60709700..336428caf97 100644 --- a/ethcore/src/verification/canon_verifier.rs +++ b/ethcore/verification/src/canon_verifier.rs @@ -19,7 +19,7 @@ use call_contract::CallContract; use client_traits::BlockInfo; use engine::Engine; -use types::{ +use common_types::{ header::Header, errors::EthcoreError as Error, }; diff --git a/ethcore/src/verification/mod.rs b/ethcore/verification/src/lib.rs similarity index 95% rename from ethcore/src/verification/mod.rs rename to ethcore/verification/src/lib.rs index 16e0e52dd1d..2f4be58dc5f 100644 --- a/ethcore/src/verification/mod.rs +++ b/ethcore/verification/src/lib.rs @@ -18,6 +18,8 @@ use call_contract::CallContract; use client_traits::BlockInfo; +// The MallocSizeOf derive looks for this in the root +use parity_util_mem as malloc_size_of; mod verification; mod verifier; @@ -29,11 +31,6 @@ pub use self::verification::FullFamilyParams; pub use self::verifier::Verifier; pub use self::queue::{BlockQueue, Config as QueueConfig}; -use self::verification::{ - verify_block_basic, - verify_block_unordered, - verify_header_params, -}; use self::canon_verifier::CanonVerifier; use self::noop_verifier::NoopVerifier; diff --git a/ethcore/src/verification/noop_verifier.rs b/ethcore/verification/src/noop_verifier.rs similarity index 98% rename from ethcore/src/verification/noop_verifier.rs rename to ethcore/verification/src/noop_verifier.rs index 3f9a6210f40..149f2798830 100644 --- a/ethcore/src/verification/noop_verifier.rs +++ b/ethcore/verification/src/noop_verifier.rs @@ -18,11 +18,11 @@ use call_contract::CallContract; use client_traits::BlockInfo; -use engine::Engine; -use types::{ +use common_types::{ header::Header, errors::EthcoreError as Error }; +use engine::Engine; use super::{verification, Verifier}; /// A no-op verifier -- this will verify everything it's given immediately. diff --git a/ethcore/src/verification/queue/kind.rs b/ethcore/verification/src/queue/kind.rs similarity index 95% rename from ethcore/src/verification/queue/kind.rs rename to ethcore/verification/src/queue/kind.rs index 937d05abd18..cc4d1167a4e 100644 --- a/ethcore/src/verification/queue/kind.rs +++ b/ethcore/verification/src/queue/kind.rs @@ -21,7 +21,7 @@ use engine::Engine; use parity_util_mem::MallocSizeOf; use ethereum_types::{H256, U256}; -use types::errors::EthcoreError as Error; +use common_types::errors::EthcoreError as Error; pub use self::blocks::Blocks; pub use self::headers::Headers; @@ -70,12 +70,13 @@ pub mod blocks { use super::{Kind, BlockLike}; use engine::Engine; - use types::{ + use common_types::{ block::PreverifiedBlock, errors::{EthcoreError as Error, BlockError}, verification::Unverified, }; - use verification::{verify_block_basic, verify_block_unordered}; + use log::{debug, warn}; + use crate::verification::{verify_block_basic, verify_block_unordered}; use ethereum_types::{H256, U256}; @@ -147,11 +148,11 @@ pub mod headers { use super::{Kind, BlockLike}; use engine::Engine; - use types::{ + use common_types::{ header::Header, errors::EthcoreError as Error, }; - use verification::verify_header_params; + use crate::verification::verify_header_params; use ethereum_types::{H256, U256}; diff --git a/ethcore/src/verification/queue/mod.rs b/ethcore/verification/src/queue/mod.rs similarity index 93% rename from ethcore/src/verification/queue/mod.rs rename to ethcore/verification/src/queue/mod.rs index 58754251dd2..70795ca7991 100644 --- a/ethcore/src/verification/queue/mod.rs +++ b/ethcore/verification/src/queue/mod.rs @@ -22,17 +22,19 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering}; use std::sync::Arc; use std::cmp; use std::collections::{VecDeque, HashSet, HashMap}; -use parity_util_mem::{MallocSizeOf, MallocSizeOfExt}; -use ethereum_types::{H256, U256}; -use parking_lot::{Condvar, Mutex, RwLock}; -use io::*; -use engine::Engine; -use client::ClientIoMessage; -use len_caching_lock::LenCachingMutex; -use types::{ +use common_types::{ + block_status::BlockStatus, + io_message::ClientIoMessage, errors::{BlockError, EthcoreError as Error, ImportError}, verification::VerificationQueueInfo as QueueInfo, }; +use ethcore_io::*; +use ethereum_types::{H256, U256}; +use engine::Engine; +use len_caching_lock::LenCachingMutex; +use log::{debug, trace}; +use parity_util_mem::{MallocSizeOf, MallocSizeOfExt}; +use parking_lot::{Condvar, Mutex, RwLock}; use self::kind::{BlockLike, Kind}; @@ -42,10 +44,10 @@ const MIN_MEM_LIMIT: usize = 16384; const MIN_QUEUE_LIMIT: usize = 512; /// Type alias for block queue convenience. -pub type BlockQueue = VerificationQueue; +pub type BlockQueue = VerificationQueue; /// Type alias for header queue convenience. -pub type HeaderQueue = VerificationQueue; +pub type HeaderQueue = VerificationQueue; /// Verification queue configuration #[derive(Debug, PartialEq, Clone)] @@ -84,7 +86,7 @@ impl Default for VerifierSettings { fn default() -> Self { VerifierSettings { scale_verifiers: false, - num_verifiers: ::num_cpus::get(), + num_verifiers: num_cpus::get(), } } } @@ -113,9 +115,8 @@ pub enum Status { Unknown, } -impl Into<::types::block_status::BlockStatus> for Status { - fn into(self) -> ::types::block_status::BlockStatus { - use ::types::block_status::BlockStatus; +impl Into for Status { + fn into(self) -> BlockStatus { match self { Status::Queued => BlockStatus::Queued, Status::Bad => BlockStatus::Bad, @@ -133,12 +134,12 @@ struct Sizes { /// A queue of items to be verified. Sits between network or other I/O and the `BlockChain`. /// Keeps them in the same order as inserted, minus invalid items. -pub struct VerificationQueue { +pub struct VerificationQueue { engine: Arc, more_to_verify: Arc, verification: Arc>, deleting: Arc, - ready_signal: Arc, + ready_signal: Arc>, empty: Arc, processing: RwLock>, // hash to difficulty ticks_since_adjustment: AtomicUsize, @@ -150,13 +151,13 @@ pub struct VerificationQueue { total_difficulty: RwLock, } -struct QueueSignal { +struct QueueSignal { deleting: Arc, signalled: AtomicBool, - message_channel: Mutex>, + message_channel: Mutex>>, } -impl QueueSignal { +impl QueueSignal { fn set_sync(&self) { // Do not signal when we are about to close if self.deleting.load(AtomicOrdering::Relaxed) { @@ -200,9 +201,9 @@ struct Verification { check_seal: bool, } -impl VerificationQueue { +impl VerificationQueue { /// Creates a new queue instance. - pub fn new(config: Config, engine: Arc, message_channel: IoChannel, check_seal: bool) -> Self { + pub fn new(config: Config, engine: Arc, message_channel: IoChannel>, check_seal: bool) -> Self { let verification = Arc::new(Verification { unverified: LenCachingMutex::new(VecDeque::new()), verifying: LenCachingMutex::new(VecDeque::new()), @@ -213,7 +214,7 @@ impl VerificationQueue { verifying: AtomicUsize::new(0), verified: AtomicUsize::new(0), }, - check_seal: check_seal, + check_seal, }); let more_to_verify = Arc::new(Condvar::new()); let deleting = Arc::new(AtomicBool::new(false)); @@ -270,19 +271,19 @@ impl VerificationQueue { } VerificationQueue { - engine: engine, - ready_signal: ready_signal, - more_to_verify: more_to_verify, - verification: verification, - deleting: deleting, + engine, + ready_signal, + more_to_verify, + verification, + deleting, processing: RwLock::new(HashMap::new()), - empty: empty, + empty, ticks_since_adjustment: AtomicUsize::new(0), max_queue_size: cmp::max(config.max_queue_size, MIN_QUEUE_LIMIT), max_mem_use: cmp::max(config.max_mem_use, MIN_MEM_LIMIT), - scale_verifiers: scale_verifiers, - verifier_handles: verifier_handles, - state: state, + scale_verifiers, + verifier_handles, + state, total_difficulty: RwLock::new(0.into()), } } @@ -291,7 +292,7 @@ impl VerificationQueue { verification: Arc>, engine: Arc, wait: Arc, - ready: Arc, + ready: Arc>, empty: Arc, state: Arc<(Mutex, Condvar)>, id: usize, @@ -373,7 +374,7 @@ impl VerificationQueue { // we're next! let mut verified = verification.verified.lock(); let mut bad = verification.bad.lock(); - VerificationQueue::drain_verifying(&mut verifying, &mut verified, &mut bad, &verification.sizes); + VerificationQueue::<_, C>::drain_verifying(&mut verifying, &mut verified, &mut bad, &verification.sizes); true } else { false @@ -388,7 +389,7 @@ impl VerificationQueue { verifying.retain(|e| e.hash != hash); if verifying.front().map_or(false, |x| x.output.is_some()) { - VerificationQueue::drain_verifying(&mut verifying, &mut verified, &mut bad, &verification.sizes); + VerificationQueue::<_, C>::drain_verifying(&mut verifying, &mut verified, &mut bad, &verification.sizes); true } else { false @@ -706,7 +707,7 @@ impl VerificationQueue { } } -impl Drop for VerificationQueue { +impl Drop for VerificationQueue { fn drop(&mut self) { trace!(target: "shutdown", "[VerificationQueue] Closing..."); self.clear(); @@ -734,11 +735,12 @@ impl Drop for VerificationQueue { #[cfg(test)] mod tests { - use io::*; + use ethcore_io::*; use super::{BlockQueue, Config, State}; - use test_helpers::{get_good_dummy_block_seq, get_good_dummy_block}; - use bytes::Bytes; - use types::{ + use ethcore::test_helpers::{get_good_dummy_block_seq, get_good_dummy_block}; + use ethcore::client::Client; + use parity_bytes::Bytes; + use common_types::{ errors::{EthcoreError, ImportError}, verification::Unverified, view, @@ -748,7 +750,7 @@ mod tests { // create a test block queue. // auto_scaling enables verifier adjustment. - fn get_test_queue(auto_scale: bool) -> BlockQueue { + fn get_test_queue(auto_scale: bool) -> BlockQueue { let spec = spec::new_test(); let engine = spec.engine; @@ -773,7 +775,7 @@ mod tests { // TODO better test let spec = spec::new_test(); let engine = spec.engine; - let _ = BlockQueue::new(Config::default(), engine, IoChannel::disconnected(), true); + let _ = BlockQueue::::new(Config::default(), engine, IoChannel::disconnected(), true); } #[test] @@ -853,7 +855,7 @@ mod tests { let engine = spec.engine; let mut config = Config::default(); config.max_mem_use = super::MIN_MEM_LIMIT; // empty queue uses about 15000 - let queue = BlockQueue::new(config, engine, IoChannel::disconnected(), true); + let queue = BlockQueue::::new(config, engine, IoChannel::disconnected(), true); assert!(!queue.queue_info().is_full()); let mut blocks = get_good_dummy_block_seq(50); for b in blocks.drain(..) { @@ -903,7 +905,7 @@ mod tests { let spec = spec::new_test(); let engine = spec.engine; let config = get_test_config(1, false); - let queue = BlockQueue::new(config, engine, IoChannel::disconnected(), true); + let queue = BlockQueue::::new(config, engine, IoChannel::disconnected(), true); assert_eq!(queue.num_verifiers(), 1); } @@ -913,7 +915,7 @@ mod tests { let spec = spec::new_test(); let engine = spec.engine; let config = get_test_config(0, false); - let queue = BlockQueue::new(config, engine, IoChannel::disconnected(), true); + let queue = BlockQueue::::new(config, engine, IoChannel::disconnected(), true); assert_eq!(queue.num_verifiers(), 1); } @@ -923,7 +925,7 @@ mod tests { let spec = spec::new_test(); let engine = spec.engine; let config = get_test_config(10_000, false); - let queue = BlockQueue::new(config, engine, IoChannel::disconnected(), true); + let queue = BlockQueue::::new(config, engine, IoChannel::disconnected(), true); let num_cpus = ::num_cpus::get(); assert_eq!(queue.num_verifiers(), num_cpus); @@ -937,7 +939,7 @@ mod tests { let spec = spec::new_test(); let engine = spec.engine; let config = get_test_config(num_cpus - 1, true); - let queue = BlockQueue::new(config, engine, IoChannel::disconnected(), true); + let queue = BlockQueue::::new(config, engine, IoChannel::disconnected(), true); queue.scale_verifiers(num_cpus); assert_eq!(queue.num_verifiers(), num_cpus); diff --git a/ethcore/src/verification/verification.rs b/ethcore/verification/src/verification.rs similarity index 98% rename from ethcore/src/verification/verification.rs rename to ethcore/verification/src/verification.rs index 4e73a37d337..f4bf48a0668 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/verification/src/verification.rs @@ -24,7 +24,7 @@ use std::collections::HashSet; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use hash::keccak; +use keccak_hash::keccak; use rlp::Rlp; use triehash::ordered_trie_root; use unexpected::{Mismatch, OutOfBounds}; @@ -33,7 +33,7 @@ use blockchain::*; use call_contract::CallContract; use client_traits::BlockInfo; use engine::Engine; -use types::{ +use common_types::{ BlockNumber, header::Header, errors::{EthcoreError as Error, BlockError}, @@ -366,15 +366,19 @@ mod tests { use std::collections::{BTreeMap, HashMap}; use std::time::{SystemTime, UNIX_EPOCH}; + use ethereum_types::{H256, BloomRef, U256, Address}; use blockchain::{BlockDetails, TransactionAddress, BlockReceipts}; - use bytes::Bytes; - use hash::keccak; + use parity_bytes::Bytes; + use keccak_hash::keccak; use engine::Engine; use ethkey::{Random, Generator}; use spec; - use test_helpers::{create_test_block_with_data, create_test_block}; - use types::{ + use ethcore::{ + client::TestBlockChainClient, + test_helpers::{create_test_block_with_data, create_test_block} + }; + use common_types::{ encoded, engines::params::CommonParams, errors::BlockError::*, @@ -383,6 +387,8 @@ mod tests { }; use rlp; use triehash::ordered_trie_root; + use machine::Machine; + use null_engine::NullEngine; fn check_ok(result: Result<(), Error>) { result.unwrap_or_else(|e| panic!("Block verification failed: {:?}", e)); @@ -518,7 +524,7 @@ mod tests { // additions that need access to state (tx filter in specific) // no existing tests need access to test, so having this not function // is fine. - let client = ::client::TestBlockChainClient::default(); + let client = TestBlockChainClient::default(); let parent = bc.block_header_data(header.parent_hash()) .ok_or(BlockError::UnknownParent(*header.parent_hash()))? .decode()?; @@ -779,11 +785,6 @@ mod tests { #[test] fn dust_protection() { - use ethkey::{Generator, Random}; - use types::transaction::{Transaction, Action}; - use machine::Machine; - use null_engine::NullEngine; - let mut params = CommonParams::default(); params.dust_protection_transition = 0; params.nonce_cap_increment = 2; diff --git a/ethcore/src/verification/verifier.rs b/ethcore/verification/src/verifier.rs similarity index 98% rename from ethcore/src/verification/verifier.rs rename to ethcore/verification/src/verifier.rs index ced7f564967..982eab68feb 100644 --- a/ethcore/src/verification/verifier.rs +++ b/ethcore/verification/src/verifier.rs @@ -18,11 +18,12 @@ use call_contract::CallContract; use client_traits::BlockInfo; -use engine::Engine; -use types::{ +use common_types::{ header::Header, errors::EthcoreError as Error, }; +use engine::Engine; + use super::verification; /// Should be used to verify blocks. diff --git a/parity/blockchain.rs b/parity/blockchain.rs index 85000e385c1..1879752589a 100644 --- a/parity/blockchain.rs +++ b/parity/blockchain.rs @@ -30,7 +30,6 @@ use client_traits::{BlockInfo, BlockChainReset, Nonce, Balance, BlockChainClient use ethcore::{ client::{DatabaseCompactionProfile, VMType}, miner::Miner, - verification::queue::VerifierSettings, }; use ethcore_service::ClientService; use cache::CacheConfig; @@ -48,6 +47,7 @@ use types::{ client_types::Mode, verification::Unverified, }; +use verification::queue::VerifierSettings; #[derive(Debug, PartialEq)] pub enum DataFormat { @@ -234,7 +234,7 @@ fn execute_import_light(cmd: ImportBlockchain) -> Result<(), String> { &cmd.cache_config, &cmd.compaction).map_err(|e| format!("Failed to open database: {:?}", e))?; - // TODO: could epoch signals be avilable at the end of the file? + // TODO: could epoch signals be available at the end of the file? let fetch = ::light::client::fetch::unavailable(); let service = LightClientService::start(config, &spec, fetch, db, cache) .map_err(|e| format!("Failed to start client: {}", e))?; diff --git a/parity/configuration.rs b/parity/configuration.rs index 007436b1e23..2066d590854 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -32,8 +32,8 @@ use ethkey::{Secret, Public}; use ethcore::client::{VMType}; use ethcore::miner::{stratum, MinerOptions}; use ethcore::snapshot::SnapshotConfiguration; -use ethcore::verification::queue::VerifierSettings; use miner::pool; +use verification::queue::VerifierSettings; use rpc::{IpcConfiguration, HttpConfiguration, WsConfiguration}; use parity_rpc::NetworkSettings; diff --git a/parity/informant.rs b/parity/informant.rs index 29a278a2a38..60ba59f7beb 100644 --- a/parity/informant.rs +++ b/parity/informant.rs @@ -23,13 +23,13 @@ use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering}; use std::time::{Instant, Duration}; use atty; -use ethcore::client::{ - ChainNotify, NewBlocks, ClientReport, Client, ClientIoMessage -}; -use client_traits::{BlockInfo, ChainInfo, BlockChainClient}; +use ethcore::client::{ChainNotify, NewBlocks, Client}; +use client_traits::{BlockInfo, ChainInfo, BlockChainClient, IoClient}; use types::{ BlockNumber, + client_types::ClientReport, ids::BlockId, + io_message::ClientIoMessage, blockchain_info::BlockChainInfo, verification::VerificationQueueInfo as BlockQueueInfo, snapshot::RestorationStatus, @@ -451,12 +451,16 @@ impl LightChainNotify for Informant { const INFO_TIMER: TimerToken = 0; -impl IoHandler for Informant { - fn initialize(&self, io: &IoContext) { +impl IoHandler> for Informant +where + T: InformantData, + C: client_traits::Tick + 'static, +{ + fn initialize(&self, io: &IoContext>) { io.register_timer(INFO_TIMER, Duration::from_secs(5)).expect("Error registering timer"); } - fn timeout(&self, _io: &IoContext, timer: TimerToken) { + fn timeout(&self, _io: &IoContext>, timer: TimerToken) { if timer == INFO_TIMER && !self.in_shutdown.load(AtomicOrdering::SeqCst) { self.tick(); } diff --git a/parity/lib.rs b/parity/lib.rs index 3d96c070048..2ea24c0bfc8 100644 --- a/parity/lib.rs +++ b/parity/lib.rs @@ -74,6 +74,7 @@ extern crate parity_updater as updater; extern crate parity_version; extern crate registrar; extern crate spec; +extern crate verification; #[macro_use] extern crate log as rlog; diff --git a/parity/run.rs b/parity/run.rs index e53e5b29394..7b88747374c 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -27,7 +27,7 @@ use ethcore::client::{Client, DatabaseCompactionProfile, VMType}; use ethcore::miner::{self, stratum, Miner, MinerService, MinerOptions}; use ethcore::snapshot::{self, SnapshotConfiguration}; use spec::SpecParams; -use ethcore::verification::queue::VerifierSettings; +use verification::queue::VerifierSettings; use ethcore_logger::{Config as LogConfig, RotatingLogger}; use ethcore_service::ClientService; use ethereum_types::Address; @@ -307,18 +307,18 @@ fn execute_light_impl(cmd: RunCmd, logger: Arc, on_client_rq // start RPCs let deps_for_rpc_apis = Arc::new(rpc_apis::LightDependencies { - signer_service: signer_service, + signer_service, client: client.clone(), sync: light_sync.clone(), net: light_sync.clone(), accounts: account_provider, - logger: logger, + logger, settings: Arc::new(cmd.net_settings), - on_demand: on_demand, + on_demand, cache: cache.clone(), transaction_queue: txq, ws_address: cmd.ws_conf.address(), - fetch: fetch, + fetch, geth_compatibility: cmd.geth_compatibility, experimental_rpcs: cmd.experimental_rpcs, executor: runtime.executor(), @@ -344,7 +344,7 @@ fn execute_light_impl(cmd: RunCmd, logger: Arc, on_client_rq LightNodeInformantData { client: client.clone(), sync: light_sync.clone(), - cache: cache, + cache, }, None, Some(rpc_stats), diff --git a/parity/snapshot.rs b/parity/snapshot.rs index 4a4c9d718b0..c9657b2d6ac 100644 --- a/parity/snapshot.rs +++ b/parity/snapshot.rs @@ -20,6 +20,7 @@ use std::time::Duration; use std::path::{Path, PathBuf}; use std::sync::Arc; +use client_traits::SnapshotClient; use hash::keccak; use ethcore::snapshot::{SnapshotConfiguration, SnapshotService as SS}; use ethcore::snapshot::io::{SnapshotReader, PackedReader, PackedWriter}; diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index fead96d71c5..ff8fa932515 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -67,6 +67,7 @@ account-state = { path = "../ethcore/account-state" } stats = { path = "../util/stats" } trace = { path = "../ethcore/trace" } vm = { path = "../ethcore/vm" } +verification = { path = "../ethcore/verification" } [dev-dependencies] client-traits = { path = "../ethcore/client-traits" } diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 648ec43bf82..5bff9358294 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -89,6 +89,7 @@ extern crate account_state; extern crate stats; extern crate tempdir; extern crate trace; +extern crate verification; extern crate vm; #[cfg(any(test, feature = "ethcore-accounts"))] diff --git a/rpc/src/v1/tests/eth.rs b/rpc/src/v1/tests/eth.rs index 45887ae9a73..062acb10d29 100644 --- a/rpc/src/v1/tests/eth.rs +++ b/rpc/src/v1/tests/eth.rs @@ -24,7 +24,7 @@ use ethcore::client::{Client, ClientConfig}; use ethcore::miner::Miner; use spec::{Genesis, Spec, self}; use ethcore::test_helpers; -use ethcore::verification::VerifierType; +use verification::VerifierType; use ethereum_types::{Address, H256, U256}; use ethjson::blockchain::BlockChain; use ethjson::spec::ForkSpec; @@ -98,7 +98,6 @@ impl EthTester { if let Ok(block) = Unverified::from_rlp(b) { let _ = tester.client.import_block(block); tester.client.flush_queue(); - tester.client.import_verified_blocks(); } }