Make ClientIoMessage generic over the Client

Cargo.toml

@@ -73,6 +73,7 @@ spec = { path = "ethcore/spec" }
 term_size = "0.3"
 textwrap = "0.9"
 toml = "0.4"
+verification = { path = "ethcore/verification" }
 
 [build-dependencies]
 rustc_version = "0.2"

ethcore/Cargo.toml

@@ -35,16 +35,14 @@ evm = { path = "evm" }
 executive-state = { path = "executive-state" }
 futures = "0.1"
 hash-db = "0.15.0"
-parity-util-mem = "0.2.0"
 itertools = "0.5"
 journaldb = { path = "../util/journaldb" }
 keccak-hash = "0.2.0"
 keccak-hasher = { path = "../util/keccak-hasher" }
 kvdb = "0.1"
-kvdb-memorydb = "0.1"
+kvdb-memorydb = { version = "0.1", optional = true }
 kvdb-rocksdb = { version = "0.1.3", optional = true }
-lazy_static = "1.3.0"
-len-caching-lock = { path = "../util/len-caching-lock" }
+lazy_static = { version = "1.3", optional = true }
 log = "0.4"
 macros = { path = "../util/macros", optional = true }
 machine = { path = "./machine" }
@@ -61,20 +59,19 @@ rand_xorshift = "0.1.1"
 rayon = "1.1"
 rlp = "0.4.0"
 rlp_derive = { path = "../util/rlp-derive" }
-rustc-hex = "1.0"
+rustc-hex = { version = "1", optional = true }
 serde = "1.0"
 serde_derive = "1.0"
 spec = { path = "spec" }
 state-db = { path = "state-db" }
-stats = { path = "../util/stats" }
 tempdir = { version = "0.3", optional = true }
-time-utils = { path = "../util/time-utils" }
 trace = { path = "trace" }
 trace-time = "0.1"
 trie-vm-factories = { path = "trie-vm-factories" }
 triehash-ethereum = { version = "0.2",  path = "../util/triehash-ethereum" }
 unexpected = { path = "../util/unexpected" }
 using_queue = { path = "../miner/using-queue" }
+verification = { path = "./verification" }
 vm = { path = "vm" }
 
 [dev-dependencies]
@@ -88,14 +85,18 @@ ethcore-accounts = { path = "../accounts" }
 ethjson = { path = "../json" }
 ethkey = { path = "../accounts/ethkey" }
 fetch = { path = "../util/fetch" }
+kvdb-memorydb = "0.1"
 kvdb-rocksdb = "0.1.3"
+lazy_static = { version = "1.3" }
 machine = { path = "./machine", features = ["test-helpers"] }
 macros = { path = "../util/macros" }
 null-engine = { path = "./engines/null-engine" }
 parity-runtime = { path = "../util/runtime" }
 pod = { path = "pod" }
 rlp_compress = { path = "../util/rlp-compress" }
+rustc-hex = "1"
 serde_json = "1.0"
+stats = { path = "../util/stats" }
 tempdir = "0.3"
 trie-standardmap = "0.15.0"
 
@@ -121,14 +122,14 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
 # EVM debug traces are printed.
 slow-blocks = []
 # Run JSON consensus tests.
-json-tests = ["env_logger", "test-helpers", "machine/test-helpers"]
+json-tests = ["env_logger", "test-helpers", "lazy_static", "machine/test-helpers"]
 # Skip JSON consensus tests with pending issues.
 ci-skip-tests = []
 # Run memory/cpu heavy tests.
 test-heavy = []
 # Compile test helpers
 # note[dvdplm]: "basic-authority/test-helpers" is needed so that `generate_dummy_client_with_spec` works
-test-helpers = ["tempdir", "kvdb-rocksdb", "blooms-db", "ethash", "ethjson", "ethkey", "macros", "pod", "basic-authority/test-helpers"]
+test-helpers = ["tempdir", "kvdb-memorydb", "kvdb-rocksdb", "blooms-db", "ethash", "ethjson", "ethkey", "macros", "pod", "rustc-hex", "basic-authority/test-helpers"]
 
 [[bench]]
 name = "builtin"

ethcore/client-traits/src/lib.rs

@@ -32,7 +32,7 @@ use common_types::{
 	client_types::Mode,
 	encoded,
 	engines::{epoch::Transition as EpochTransition, machine::Executed},
-	errors::EthcoreResult,
+	errors::{EthcoreError, EthcoreResult},
 	filter::Filter,
 	header::Header,
 	ids::{BlockId, TransactionId, TraceId, UncleId},
@@ -56,6 +56,8 @@ use trace::{
 };
 use vm::{LastHashes, Schedule};
 
+use common_types::snapshot::Progress;
+
 /// State information to be used during client query
 pub enum StateOrBlock {
 	/// State to be used, may be pending
@@ -168,11 +170,14 @@ pub trait EngineClient: Sync + Send + ChainInfo {
 	fn block_header(&self, id: BlockId) -> Option<encoded::Header>;
 }
 
-// FIXME Why these methods belong to BlockChainClient and not MiningBlockChainClient?
 /// Provides methods to import block into blockchain
 pub trait ImportBlock {
 	/// Import a block into the blockchain.
 	fn import_block(&self, block: Unverified) -> EthcoreResult<H256>;
+
+	/// Triggered by a message from a block queue when the block is ready for insertion.
+	/// Returns the number of blocks imported.
+	fn import_verified_blocks(&self) -> usize;
 }
 
 /// IO operations that should off-load heavy work to another thread.
@@ -187,6 +192,14 @@ pub trait IoClient: Sync + Send {
 	fn queue_consensus_message(&self, message: Bytes);
 }
 
+/// Implement this for clients that need logic to decide when/how to advance.
+pub trait Tick {
+	/// Tick the client
+	fn tick(&self, _prevent_sleep: bool) {}
+}
+
+impl Tick for () {}
+
 /// Provides recently seen bad blocks.
 pub trait BadBlocks {
 	/// Returns a list of blocks that were recently not imported because they were invalid.
@@ -377,6 +390,9 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra
 pub trait BlockChainReset {
 	/// reset to best_block - n
 	fn reset(&self, num: u32) -> Result<(), String>;
+
+	/// Number of eras kept in a journal before they are pruned
+	fn pruning_history(&self) -> u64;
 }
 
 
@@ -423,3 +439,38 @@ pub trait ProvingBlockChainClient: BlockChainClient {
 	/// Get an epoch change signal by block hash.
 	fn epoch_signal(&self, hash: H256) -> Option<Vec<u8>>;
 }
+
+/// External database restoration handler
+pub trait DatabaseRestore: Send + Sync {
+	/// Restart with a new backend. Takes ownership of passed database and moves it to a new location.
+	fn restore_db(&self, new_db: &str) -> Result<(), EthcoreError>;
+}
+
+/// Snapshot related functionality
+pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + BlockChainReset {
+	/// Take a snapshot at the given block.
+	/// If the ID given is "latest", this will default to 1000 blocks behind.
+	fn take_snapshot<W: SnapshotWriter + Send>(
+		&self,
+		writer: W,
+		at: BlockId,
+		p: &Progress,
+	) -> Result<(), EthcoreError>;
+}
+
+
+// todo[dvdplm] move this back to snapshot once extracted from ethcore
+/// Something which can write snapshots.
+/// Writing the same chunk multiple times will lead to implementation-defined
+/// behavior, and is not advised.
+pub trait SnapshotWriter {
+	/// Write a compressed state chunk.
+	fn write_state_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>;
+
+	/// Write a compressed block chunk.
+	fn write_block_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>;
+
+	/// Complete writing. The manifest's chunk lists must be consistent
+	/// with the chunks written.
+	fn finish(self, manifest: common_types::snapshot::ManifestData) -> std::io::Result<()> where Self: Sized;
+}

ethcore/light/Cargo.toml

@@ -13,7 +13,6 @@ client-traits = { path = "../client-traits" }
 common-types = { path = "../types" }
 derive_more = "0.14.0"
 engine = { path = "../engine" }
-ethcore = { path = ".."}
 ethcore-db = { path = "../db" }
 ethcore-blockchain = { path = "../blockchain" }
 ethereum-types = "0.6.0"
@@ -49,6 +48,7 @@ kvdb = "0.1"
 memory-cache = { path = "../../util/memory-cache" }
 error-chain = { version = "0.12", default-features = false }
 journaldb = { path = "../../util/journaldb" }
+verification = { path = "../verification" }
 
 [dev-dependencies]
 ethcore = { path = "..", features = ["test-helpers"] }

ethcore/light/src/client/mod.rs

@@ -19,8 +19,7 @@
 use std::sync::{Weak, Arc};
 
 use engine::{Engine, EpochChange, Proof};
-use ethcore::client::{ClientReport, ClientIoMessage};
-use ethcore::verification::queue::{self, HeaderQueue};
+use verification::queue::{self, HeaderQueue};
 use spec::{Spec, SpecHardcodedSync};
 use io::IoChannel;
 use parking_lot::{Mutex, RwLock};
@@ -30,12 +29,14 @@ use common_types::{
 	BlockNumber,
 	block_status::BlockStatus,
 	blockchain_info::BlockChainInfo,
+	client_types::ClientReport,
 	encoded,
 	engines::epoch::{Transition as EpochTransition, PendingTransition},
 	errors::EthcoreError as Error,
 	errors::EthcoreResult,
 	header::Header,
 	ids::BlockId,
+	io_message::ClientIoMessage,
 	verification::VerificationQueueInfo as BlockQueueInfo,
 };
 use kvdb::KeyValueDB;
@@ -162,7 +163,7 @@ impl<T: LightChainClient> AsLightClient for T {
 
 /// Light client implementation.
 pub struct Client<T> {
-	queue: HeaderQueue,
+	queue: HeaderQueue<()>,
 	engine: Arc<dyn Engine>,
 	chain: HeaderChain,
 	report: RwLock<ClientReport>,
@@ -183,7 +184,7 @@ impl<T: ChainDataFetcher> Client<T> {
 		chain_col: Option<u32>,
 		spec: &Spec,
 		fetcher: T,
-		io_channel: IoChannel<ClientIoMessage>,
+		io_channel: IoChannel<ClientIoMessage<()>>,
 		cache: Arc<Mutex<Cache>>
 	) -> Result<Self, Error> {
 		Ok(Self {
@@ -649,3 +650,5 @@ impl<T: ChainDataFetcher> client_traits::EngineClient for Client<T> {
 		Client::block_header(self, id)
 	}
 }
+
+impl<T> client_traits::Tick for Client<T> {}