diff --git a/ethcore/private-tx/src/lib.rs b/ethcore/private-tx/src/lib.rs index d59c52308dd..12d063974e4 100644 --- a/ethcore/private-tx/src/lib.rs +++ b/ethcore/private-tx/src/lib.rs @@ -82,7 +82,8 @@ use ethcore::executed::{Executed}; use transaction::{SignedTransaction, Transaction, Action, UnverifiedTransaction}; use ethcore::{contract_address as ethcore_contract_address}; use ethcore::client::{ - Client, ChainNotify, ChainRoute, ChainMessageType, ClientIoMessage, BlockId, CallContract + Client, ChainNotify, ChainRoute, ChainMessageType, ClientIoMessage, BlockId, + CallContract, Call, BlockInfo }; use ethcore::account_provider::AccountProvider; use ethcore::miner::{self, Miner, MinerService, pool_client::NonceCache}; @@ -550,19 +551,34 @@ impl Provider where { let state = self.client.state_at(block).ok_or(ErrorKind::StatePruned)?; let nonce = state.nonce(&sender)?; let executed = self.execute_private(source, TransactOptions::with_no_tracing(), block)?; - let gas: u64 = 650000 + - validators.len() as u64 * 30000 + - executed.code.as_ref().map_or(0, |c| c.len() as u64) * 8000 + - executed.state.len() as u64 * 8000; - Ok((Transaction { + let header = self.client.block_header(block) + .ok_or(ErrorKind::StatePruned) + .and_then(|h| h.decode().map_err(|_| ErrorKind::StateIncorrect).into())?; + let (executed_code, executed_state) = (executed.code.unwrap_or_default(), executed.state); + let tx_data = Self::generate_constructor(validators, executed_code.clone(), executed_state.clone()); + let mut tx = Transaction { nonce: nonce, action: Action::Create, - gas: gas.into(), + gas: u64::max_value().into(), gas_price: gas_price, value: source.value, - data: Self::generate_constructor(validators, executed.code.unwrap_or_default(), executed.state) - }, - executed.contract_address)) + data: tx_data, + }; + tx.gas = match self.client.estimate_gas(&tx.clone().fake_sign(sender), &state, &header) { + Ok(estimated_gas) => estimated_gas, + Err(_) => self.estimate_tx_gas(validators, &executed_code, &executed_state, &[]), + }; + + Ok((tx, executed.contract_address)) + } + + fn estimate_tx_gas(&self, validators: &[Address], code: &Bytes, state: &Bytes, signatures: &[Signature]) -> U256 { + let default_gas = 650000 + + validators.len() as u64 * 30000 + + code.len() as u64 * 8000 + + signatures.len() as u64 * 50000 + + state.len() as u64 * 8000; + default_gas.into() } /// Create encrypted public contract deployment transaction. Returns updated encrypted state. @@ -576,7 +592,7 @@ impl Provider where { /// Create encrypted public transaction from private transaction. pub fn public_transaction(&self, state: Bytes, source: &SignedTransaction, signatures: &[Signature], nonce: U256, gas_price: U256) -> Result { - let gas: u64 = 650000 + state.len() as u64 * 8000 + signatures.len() as u64 * 50000; + let gas = self.estimate_tx_gas(&[], &Vec::new(), &state, signatures); Ok(Transaction { nonce: nonce, action: source.action.clone(), diff --git a/ethcore/res/tendermint.json b/ethcore/res/tendermint.json deleted file mode 100644 index 3c60097a4e4..00000000000 --- a/ethcore/res/tendermint.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "name": "TestBFT", - "engine": { - "tendermint": { - "params": { - "validators" : { - "list": [ - "0x82a978b3f5962a5b0957d9ee9eef472ee55b42f1", - "0x7d577a597b2742b498cb5cf0c26cdcd726d39e6e" - ] - }, - "timeoutPropose": 10000, - "timeoutPrevote": 10000, - "timeoutPrecommit": 10000, - "timeoutCommit": 10000 - } - } - }, - "params": { - "gasLimitBoundDivisor": "0x0400", - "accountStartNonce": "0x0", - "maximumExtraDataSize": "0x20", - "minGasLimit": "0x1388", - "networkID" : "0x2323", - "eip140Transition": "0x0", - "eip211Transition": "0x0", - "eip214Transition": "0x0", - "eip658Transition": "0x0" - }, - "genesis": { - "seal": { - "tendermint": { - "round": "0x0", - "proposal": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "precommits": [ - "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" - ] - } - }, - "difficulty": "0x20000", - "author": "0x0000000000000000000000000000000000000000", - "timestamp": "0x00", - "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "extraData": "0x", - "gasLimit": "0x222222" - }, - "accounts": { - "0000000000000000000000000000000000000001": { "balance": "1", "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } }, - "0000000000000000000000000000000000000002": { "balance": "1", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } }, - "0000000000000000000000000000000000000003": { "balance": "1", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } }, - "0000000000000000000000000000000000000004": { "balance": "1", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } }, - "0000000000000000000000000000000000000005": { "balance": "1", "builtin": { "name": "modexp", "activate_at": 0, "pricing": { "modexp": { "divisor": 20 } } } }, - "0000000000000000000000000000000000000006": { "balance": "1", "builtin": { "name": "alt_bn128_add", "activate_at": 0, "pricing": { "linear": { "base": 500, "word": 0 } } } }, - "0000000000000000000000000000000000000007": { "balance": "1", "builtin": { "name": "alt_bn128_mul", "activate_at": 0, "pricing": { "linear": { "base": 40000, "word": 0 } } } }, - "0000000000000000000000000000000000000008": { "balance": "1", "builtin": { "name": "alt_bn128_pairing", "activate_at": 0, "pricing": { "alt_bn128_pairing": { "base": 100000, "pair": 80000 } } } }, - "9cce34f7ab185c7aba1b7c8140d620b4bda941d6": { "balance": "1606938044258990275541962092341162602522202993782792835301376" } - } -} diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs index 1763b8fd5ed..77429a4e56e 100644 --- a/ethcore/service/src/service.rs +++ b/ethcore/service/src/service.rs @@ -106,7 +106,13 @@ impl ClientService { info!("Configured for {} using {} engine", Colour::White.bold().paint(spec.name.clone()), Colour::Yellow.bold().paint(spec.engine.name())); let pruning = config.pruning; - let client = Client::new(config, &spec, blockchain_db.clone(), miner.clone(), io_service.channel())?; + let client = Client::new( + config, + &spec, + blockchain_db.clone(), + miner.clone(), + io_service.channel(), + )?; miner.set_io_channel(io_service.channel()); miner.set_in_chain_checker(&client.clone()); diff --git a/ethcore/src/client/chain_notify.rs b/ethcore/src/client/chain_notify.rs index ebfe7bdef5c..3d576ae12aa 100644 --- a/ethcore/src/client/chain_notify.rs +++ b/ethcore/src/client/chain_notify.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use bytes::Bytes; -use ethereum_types::H256; +use ethereum_types::{H256, U256}; use transaction::UnverifiedTransaction; use blockchain::ImportRoute; use std::time::Duration; @@ -141,7 +141,15 @@ pub trait ChainNotify : Send + Sync { } /// fires when chain broadcasts a message - fn broadcast(&self, _message_type: ChainMessageType) {} + fn broadcast(&self, _message_type: ChainMessageType) { + // does nothing by default + } + + /// fires when new block is about to be imported + /// implementations should be light + fn block_pre_import(&self, _bytes: &Bytes, _hash: &H256, _difficulty: &U256) { + // does nothing by default + } /// fires when new transactions are received from a peer fn transactions_received(&self, diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index d5ae9d80f6a..00dc9cab53e 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -881,7 +881,7 @@ impl Client { /// Flush the block import queue. pub fn flush_queue(&self) { self.importer.block_queue.flush(); - while !self.importer.block_queue.queue_info().is_empty() { + while !self.importer.block_queue.is_empty() { self.import_verified_blocks(); } } @@ -1423,8 +1423,21 @@ impl ImportBlock for Client { bail!(EthcoreErrorKind::Block(BlockError::UnknownParent(unverified.parent_hash()))); } + let raw = if self.importer.block_queue.is_empty() { + Some(( + unverified.bytes.clone(), + unverified.header.hash(), + *unverified.header.difficulty(), + )) + } else { None }; + match self.importer.block_queue.import(unverified) { - Ok(res) => Ok(res), + Ok(hash) => { + if let Some((raw, hash, difficulty)) = raw { + self.notify(move |n| n.block_pre_import(&raw, &hash, &difficulty)); + } + Ok(hash) + }, // we only care about block errors (not import errors) Err((block, EthcoreError(EthcoreErrorKind::Block(err), _))) => { self.importer.bad_blocks.report(block.bytes, format!("{:?}", err)); @@ -1878,6 +1891,10 @@ impl BlockChainClient for Client { self.importer.block_queue.queue_info() } + fn is_queue_empty(&self) -> bool { + self.importer.block_queue.is_empty() + } + fn clear_queue(&self) { self.importer.block_queue.clear(); } @@ -2288,7 +2305,11 @@ impl ScheduleInfo for Client { impl ImportSealedBlock for Client { fn import_sealed_block(&self, block: SealedBlock) -> EthcoreResult { let start = Instant::now(); + let raw = block.rlp_bytes(); let header = block.header().clone(); + let hash = header.hash(); + self.notify(|n| n.block_pre_import(&raw, &hash, header.difficulty())); + let route = { // Do a super duper basic verification to detect potential bugs if let Err(e) = self.engine.verify_block_basic(&header) { @@ -2306,15 +2327,14 @@ impl ImportSealedBlock for Client { let block_data = block.rlp_bytes(); let route = self.importer.commit_block(block, &header, encoded::Block::new(block_data), self); - trace!(target: "client", "Imported sealed block #{} ({})", header.number(), header.hash()); + trace!(target: "client", "Imported sealed block #{} ({})", header.number(), hash); self.state_db.write().sync_cache(&route.enacted, &route.retracted, false); route }; - let h = header.hash(); let route = ChainRoute::from([route].as_ref()); self.importer.miner.chain_new_blocks( self, - &[h], + &[hash], &[], route.enacted(), route.retracted(), @@ -2322,16 +2342,16 @@ impl ImportSealedBlock for Client { ); self.notify(|notify| { notify.new_blocks( - vec![h], + vec![hash], vec![], route.clone(), - vec![h], + vec![hash], vec![], start.elapsed(), ); }); self.db.read().key_value().flush().expect("DB flush failed."); - Ok(h) + Ok(hash) } } diff --git a/ethcore/src/client/traits.rs b/ethcore/src/client/traits.rs index 5b78a54b34e..55d527013ec 100644 --- a/ethcore/src/client/traits.rs +++ b/ethcore/src/client/traits.rs @@ -300,6 +300,11 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra /// Get block queue information. fn queue_info(&self) -> BlockQueueInfo; + /// Returns true if block queue is empty. + fn is_queue_empty(&self) -> bool { + self.queue_info().is_empty() + } + /// Clear block queue and abort all import activity. fn clear_queue(&self); diff --git a/ethcore/src/engines/mod.rs b/ethcore/src/engines/mod.rs index 4e583cbaf77..4e17b59d503 100644 --- a/ethcore/src/engines/mod.rs +++ b/ethcore/src/engines/mod.rs @@ -22,10 +22,7 @@ mod clique; mod instant_seal; mod null_engine; mod signer; -mod tendermint; -mod transition; mod validator_set; -mod vote_collector; pub mod block_reward; pub mod epoch; @@ -35,7 +32,6 @@ pub use self::basic_authority::BasicAuthority; pub use self::epoch::{EpochVerifier, Transition as EpochTransition}; pub use self::instant_seal::{InstantSeal, InstantSealParams}; pub use self::null_engine::NullEngine; -pub use self::tendermint::Tendermint; pub use self::clique::Clique; use std::sync::{Weak, Arc}; diff --git a/ethcore/src/engines/signer.rs b/ethcore/src/engines/signer.rs index 670c0959c81..c28e44451c5 100644 --- a/ethcore/src/engines/signer.rs +++ b/ethcore/src/engines/signer.rs @@ -57,11 +57,6 @@ impl EngineSigner { self.address.clone() } - /// Check if the given address is the signing address. - pub fn is_address(&self, address: &Address) -> bool { - self.address.map_or(false, |a| a == *address) - } - /// Check if the signing address was set. pub fn is_some(&self) -> bool { self.address.is_some() diff --git a/ethcore/src/engines/tendermint/message.rs b/ethcore/src/engines/tendermint/message.rs deleted file mode 100644 index 137148667de..00000000000 --- a/ethcore/src/engines/tendermint/message.rs +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! Tendermint message handling. - -use bytes::Bytes; -use error::Error; -use ethereum_types::{H256, H520, Address}; -use ethkey::{recover, public_to_address}; -use hash::keccak; -use header::Header; -use rlp::{Rlp, RlpStream, Encodable, Decodable, DecoderError}; -use std::cmp; -use super::{Height, View, BlockHash, Step}; -use super::super::vote_collector::Message; - -/// Message transmitted between consensus participants. -#[derive(Debug, PartialEq, Eq, Clone, Hash, Default)] -pub struct ConsensusMessage { - pub vote_step: VoteStep, - pub block_hash: Option, - pub signature: H520, -} - -/// Complete step of the consensus process. -#[derive(Debug, PartialEq, Eq, Clone, Hash)] -pub struct VoteStep { - pub height: Height, - pub view: View, - pub step: Step, -} - -impl VoteStep { - pub fn new(height: Height, view: View, step: Step) -> Self { - VoteStep { height: height, view: view, step: step } - } - - pub fn is_height(&self, height: Height) -> bool { - self.height == height - } - - pub fn is_view(&self, height: Height, view: View) -> bool { - self.height == height && self.view == view - } -} - -/// Header consensus view. -pub fn consensus_view(header: &Header) -> Result { - let view_rlp = header.seal().get(0).expect("seal passed basic verification; seal has 3 fields; qed"); - Rlp::new(view_rlp.as_slice()).as_val() -} - -/// Proposal signature. -pub fn proposal_signature(header: &Header) -> Result { - Rlp::new(header.seal().get(1).expect("seal passed basic verification; seal has 3 fields; qed").as_slice()).as_val() -} - -impl Message for ConsensusMessage { - type Round = VoteStep; - - fn signature(&self) -> H520 { self.signature } - - fn block_hash(&self) -> Option { self.block_hash } - - fn round(&self) -> &VoteStep { &self.vote_step } - - fn is_broadcastable(&self) -> bool { self.vote_step.step.is_pre() } -} - -impl ConsensusMessage { - pub fn new(signature: H520, height: Height, view: View, step: Step, block_hash: Option) -> Self { - ConsensusMessage { - signature: signature, - block_hash: block_hash, - vote_step: VoteStep::new(height, view, step), - } - } - - pub fn new_proposal(header: &Header) -> Result { - Ok(ConsensusMessage { - signature: proposal_signature(header)?, - vote_step: VoteStep::new(header.number() as Height, consensus_view(header)?, Step::Propose), - block_hash: Some(header.bare_hash()), - }) - } - - pub fn verify(&self) -> Result { - let full_rlp = ::rlp::encode(self); - let block_info = Rlp::new(&full_rlp).at(1)?; - let public_key = recover(&self.signature.into(), &keccak(block_info.as_raw()))?; - Ok(public_to_address(&public_key)) - } -} - -impl Default for VoteStep { - fn default() -> Self { - VoteStep::new(0, 0, Step::Propose) - } -} - -impl PartialOrd for VoteStep { - fn partial_cmp(&self, m: &VoteStep) -> Option { - Some(self.cmp(m)) - } -} - -impl Ord for VoteStep { - fn cmp(&self, m: &VoteStep) -> cmp::Ordering { - if self.height != m.height { - self.height.cmp(&m.height) - } else if self.view != m.view { - self.view.cmp(&m.view) - } else { - self.step.number().cmp(&m.step.number()) - } - } -} - -impl Step { - fn number(&self) -> u8 { - match *self { - Step::Propose => 0, - Step::Prevote => 1, - Step::Precommit => 2, - Step::Commit => 3, - } - } -} - -impl Decodable for Step { - fn decode(rlp: &Rlp) -> Result { - match rlp.as_val()? { - 0u8 => Ok(Step::Propose), - 1 => Ok(Step::Prevote), - 2 => Ok(Step::Precommit), - _ => Err(DecoderError::Custom("Invalid step.")), - } - } -} - -impl Encodable for Step { - fn rlp_append(&self, s: &mut RlpStream) { - s.append_internal(&self.number()); - } -} - -/// (signature, (height, view, step, block_hash)) -impl Decodable for ConsensusMessage { - fn decode(rlp: &Rlp) -> Result { - let m = rlp.at(1)?; - let block_message: H256 = m.val_at(3)?; - Ok(ConsensusMessage { - vote_step: VoteStep::new(m.val_at(0)?, m.val_at(1)?, m.val_at(2)?), - block_hash: match block_message.is_zero() { - true => None, - false => Some(block_message), - }, - signature: rlp.val_at(0)?, - }) - } -} - -impl Encodable for ConsensusMessage { - fn rlp_append(&self, s: &mut RlpStream) { - let info = message_info_rlp(&self.vote_step, self.block_hash); - s.begin_list(2) - .append(&self.signature) - .append_raw(&info, 1); - } -} - -pub fn message_info_rlp(vote_step: &VoteStep, block_hash: Option) -> Bytes { - let mut s = RlpStream::new_list(4); - s.append(&vote_step.height).append(&vote_step.view).append(&vote_step.step).append(&block_hash.unwrap_or_else(H256::zero)); - s.out() -} - -pub fn message_full_rlp(signature: &H520, vote_info: &Bytes) -> Bytes { - let mut s = RlpStream::new_list(2); - s.append(signature).append_raw(vote_info, 1); - s.out() -} - -pub fn message_hash(vote_step: VoteStep, block_hash: H256) -> H256 { - keccak(message_info_rlp(&vote_step, Some(block_hash))) -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - use hash::keccak; - use rlp::*; - use account_provider::AccountProvider; - use header::Header; - use super::super::Step; - use super::*; - - #[test] - fn encode_step() { - let step = Step::Precommit; - - let mut s = RlpStream::new_list(2); - s.append(&step); - assert!(!s.is_finished(), "List shouldn't finished yet"); - s.append(&step); - assert!(s.is_finished(), "List should be finished now"); - s.out(); - } - - #[test] - fn encode_decode() { - let message = ConsensusMessage { - signature: H520::default(), - vote_step: VoteStep { - height: 10, - view: 123, - step: Step::Precommit, - }, - block_hash: Some(keccak("1")), - }; - let raw_rlp = ::rlp::encode(&message); - let rlp = Rlp::new(&raw_rlp); - assert_eq!(Ok(message), rlp.as_val()); - - let message = ConsensusMessage { - signature: H520::default(), - vote_step: VoteStep { - height: 1314, - view: 0, - step: Step::Prevote, - }, - block_hash: None - }; - let raw_rlp = ::rlp::encode(&message); - let rlp = Rlp::new(&raw_rlp); - assert_eq!(Ok(message), rlp.as_val()); - } - - #[test] - fn generate_and_verify() { - let tap = Arc::new(AccountProvider::transient_provider()); - let addr = tap.insert_account(keccak("0").into(), &"0".into()).unwrap(); - tap.unlock_account_permanently(addr, "0".into()).unwrap(); - - let mi = message_info_rlp(&VoteStep::new(123, 2, Step::Precommit), Some(H256::default())); - - let raw_rlp = message_full_rlp(&tap.sign(addr, None, keccak(&mi)).unwrap().into(), &mi); - - let rlp = Rlp::new(&raw_rlp); - let message: ConsensusMessage = rlp.as_val().unwrap(); - match message.verify() { Ok(a) if a == addr => {}, _ => panic!(), }; - } - - #[test] - fn proposal_message() { - let mut header = Header::default(); - let seal = vec![ - ::rlp::encode(&0u8), - ::rlp::encode(&H520::default()), - Vec::new() - ]; - - header.set_seal(seal); - let message = ConsensusMessage::new_proposal(&header).unwrap(); - assert_eq!( - message, - ConsensusMessage { - signature: Default::default(), - vote_step: VoteStep { - height: 0, - view: 0, - step: Step::Propose, - }, - block_hash: Some(header.bare_hash()) - } - ); - } - - #[test] - fn step_ordering() { - assert!(VoteStep::new(10, 123, Step::Precommit) < VoteStep::new(11, 123, Step::Precommit)); - assert!(VoteStep::new(10, 123, Step::Propose) < VoteStep::new(11, 123, Step::Precommit)); - assert!(VoteStep::new(10, 122, Step::Propose) < VoteStep::new(11, 123, Step::Propose)); - } -} diff --git a/ethcore/src/engines/tendermint/mod.rs b/ethcore/src/engines/tendermint/mod.rs deleted file mode 100644 index a3710222ad2..00000000000 --- a/ethcore/src/engines/tendermint/mod.rs +++ /dev/null @@ -1,1144 +0,0 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -/// Tendermint BFT consensus engine with round robin proof-of-authority. -/// At each blockchain `Height` there can be multiple `View`s of voting. -/// Signatures always sign `Height`, `View`, `Step` and `BlockHash` which is a block hash without seal. -/// First a block with `Seal::Proposal` is issued by the designated proposer. -/// Next the `View` proceeds through `Prevote` and `Precommit` `Step`s. -/// Block is issued when there is enough `Precommit` votes collected on a particular block at the end of a `View`. -/// Once enough votes have been gathered the proposer issues that block in the `Commit` step. - -mod message; -mod params; - -use std::sync::{Weak, Arc}; -use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; -use std::collections::HashSet; -use hash::keccak; -use ethereum_types::{H256, H520, U128, U256, Address}; -use parking_lot::RwLock; -use unexpected::{OutOfBounds, Mismatch}; -use client::EngineClient; -use bytes::Bytes; -use error::{Error, BlockError}; -use header::{Header, BlockNumber, ExtendedHeader}; -use rlp::Rlp; -use ethkey::{self, Password, Message, Signature}; -use account_provider::AccountProvider; -use block::*; -use engines::{Engine, Seal, EngineError, ConstructedVerifier}; -use engines::block_reward::{self, RewardKind}; -use io::IoService; -use super::signer::EngineSigner; -use super::validator_set::{ValidatorSet, SimpleList}; -use super::transition::TransitionHandler; -use super::vote_collector::VoteCollector; -use self::message::*; -use self::params::TendermintParams; -use machine::{AuxiliaryData, EthereumMachine}; - -#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] -pub enum Step { - Propose, - Prevote, - Precommit, - Commit -} - -impl Step { - pub fn is_pre(self) -> bool { - match self { - Step::Prevote | Step::Precommit => true, - _ => false, - } - } -} - -pub type Height = usize; -pub type View = usize; -pub type BlockHash = H256; - -/// Engine using `Tendermint` consensus algorithm, suitable for EVM chain. -pub struct Tendermint { - step_service: IoService, - client: RwLock>>, - /// Blockchain height. - height: AtomicUsize, - /// Consensus view. - view: AtomicUsize, - /// Consensus step. - step: RwLock, - /// Vote accumulator. - votes: VoteCollector, - /// Used to sign messages and proposals. - signer: RwLock, - /// Message for the last PoLC. - lock_change: RwLock>, - /// Last lock view. - last_lock: AtomicUsize, - /// Bare hash of the proposed block, used for seal submission. - proposal: RwLock>, - /// Hash of the proposal parent block. - proposal_parent: RwLock, - /// Last block proposed by this validator. - last_proposed: RwLock, - /// Set used to determine the current validators. - validators: Box, - /// Reward per block, in base units. - block_reward: U256, - /// ethereum machine descriptor - machine: EthereumMachine, -} - -struct EpochVerifier - where F: Fn(&Signature, &Message) -> Result + Send + Sync -{ - subchain_validators: SimpleList, - recover: F -} - -impl super::EpochVerifier for EpochVerifier - where F: Fn(&Signature, &Message) -> Result + Send + Sync -{ - fn verify_light(&self, header: &Header) -> Result<(), Error> { - let message = header.bare_hash(); - - let mut addresses = HashSet::new(); - let ref header_signatures_field = header.seal().get(2).ok_or(BlockError::InvalidSeal)?; - for rlp in Rlp::new(header_signatures_field).iter() { - let signature: H520 = rlp.as_val()?; - let address = (self.recover)(&signature.into(), &message)?; - - if !self.subchain_validators.contains(header.parent_hash(), &address) { - return Err(EngineError::NotAuthorized(address.to_owned()).into()); - } - addresses.insert(address); - } - - let n = addresses.len(); - let threshold = self.subchain_validators.len() * 2/3; - if n > threshold { - Ok(()) - } else { - Err(EngineError::BadSealFieldSize(OutOfBounds { - min: Some(threshold), - max: None, - found: n - }).into()) - } - } - - fn check_finality_proof(&self, proof: &[u8]) -> Option> { - match ::rlp::decode(proof) { - Ok(header) => self.verify_light(&header).ok().map(|_| vec![header.hash()]), - Err(_) => None - } - } -} - -fn combine_proofs(signal_number: BlockNumber, set_proof: &[u8], finality_proof: &[u8]) -> Vec { - let mut stream = ::rlp::RlpStream::new_list(3); - stream.append(&signal_number).append(&set_proof).append(&finality_proof); - stream.out() -} - -fn destructure_proofs(combined: &[u8]) -> Result<(BlockNumber, &[u8], &[u8]), Error> { - let rlp = Rlp::new(combined); - Ok(( - rlp.at(0)?.as_val()?, - rlp.at(1)?.data()?, - rlp.at(2)?.data()?, - )) -} - -impl Tendermint { - /// Create a new instance of Tendermint engine - pub fn new(our_params: TendermintParams, machine: EthereumMachine) -> Result, Error> { - let engine = Arc::new( - Tendermint { - client: RwLock::new(None), - step_service: IoService::::start()?, - height: AtomicUsize::new(1), - view: AtomicUsize::new(0), - step: RwLock::new(Step::Propose), - votes: Default::default(), - signer: Default::default(), - lock_change: RwLock::new(None), - last_lock: AtomicUsize::new(0), - proposal: RwLock::new(None), - proposal_parent: Default::default(), - last_proposed: Default::default(), - validators: our_params.validators, - block_reward: our_params.block_reward, - machine: machine, - }); - - let handler = TransitionHandler::new(Arc::downgrade(&engine) as Weak>, Box::new(our_params.timeouts)); - engine.step_service.register_handler(Arc::new(handler))?; - - Ok(engine) - } - - fn update_sealing(&self) { - if let Some(ref weak) = *self.client.read() { - if let Some(c) = weak.upgrade() { - c.update_sealing(); - } - } - } - - fn submit_seal(&self, block_hash: H256, seal: Vec) { - if let Some(ref weak) = *self.client.read() { - if let Some(c) = weak.upgrade() { - c.submit_seal(block_hash, seal); - } - } - } - - fn broadcast_message(&self, message: Bytes) { - if let Some(ref weak) = *self.client.read() { - if let Some(c) = weak.upgrade() { - c.broadcast_consensus_message(message); - } - } - } - - fn generate_message(&self, block_hash: Option) -> Option { - let h = self.height.load(AtomicOrdering::SeqCst); - let r = self.view.load(AtomicOrdering::SeqCst); - let s = *self.step.read(); - let vote_info = message_info_rlp(&VoteStep::new(h, r, s), block_hash); - match (self.signer.read().address(), self.sign(keccak(&vote_info)).map(Into::into)) { - (Some(validator), Ok(signature)) => { - let message_rlp = message_full_rlp(&signature, &vote_info); - let message = ConsensusMessage::new(signature, h, r, s, block_hash); - self.votes.vote(message.clone(), validator); - debug!(target: "engine", "Generated {:?} as {}.", message, validator); - self.handle_valid_message(&message); - - Some(message_rlp) - }, - (None, _) => { - trace!(target: "engine", "No message, since there is no engine signer."); - None - }, - (Some(v), Err(e)) => { - trace!(target: "engine", "{} could not sign the message {}", v, e); - None - }, - } - } - - fn generate_and_broadcast_message(&self, block_hash: Option) { - if let Some(message) = self.generate_message(block_hash) { - self.broadcast_message(message); - } - } - - /// Broadcast all messages since last issued block to get the peers up to speed. - fn broadcast_old_messages(&self) { - for m in self.votes.get_up_to(&VoteStep::new(self.height.load(AtomicOrdering::SeqCst), self.view.load(AtomicOrdering::SeqCst), Step::Precommit)).into_iter() { - self.broadcast_message(m); - } - } - - fn to_next_height(&self, height: Height) { - let new_height = height + 1; - debug!(target: "engine", "Received a Commit, transitioning to height {}.", new_height); - self.last_lock.store(0, AtomicOrdering::SeqCst); - self.height.store(new_height, AtomicOrdering::SeqCst); - self.view.store(0, AtomicOrdering::SeqCst); - *self.lock_change.write() = None; - *self.proposal.write() = None; - } - - /// Use via step_service to transition steps. - fn to_step(&self, step: Step) { - if let Err(io_err) = self.step_service.send_message(step) { - warn!(target: "engine", "Could not proceed to step {}.", io_err) - } - *self.step.write() = step; - match step { - Step::Propose => { - self.update_sealing() - }, - Step::Prevote => { - let block_hash = match *self.lock_change.read() { - Some(ref m) if !self.should_unlock(m.vote_step.view) => m.block_hash, - _ => self.proposal.read().clone(), - }; - self.generate_and_broadcast_message(block_hash); - }, - Step::Precommit => { - trace!(target: "engine", "to_step: Precommit."); - let block_hash = match *self.lock_change.read() { - Some(ref m) if self.is_view(m) && m.block_hash.is_some() => { - trace!(target: "engine", "Setting last lock: {}", m.vote_step.view); - self.last_lock.store(m.vote_step.view, AtomicOrdering::SeqCst); - m.block_hash - }, - _ => None, - }; - self.generate_and_broadcast_message(block_hash); - }, - Step::Commit => { - trace!(target: "engine", "to_step: Commit."); - }, - } - } - - fn is_authority(&self, address: &Address) -> bool { - self.validators.contains(&*self.proposal_parent.read(), address) - } - - fn check_above_threshold(&self, n: usize) -> Result<(), EngineError> { - let threshold = self.validators.count(&*self.proposal_parent.read()) * 2/3; - if n > threshold { - Ok(()) - } else { - Err(EngineError::BadSealFieldSize(OutOfBounds { - min: Some(threshold), - max: None, - found: n - })) - } - } - - /// Find the designated for the given view. - fn view_proposer(&self, bh: &H256, height: Height, view: View) -> Address { - let proposer_nonce = height + view; - trace!(target: "engine", "Proposer nonce: {}", proposer_nonce); - self.validators.get(bh, proposer_nonce) - } - - /// Check if address is a proposer for given view. - fn check_view_proposer(&self, bh: &H256, height: Height, view: View, address: &Address) -> Result<(), EngineError> { - let proposer = self.view_proposer(bh, height, view); - if proposer == *address { - Ok(()) - } else { - Err(EngineError::NotProposer(Mismatch { expected: proposer, found: address.clone() })) - } - } - - /// Check if current signer is the current proposer. - fn is_signer_proposer(&self, bh: &H256) -> bool { - let proposer = self.view_proposer(bh, self.height.load(AtomicOrdering::SeqCst), self.view.load(AtomicOrdering::SeqCst)); - self.signer.read().is_address(&proposer) - } - - fn is_height(&self, message: &ConsensusMessage) -> bool { - message.vote_step.is_height(self.height.load(AtomicOrdering::SeqCst)) - } - - fn is_view(&self, message: &ConsensusMessage) -> bool { - message.vote_step.is_view(self.height.load(AtomicOrdering::SeqCst), self.view.load(AtomicOrdering::SeqCst)) - } - - fn increment_view(&self, n: View) { - trace!(target: "engine", "increment_view: New view."); - self.view.fetch_add(n, AtomicOrdering::SeqCst); - } - - fn should_unlock(&self, lock_change_view: View) -> bool { - self.last_lock.load(AtomicOrdering::SeqCst) < lock_change_view - && lock_change_view < self.view.load(AtomicOrdering::SeqCst) - } - - fn has_enough_any_votes(&self) -> bool { - let step_votes = self.votes.count_round_votes(&VoteStep::new(self.height.load(AtomicOrdering::SeqCst), self.view.load(AtomicOrdering::SeqCst), *self.step.read())); - self.check_above_threshold(step_votes).is_ok() - } - - fn has_enough_future_step_votes(&self, vote_step: &VoteStep) -> bool { - if vote_step.view > self.view.load(AtomicOrdering::SeqCst) { - let step_votes = self.votes.count_round_votes(vote_step); - self.check_above_threshold(step_votes).is_ok() - } else { - false - } - } - - fn has_enough_aligned_votes(&self, message: &ConsensusMessage) -> bool { - let aligned_count = self.votes.count_aligned_votes(&message); - self.check_above_threshold(aligned_count).is_ok() - } - - fn handle_valid_message(&self, message: &ConsensusMessage) { - let ref vote_step = message.vote_step; - let is_newer_than_lock = match *self.lock_change.read() { - Some(ref lock) => vote_step > &lock.vote_step, - None => true, - }; - let lock_change = is_newer_than_lock - && vote_step.step == Step::Prevote - && message.block_hash.is_some() - && self.has_enough_aligned_votes(message); - if lock_change { - trace!(target: "engine", "handle_valid_message: Lock change."); - *self.lock_change.write() = Some(message.clone()); - } - // Check if it can affect the step transition. - if self.is_height(message) { - let next_step = match *self.step.read() { - Step::Precommit if message.block_hash.is_none() && self.has_enough_aligned_votes(message) => { - self.increment_view(1); - Some(Step::Propose) - }, - Step::Precommit if self.has_enough_aligned_votes(message) => { - let bh = message.block_hash.expect("previous guard ensures is_some; qed"); - if *self.last_proposed.read() == bh { - // Commit the block using a complete signature set. - // Generate seal and remove old votes. - let precommits = self.votes.round_signatures(vote_step, &bh); - trace!(target: "engine", "Collected seal: {:?}", precommits); - let seal = vec![ - ::rlp::encode(&vote_step.view), - ::rlp::NULL_RLP.to_vec(), - ::rlp::encode_list(&precommits) - ]; - self.submit_seal(bh, seal); - self.votes.throw_out_old(&vote_step); - } - self.to_next_height(self.height.load(AtomicOrdering::SeqCst)); - Some(Step::Commit) - }, - Step::Precommit if self.has_enough_future_step_votes(&vote_step) => { - self.increment_view(vote_step.view - self.view.load(AtomicOrdering::SeqCst)); - Some(Step::Precommit) - }, - // Avoid counting votes twice. - Step::Prevote if lock_change => Some(Step::Precommit), - Step::Prevote if self.has_enough_aligned_votes(message) => Some(Step::Precommit), - Step::Prevote if self.has_enough_future_step_votes(&vote_step) => { - self.increment_view(vote_step.view - self.view.load(AtomicOrdering::SeqCst)); - Some(Step::Prevote) - }, - _ => None, - }; - - if let Some(step) = next_step { - trace!(target: "engine", "Transition to {:?} triggered.", step); - self.to_step(step); - } - } - } -} - -impl Engine for Tendermint { - fn name(&self) -> &str { "Tendermint" } - - /// (consensus view, proposal signature, authority signatures) - fn seal_fields(&self, _header: &Header) -> usize { 3 } - - fn machine(&self) -> &EthereumMachine { &self.machine } - - fn maximum_uncle_count(&self, _block: BlockNumber) -> usize { 0 } - - fn maximum_uncle_age(&self) -> usize { 0 } - - fn populate_from_parent(&self, header: &mut Header, parent: &Header) { - // Chain scoring: total weight is sqrt(U256::max_value())*height - view - let new_difficulty = U256::from(U128::max_value()) - + consensus_view(parent).expect("Header has been verified; qed") - - self.view.load(AtomicOrdering::SeqCst); - - header.set_difficulty(new_difficulty); - } - - /// Should this node participate. - fn seals_internally(&self) -> Option { - Some(self.signer.read().is_some()) - } - - /// Attempt to seal generate a proposal seal. - /// - /// This operation is synchronous and may (quite reasonably) not be available, in which case - /// `Seal::None` will be returned. - fn generate_seal(&self, block: &ExecutedBlock, _parent: &Header) -> Seal { - let header = block.header(); - let author = header.author(); - // Only proposer can generate seal if None was generated. - if !self.is_signer_proposer(header.parent_hash()) || self.proposal.read().is_some() { - return Seal::None; - } - - let height = header.number() as Height; - let view = self.view.load(AtomicOrdering::SeqCst); - let bh = Some(header.bare_hash()); - let vote_info = message_info_rlp(&VoteStep::new(height, view, Step::Propose), bh.clone()); - if let Ok(signature) = self.sign(keccak(&vote_info)).map(Into::into) { - // Insert Propose vote. - debug!(target: "engine", "Submitting proposal {} at height {} view {}.", header.bare_hash(), height, view); - self.votes.vote(ConsensusMessage::new(signature, height, view, Step::Propose, bh), *author); - // Remember the owned block. - *self.last_proposed.write() = header.bare_hash(); - // Remember proposal for later seal submission. - *self.proposal.write() = bh; - *self.proposal_parent.write() = header.parent_hash().clone(); - Seal::Proposal(vec![ - ::rlp::encode(&view), - ::rlp::encode(&signature), - ::rlp::EMPTY_LIST_RLP.to_vec() - ]) - } else { - warn!(target: "engine", "generate_seal: FAIL: accounts secret key unavailable"); - Seal::None - } - } - - fn handle_message(&self, rlp: &[u8]) -> Result<(), EngineError> { - fn fmt_err(x: T) -> EngineError { - EngineError::MalformedMessage(format!("{:?}", x)) - } - - let rlp = Rlp::new(rlp); - let message: ConsensusMessage = rlp.as_val().map_err(fmt_err)?; - if !self.votes.is_old_or_known(&message) { - let msg_hash = keccak(rlp.at(1).map_err(fmt_err)?.as_raw()); - let sender = ethkey::public_to_address( - ðkey::recover(&message.signature.into(), &msg_hash).map_err(fmt_err)? - ); - - if !self.is_authority(&sender) { - return Err(EngineError::NotAuthorized(sender)); - } - self.broadcast_message(rlp.as_raw().to_vec()); - if let Some(double) = self.votes.vote(message.clone(), sender) { - let height = message.vote_step.height as BlockNumber; - self.validators.report_malicious(&sender, height, height, ::rlp::encode(&double)); - return Err(EngineError::DoubleVote(sender)); - } - trace!(target: "engine", "Handling a valid {:?} from {}.", message, sender); - self.handle_valid_message(&message); - } - Ok(()) - } - - fn on_new_block(&self, block: &mut ExecutedBlock, epoch_begin: bool, _ancestry: &mut Iterator) -> Result<(), Error> { - if !epoch_begin { return Ok(()) } - - // genesis is never a new block, but might as well check. - let header = block.header().clone(); - let first = header.number() == 0; - - let mut call = |to, data| { - let result = self.machine.execute_as_system( - block, - to, - U256::max_value(), // unbounded gas? maybe make configurable. - Some(data), - ); - - result.map_err(|e| format!("{}", e)) - }; - - self.validators.on_epoch_begin(first, &header, &mut call) - } - - /// Apply the block reward on finalisation of the block. - fn on_close_block(&self, block: &mut ExecutedBlock) -> Result<(), Error>{ - let author = *block.header().author(); - - block_reward::apply_block_rewards( - &[(author, RewardKind::Author, self.block_reward)], - block, - &self.machine, - ) - } - - fn verify_local_seal(&self, _header: &Header) -> Result<(), Error> { - Ok(()) - } - - fn verify_block_basic(&self, header: &Header) -> Result<(), Error> { - let seal_length = header.seal().len(); - let expected_seal_fields = self.seal_fields(header); - if seal_length == expected_seal_fields { - // Either proposal or commit. - if (header.seal()[1] == ::rlp::NULL_RLP) - != (header.seal()[2] == ::rlp::EMPTY_LIST_RLP) { - Ok(()) - } else { - warn!(target: "engine", "verify_block_basic: Block is neither a Commit nor Proposal."); - Err(BlockError::InvalidSeal.into()) - } - } else { - Err(BlockError::InvalidSealArity( - Mismatch { expected: expected_seal_fields, found: seal_length } - ).into()) - } - } - - fn verify_block_external(&self, header: &Header) -> Result<(), Error> { - if let Ok(proposal) = ConsensusMessage::new_proposal(header) { - let proposer = proposal.verify()?; - if !self.is_authority(&proposer) { - return Err(EngineError::NotAuthorized(proposer).into()); - } - self.check_view_proposer( - header.parent_hash(), - proposal.vote_step.height, - proposal.vote_step.view, - &proposer - ).map_err(Into::into) - } else { - let vote_step = VoteStep::new(header.number() as usize, consensus_view(header)?, Step::Precommit); - let precommit_hash = message_hash(vote_step.clone(), header.bare_hash()); - let ref signatures_field = header.seal().get(2).expect("block went through verify_block_basic; block has .seal_fields() fields; qed"); - let mut origins = HashSet::new(); - for rlp in Rlp::new(signatures_field).iter() { - let precommit = ConsensusMessage { - signature: rlp.as_val()?, - block_hash: Some(header.bare_hash()), - vote_step: vote_step.clone(), - }; - let address = match self.votes.get(&precommit) { - Some(a) => a, - None => ethkey::public_to_address(ðkey::recover(&precommit.signature.into(), &precommit_hash)?), - }; - if !self.validators.contains(header.parent_hash(), &address) { - return Err(EngineError::NotAuthorized(address.to_owned()).into()); - } - - if !origins.insert(address) { - warn!(target: "engine", "verify_block_unordered: Duplicate signature from {} on the seal.", address); - return Err(BlockError::InvalidSeal.into()); - } - } - - self.check_above_threshold(origins.len()).map_err(Into::into) - } - } - - fn signals_epoch_end(&self, header: &Header, aux: AuxiliaryData) - -> super::EpochChange - { - let first = header.number() == 0; - self.validators.signals_epoch_end(first, header, aux) - } - - fn is_epoch_end( - &self, - chain_head: &Header, - _finalized: &[H256], - _chain: &super::Headers
, - transition_store: &super::PendingTransitionStore, - ) -> Option> { - let first = chain_head.number() == 0; - - if let Some(change) = self.validators.is_epoch_end(first, chain_head) { - let change = combine_proofs(chain_head.number(), &change, &[]); - return Some(change) - } else if let Some(pending) = transition_store(chain_head.hash()) { - let signal_number = chain_head.number(); - let finality_proof = ::rlp::encode(chain_head); - return Some(combine_proofs(signal_number, &pending.proof, &finality_proof)) - } - - None - } - - fn is_epoch_end_light( - &self, - chain_head: &Header, - chain: &super::Headers
, - transition_store: &super::PendingTransitionStore, - ) -> Option> { - self.is_epoch_end(chain_head, &[], chain, transition_store) - } - - fn epoch_verifier<'a>(&self, _header: &Header, proof: &'a [u8]) -> ConstructedVerifier<'a, EthereumMachine> { - let (signal_number, set_proof, finality_proof) = match destructure_proofs(proof) { - Ok(x) => x, - Err(e) => return ConstructedVerifier::Err(e), - }; - - let first = signal_number == 0; - match self.validators.epoch_set(first, &self.machine, signal_number, set_proof) { - Ok((list, finalize)) => { - let verifier = Box::new(EpochVerifier { - subchain_validators: list, - recover: |signature: &Signature, message: &Message| { - Ok(ethkey::public_to_address(ðkey::recover(&signature, &message)?)) - }, - }); - - match finalize { - Some(finalize) => ConstructedVerifier::Unconfirmed(verifier, finality_proof, finalize), - None => ConstructedVerifier::Trusted(verifier), - } - } - Err(e) => ConstructedVerifier::Err(e), - } - } - - fn set_signer(&self, ap: Arc, address: Address, password: Password) { - { - self.signer.write().set(ap, address, password); - } - self.to_step(Step::Propose); - } - - fn sign(&self, hash: H256) -> Result { - Ok(self.signer.read().sign(hash)?) - } - - fn snapshot_components(&self) -> Option> { - Some(Box::new(::snapshot::PoaSnapshot)) - } - - fn stop(&self) { - } - - fn is_proposal(&self, header: &Header) -> bool { - let signatures_len = header.seal()[2].len(); - // Signatures have to be an empty list rlp. - if signatures_len != 1 { - // New Commit received, skip to next height. - trace!(target: "engine", "Received a commit: {:?}.", header.number()); - self.to_next_height(header.number() as usize); - self.to_step(Step::Commit); - return false; - } - let proposal = ConsensusMessage::new_proposal(header).expect("block went through full verification; this Engine verifies new_proposal creation; qed"); - let proposer = proposal.verify().expect("block went through full verification; this Engine tries verify; qed"); - debug!(target: "engine", "Received a new proposal {:?} from {}.", proposal.vote_step, proposer); - if self.is_view(&proposal) { - *self.proposal.write() = proposal.block_hash.clone(); - *self.proposal_parent.write() = header.parent_hash().clone(); - } - self.votes.vote(proposal, proposer); - true - } - - /// Equivalent to a timeout: to be used for tests. - fn step(&self) { - let next_step = match *self.step.read() { - Step::Propose => { - trace!(target: "engine", "Propose timeout."); - if self.proposal.read().is_none() { - // Report the proposer if no proposal was received. - let height = self.height.load(AtomicOrdering::SeqCst); - let current_proposer = self.view_proposer(&*self.proposal_parent.read(), height, self.view.load(AtomicOrdering::SeqCst)); - self.validators.report_benign(¤t_proposer, height as BlockNumber, height as BlockNumber); - } - Step::Prevote - }, - Step::Prevote if self.has_enough_any_votes() => { - trace!(target: "engine", "Prevote timeout."); - Step::Precommit - }, - Step::Prevote => { - trace!(target: "engine", "Prevote timeout without enough votes."); - self.broadcast_old_messages(); - Step::Prevote - }, - Step::Precommit if self.has_enough_any_votes() => { - trace!(target: "engine", "Precommit timeout."); - self.increment_view(1); - Step::Propose - }, - Step::Precommit => { - trace!(target: "engine", "Precommit timeout without enough votes."); - self.broadcast_old_messages(); - Step::Precommit - }, - Step::Commit => { - trace!(target: "engine", "Commit timeout."); - Step::Propose - }, - }; - self.to_step(next_step); - } - - fn register_client(&self, client: Weak) { - if let Some(c) = client.upgrade() { - self.height.store(c.chain_info().best_block_number as usize + 1, AtomicOrdering::SeqCst); - } - *self.client.write() = Some(client.clone()); - self.validators.register_client(client); - } - - fn fork_choice(&self, new: &ExtendedHeader, current: &ExtendedHeader) -> super::ForkChoice { - super::total_difficulty_fork_choice(new, current) - } -} - -#[cfg(test)] -mod tests { - use std::str::FromStr; - use rustc_hex::FromHex; - use ethereum_types::Address; - use bytes::Bytes; - use block::*; - use error::{Error, ErrorKind, BlockError}; - use header::Header; - use client::ChainInfo; - use miner::MinerService; - use test_helpers::{ - TestNotify, get_temp_state_db, generate_dummy_client, - generate_dummy_client_with_spec_and_accounts - }; - use account_provider::AccountProvider; - use spec::Spec; - use engines::{EthEngine, EngineError, Seal}; - use engines::epoch::EpochVerifier; - use super::*; - - /// Accounts inserted with "0" and "1" are validators. First proposer is "0". - fn setup() -> (Spec, Arc) { - let tap = Arc::new(AccountProvider::transient_provider()); - let spec = Spec::new_test_tendermint(); - (spec, tap) - } - - fn propose_default(spec: &Spec, proposer: Address) -> (ClosedBlock, Vec) { - let db = get_temp_state_db(); - let db = spec.ensure_db_good(db, &Default::default()).unwrap(); - let genesis_header = spec.genesis_header(); - let last_hashes = Arc::new(vec![genesis_header.hash()]); - let b = OpenBlock::new(spec.engine.as_ref(), Default::default(), false, db.boxed_clone(), &genesis_header, last_hashes, proposer, (3141562.into(), 31415620.into()), vec![], false, &mut Vec::new().into_iter()).unwrap(); - let b = b.close().unwrap(); - if let Seal::Proposal(seal) = spec.engine.generate_seal(b.block(), &genesis_header) { - (b, seal) - } else { - panic!() - } - } - - fn vote(engine: &EthEngine, signer: F, height: usize, view: usize, step: Step, block_hash: Option) -> Bytes where F: FnOnce(H256) -> Result { - let mi = message_info_rlp(&VoteStep::new(height, view, step), block_hash); - let m = message_full_rlp(&signer(keccak(&mi)).unwrap().into(), &mi); - engine.handle_message(&m).unwrap(); - m - } - - fn proposal_seal(tap: &Arc, header: &Header, view: View) -> Vec { - let author = header.author(); - let vote_info = message_info_rlp(&VoteStep::new(header.number() as Height, view, Step::Propose), Some(header.bare_hash())); - let signature = tap.sign(*author, None, keccak(vote_info)).unwrap(); - vec![ - ::rlp::encode(&view), - ::rlp::encode(&H520::from(signature)), - ::rlp::EMPTY_LIST_RLP.to_vec() - ] - } - - fn insert_and_unlock(tap: &Arc, acc: &str) -> Address { - let addr = tap.insert_account(keccak(acc).into(), &acc.into()).unwrap(); - tap.unlock_account_permanently(addr, acc.into()).unwrap(); - addr - } - - fn insert_and_register(tap: &Arc, engine: &EthEngine, acc: &str) -> Address { - let addr = insert_and_unlock(tap, acc); - engine.set_signer(tap.clone(), addr.clone(), acc.into()); - addr - } - - #[test] - fn has_valid_metadata() { - let engine = Spec::new_test_tendermint().engine; - assert!(!engine.name().is_empty()); - } - - #[test] - fn can_return_schedule() { - let engine = Spec::new_test_tendermint().engine; - let schedule = engine.schedule(10000000); - - assert!(schedule.stack_limit > 0); - } - - #[test] - fn verification_fails_on_short_seal() { - let engine = Spec::new_test_tendermint().engine; - let header = Header::default(); - - let verify_result = engine.verify_block_basic(&header); - - match verify_result { - Err(Error(ErrorKind::Block(BlockError::InvalidSealArity(_)), _)) => {}, - Err(_) => { panic!("should be block seal-arity mismatch error (got {:?})", verify_result); }, - _ => { panic!("Should be error, got Ok"); }, - } - } - - #[test] - fn allows_correct_proposer() { - let (spec, tap) = setup(); - let engine = spec.engine; - - let mut parent_header: Header = Header::default(); - parent_header.set_gas_limit(U256::from_str("222222").unwrap()); - - let mut header = Header::default(); - header.set_number(1); - header.set_gas_limit(U256::from_str("222222").unwrap()); - let validator = insert_and_unlock(&tap, "1"); - header.set_author(validator); - let seal = proposal_seal(&tap, &header, 0); - header.set_seal(seal); - // Good proposer. - assert!(engine.verify_block_external(&header).is_ok()); - - let validator = insert_and_unlock(&tap, "0"); - header.set_author(validator); - let seal = proposal_seal(&tap, &header, 0); - header.set_seal(seal); - // Bad proposer. - match engine.verify_block_external(&header) { - Err(Error(ErrorKind::Engine(EngineError::NotProposer(_)), _)) => {}, - _ => panic!(), - } - - let random = insert_and_unlock(&tap, "101"); - header.set_author(random); - let seal = proposal_seal(&tap, &header, 0); - header.set_seal(seal); - // Not authority. - match engine.verify_block_external(&header) { - Err(Error(ErrorKind::Engine(EngineError::NotAuthorized(_)), _)) => {}, - _ => panic!(), - }; - engine.stop(); - } - - #[test] - fn seal_signatures_checking() { - let (spec, tap) = setup(); - let engine = spec.engine; - - let mut parent_header: Header = Header::default(); - parent_header.set_gas_limit(U256::from_str("222222").unwrap()); - - let mut header = Header::default(); - header.set_number(2); - header.set_gas_limit(U256::from_str("222222").unwrap()); - let proposer = insert_and_unlock(&tap, "1"); - header.set_author(proposer); - let mut seal = proposal_seal(&tap, &header, 0); - - let vote_info = message_info_rlp(&VoteStep::new(2, 0, Step::Precommit), Some(header.bare_hash())); - let signature1 = tap.sign(proposer, None, keccak(&vote_info)).unwrap(); - - seal[1] = ::rlp::NULL_RLP.to_vec(); - seal[2] = ::rlp::encode_list(&vec![H520::from(signature1.clone())]); - header.set_seal(seal.clone()); - - // One good signature is not enough. - match engine.verify_block_external(&header) { - Err(Error(ErrorKind::Engine(EngineError::BadSealFieldSize(_)), _)) => {}, - _ => panic!(), - } - - let voter = insert_and_unlock(&tap, "0"); - let signature0 = tap.sign(voter, None, keccak(&vote_info)).unwrap(); - - seal[2] = ::rlp::encode_list(&vec![H520::from(signature1.clone()), H520::from(signature0.clone())]); - header.set_seal(seal.clone()); - - assert!(engine.verify_block_external(&header).is_ok()); - - let bad_voter = insert_and_unlock(&tap, "101"); - let bad_signature = tap.sign(bad_voter, None, keccak(vote_info)).unwrap(); - - seal[2] = ::rlp::encode_list(&vec![H520::from(signature1), H520::from(bad_signature)]); - header.set_seal(seal); - - // One good and one bad signature. - match engine.verify_block_external(&header) { - Err(Error(ErrorKind::Engine(EngineError::NotAuthorized(_)), _)) => {}, - _ => panic!(), - }; - engine.stop(); - } - - #[test] - fn can_generate_seal() { - let (spec, tap) = setup(); - - let proposer = insert_and_register(&tap, spec.engine.as_ref(), "1"); - - let (b, seal) = propose_default(&spec, proposer); - assert!(b.lock().try_seal(spec.engine.as_ref(), seal).is_ok()); - } - - #[test] - fn can_recognize_proposal() { - let (spec, tap) = setup(); - - let proposer = insert_and_register(&tap, spec.engine.as_ref(), "1"); - - let (b, seal) = propose_default(&spec, proposer); - let sealed = b.lock().seal(spec.engine.as_ref(), seal).unwrap(); - assert!(spec.engine.is_proposal(sealed.header())); - } - - #[test] - fn relays_messages() { - let (spec, tap) = setup(); - let engine = spec.engine.clone(); - - let v0 = insert_and_unlock(&tap, "0"); - let v1 = insert_and_register(&tap, engine.as_ref(), "1"); - - let h = 1; - let r = 0; - - // Propose - let (b, _) = propose_default(&spec, v1.clone()); - let proposal = Some(b.header().bare_hash()); - - let client = generate_dummy_client(0); - let notify = Arc::new(TestNotify::default()); - client.add_notify(notify.clone()); - engine.register_client(Arc::downgrade(&client) as _); - - let prevote_current = vote(engine.as_ref(), |mh| tap.sign(v0, None, mh).map(H520::from), h, r, Step::Prevote, proposal); - - let precommit_current = vote(engine.as_ref(), |mh| tap.sign(v0, None, mh).map(H520::from), h, r, Step::Precommit, proposal); - - let prevote_future = vote(engine.as_ref(), |mh| tap.sign(v0, None, mh).map(H520::from), h + 1, r, Step::Prevote, proposal); - - // Relays all valid present and future messages. - assert!(notify.messages.read().contains(&prevote_current)); - assert!(notify.messages.read().contains(&precommit_current)); - assert!(notify.messages.read().contains(&prevote_future)); - } - - #[test] - fn seal_submission() { - use ethkey::{Generator, Random}; - use transaction::{Transaction, Action}; - - let tap = Arc::new(AccountProvider::transient_provider()); - // Accounts for signing votes. - let v0 = insert_and_unlock(&tap, "0"); - let v1 = insert_and_unlock(&tap, "1"); - let client = generate_dummy_client_with_spec_and_accounts(Spec::new_test_tendermint, Some(tap.clone())); - let engine = client.engine(); - - client.miner().set_author(v1.clone(), Some("1".into())).unwrap(); - - let notify = Arc::new(TestNotify::default()); - client.add_notify(notify.clone()); - engine.register_client(Arc::downgrade(&client) as _); - - let keypair = Random.generate().unwrap(); - let transaction = Transaction { - action: Action::Create, - value: U256::zero(), - data: "3331600055".from_hex().unwrap(), - gas: U256::from(100_000), - gas_price: U256::zero(), - nonce: U256::zero(), - }.sign(keypair.secret(), None); - client.miner().import_own_transaction(client.as_ref(), transaction.into()).unwrap(); - - // Propose - let proposal = Some(client.miner().pending_block(0).unwrap().header.bare_hash()); - // Propose timeout - engine.step(); - - let h = 1; - let r = 0; - - // Prevote. - vote(engine, |mh| tap.sign(v1, None, mh).map(H520::from), h, r, Step::Prevote, proposal); - vote(engine, |mh| tap.sign(v0, None, mh).map(H520::from), h, r, Step::Prevote, proposal); - vote(engine, |mh| tap.sign(v1, None, mh).map(H520::from), h, r, Step::Precommit, proposal); - - assert_eq!(client.chain_info().best_block_number, 0); - // Last precommit. - vote(engine, |mh| tap.sign(v0, None, mh).map(H520::from), h, r, Step::Precommit, proposal); - assert_eq!(client.chain_info().best_block_number, 1); - } - - #[test] - fn epoch_verifier_verify_light() { - use ethkey::Error as EthkeyError; - - let (spec, tap) = setup(); - let engine = spec.engine; - - let mut parent_header: Header = Header::default(); - parent_header.set_gas_limit(U256::from_str("222222").unwrap()); - - let mut header = Header::default(); - header.set_number(2); - header.set_gas_limit(U256::from_str("222222").unwrap()); - let proposer = insert_and_unlock(&tap, "1"); - header.set_author(proposer); - let mut seal = proposal_seal(&tap, &header, 0); - - let vote_info = message_info_rlp(&VoteStep::new(2, 0, Step::Precommit), Some(header.bare_hash())); - let signature1 = tap.sign(proposer, None, keccak(&vote_info)).unwrap(); - - let voter = insert_and_unlock(&tap, "0"); - let signature0 = tap.sign(voter, None, keccak(&vote_info)).unwrap(); - - seal[1] = ::rlp::NULL_RLP.to_vec(); - seal[2] = ::rlp::encode_list(&vec![H520::from(signature1.clone())]); - header.set_seal(seal.clone()); - - let epoch_verifier = super::EpochVerifier { - subchain_validators: SimpleList::new(vec![proposer.clone(), voter.clone()]), - recover: { - let signature1 = signature1.clone(); - let signature0 = signature0.clone(); - let proposer = proposer.clone(); - let voter = voter.clone(); - move |s: &Signature, _: &Message| { - if *s == signature1 { - Ok(proposer) - } else if *s == signature0 { - Ok(voter) - } else { - Err(ErrorKind::Ethkey(EthkeyError::InvalidSignature).into()) - } - } - }, - }; - - // One good signature is not enough. - match epoch_verifier.verify_light(&header) { - Err(Error(ErrorKind::Engine(EngineError::BadSealFieldSize(_)), _)) => {}, - _ => panic!(), - } - - seal[2] = ::rlp::encode_list(&vec![H520::from(signature1.clone()), H520::from(signature0.clone())]); - header.set_seal(seal.clone()); - - assert!(epoch_verifier.verify_light(&header).is_ok()); - - let bad_voter = insert_and_unlock(&tap, "101"); - let bad_signature = tap.sign(bad_voter, None, keccak(&vote_info)).unwrap(); - - seal[2] = ::rlp::encode_list(&vec![H520::from(signature1), H520::from(bad_signature)]); - header.set_seal(seal); - - // One good and one bad signature. - match epoch_verifier.verify_light(&header) { - Err(Error(ErrorKind::Ethkey(EthkeyError::InvalidSignature), _)) => {}, - _ => panic!(), - }; - - engine.stop(); - } -} diff --git a/ethcore/src/engines/tendermint/params.rs b/ethcore/src/engines/tendermint/params.rs deleted file mode 100644 index fbd3839cadd..00000000000 --- a/ethcore/src/engines/tendermint/params.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! Tendermint specific parameters. - -use ethjson; -use std::time::Duration; -use ethereum_types::U256; -use super::super::validator_set::{ValidatorSet, new_validator_set}; -use super::super::transition::Timeouts; -use super::Step; - -/// `Tendermint` params. -pub struct TendermintParams { - /// List of validators. - pub validators: Box, - /// Timeout durations for different steps. - pub timeouts: TendermintTimeouts, - /// Reward per block in base units. - pub block_reward: U256, -} - -/// Base timeout of each step in ms. -#[derive(Debug, Clone)] -pub struct TendermintTimeouts { - pub propose: Duration, - pub prevote: Duration, - pub precommit: Duration, - pub commit: Duration, -} - -impl Default for TendermintTimeouts { - fn default() -> Self { - TendermintTimeouts { - propose: Duration::from_millis(1000), - prevote: Duration::from_millis(1000), - precommit: Duration::from_millis(1000), - commit: Duration::from_millis(1000), - } - } -} - -impl Timeouts for TendermintTimeouts { - fn initial(&self) -> Duration { - self.propose - } - - fn timeout(&self, step: &Step) -> Duration { - match *step { - Step::Propose => self.propose, - Step::Prevote => self.prevote, - Step::Precommit => self.precommit, - Step::Commit => self.commit, - } - } -} - -fn to_duration(ms: ethjson::uint::Uint) -> Duration { - let ms: usize = ms.into(); - Duration::from_millis(ms as u64) -} - -impl From for TendermintParams { - fn from(p: ethjson::spec::TendermintParams) -> Self { - let dt = TendermintTimeouts::default(); - TendermintParams { - validators: new_validator_set(p.validators), - timeouts: TendermintTimeouts { - propose: p.timeout_propose.map_or(dt.propose, to_duration), - prevote: p.timeout_prevote.map_or(dt.prevote, to_duration), - precommit: p.timeout_precommit.map_or(dt.precommit, to_duration), - commit: p.timeout_commit.map_or(dt.commit, to_duration), - }, - block_reward: p.block_reward.map_or(U256::default(), Into::into), - } - } -} diff --git a/ethcore/src/engines/transition.rs b/ethcore/src/engines/transition.rs deleted file mode 100644 index ddc9a70628f..00000000000 --- a/ethcore/src/engines/transition.rs +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! Engine timeout transitioning calls `Engine.step()` on timeout. - -use std::sync::Weak; -use std::time::Duration; -use io::{IoContext, IoHandler, TimerToken}; -use engines::Engine; -use parity_machine::Machine; - -/// Timeouts lookup -pub trait Timeouts: Send + Sync { - /// Return the first timeout. - fn initial(&self) -> Duration; - - /// Get a timeout based on step. - fn timeout(&self, step: &S) -> Duration; -} - -/// Timeout transition handling. -pub struct TransitionHandler { - engine: Weak>, - timeouts: Box>, -} - -impl TransitionHandler where S: Sync + Send + Clone { - /// New step caller by timeouts. - pub fn new(engine: Weak>, timeouts: Box>) -> Self { - TransitionHandler { - engine: engine, - timeouts: timeouts, - } - } -} - -/// Timer token representing the consensus step timeouts. -pub const ENGINE_TIMEOUT_TOKEN: TimerToken = 23; - -fn set_timeout(io: &IoContext, timeout: Duration) { - io.register_timer_once(ENGINE_TIMEOUT_TOKEN, timeout) - .unwrap_or_else(|e| warn!(target: "engine", "Failed to set consensus step timeout: {}.", e)) -} - -impl IoHandler for TransitionHandler - where S: Sync + Send + Clone + 'static, M: Machine -{ - fn initialize(&self, io: &IoContext) { - let initial = self.timeouts.initial(); - trace!(target: "engine", "Setting the initial timeout to {:?}.", initial); - set_timeout(io, initial); - } - - /// Call step after timeout. - fn timeout(&self, _io: &IoContext, timer: TimerToken) { - if timer == ENGINE_TIMEOUT_TOKEN { - if let Some(engine) = self.engine.upgrade() { - engine.step(); - } - } - } - - /// Set a new timer on message. - fn message(&self, io: &IoContext, next: &S) { - if let Err(io_err) = io.clear_timer(ENGINE_TIMEOUT_TOKEN) { - warn!(target: "engine", "Could not remove consensus timer {}.", io_err) - } - set_timeout(io, self.timeouts.timeout(next)); - } -} diff --git a/ethcore/src/engines/vote_collector.rs b/ethcore/src/engines/vote_collector.rs deleted file mode 100644 index f416d0c3f72..00000000000 --- a/ethcore/src/engines/vote_collector.rs +++ /dev/null @@ -1,349 +0,0 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! Collects votes on hashes at each Message::Round. - -use std::fmt::Debug; -use std::collections::{BTreeMap, HashSet, HashMap}; -use std::hash::Hash; -use ethereum_types::{H256, H520, Address}; -use parking_lot:: RwLock; -use bytes::Bytes; -use rlp::{Encodable, RlpStream}; - -pub trait Message: Clone + PartialEq + Eq + Hash + Encodable + Debug { - type Round: Clone + PartialEq + Eq + Hash + Default + Debug + Ord; - - fn signature(&self) -> H520; - - fn block_hash(&self) -> Option; - - fn round(&self) -> &Self::Round; - - fn is_broadcastable(&self) -> bool; -} - -/// Storing all Proposals, Prevotes and Precommits. -#[derive(Debug)] -pub struct VoteCollector { - votes: RwLock>>, -} - -#[derive(Debug, Default)] -struct StepCollector { - voted: HashMap, - block_votes: HashMap, HashMap>, - messages: HashSet, -} - -#[derive(Debug)] -pub struct DoubleVote { - author: Address, - vote_one: M, - vote_two: M, -} - -impl Encodable for DoubleVote { - fn rlp_append(&self, s: &mut RlpStream) { - s.begin_list(2) - .append(&self.vote_one) - .append(&self.vote_two); - } -} - -impl StepCollector { - /// Returns Some(&Address) when validator is double voting. - fn insert(&mut self, message: M, address: Address) -> Option> { - // Do nothing when message was seen. - if self.messages.insert(message.clone()) { - if let Some(previous) = self.voted.insert(address, message.clone()) { - // Bad validator sent a different message. - return Some(DoubleVote { - author: address, - vote_one: previous, - vote_two: message - }); - } else { - self - .block_votes - .entry(message.block_hash()) - .or_insert_with(HashMap::new) - .insert(message.signature(), address); - } - } - None - } - - /// Count all votes for the given block hash at this round. - fn count_block(&self, block_hash: &Option) -> usize { - self.block_votes.get(block_hash).map_or(0, HashMap::len) - } - - /// Count all votes collected for the given round. - fn count(&self) -> usize { - self.block_votes.values().map(HashMap::len).sum() - } -} - -#[derive(Debug)] -pub struct SealSignatures { - pub proposal: H520, - pub votes: Vec, -} - -impl PartialEq for SealSignatures { - fn eq(&self, other: &SealSignatures) -> bool { - self.proposal == other.proposal - && self.votes.iter().collect::>() == other.votes.iter().collect::>() - } -} - -impl Eq for SealSignatures {} - -impl Default for VoteCollector { - fn default() -> Self { - let mut collector = BTreeMap::new(); - // Insert dummy entry to fulfill invariant: "only messages newer than the oldest are inserted". - collector.insert(Default::default(), Default::default()); - VoteCollector { votes: RwLock::new(collector) } - } -} - -impl VoteCollector { - /// Insert vote if it is newer than the oldest one. - pub fn vote(&self, message: M, voter: Address) -> Option> { - self - .votes - .write() - .entry(message.round().clone()) - .or_insert_with(Default::default) - .insert(message, voter) - } - - /// Checks if the message should be ignored. - pub fn is_old_or_known(&self, message: &M) -> bool { - self - .votes - .read() - .get(&message.round()) - .map_or(false, |c| { - let is_known = c.messages.contains(message); - if is_known { trace!(target: "engine", "Known message: {:?}.", message); } - is_known - }) - || { - let guard = self.votes.read(); - let is_old = guard.keys().next().map_or(true, |oldest| message.round() <= oldest); - if is_old { trace!(target: "engine", "Old message {:?}.", message); } - is_old - } - } - - /// Throws out messages older than message, leaves message as marker for the oldest. - pub fn throw_out_old(&self, vote_round: &M::Round) { - let mut guard = self.votes.write(); - let new_collector = guard.split_off(vote_round); - *guard = new_collector; - } - - /// Collects the signatures for a given round and hash. - pub fn round_signatures(&self, round: &M::Round, block_hash: &H256) -> Vec { - let guard = self.votes.read(); - guard - .get(round) - .and_then(|c| c.block_votes.get(&Some(*block_hash))) - .map(|votes| votes.keys().cloned().collect()) - .unwrap_or_else(Vec::new) - } - - /// Count votes which agree with the given message. - pub fn count_aligned_votes(&self, message: &M) -> usize { - self - .votes - .read() - .get(&message.round()) - .map_or(0, |m| m.count_block(&message.block_hash())) - } - - /// Count all votes collected for a given round. - pub fn count_round_votes(&self, vote_round: &M::Round) -> usize { - self.votes.read().get(vote_round).map_or(0, StepCollector::count) - } - - /// Get all messages older than the round. - pub fn get_up_to(&self, round: &M::Round) -> Vec { - let guard = self.votes.read(); - guard - .iter() - .take_while(|&(r, _)| r <= round) - .map(|(_, c)| c.messages.iter().filter(|m| m.is_broadcastable()).map(|m| ::rlp::encode(m).to_vec()).collect::>()) - .fold(Vec::new(), |mut acc, mut messages| { acc.append(&mut messages); acc }) - } - - /// Retrieve address from which the message was sent from cache. - pub fn get(&self, message: &M) -> Option
{ - let guard = self.votes.read(); - guard.get(&message.round()).and_then(|c| c.block_votes.get(&message.block_hash())).and_then(|origins| origins.get(&message.signature()).cloned()) - } -} - -#[cfg(test)] -mod tests { - use hash::keccak; - use ethereum_types::{H160, H256}; - use rlp::*; - use super::*; - - #[derive(Debug, PartialEq, Eq, Clone, Hash, Default)] - struct TestMessage { - step: TestStep, - block_hash: Option, - signature: H520, - } - - type TestStep = u64; - - impl Message for TestMessage { - type Round = TestStep; - - fn signature(&self) -> H520 { self.signature } - - fn block_hash(&self) -> Option { self.block_hash } - - fn round(&self) -> &TestStep { &self.step } - - fn is_broadcastable(&self) -> bool { true } - } - - impl Encodable for TestMessage { - fn rlp_append(&self, s: &mut RlpStream) { - s.begin_list(3) - .append(&self.signature) - .append(&self.step) - .append(&self.block_hash.unwrap_or_else(H256::zero)); - } - } - - fn random_vote(collector: &VoteCollector, signature: H520, step: TestStep, block_hash: Option) -> bool { - full_vote(collector, signature, step, block_hash, H160::random()) - } - - fn full_vote(collector: &VoteCollector, signature: H520, step: TestStep, block_hash: Option, address: Address) -> bool { - collector.vote(TestMessage { signature: signature, step: step, block_hash: block_hash }, address).is_none() - } - - #[test] - fn seal_retrieval() { - let collector = VoteCollector::default(); - let bh = Some(keccak("1")); - let mut signatures = Vec::new(); - for _ in 0..5 { - signatures.push(H520::random()); - } - let propose_round = 3; - let commit_round = 5; - // Wrong round. - random_vote(&collector, signatures[4].clone(), 1, bh.clone()); - // Good proposal - random_vote(&collector, signatures[0].clone(), propose_round.clone(), bh.clone()); - // Wrong block proposal. - random_vote(&collector, signatures[0].clone(), propose_round.clone(), Some(keccak("0"))); - // Wrong block commit. - random_vote(&collector, signatures[3].clone(), commit_round.clone(), Some(keccak("0"))); - // Wrong round. - random_vote(&collector, signatures[0].clone(), 6, bh.clone()); - // Wrong round. - random_vote(&collector, signatures[0].clone(), 4, bh.clone()); - // Relevant commit. - random_vote(&collector, signatures[2].clone(), commit_round.clone(), bh.clone()); - // Replicated vote. - random_vote(&collector, signatures[2].clone(), commit_round.clone(), bh.clone()); - // Wrong round. - random_vote(&collector, signatures[4].clone(), 6, bh.clone()); - // Relevant precommit. - random_vote(&collector, signatures[1].clone(), commit_round.clone(), bh.clone()); - // Wrong round, same signature. - random_vote(&collector, signatures[1].clone(), 7, bh.clone()); - - assert_eq!(signatures[0..1].to_vec(), collector.round_signatures(&propose_round, &bh.unwrap())); - assert_eq!(signatures[1..3].iter().collect::>(), collector.round_signatures(&commit_round, &bh.unwrap()).iter().collect::>()); - } - - #[test] - fn count_votes() { - let collector = VoteCollector::default(); - let round1 = 1; - let round3 = 3; - // good 1 - random_vote(&collector, H520::random(), round1, Some(keccak("0"))); - random_vote(&collector, H520::random(), 0, Some(keccak("0"))); - // good 3 - random_vote(&collector, H520::random(), round3, Some(keccak("0"))); - random_vote(&collector, H520::random(), 2, Some(keccak("0"))); - // good prevote - random_vote(&collector, H520::random(), round1, Some(keccak("1"))); - // good prevote - let same_sig = H520::random(); - random_vote(&collector, same_sig.clone(), round1, Some(keccak("1"))); - random_vote(&collector, same_sig, round1, Some(keccak("1"))); - // good precommit - random_vote(&collector, H520::random(), round3, Some(keccak("1"))); - // good prevote - random_vote(&collector, H520::random(), round1, Some(keccak("0"))); - random_vote(&collector, H520::random(), 4, Some(keccak("2"))); - - assert_eq!(collector.count_round_votes(&round1), 4); - assert_eq!(collector.count_round_votes(&round3), 2); - - let message = TestMessage { - signature: H520::default(), - step: round1, - block_hash: Some(keccak("1")) - }; - assert_eq!(collector.count_aligned_votes(&message), 2); - } - - #[test] - fn remove_old() { - let collector = VoteCollector::default(); - let vote = |round, hash| { - random_vote(&collector, H520::random(), round, hash); - }; - vote(6, Some(keccak("0"))); - vote(3, Some(keccak("0"))); - vote(7, Some(keccak("0"))); - vote(8, Some(keccak("1"))); - vote(1, Some(keccak("1"))); - - collector.throw_out_old(&7); - assert_eq!(collector.count_round_votes(&1), 0); - assert_eq!(collector.count_round_votes(&3), 0); - assert_eq!(collector.count_round_votes(&6), 0); - assert_eq!(collector.count_round_votes(&7), 1); - assert_eq!(collector.count_round_votes(&8), 1); - } - - #[test] - fn malicious_authority() { - let collector = VoteCollector::default(); - let round = 3; - // Vote is inserted fine. - assert!(full_vote(&collector, H520::random(), round, Some(keccak("0")), Address::default())); - // Returns the double voting address. - assert!(!full_vote(&collector, H520::random(), round, Some(keccak("1")), Address::default())); - assert_eq!(collector.count_round_votes(&round), 1); - } -} diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 39dac1f2b21..c73887800c8 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -576,7 +576,7 @@ impl Miner { trace!(target: "miner", "requires_reseal: sealing enabled"); // Disable sealing if there were no requests for SEALING_TIMEOUT_IN_BLOCKS - let had_requests = sealing.last_request.map(|last_request| + let had_requests = sealing.last_request.map(|last_request| best_block.saturating_sub(last_request) <= SEALING_TIMEOUT_IN_BLOCKS ).unwrap_or(false); diff --git a/ethcore/src/spec/spec.rs b/ethcore/src/spec/spec.rs index e6e91137889..d6e1f92d949 100644 --- a/ethcore/src/spec/spec.rs +++ b/ethcore/src/spec/spec.rs @@ -35,7 +35,7 @@ use builtin::Builtin; use encoded; use engines::{ EthEngine, NullEngine, InstantSeal, InstantSealParams, BasicAuthority, Clique, - AuthorityRound, Tendermint, DEFAULT_BLOCKHASH_CONTRACT + AuthorityRound, DEFAULT_BLOCKHASH_CONTRACT }; use error::Error; use executive::Executive; @@ -501,7 +501,6 @@ fn load_from(spec_params: SpecParams, s: ethjson::spec::Spec) -> Result AuthorityRound::new(authority_round.params.into(), machine) .expect("Failed to start AuthorityRound consensus engine."), - ethjson::spec::Engine::Tendermint(tendermint) => Tendermint::new(tendermint.params.into(), machine) - .expect("Failed to start the Tendermint consensus engine."), } } @@ -957,14 +954,6 @@ impl Spec { load_bundled!("authority_round_block_reward_contract") } - /// Create a new Spec with Tendermint consensus which does internal sealing (not requiring - /// work). - /// Account keccak("0") and keccak("1") are a authorities. - #[cfg(any(test, feature = "test-helpers"))] - pub fn new_test_tendermint() -> Self { - load_bundled!("tendermint") - } - /// TestList.sol used in both specs: https://github.com/paritytech/contracts/pull/30/files /// Accounts with secrets keccak("0") and keccak("1") are initially the validators. /// Create a new Spec with BasicAuthority which uses a contract at address 5 to determine diff --git a/ethcore/src/verification/queue/mod.rs b/ethcore/src/verification/queue/mod.rs index 9b1597439b0..b9242f47b06 100644 --- a/ethcore/src/verification/queue/mod.rs +++ b/ethcore/src/verification/queue/mod.rs @@ -583,6 +583,13 @@ impl VerificationQueue { result } + /// Returns true if there is nothing currently in the queue. + /// TODO [ToDr] Optimize to avoid locking + pub fn is_empty(&self) -> bool { + let v = &self.verification; + v.unverified.lock().is_empty() && v.verifying.lock().is_empty() && v.verified.lock().is_empty() + } + /// Get queue status. pub fn queue_info(&self) -> QueueInfo { use std::mem::size_of; diff --git a/ethcore/sync/src/api.rs b/ethcore/sync/src/api.rs index 4296a4c2d07..7474d79b37f 100644 --- a/ethcore/sync/src/api.rs +++ b/ethcore/sync/src/api.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use std::sync::Arc; +use std::sync::{Arc, mpsc, atomic}; use std::collections::{HashMap, BTreeMap}; use std::io; use std::ops::Range; @@ -33,10 +33,10 @@ use ethcore::client::{BlockChainClient, ChainNotify, ChainRoute, ChainMessageTyp use ethcore::snapshot::SnapshotService; use ethcore::header::BlockNumber; use sync_io::NetSyncIo; -use chain::{ChainSync, SyncStatus as EthSyncStatus}; +use chain::{ChainSyncApi, SyncStatus as EthSyncStatus}; use std::net::{SocketAddr, AddrParseError}; use std::str::FromStr; -use parking_lot::RwLock; +use parking_lot::{RwLock, Mutex}; use chain::{ETH_PROTOCOL_VERSION_63, ETH_PROTOCOL_VERSION_62, PAR_PROTOCOL_VERSION_1, PAR_PROTOCOL_VERSION_2, PAR_PROTOCOL_VERSION_3, PRIVATE_TRANSACTION_PACKET, SIGNED_PRIVATE_TRANSACTION_PACKET}; @@ -228,6 +228,37 @@ impl AttachedProtocol { } } +/// A prioritized tasks run in a specialised timer. +/// Every task should be completed within a hard deadline, +/// if it's not it's either cancelled or split into multiple tasks. +/// NOTE These tasks might not complete at all, so anything +/// that happens here should work even if the task is cancelled. +#[derive(Debug)] +pub enum PriorityTask { + /// Propagate given block + PropagateBlock { + /// When the task was initiated + started: ::std::time::Instant, + /// Raw block RLP to propagate + block: Bytes, + /// Block hash + hash: H256, + /// Blocks difficulty + difficulty: U256, + }, + /// Propagate a list of transactions + PropagateTransactions(::std::time::Instant, Arc), +} +impl PriorityTask { + /// Mark the task as being processed, right after it's retrieved from the queue. + pub fn starting(&self) { + match *self { + PriorityTask::PropagateTransactions(_, ref is_ready) => is_ready.store(true, atomic::Ordering::SeqCst), + _ => {}, + } + } +} + /// EthSync initialization parameters. pub struct Params { /// Configuration. @@ -260,6 +291,8 @@ pub struct EthSync { subprotocol_name: [u8; 3], /// Light subprotocol name. light_subprotocol_name: [u8; 3], + /// Priority tasks notification channel + priority_tasks: Mutex>, } fn light_params( @@ -312,13 +345,19 @@ impl EthSync { }) }; - let chain_sync = ChainSync::new(params.config, &*params.chain, params.private_tx_handler.clone()); + let (priority_tasks_tx, priority_tasks_rx) = mpsc::channel(); + let sync = ChainSyncApi::new( + params.config, + &*params.chain, + params.private_tx_handler.clone(), + priority_tasks_rx, + ); let service = NetworkService::new(params.network_config.clone().into_basic()?, connection_filter)?; let sync = Arc::new(EthSync { network: service, eth_handler: Arc::new(SyncProtocolHandler { - sync: RwLock::new(chain_sync), + sync, chain: params.chain, snapshot_service: params.snapshot_service, overlay: RwLock::new(HashMap::new()), @@ -327,26 +366,32 @@ impl EthSync { subprotocol_name: params.config.subprotocol_name, light_subprotocol_name: params.config.light_subprotocol_name, attached_protos: params.attached_protos, + priority_tasks: Mutex::new(priority_tasks_tx), }); Ok(sync) } + + /// Priority tasks producer + pub fn priority_tasks(&self) -> mpsc::Sender { + self.priority_tasks.lock().clone() + } } impl SyncProvider for EthSync { /// Get sync status fn status(&self) -> EthSyncStatus { - self.eth_handler.sync.read().status() + self.eth_handler.sync.status() } /// Get sync peers fn peers(&self) -> Vec { self.network.with_context_eval(self.subprotocol_name, |ctx| { let peer_ids = self.network.connected_peers(); - let eth_sync = self.eth_handler.sync.read(); let light_proto = self.light_proto.as_ref(); - peer_ids.into_iter().filter_map(|peer_id| { + let peer_info = self.eth_handler.sync.peer_info(&peer_ids); + peer_ids.into_iter().zip(peer_info).filter_map(|(peer_id, peer_info)| { let session_info = match ctx.session_info(peer_id) { None => return None, Some(info) => info, @@ -358,7 +403,7 @@ impl SyncProvider for EthSync { capabilities: session_info.peer_capabilities.into_iter().map(|c| c.to_string()).collect(), remote_address: session_info.remote_address, local_address: session_info.local_address, - eth_info: eth_sync.peer_info(&peer_id), + eth_info: peer_info, pip_info: light_proto.as_ref().and_then(|lp| lp.peer_status(peer_id)).map(Into::into), }) }).collect() @@ -370,17 +415,17 @@ impl SyncProvider for EthSync { } fn transactions_stats(&self) -> BTreeMap { - let sync = self.eth_handler.sync.read(); - sync.transactions_stats() - .iter() - .map(|(hash, stats)| (*hash, stats.into())) - .collect() + self.eth_handler.sync.transactions_stats() } } const PEERS_TIMER: TimerToken = 0; -const SYNC_TIMER: TimerToken = 1; -const TX_TIMER: TimerToken = 2; +const MAINTAIN_SYNC_TIMER: TimerToken = 1; +const CONTINUE_SYNC_TIMER: TimerToken = 2; +const TX_TIMER: TimerToken = 3; +const PRIORITY_TIMER: TimerToken = 4; + +pub(crate) const PRIORITY_TIMER_INTERVAL: Duration = Duration::from_millis(250); struct SyncProtocolHandler { /// Shared blockchain client. @@ -388,7 +433,7 @@ struct SyncProtocolHandler { /// Shared snapshot service. snapshot_service: Arc, /// Sync strategy - sync: RwLock, + sync: ChainSyncApi, /// Chain overlay used to cache data such as fork block. overlay: RwLock>, } @@ -397,13 +442,16 @@ impl NetworkProtocolHandler for SyncProtocolHandler { fn initialize(&self, io: &NetworkContext) { if io.subprotocol_name() != WARP_SYNC_PROTOCOL_ID { io.register_timer(PEERS_TIMER, Duration::from_millis(700)).expect("Error registering peers timer"); - io.register_timer(SYNC_TIMER, Duration::from_millis(1100)).expect("Error registering sync timer"); + io.register_timer(MAINTAIN_SYNC_TIMER, Duration::from_millis(1100)).expect("Error registering sync timer"); + io.register_timer(CONTINUE_SYNC_TIMER, Duration::from_millis(2500)).expect("Error registering sync timer"); io.register_timer(TX_TIMER, Duration::from_millis(1300)).expect("Error registering transactions timer"); + + io.register_timer(PRIORITY_TIMER, PRIORITY_TIMER_INTERVAL).expect("Error registering peers timer"); } } fn read(&self, io: &NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) { - ChainSync::dispatch_packet(&self.sync, &mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay), *peer, packet_id, data); + self.sync.dispatch_packet(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay), *peer, packet_id, data); } fn connected(&self, io: &NetworkContext, peer: &PeerId) { @@ -428,16 +476,28 @@ impl NetworkProtocolHandler for SyncProtocolHandler { let mut io = NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay); match timer { PEERS_TIMER => self.sync.write().maintain_peers(&mut io), - SYNC_TIMER => self.sync.write().maintain_sync(&mut io), - TX_TIMER => { - self.sync.write().propagate_new_transactions(&mut io); - }, + MAINTAIN_SYNC_TIMER => self.sync.write().maintain_sync(&mut io), + CONTINUE_SYNC_TIMER => self.sync.write().continue_sync(&mut io), + TX_TIMER => self.sync.write().propagate_new_transactions(&mut io), + PRIORITY_TIMER => self.sync.process_priority_queue(&mut io), _ => warn!("Unknown timer {} triggered.", timer), } } } impl ChainNotify for EthSync { + fn block_pre_import(&self, bytes: &Bytes, hash: &H256, difficulty: &U256) { + let task = PriorityTask::PropagateBlock { + started: ::std::time::Instant::now(), + block: bytes.clone(), + hash: *hash, + difficulty: *difficulty, + }; + if let Err(e) = self.priority_tasks.lock().send(task) { + warn!(target: "sync", "Unexpected error during priority block propagation: {:?}", e); + } + } + fn new_blocks(&self, imported: Vec, invalid: Vec, diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index e6702974349..e1518d4f522 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -29,7 +29,6 @@ use rlp::Rlp; use snapshot::ChunkType; use std::cmp; use std::mem; -use std::collections::HashSet; use std::time::Instant; use sync_io::SyncIo; @@ -58,7 +57,6 @@ use super::{ SNAPSHOT_DATA_PACKET, SNAPSHOT_MANIFEST_PACKET, STATUS_PACKET, - TRANSACTIONS_PACKET, }; /// The Chain Sync Handler: handles responses from peers @@ -67,14 +65,9 @@ pub struct SyncHandler; impl SyncHandler { /// Handle incoming packet from peer pub fn on_packet(sync: &mut ChainSync, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - if packet_id != STATUS_PACKET && !sync.peers.contains_key(&peer) { - debug!(target:"sync", "Unexpected packet {} from unregistered peer: {}:{}", packet_id, peer, io.peer_info(peer)); - return; - } let rlp = Rlp::new(data); let result = match packet_id { STATUS_PACKET => SyncHandler::on_peer_status(sync, io, peer, &rlp), - TRANSACTIONS_PACKET => SyncHandler::on_peer_transactions(sync, io, peer, &rlp), BLOCK_HEADERS_PACKET => SyncHandler::on_peer_block_headers(sync, io, peer, &rlp), BLOCK_BODIES_PACKET => SyncHandler::on_peer_block_bodies(sync, io, peer, &rlp), RECEIPTS_PACKET => SyncHandler::on_peer_block_receipts(sync, io, peer, &rlp), @@ -104,15 +97,12 @@ impl SyncHandler { sync.sync_peer(io, peer, false); }, } - // give tasks to other peers - sync.continue_sync(io); } /// Called when peer sends us new consensus packet - pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) { trace!(target: "sync", "Received consensus packet from {:?}", peer_id); io.chain().queue_consensus_message(r.as_raw().to_vec()); - Ok(()) } /// Called by peer when it is disconnecting @@ -578,8 +568,8 @@ impl SyncHandler { asking_blocks: Vec::new(), asking_hash: None, ask_time: Instant::now(), - last_sent_transactions: HashSet::new(), - last_sent_private_transactions: HashSet::new(), + last_sent_transactions: Default::default(), + last_sent_private_transactions: Default::default(), expired: false, confirmation: if sync.fork_block.is_none() { ForkConfirmation::Confirmed } else { ForkConfirmation::Unconfirmed }, asking_snapshot_data: None, @@ -635,7 +625,7 @@ impl SyncHandler { } /// Called when peer sends us new transactions - fn on_peer_transactions(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), DownloaderImportError> { + pub fn on_peer_transactions(sync: &ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { // Accept transactions only when fully synced if !io.is_chain_queue_empty() || (sync.state != SyncState::Idle && sync.state != SyncState::NewBlocks) { trace!(target: "sync", "{} Ignoring transactions while syncing", peer_id); diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index e0fc8ecddb6..a01b25528f0 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -92,17 +92,17 @@ mod propagator; mod requester; mod supplier; -use std::sync::Arc; -use std::collections::{HashSet, HashMap}; +use std::sync::{Arc, mpsc}; +use std::collections::{HashSet, HashMap, BTreeMap}; use std::cmp; use std::time::{Duration, Instant}; use hash::keccak; use heapsize::HeapSizeOf; use ethereum_types::{H256, U256}; -use fastmap::H256FastMap; -use parking_lot::RwLock; +use fastmap::{H256FastMap, H256FastSet}; +use parking_lot::{Mutex, RwLock, RwLockWriteGuard}; use bytes::Bytes; -use rlp::{Rlp, RlpStream, DecoderError}; +use rlp::{RlpStream, DecoderError}; use network::{self, PeerId, PacketId}; use ethcore::header::{BlockNumber}; use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo, BlockQueueInfo}; @@ -112,7 +112,7 @@ use super::{WarpSync, SyncConfig}; use block_sync::{BlockDownloader, DownloadAction}; use rand::Rng; use snapshot::{Snapshot}; -use api::{EthProtocolInfo as PeerInfoDigest, WARP_SYNC_PROTOCOL_ID}; +use api::{EthProtocolInfo as PeerInfoDigest, WARP_SYNC_PROTOCOL_ID, PriorityTask}; use private_tx::PrivateTxHandler; use transactions_stats::{TransactionsStats, Stats as TransactionStats}; use transaction::UnverifiedTransaction; @@ -120,7 +120,7 @@ use transaction::UnverifiedTransaction; use self::handler::SyncHandler; use self::propagator::SyncPropagator; use self::requester::SyncRequester; -use self::supplier::SyncSupplier; +pub(crate) use self::supplier::SyncSupplier; known_heap_size!(0, PeerInfo); @@ -187,6 +187,11 @@ const FORK_HEADER_TIMEOUT: Duration = Duration::from_secs(3); const SNAPSHOT_MANIFEST_TIMEOUT: Duration = Duration::from_secs(5); const SNAPSHOT_DATA_TIMEOUT: Duration = Duration::from_secs(120); +/// Defines how much time we have to complete priority transaction or block propagation. +/// after the deadline is reached the task is considered finished +/// (so we might sent only to some part of the peers we originally intended to send to) +const PRIORITY_TASK_DEADLINE: Duration = Duration::from_millis(100); + #[derive(Copy, Clone, Eq, PartialEq, Debug)] /// Sync state pub enum SyncState { @@ -323,9 +328,9 @@ pub struct PeerInfo { /// Request timestamp ask_time: Instant, /// Holds a set of transactions recently sent to this peer to avoid spamming. - last_sent_transactions: HashSet, + last_sent_transactions: H256FastSet, /// Holds a set of private transactions and their signatures recently sent to this peer to avoid spamming. - last_sent_private_transactions: HashSet, + last_sent_private_transactions: H256FastSet, /// Pending request is expired and result should be ignored expired: bool, /// Peer fork confirmation status @@ -375,6 +380,217 @@ pub mod random { pub type RlpResponseResult = Result, PacketDecodeError>; pub type Peers = HashMap; +/// Thread-safe wrapper for `ChainSync`. +/// +/// NOTE always lock in order of fields declaration +pub struct ChainSyncApi { + /// Priority tasks queue + priority_tasks: Mutex>, + /// The rest of sync data + sync: RwLock, +} + +impl ChainSyncApi { + /// Creates new `ChainSyncApi` + pub fn new( + config: SyncConfig, + chain: &BlockChainClient, + private_tx_handler: Arc, + priority_tasks: mpsc::Receiver, + ) -> Self { + ChainSyncApi { + sync: RwLock::new(ChainSync::new(config, chain, private_tx_handler)), + priority_tasks: Mutex::new(priority_tasks), + } + } + + /// Gives `write` access to underlying `ChainSync` + pub fn write(&self) -> RwLockWriteGuard { + self.sync.write() + } + + /// Returns info about given list of peers + pub fn peer_info(&self, ids: &[PeerId]) -> Vec> { + let sync = self.sync.read(); + ids.iter().map(|id| sync.peer_info(id)).collect() + } + + /// Returns synchonization status + pub fn status(&self) -> SyncStatus { + self.sync.read().status() + } + + /// Returns transactions propagation statistics + pub fn transactions_stats(&self) -> BTreeMap { + self.sync.read().transactions_stats() + .iter() + .map(|(hash, stats)| (*hash, stats.into())) + .collect() + } + + /// Dispatch incoming requests and responses + pub fn dispatch_packet(&self, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { + SyncSupplier::dispatch_packet(&self.sync, io, peer, packet_id, data) + } + + /// Process a priority propagation queue. + /// This task is run from a timer and should be time constrained. + /// Hence we set up a deadline for the execution and cancel the task if the deadline is exceeded. + /// + /// NOTE This method should only handle stuff that can be canceled and would reach other peers + /// by other means. + pub fn process_priority_queue(&self, io: &mut SyncIo) { + fn check_deadline(deadline: Instant) -> Option { + let now = Instant::now(); + if now > deadline { + None + } else { + Some(deadline - now) + } + } + + // deadline to get the task from the queue + let deadline = Instant::now() + ::api::PRIORITY_TIMER_INTERVAL; + let mut work = || { + let task = { + let tasks = self.priority_tasks.try_lock_until(deadline)?; + let left = check_deadline(deadline)?; + tasks.recv_timeout(left).ok()? + }; + task.starting(); + // wait for the sync lock until deadline, + // note we might drop the task here if we won't manage to acquire the lock. + let mut sync = self.sync.try_write_until(deadline)?; + // since we already have everything let's use a different deadline + // to do the rest of the job now, so that previous work is not wasted. + let deadline = Instant::now() + PRIORITY_TASK_DEADLINE; + let as_ms = move |prev| { + let dur: Duration = Instant::now() - prev; + dur.as_secs() * 1_000 + dur.subsec_millis() as u64 + }; + match task { + // NOTE We can't simply use existing methods, + // cause the block is not in the DB yet. + PriorityTask::PropagateBlock { started, block, hash, difficulty } => { + // try to send to peers that are on the same block as us + // (they will most likely accept the new block). + let chain_info = io.chain().chain_info(); + let total_difficulty = chain_info.total_difficulty + difficulty; + let rlp = ChainSync::create_block_rlp(&block, total_difficulty); + for peers in sync.get_peers(&chain_info, PeerState::SameBlock).chunks(10) { + check_deadline(deadline)?; + for peer in peers { + SyncPropagator::send_packet(io, *peer, NEW_BLOCK_PACKET, rlp.clone()); + if let Some(ref mut peer) = sync.peers.get_mut(peer) { + peer.latest_hash = hash; + } + } + } + debug!(target: "sync", "Finished block propagation, took {}ms", as_ms(started)); + }, + PriorityTask::PropagateTransactions(time, _) => { + SyncPropagator::propagate_new_transactions(&mut sync, io, || { + check_deadline(deadline).is_some() + }); + debug!(target: "sync", "Finished transaction propagation, took {}ms", as_ms(time)); + }, + } + + Some(()) + }; + + // Process as many items as we can until the deadline is reached. + loop { + if work().is_none() { + return; + } + } + } +} + +// Static methods +impl ChainSync { + /// creates rlp to send for the tree defined by 'from' and 'to' hashes + fn create_new_hashes_rlp(chain: &BlockChainClient, from: &H256, to: &H256) -> Option { + match chain.tree_route(from, to) { + Some(route) => { + let uncles = chain.find_uncles(from).unwrap_or_else(Vec::new); + match route.blocks.len() { + 0 => None, + _ => { + let mut blocks = route.blocks; + blocks.extend(uncles); + let mut rlp_stream = RlpStream::new_list(blocks.len()); + for block_hash in blocks { + let mut hash_rlp = RlpStream::new_list(2); + let number = chain.block_header(BlockId::Hash(block_hash.clone())) + .expect("chain.tree_route and chain.find_uncles only return hahses of blocks that are in the blockchain. qed.").number(); + hash_rlp.append(&block_hash); + hash_rlp.append(&number); + rlp_stream.append_raw(hash_rlp.as_raw(), 1); + } + Some(rlp_stream.out()) + } + } + }, + None => None + } + } + + /// creates rlp from block bytes and total difficulty + fn create_block_rlp(bytes: &Bytes, total_difficulty: U256) -> Bytes { + let mut rlp_stream = RlpStream::new_list(2); + rlp_stream.append_raw(bytes, 1); + rlp_stream.append(&total_difficulty); + rlp_stream.out() + } + + /// creates latest block rlp for the given client + fn create_latest_block_rlp(chain: &BlockChainClient) -> Bytes { + Self::create_block_rlp( + &chain.block(BlockId::Hash(chain.chain_info().best_block_hash)) + .expect("Best block always exists").into_inner(), + chain.chain_info().total_difficulty + ) + } + + /// creates given hash block rlp for the given client + fn create_new_block_rlp(chain: &BlockChainClient, hash: &H256) -> Bytes { + Self::create_block_rlp( + &chain.block(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed").into_inner(), + chain.block_total_difficulty(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed.") + ) + } + + fn select_random_peers(peers: &[PeerId]) -> Vec { + // take sqrt(x) peers + let mut peers = peers.to_vec(); + let mut count = (peers.len() as f64).powf(0.5).round() as usize; + count = cmp::min(count, MAX_PEERS_PROPAGATION); + count = cmp::max(count, MIN_PEERS_PROPAGATION); + random::new().shuffle(&mut peers); + peers.truncate(count); + peers + } + + fn get_init_state(warp_sync: WarpSync, chain: &BlockChainClient) -> SyncState { + let best_block = chain.chain_info().best_block_number; + match warp_sync { + WarpSync::Enabled => SyncState::WaitingPeers, + WarpSync::OnlyAndAfter(block) if block > best_block => SyncState::WaitingPeers, + _ => SyncState::Idle, + } + } +} + +/// A peer query method for getting a list of peers +enum PeerState { + /// Peer is on different hash than us + Lagging, + /// Peer is on the same block as us + SameBlock +} + /// Blockchain sync handler. /// See module documentation for more details. pub struct ChainSync { @@ -417,10 +633,14 @@ pub struct ChainSync { impl ChainSync { /// Create a new instance of syncing strategy. - pub fn new(config: SyncConfig, chain: &BlockChainClient, private_tx_handler: Arc) -> ChainSync { + pub fn new( + config: SyncConfig, + chain: &BlockChainClient, + private_tx_handler: Arc, + ) -> Self { let chain_info = chain.chain_info(); let best_block = chain.chain_info().best_block_number; - let state = ChainSync::get_init_state(config.warp_sync, chain); + let state = Self::get_init_state(config.warp_sync, chain); let mut sync = ChainSync { state, @@ -445,15 +665,6 @@ impl ChainSync { sync } - fn get_init_state(warp_sync: WarpSync, chain: &BlockChainClient) -> SyncState { - let best_block = chain.chain_info().best_block_number; - match warp_sync { - WarpSync::Enabled => SyncState::WaitingPeers, - WarpSync::OnlyAndAfter(block) if block > best_block => SyncState::WaitingPeers, - _ => SyncState::Idle, - } - } - /// Returns synchonization status pub fn status(&self) -> SyncStatus { let last_imported_number = self.new_blocks.last_imported_block_number(); @@ -521,7 +732,7 @@ impl ChainSync { } } } - self.state = state.unwrap_or_else(|| ChainSync::get_init_state(self.warp_sync, io.chain())); + self.state = state.unwrap_or_else(|| Self::get_init_state(self.warp_sync, io.chain())); // Reactivate peers only if some progress has been made // since the last sync round of if starting fresh. self.active_peers = self.peers.keys().cloned().collect(); @@ -655,37 +866,35 @@ impl ChainSync { } /// Resume downloading - fn continue_sync(&mut self, io: &mut SyncIo) { - // Collect active peers that can sync - let confirmed_peers: Vec<(PeerId, u8)> = self.peers.iter().filter_map(|(peer_id, peer)| - if peer.can_sync() { - Some((*peer_id, peer.protocol_version)) - } else { - None - } - ).collect(); - - trace!( - target: "sync", - "Syncing with peers: {} active, {} confirmed, {} total", - self.active_peers.len(), confirmed_peers.len(), self.peers.len() - ); - + pub fn continue_sync(&mut self, io: &mut SyncIo) { if self.state == SyncState::Waiting { trace!(target: "sync", "Waiting for the block queue"); } else if self.state == SyncState::SnapshotWaiting { trace!(target: "sync", "Waiting for the snapshot restoration"); } else { - let mut peers: Vec<(PeerId, u8)> = confirmed_peers.iter().filter(|&&(peer_id, _)| - self.active_peers.contains(&peer_id) - ).map(|v| *v).collect(); + // Collect active peers that can sync + let mut peers: Vec<(PeerId, u8)> = self.peers.iter().filter_map(|(peer_id, peer)| + if peer.can_sync() && peer.asking == PeerAsking::Nothing && self.active_peers.contains(&peer_id) { + Some((*peer_id, peer.protocol_version)) + } else { + None + } + ).collect(); + + if peers.len() > 0 { + trace!( + target: "sync", + "Syncing with peers: {} active, {} available, {} total", + self.active_peers.len(), peers.len(), self.peers.len() + ); - random::new().shuffle(&mut peers); //TODO: sort by rating - // prefer peers with higher protocol version - peers.sort_by(|&(_, ref v1), &(_, ref v2)| v1.cmp(v2)); + random::new().shuffle(&mut peers); // TODO (#646): sort by rating + // prefer peers with higher protocol version + peers.sort_by(|&(_, ref v1), &(_, ref v2)| v1.cmp(v2)); - for (peer_id, _) in peers { - self.sync_peer(io, peer_id, false); + for (peer_id, _) in peers { + self.sync_peer(io, peer_id, false); + } } } @@ -1004,67 +1213,24 @@ impl ChainSync { } } - /// creates rlp to send for the tree defined by 'from' and 'to' hashes - fn create_new_hashes_rlp(chain: &BlockChainClient, from: &H256, to: &H256) -> Option { - match chain.tree_route(from, to) { - Some(route) => { - let uncles = chain.find_uncles(from).unwrap_or_else(Vec::new); - match route.blocks.len() { - 0 => None, - _ => { - let mut blocks = route.blocks; - blocks.extend(uncles); - let mut rlp_stream = RlpStream::new_list(blocks.len()); - for block_hash in blocks { - let mut hash_rlp = RlpStream::new_list(2); - let number = chain.block_header(BlockId::Hash(block_hash.clone())) - .expect("chain.tree_route and chain.find_uncles only return hahses of blocks that are in the blockchain. qed.").number(); - hash_rlp.append(&block_hash); - hash_rlp.append(&number); - rlp_stream.append_raw(hash_rlp.as_raw(), 1); - } - Some(rlp_stream.out()) - } - } - }, - None => None - } - } - - /// creates rlp from block bytes and total difficulty - fn create_block_rlp(bytes: &Bytes, total_difficulty: U256) -> Bytes { - let mut rlp_stream = RlpStream::new_list(2); - rlp_stream.append_raw(bytes, 1); - rlp_stream.append(&total_difficulty); - rlp_stream.out() - } - - /// creates latest block rlp for the given client - fn create_latest_block_rlp(chain: &BlockChainClient) -> Bytes { - ChainSync::create_block_rlp( - &chain.block(BlockId::Hash(chain.chain_info().best_block_hash)) - .expect("Best block always exists").into_inner(), - chain.chain_info().total_difficulty - ) - } - - /// creates given hash block rlp for the given client - fn create_new_block_rlp(chain: &BlockChainClient, hash: &H256) -> Bytes { - ChainSync::create_block_rlp( - &chain.block(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed").into_inner(), - chain.block_total_difficulty(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed.") - ) + /// returns peer ids that have different block than our chain + fn get_lagging_peers(&self, chain_info: &BlockChainInfo) -> Vec { + self.get_peers(chain_info, PeerState::Lagging) } - /// returns peer ids that have different blocks than our chain - fn get_lagging_peers(&mut self, chain_info: &BlockChainInfo) -> Vec { + /// returns peer ids that have different or the same blocks than our chain + fn get_peers(&self, chain_info: &BlockChainInfo, peers: PeerState) -> Vec { let latest_hash = chain_info.best_block_hash; self .peers - .iter_mut() + .iter() .filter_map(|(&id, ref mut peer_info)| { trace!(target: "sync", "Checking peer our best {} their best {}", latest_hash, peer_info.latest_hash); - if peer_info.latest_hash != latest_hash { + let matches = match peers { + PeerState::Lagging => peer_info.latest_hash != latest_hash, + PeerState::SameBlock => peer_info.latest_hash == latest_hash, + }; + if matches { Some(id) } else { None @@ -1073,17 +1239,6 @@ impl ChainSync { .collect::>() } - fn select_random_peers(peers: &[PeerId]) -> Vec { - // take sqrt(x) peers - let mut peers = peers.to_vec(); - let mut count = (peers.len() as f64).powf(0.5).round() as usize; - count = cmp::min(count, MAX_PEERS_PROPAGATION); - count = cmp::max(count, MIN_PEERS_PROPAGATION); - random::new().shuffle(&mut peers); - peers.truncate(count); - peers - } - fn get_consensus_peers(&self) -> Vec { self.peers.iter().filter_map(|(id, p)| if p.protocol_version >= PAR_PROTOCOL_VERSION_2.0 { Some(*id) } else { None }).collect() } @@ -1132,21 +1287,10 @@ impl ChainSync { } } - /// Dispatch incoming requests and responses - pub fn dispatch_packet(sync: &RwLock, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - SyncSupplier::dispatch_packet(sync, io, peer, packet_id, data) - } - pub fn on_packet(&mut self, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - debug!(target: "sync", "{} -> Dispatching packet: {}", peer, packet_id); SyncHandler::on_packet(self, io, peer, packet_id, data); } - /// Called when peer sends us new consensus packet - pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - SyncHandler::on_consensus_packet(io, peer_id, r) - } - /// Called by peer when it is disconnecting pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: PeerId) { SyncHandler::on_peer_aborting(self, io, peer); @@ -1158,8 +1302,16 @@ impl ChainSync { } /// propagates new transactions to all peers - pub fn propagate_new_transactions(&mut self, io: &mut SyncIo) -> usize { - SyncPropagator::propagate_new_transactions(self, io) + pub fn propagate_new_transactions(&mut self, io: &mut SyncIo) { + let deadline = Instant::now() + Duration::from_millis(500); + SyncPropagator::propagate_new_transactions(self, io, || { + if deadline > Instant::now() { + true + } else { + debug!(target: "sync", "Wasn't able to finish transaction propagation within a deadline."); + false + } + }); } /// Broadcast consensus message to peers. @@ -1175,7 +1327,7 @@ impl ChainSync { #[cfg(test)] pub mod tests { - use std::collections::{HashSet, VecDeque}; + use std::collections::{VecDeque}; use ethkey; use network::PeerId; use tests::helpers::{TestIo}; @@ -1291,8 +1443,8 @@ pub mod tests { asking_blocks: Vec::new(), asking_hash: None, ask_time: Instant::now(), - last_sent_transactions: HashSet::new(), - last_sent_private_transactions: HashSet::new(), + last_sent_transactions: Default::default(), + last_sent_private_transactions: Default::default(), expired: false, confirmation: super::ForkConfirmation::Confirmed, snapshot_number: None, @@ -1307,7 +1459,7 @@ pub mod tests { fn finds_lagging_peers() { let mut client = TestBlockChainClient::new(); client.add_blocks(100, EachBlockWith::Uncle); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10), &client); + let sync = dummy_sync_with_peer(client.block_hash_delta_minus(10), &client); let chain_info = client.chain_info(); let lagging_peers = sync.get_lagging_peers(&chain_info); @@ -1447,3 +1599,4 @@ pub mod tests { assert_eq!(status.status.transaction_count, 0); } } + diff --git a/ethcore/sync/src/chain/propagator.rs b/ethcore/sync/src/chain/propagator.rs index 82b592c4ba0..689ccfc02b1 100644 --- a/ethcore/sync/src/chain/propagator.rs +++ b/ethcore/sync/src/chain/propagator.rs @@ -18,6 +18,7 @@ use bytes::Bytes; use ethereum_types::H256; use ethcore::client::BlockChainInfo; use ethcore::header::BlockNumber; +use fastmap::H256FastSet; use network::{PeerId, PacketId}; use rand::Rng; use rlp::{Encodable, RlpStream}; @@ -69,49 +70,51 @@ impl SyncPropagator { /// propagates latest block to a set of peers pub fn propagate_blocks(sync: &mut ChainSync, chain_info: &BlockChainInfo, io: &mut SyncIo, blocks: &[H256], peers: &[PeerId]) -> usize { trace!(target: "sync", "Sending NewBlocks to {:?}", peers); - let mut sent = 0; - for peer_id in peers { - if blocks.is_empty() { - let rlp = ChainSync::create_latest_block_rlp(io.chain()); - SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp); - } else { - for h in blocks { - let rlp = ChainSync::create_new_block_rlp(io.chain(), h); - SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp); + let sent = peers.len(); + let mut send_packet = |io: &mut SyncIo, rlp: Bytes| { + for peer_id in peers { + SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp.clone()); + if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { + peer.latest_hash = chain_info.best_block_hash.clone(); } } - if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { - peer.latest_hash = chain_info.best_block_hash.clone(); + }; + + if blocks.is_empty() { + let rlp = ChainSync::create_latest_block_rlp(io.chain()); + send_packet(io, rlp); + } else { + for h in blocks { + let rlp = ChainSync::create_new_block_rlp(io.chain(), h); + send_packet(io, rlp); } - sent += 1; } + sent } /// propagates new known hashes to all peers pub fn propagate_new_hashes(sync: &mut ChainSync, chain_info: &BlockChainInfo, io: &mut SyncIo, peers: &[PeerId]) -> usize { trace!(target: "sync", "Sending NewHashes to {:?}", peers); - let mut sent = 0; let last_parent = *io.chain().best_block_header().parent_hash(); + let best_block_hash = chain_info.best_block_hash; + let rlp = match ChainSync::create_new_hashes_rlp(io.chain(), &last_parent, &best_block_hash) { + Some(rlp) => rlp, + None => return 0 + }; + + let sent = peers.len(); for peer_id in peers { - sent += match ChainSync::create_new_hashes_rlp(io.chain(), &last_parent, &chain_info.best_block_hash) { - Some(rlp) => { - { - if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { - peer.latest_hash = chain_info.best_block_hash.clone(); - } - } - SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_HASHES_PACKET, rlp); - 1 - }, - None => 0 + if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { + peer.latest_hash = best_block_hash; } + SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_HASHES_PACKET, rlp.clone()); } sent } /// propagates new transactions to all peers - pub fn propagate_new_transactions(sync: &mut ChainSync, io: &mut SyncIo) -> usize { + pub fn propagate_new_transactions bool>(sync: &mut ChainSync, io: &mut SyncIo, mut should_continue: F) -> usize { // Early out if nobody to send to. if sync.peers.is_empty() { return 0; @@ -122,6 +125,10 @@ impl SyncPropagator { return 0; } + if !should_continue() { + return 0; + } + let (transactions, service_transactions): (Vec<_>, Vec<_>) = transactions.iter() .map(|tx| tx.signed()) .partition(|tx| !tx.gas_price.is_zero()); @@ -130,24 +137,34 @@ impl SyncPropagator { let mut affected_peers = HashSet::new(); if !transactions.is_empty() { let peers = SyncPropagator::select_peers_for_transactions(sync, |_| true); - affected_peers = SyncPropagator::propagate_transactions_to_peers(sync, io, peers, transactions); + affected_peers = SyncPropagator::propagate_transactions_to_peers( + sync, io, peers, transactions, &mut should_continue, + ); } // most of times service_transactions will be empty // => there's no need to merge packets if !service_transactions.is_empty() { let service_transactions_peers = SyncPropagator::select_peers_for_transactions(sync, |peer_id| accepts_service_transaction(&io.peer_info(*peer_id))); - let service_transactions_affected_peers = SyncPropagator::propagate_transactions_to_peers(sync, io, service_transactions_peers, service_transactions); + let service_transactions_affected_peers = SyncPropagator::propagate_transactions_to_peers( + sync, io, service_transactions_peers, service_transactions, &mut should_continue + ); affected_peers.extend(&service_transactions_affected_peers); } affected_peers.len() } - fn propagate_transactions_to_peers(sync: &mut ChainSync, io: &mut SyncIo, peers: Vec, transactions: Vec<&SignedTransaction>) -> HashSet { + fn propagate_transactions_to_peers bool>( + sync: &mut ChainSync, + io: &mut SyncIo, + peers: Vec, + transactions: Vec<&SignedTransaction>, + mut should_continue: F, + ) -> HashSet { let all_transactions_hashes = transactions.iter() .map(|tx| tx.hash()) - .collect::>(); + .collect::(); let all_transactions_rlp = { let mut packet = RlpStream::new_list(transactions.len()); for tx in &transactions { packet.append(&**tx); } @@ -157,102 +174,104 @@ impl SyncPropagator { // Clear old transactions from stats sync.transactions_stats.retain(&all_transactions_hashes); - // sqrt(x)/x scaled to max u32 - let block_number = io.chain().chain_info().best_block_number; + let send_packet = |io: &mut SyncIo, peer_id: PeerId, sent: usize, rlp: Bytes| { + let size = rlp.len(); + SyncPropagator::send_packet(io, peer_id, TRANSACTIONS_PACKET, rlp); + trace!(target: "sync", "{:02} <- Transactions ({} entries; {} bytes)", peer_id, sent, size); + }; - let lucky_peers = { - peers.into_iter() - .filter_map(|peer_id| { - let stats = &mut sync.transactions_stats; - let peer_info = sync.peers.get_mut(&peer_id) - .expect("peer_id is form peers; peers is result of select_peers_for_transactions; select_peers_for_transactions selects peers from self.peers; qed"); - - // Send all transactions - if peer_info.last_sent_transactions.is_empty() { - // update stats - for hash in &all_transactions_hashes { - let id = io.peer_session_info(peer_id).and_then(|info| info.id); - stats.propagated(hash, id, block_number); - } - peer_info.last_sent_transactions = all_transactions_hashes.clone(); - return Some((peer_id, all_transactions_hashes.len(), all_transactions_rlp.clone())); - } + let block_number = io.chain().chain_info().best_block_number; + let mut sent_to_peers = HashSet::new(); + let mut max_sent = 0; - // Get hashes of all transactions to send to this peer - let to_send = all_transactions_hashes.difference(&peer_info.last_sent_transactions) - .cloned() - .collect::>(); - if to_send.is_empty() { - return None; - } + // for every peer construct and send transactions packet + for peer_id in peers { + if !should_continue() { + debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, sent_to_peers.len()); + return sent_to_peers; + } - // Construct RLP - let (packet, to_send) = { - let mut to_send = to_send; - let mut packet = RlpStream::new(); - packet.begin_unbounded_list(); - let mut pushed = 0; - for tx in &transactions { - let hash = tx.hash(); - if to_send.contains(&hash) { - let mut transaction = RlpStream::new(); - tx.rlp_append(&mut transaction); - let appended = packet.append_raw_checked(&transaction.drain(), 1, MAX_TRANSACTION_PACKET_SIZE); - if !appended { - // Maximal packet size reached just proceed with sending - debug!(target: "sync", "Transaction packet size limit reached. Sending incomplete set of {}/{} transactions.", pushed, to_send.len()); - to_send = to_send.into_iter().take(pushed).collect(); - break; - } - pushed += 1; - } - } - packet.complete_unbounded_list(); - (packet, to_send) - }; + let stats = &mut sync.transactions_stats; + let peer_info = sync.peers.get_mut(&peer_id) + .expect("peer_id is form peers; peers is result of select_peers_for_transactions; select_peers_for_transactions selects peers from self.peers; qed"); - // Update stats + // Send all transactions, if the peer doesn't know about anything + if peer_info.last_sent_transactions.is_empty() { + // update stats + for hash in &all_transactions_hashes { let id = io.peer_session_info(peer_id).and_then(|info| info.id); - for hash in &to_send { - // update stats - stats.propagated(hash, id, block_number); - } + stats.propagated(hash, id, block_number); + } + peer_info.last_sent_transactions = all_transactions_hashes.clone(); - peer_info.last_sent_transactions = all_transactions_hashes - .intersection(&peer_info.last_sent_transactions) - .chain(&to_send) - .cloned() - .collect(); - Some((peer_id, to_send.len(), packet.out())) - }) - .collect::>() - }; + send_packet(io, peer_id, all_transactions_hashes.len(), all_transactions_rlp.clone()); + sent_to_peers.insert(peer_id); + max_sent = cmp::max(max_sent, all_transactions_hashes.len()); + continue; + } + + // Get hashes of all transactions to send to this peer + let to_send = all_transactions_hashes.difference(&peer_info.last_sent_transactions) + .cloned() + .collect::>(); + if to_send.is_empty() { + continue; + } + + // Construct RLP + let (packet, to_send) = { + let mut to_send = to_send; + let mut packet = RlpStream::new(); + packet.begin_unbounded_list(); + let mut pushed = 0; + for tx in &transactions { + let hash = tx.hash(); + if to_send.contains(&hash) { + let mut transaction = RlpStream::new(); + tx.rlp_append(&mut transaction); + let appended = packet.append_raw_checked(&transaction.drain(), 1, MAX_TRANSACTION_PACKET_SIZE); + if !appended { + // Maximal packet size reached just proceed with sending + debug!(target: "sync", "Transaction packet size limit reached. Sending incomplete set of {}/{} transactions.", pushed, to_send.len()); + to_send = to_send.into_iter().take(pushed).collect(); + break; + } + pushed += 1; + } + } + packet.complete_unbounded_list(); + (packet, to_send) + }; - // Send RLPs - let mut peers = HashSet::new(); - if lucky_peers.len() > 0 { - let mut max_sent = 0; - let lucky_peers_len = lucky_peers.len(); - for (peer_id, sent, rlp) in lucky_peers { - peers.insert(peer_id); - let size = rlp.len(); - SyncPropagator::send_packet(io, peer_id, TRANSACTIONS_PACKET, rlp); - trace!(target: "sync", "{:02} <- Transactions ({} entries; {} bytes)", peer_id, sent, size); - max_sent = cmp::max(max_sent, sent); + // Update stats + let id = io.peer_session_info(peer_id).and_then(|info| info.id); + for hash in &to_send { + // update stats + stats.propagated(hash, id, block_number); } - debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, lucky_peers_len); + + peer_info.last_sent_transactions = all_transactions_hashes + .intersection(&peer_info.last_sent_transactions) + .chain(&to_send) + .cloned() + .collect(); + send_packet(io, peer_id, to_send.len(), packet.out()); + sent_to_peers.insert(peer_id); + max_sent = cmp::max(max_sent, to_send.len()); + } - peers + debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, sent_to_peers.len()); + sent_to_peers } pub fn propagate_latest_blocks(sync: &mut ChainSync, io: &mut SyncIo, sealed: &[H256]) { let chain_info = io.chain().chain_info(); if (((chain_info.best_block_number as i64) - (sync.last_sent_block_number as i64)).abs() as BlockNumber) < MAX_PEER_LAG_PROPAGATION { - let mut peers = sync.get_lagging_peers(&chain_info); + let peers = sync.get_lagging_peers(&chain_info); if sealed.is_empty() { let hashes = SyncPropagator::propagate_new_hashes(sync, &chain_info, io, &peers); - peers = ChainSync::select_random_peers(&peers); + let peers = ChainSync::select_random_peers(&peers); let blocks = SyncPropagator::propagate_blocks(sync, &chain_info, io, sealed, &peers); if blocks != 0 || hashes != 0 { trace!(target: "sync", "Sent latest {} blocks and {} hashes to peers.", blocks, hashes); @@ -318,7 +337,7 @@ impl SyncPropagator { } /// Generic packet sender - fn send_packet(sync: &mut SyncIo, peer_id: PeerId, packet_id: PacketId, packet: Bytes) { + pub fn send_packet(sync: &mut SyncIo, peer_id: PeerId, packet_id: PacketId, packet: Bytes) { if let Err(e) = sync.send(peer_id, packet_id, packet) { debug!(target:"sync", "Error sending packet: {:?}", e); sync.disconnect_peer(peer_id); @@ -419,8 +438,8 @@ mod tests { asking_blocks: Vec::new(), asking_hash: None, ask_time: Instant::now(), - last_sent_transactions: HashSet::new(), - last_sent_private_transactions: HashSet::new(), + last_sent_transactions: Default::default(), + last_sent_private_transactions: Default::default(), expired: false, confirmation: ForkConfirmation::Confirmed, snapshot_number: None, @@ -447,13 +466,13 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // Try to propagate same transactions for the second time - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // Even after new block transactions should not be propagated twice sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); // Try to propagate same transactions for the third time - let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // 1 message should be send assert_eq!(1, io.packets.len()); @@ -474,7 +493,7 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); io.chain.insert_transaction_to_queue(); // New block import should not trigger propagation. // (we only propagate on timeout) @@ -498,10 +517,10 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); // Try to propagate same transactions for the second time - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); assert_eq!(0, io.packets.len()); assert_eq!(0, peer_count); @@ -519,7 +538,7 @@ mod tests { // should sent some { let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); assert_eq!(1, io.packets.len()); assert_eq!(1, peer_count); } @@ -528,9 +547,9 @@ mod tests { let (peer_count2, peer_count3) = { let mut io = TestIo::new(&mut client, &ss, &queue, None); // Propagate new transactions - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // And now the peer should have all transactions - let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); (peer_count2, peer_count3) }; @@ -553,7 +572,7 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); let stats = sync.transactions_stats(); assert_eq!(stats.len(), 1, "Should maintain stats for single transaction.") @@ -583,7 +602,7 @@ mod tests { io.peers_info.insert(4, "Parity-Ethereum/v2.7.3-ABCDEFGH".to_owned()); // and new service transaction is propagated to peers - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // peer#2 && peer#4 are receiving service transaction assert!(io.packets.iter().any(|p| p.packet_id == 0x02 && p.recipient == 2)); // TRANSACTIONS_PACKET @@ -607,7 +626,7 @@ mod tests { io.peers_info.insert(1, "Parity-Ethereum/v2.6".to_owned()); // and service + non-service transactions are propagated to peers - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // two separate packets for peer are queued: // 1) with non-service-transaction diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 4bce0ef9850..eaee584cad0 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -27,6 +27,7 @@ use sync_io::SyncIo; use super::{ ChainSync, + SyncHandler, RlpResponseResult, PacketDecodeError, BLOCK_BODIES_PACKET, @@ -47,6 +48,8 @@ use super::{ RECEIPTS_PACKET, SNAPSHOT_DATA_PACKET, SNAPSHOT_MANIFEST_PACKET, + STATUS_PACKET, + TRANSACTIONS_PACKET, }; /// The Chain Sync Supplier: answers requests from peers with available data @@ -56,6 +59,7 @@ impl SyncSupplier { /// Dispatch incoming requests and responses pub fn dispatch_packet(sync: &RwLock, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { let rlp = Rlp::new(data); + let result = match packet_id { GET_BLOCK_BODIES_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, SyncSupplier::return_block_bodies, @@ -80,9 +84,39 @@ impl SyncSupplier { GET_SNAPSHOT_DATA_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, SyncSupplier::return_snapshot_data, |e| format!("Error sending snapshot data: {:?}", e)), - CONSENSUS_DATA_PACKET => ChainSync::on_consensus_packet(io, peer, &rlp), - _ => { + + STATUS_PACKET => { sync.write().on_packet(io, peer, packet_id, data); + Ok(()) + }, + // Packets that require the peer to be confirmed + _ => { + if !sync.read().peers.contains_key(&peer) { + debug!(target:"sync", "Unexpected packet {} from unregistered peer: {}:{}", packet_id, peer, io.peer_info(peer)); + return; + } + debug!(target: "sync", "{} -> Dispatching packet: {}", peer, packet_id); + + match packet_id { + CONSENSUS_DATA_PACKET => { + SyncHandler::on_consensus_packet(io, peer, &rlp) + }, + TRANSACTIONS_PACKET => { + let res = { + let sync_ro = sync.read(); + SyncHandler::on_peer_transactions(&*sync_ro, io, peer, &rlp) + }; + if res.is_err() { + // peer sent invalid data, disconnect. + io.disable_peer(peer); + sync.write().deactivate_peer(io, peer); + } + }, + _ => { + sync.write().on_packet(io, peer, packet_id, data); + } + } + Ok(()) } }; @@ -404,7 +438,7 @@ mod test { io.sender = Some(2usize); - ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_NODE_DATA_PACKET, &node_request); + SyncSupplier::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_NODE_DATA_PACKET, &node_request); assert_eq!(1, io.packets.len()); } @@ -446,7 +480,7 @@ mod test { assert_eq!(603, rlp_result.unwrap().1.out().len()); io.sender = Some(2usize); - ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_RECEIPTS_PACKET, &receipts_request); + SyncSupplier::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_RECEIPTS_PACKET, &receipts_request); assert_eq!(1, io.packets.len()); } } diff --git a/ethcore/sync/src/sync_io.rs b/ethcore/sync/src/sync_io.rs index c7704724c66..a5e9f7b2f44 100644 --- a/ethcore/sync/src/sync_io.rs +++ b/ethcore/sync/src/sync_io.rs @@ -52,7 +52,7 @@ pub trait SyncIo { fn protocol_version(&self, protocol: &ProtocolId, peer_id: PeerId) -> u8; /// Returns if the chain block queue empty fn is_chain_queue_empty(&self) -> bool { - self.chain().queue_info().is_empty() + self.chain().is_queue_empty() } /// Check if the session is expired fn is_expired(&self) -> bool; diff --git a/ethcore/sync/src/tests/consensus.rs b/ethcore/sync/src/tests/consensus.rs index 40a4edef394..4a6871d16b3 100644 --- a/ethcore/sync/src/tests/consensus.rs +++ b/ethcore/sync/src/tests/consensus.rs @@ -125,81 +125,3 @@ fn authority_round() { assert_eq!(ci1.best_block_number, 5); assert_eq!(ci0.best_block_hash, ci1.best_block_hash); } - -#[test] -fn tendermint() { - let s0 = KeyPair::from_secret_slice(&keccak("1")).unwrap(); - let s1 = KeyPair::from_secret_slice(&keccak("0")).unwrap(); - let ap = Arc::new(AccountProvider::transient_provider()); - ap.insert_account(s0.secret().clone(), &"".into()).unwrap(); - ap.insert_account(s1.secret().clone(), &"".into()).unwrap(); - - let chain_id = Spec::new_test_tendermint().chain_id(); - let mut net = TestNet::with_spec_and_accounts(2, SyncConfig::default(), Spec::new_test_tendermint, Some(ap)); - let io_handler0: Arc> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone())); - let io_handler1: Arc> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone())); - // Push transaction to both clients. Only one of them issues a proposal. - net.peer(0).miner.set_author(s0.address(), Some("".into())).unwrap(); - trace!(target: "poa", "Peer 0 is {}.", s0.address()); - net.peer(1).miner.set_author(s1.address(), Some("".into())).unwrap(); - trace!(target: "poa", "Peer 1 is {}.", s1.address()); - net.peer(0).chain.engine().register_client(Arc::downgrade(&net.peer(0).chain) as _); - net.peer(1).chain.engine().register_client(Arc::downgrade(&net.peer(1).chain) as _); - net.peer(0).chain.set_io_channel(IoChannel::to_handler(Arc::downgrade(&io_handler0))); - net.peer(1).chain.set_io_channel(IoChannel::to_handler(Arc::downgrade(&io_handler1))); - // Exhange statuses - net.sync(); - // Propose - net.peer(0).miner.import_own_transaction(&*net.peer(0).chain, new_tx(s0.secret(), 0.into(), chain_id)).unwrap(); - net.sync(); - // Propose timeout, synchronous for now - net.peer(0).chain.engine().step(); - net.peer(1).chain.engine().step(); - // Prevote, precommit and commit - net.sync(); - - assert_eq!(net.peer(0).chain.chain_info().best_block_number, 1); - assert_eq!(net.peer(1).chain.chain_info().best_block_number, 1); - - net.peer(1).miner.import_own_transaction(&*net.peer(1).chain, new_tx(s1.secret(), 0.into(), chain_id)).unwrap(); - // Commit timeout - net.peer(0).chain.engine().step(); - net.peer(1).chain.engine().step(); - // Propose - net.sync(); - // Propose timeout - net.peer(0).chain.engine().step(); - net.peer(1).chain.engine().step(); - // Prevote, precommit and commit - net.sync(); - assert_eq!(net.peer(0).chain.chain_info().best_block_number, 2); - assert_eq!(net.peer(1).chain.chain_info().best_block_number, 2); - - net.peer(0).miner.import_own_transaction(&*net.peer(0).chain, new_tx(s0.secret(), 1.into(), chain_id)).unwrap(); - net.peer(1).miner.import_own_transaction(&*net.peer(1).chain, new_tx(s1.secret(), 1.into(), chain_id)).unwrap(); - // Peers get disconnected. - // Commit - net.peer(0).chain.engine().step(); - net.peer(1).chain.engine().step(); - // Propose - net.peer(0).chain.engine().step(); - net.peer(1).chain.engine().step(); - net.peer(0).miner.import_own_transaction(&*net.peer(0).chain, new_tx(s0.secret(), 2.into(), chain_id)).unwrap(); - net.peer(1).miner.import_own_transaction(&*net.peer(1).chain, new_tx(s1.secret(), 2.into(), chain_id)).unwrap(); - // Send different prevotes - net.sync(); - // Prevote timeout - net.peer(0).chain.engine().step(); - net.peer(1).chain.engine().step(); - // Precommit and commit - net.sync(); - // Propose timeout - net.peer(0).chain.engine().step(); - net.peer(1).chain.engine().step(); - net.sync(); - let ci0 = net.peer(0).chain.chain_info(); - let ci1 = net.peer(1).chain.chain_info(); - assert_eq!(ci0.best_block_number, 3); - assert_eq!(ci1.best_block_number, 3); - assert_eq!(ci0.best_block_hash, ci1.best_block_hash); -} diff --git a/ethcore/sync/src/tests/helpers.rs b/ethcore/sync/src/tests/helpers.rs index d75d71ea90a..3eac91a0db1 100644 --- a/ethcore/sync/src/tests/helpers.rs +++ b/ethcore/sync/src/tests/helpers.rs @@ -33,7 +33,7 @@ use ethcore::test_helpers; use sync_io::SyncIo; use io::{IoChannel, IoContext, IoHandler}; use api::WARP_SYNC_PROTOCOL_ID; -use chain::{ChainSync, ETH_PROTOCOL_VERSION_63, PAR_PROTOCOL_VERSION_3, PRIVATE_TRANSACTION_PACKET, SIGNED_PRIVATE_TRANSACTION_PACKET}; +use chain::{ChainSync, ETH_PROTOCOL_VERSION_63, PAR_PROTOCOL_VERSION_3, PRIVATE_TRANSACTION_PACKET, SIGNED_PRIVATE_TRANSACTION_PACKET, SyncSupplier}; use SyncConfig; use private_tx::SimplePrivateTxHandler; @@ -271,7 +271,7 @@ impl Peer for EthPeer { fn receive_message(&self, from: PeerId, msg: TestPacket) -> HashSet { let mut io = TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, Some(from)); - ChainSync::dispatch_packet(&self.sync, &mut io, from, msg.packet_id, &msg.data); + SyncSupplier::dispatch_packet(&self.sync, &mut io, from, msg.packet_id, &msg.data); self.chain.flush(); io.to_disconnect.clone() } @@ -286,10 +286,12 @@ impl Peer for EthPeer { } fn sync_step(&self) { + let mut io = TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None); self.chain.flush(); - self.sync.write().maintain_peers(&mut TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None)); - self.sync.write().maintain_sync(&mut TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None)); - self.sync.write().propagate_new_transactions(&mut TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None)); + self.sync.write().maintain_peers(&mut io); + self.sync.write().maintain_sync(&mut io); + self.sync.write().continue_sync(&mut io); + self.sync.write().propagate_new_transactions(&mut io); } fn restart_sync(&self) { diff --git a/ethcore/sync/src/transactions_stats.rs b/ethcore/sync/src/transactions_stats.rs index 4d11dcf6809..7d5e2ca4a86 100644 --- a/ethcore/sync/src/transactions_stats.rs +++ b/ethcore/sync/src/transactions_stats.rs @@ -15,6 +15,7 @@ // along with Parity. If not, see . use api::TransactionStats; +use std::hash::BuildHasher; use std::collections::{HashSet, HashMap}; use ethereum_types::{H256, H512}; use fastmap::H256FastMap; @@ -74,7 +75,7 @@ impl TransactionsStats { } /// Retains only transactions present in given `HashSet`. - pub fn retain(&mut self, hashes: &HashSet) { + pub fn retain(&mut self, hashes: &HashSet) { let to_remove = self.pending_transactions.keys() .filter(|hash| !hashes.contains(hash)) .cloned() diff --git a/json/src/spec/engine.rs b/json/src/spec/engine.rs index e18220367d2..a9507545b6e 100644 --- a/json/src/spec/engine.rs +++ b/json/src/spec/engine.rs @@ -16,7 +16,7 @@ //! Engine deserialization. -use super::{Ethash, BasicAuthority, AuthorityRound, Tendermint, NullEngine, InstantSeal, Clique}; +use super::{Ethash, BasicAuthority, AuthorityRound, NullEngine, InstantSeal, Clique}; /// Engine deserialization. #[derive(Debug, PartialEq, Deserialize)] @@ -34,11 +34,8 @@ pub enum Engine { BasicAuthority(BasicAuthority), /// AuthorityRound engine. AuthorityRound(AuthorityRound), - /// Tendermint engine. - #[serde(rename="tendermint")] - Tendermint(Tendermint), /// Clique engine. - #[serde(rename="clique")] + #[serde(rename="Clique")] Clique(Clique) } @@ -49,19 +46,6 @@ mod tests { #[test] fn engine_deserialization() { - let s = r#"{ - "clique": { - "params": { - } - } - }"#; - - let deserialized: Engine = serde_json::from_str(s).unwrap(); - match deserialized { - Engine::Clique(_) => {}, // unit test in its own file. - _ => panic!(), - } - let s = r#"{ "null": { "params": { @@ -150,20 +134,5 @@ mod tests { Engine::AuthorityRound(_) => {}, // AuthorityRound is unit tested in its own file. _ => panic!(), }; - - let s = r#"{ - "tendermint": { - "params": { - "validators": { - "list": ["0xc6d9d2cd449a754c494264e1809c50e34d64562b"] - } - } - } - }"#; - let deserialized: Engine = serde_json::from_str(s).unwrap(); - match deserialized { - Engine::Tendermint(_) => {}, // Tendermint is unit tested in its own file. - _ => panic!(), - }; } } diff --git a/json/src/spec/mod.rs b/json/src/spec/mod.rs index 19f9cd29f83..6e38991ada3 100644 --- a/json/src/spec/mod.rs +++ b/json/src/spec/mod.rs @@ -28,7 +28,6 @@ pub mod ethash; pub mod validator_set; pub mod basic_authority; pub mod authority_round; -pub mod tendermint; pub mod null_engine; pub mod instant_seal; pub mod hardcoded_sync; @@ -46,7 +45,6 @@ pub use self::ethash::{Ethash, EthashParams, BlockReward}; pub use self::validator_set::ValidatorSet; pub use self::basic_authority::{BasicAuthority, BasicAuthorityParams}; pub use self::authority_round::{AuthorityRound, AuthorityRoundParams}; -pub use self::tendermint::{Tendermint, TendermintParams}; pub use self::clique::{Clique, CliqueParams}; pub use self::null_engine::{NullEngine, NullEngineParams}; pub use self::instant_seal::{InstantSeal, InstantSealParams}; diff --git a/json/src/spec/tendermint.rs b/json/src/spec/tendermint.rs deleted file mode 100644 index 4cea89edfac..00000000000 --- a/json/src/spec/tendermint.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! Tendermint params deserialization. - -use uint::Uint; -use super::ValidatorSet; - -/// Tendermint params deserialization. -#[derive(Debug, PartialEq, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -pub struct TendermintParams { - /// Valid validators. - pub validators: ValidatorSet, - /// Propose step timeout in milliseconds. - pub timeout_propose: Option, - /// Prevote step timeout in milliseconds. - pub timeout_prevote: Option, - /// Precommit step timeout in milliseconds. - pub timeout_precommit: Option, - /// Commit step timeout in milliseconds. - pub timeout_commit: Option, - /// Reward per block. - pub block_reward: Option, -} - -/// Tendermint engine deserialization. -#[derive(Debug, PartialEq, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct Tendermint { - /// Ethash params. - pub params: TendermintParams, -} - -#[cfg(test)] -mod tests { - use serde_json; - use ethereum_types::H160; - use hash::Address; - use spec::tendermint::Tendermint; - use spec::validator_set::ValidatorSet; - - #[test] - fn tendermint_deserialization() { - let s = r#"{ - "params": { - "validators": { - "list": ["0xc6d9d2cd449a754c494264e1809c50e34d64562b"] - } - } - }"#; - - let deserialized: Tendermint = serde_json::from_str(s).unwrap(); - let vs = ValidatorSet::List(vec![Address(H160::from("0xc6d9d2cd449a754c494264e1809c50e34d64562b"))]); - assert_eq!(deserialized.params.validators, vs); - } -} diff --git a/miner/src/pool/listener.rs b/miner/src/pool/listener.rs index 5776ba84532..f5e76e15d52 100644 --- a/miner/src/pool/listener.rs +++ b/miner/src/pool/listener.rs @@ -50,6 +50,10 @@ impl Notifier { /// Notify listeners about all currently pending transactions. pub fn notify(&mut self) { + if self.pending.is_empty() { + return; + } + for l in &self.listeners { (l)(&self.pending); } diff --git a/parity/modules.rs b/parity/modules.rs index e12e8ee4583..ac84aea5f21 100644 --- a/parity/modules.rs +++ b/parity/modules.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use std::sync::Arc; +use std::sync::{Arc, mpsc}; use ethcore::client::BlockChainClient; use sync::{self, AttachedProtocol, SyncConfig, NetworkConfiguration, Params, ConnectionFilter}; @@ -25,12 +25,17 @@ pub use sync::{EthSync, SyncProvider, ManageNetwork, PrivateTxHandler}; pub use ethcore::client::ChainNotify; use ethcore_logger::Config as LogConfig; -pub type SyncModules = (Arc, Arc, Arc); +pub type SyncModules = ( + Arc, + Arc, + Arc, + mpsc::Sender, +); pub fn sync( - sync_cfg: SyncConfig, - net_cfg: NetworkConfiguration, - client: Arc, + config: SyncConfig, + network_config: NetworkConfiguration, + chain: Arc, snapshot_service: Arc, private_tx_handler: Arc, provider: Arc, @@ -39,15 +44,20 @@ pub fn sync( connection_filter: Option>, ) -> Result { let eth_sync = EthSync::new(Params { - config: sync_cfg, - chain: client, - provider: provider, - snapshot_service: snapshot_service, + config, + chain, + provider, + snapshot_service, private_tx_handler, - network_config: net_cfg, - attached_protos: attached_protos, + network_config, + attached_protos, }, connection_filter)?; - Ok((eth_sync.clone() as Arc, eth_sync.clone() as Arc, eth_sync.clone() as Arc)) + Ok(( + eth_sync.clone() as Arc, + eth_sync.clone() as Arc, + eth_sync.clone() as Arc, + eth_sync.priority_tasks() + )) } diff --git a/parity/run.rs b/parity/run.rs index 62323cb4e10..45c4c5495bb 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use std::any::Any; -use std::sync::{Arc, Weak}; +use std::sync::{Arc, Weak, atomic}; use std::time::{Duration, Instant}; use std::thread; @@ -480,7 +480,6 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: cmd.gas_pricer_conf.to_gas_pricer(fetch.clone(), runtime.executor()), &spec, Some(account_provider.clone()), - )); miner.set_author(cmd.miner_extras.author, None).expect("Fails only if password is Some; password is None; qed"); miner.set_gas_range_target(cmd.miner_extras.gas_range_target); @@ -637,7 +636,7 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: }; // create sync object - let (sync_provider, manage_network, chain_notify) = modules::sync( + let (sync_provider, manage_network, chain_notify, priority_tasks) = modules::sync( sync_config, net_conf.clone().into(), client.clone(), @@ -651,6 +650,18 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: service.add_notify(chain_notify.clone()); + // Propagate transactions as soon as they are imported. + let tx = ::parking_lot::Mutex::new(priority_tasks); + let is_ready = Arc::new(atomic::AtomicBool::new(true)); + miner.add_transactions_listener(Box::new(move |_hashes| { + // we want to have only one PendingTransactions task in the queue. + if is_ready.compare_and_swap(true, false, atomic::Ordering::SeqCst) { + let task = ::sync::PriorityTask::PropagateTransactions(Instant::now(), is_ready.clone()); + // we ignore error cause it means that we are closing + let _ = tx.lock().send(task); + } + })); + // provider not added to a notification center is effectively disabled // TODO [debris] refactor it later on if cmd.private_tx_enabled { @@ -737,7 +748,7 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: let secretstore_deps = secretstore::Dependencies { client: client.clone(), sync: sync_provider.clone(), - miner: miner, + miner: miner.clone(), account_provider: account_provider, accounts_passwords: &passwords, }; diff --git a/rpc/src/v1/helpers/mod.rs b/rpc/src/v1/helpers/mod.rs index 0259818afa4..c770836fb47 100644 --- a/rpc/src/v1/helpers/mod.rs +++ b/rpc/src/v1/helpers/mod.rs @@ -36,8 +36,10 @@ mod signing_queue; mod subscribers; mod subscription_manager; mod work; +mod signature; pub use self::dispatch::{Dispatcher, FullDispatcher, LightDispatcher}; +pub use self::signature::verify_signature; pub use self::network_settings::NetworkSettings; pub use self::poll_manager::PollManager; pub use self::poll_filter::{PollFilter, SyncPollFilter, limit_logs}; diff --git a/rpc/src/v1/helpers/signature.rs b/rpc/src/v1/helpers/signature.rs new file mode 100644 index 00000000000..47664fc74bf --- /dev/null +++ b/rpc/src/v1/helpers/signature.rs @@ -0,0 +1,182 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use ethkey::{recover, public_to_address, Signature}; +use jsonrpc_core::Result; +use v1::types::{Bytes, RecoveredAccount, H256, U64}; +use v1::helpers::errors; +use v1::helpers::dispatch::eth_data_hash; +use hash::keccak; + +/// helper method for parity_verifySignature +pub fn verify_signature( + is_prefixed: bool, + message: Bytes, + r: H256, + s: H256, + v: U64, + chain_id: Option +) -> Result { + let hash = if is_prefixed { + eth_data_hash(message.0) + } else { + keccak(message.0) + }; + let v: u64 = v.into(); + let is_valid_for_current_chain = match (chain_id, v) { + (None, v) if v == 0 || v == 1 => true, + (Some(chain_id), v) if v >= 35 => (v - 35) / 2 == chain_id, + _ => false, + }; + + let v = if v >= 35 { (v - 1) % 2 } else { v }; + + let signature = Signature::from_rsv(&r.into(), &s.into(), v as u8); + let public_key = recover(&signature, &hash).map_err(errors::encryption)?; + let address = public_to_address(&public_key); + Ok(RecoveredAccount { address, public_key, is_valid_for_current_chain }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use ethcore::account_provider::AccountProvider; + use v1::types::H160; + + pub fn add_chain_replay_protection(v: u64, chain_id: Option) -> u64 { + v + if let Some(n) = chain_id { 35 + n * 2 } else { 0 } + } + + struct TestCase { + should_prefix: bool, + signing_chain_id: Option, + rpc_chain_id: Option, + is_valid_for_current_chain: bool, + } + + /// mocked signer + fn sign(should_prefix: bool, data: Vec, signing_chain_id: Option) -> (H160, [u8; 32], [u8; 32], U64) { + let hash = if should_prefix { eth_data_hash(data) } else { keccak(data) }; + let accounts = Arc::new(AccountProvider::transient_provider()); + let address = accounts.new_account(&"password123".into()).unwrap(); + let sig = accounts.sign(address, Some("password123".into()), hash).unwrap(); + let (r, s, v) = (sig.r(), sig.s(), sig.v()); + let v = add_chain_replay_protection(v as u64, signing_chain_id); + let (r_buf, s_buf) = { + let (mut r_buf, mut s_buf) = ([0u8; 32], [0u8; 32]); + r_buf.copy_from_slice(r); + s_buf.copy_from_slice(s); + (r_buf, s_buf) + }; + (address.into(), r_buf, s_buf, v.into()) + } + + fn run_test(test_case: TestCase) { + let TestCase { should_prefix, signing_chain_id, rpc_chain_id, is_valid_for_current_chain } = test_case; + let data = vec![5u8]; + + let (address, r, s, v) = sign(should_prefix, data.clone(), signing_chain_id); + let account = verify_signature(should_prefix, data.into(), r.into(), s.into(), v, rpc_chain_id).unwrap(); + + assert_eq!(account.address, address.into()); + assert_eq!(account.is_valid_for_current_chain, is_valid_for_current_chain) + } + + #[test] + fn test_verify_signature_prefixed_mainnet() { + run_test(TestCase { + should_prefix: true, + signing_chain_id: Some(1), + rpc_chain_id: Some(1), + is_valid_for_current_chain: true, + }) + } + + #[test] + fn test_verify_signature_not_prefixed_mainnet() { + run_test(TestCase { + should_prefix: false, + signing_chain_id: Some(1), + rpc_chain_id: Some(1), + is_valid_for_current_chain: true, + }) + } + + #[test] + fn test_verify_signature_incompatible_chain_id() { + run_test(TestCase { + should_prefix: false, + signing_chain_id: Some(65), + rpc_chain_id: Some(1), + is_valid_for_current_chain: false, + }); + run_test(TestCase { + should_prefix: true, + signing_chain_id: Some(65), + rpc_chain_id: Some(1), + is_valid_for_current_chain: false, + }); + } + + #[test] + fn test_verify_signature_no_signing_chain_id() { + run_test(TestCase { + should_prefix: false, + signing_chain_id: None, + rpc_chain_id: Some(1), + is_valid_for_current_chain: false, + }); + run_test(TestCase { + should_prefix: true, + signing_chain_id: None, + rpc_chain_id: Some(1), + is_valid_for_current_chain: false, + }); + } + + #[test] + fn test_verify_signature_no_rpc_chain_id() { + run_test(TestCase { + should_prefix: false, + signing_chain_id: Some(1), + rpc_chain_id: None, + is_valid_for_current_chain: false, + }); + run_test(TestCase { + should_prefix: true, + signing_chain_id: Some(1), + rpc_chain_id: None, + is_valid_for_current_chain: false, + }); + } + + #[test] + fn test_verify_signature_no_chain_replay_protection() { + run_test(TestCase { + should_prefix: false, + signing_chain_id: None, + rpc_chain_id: None, + is_valid_for_current_chain: true, + }); + run_test(TestCase { + should_prefix: true, + signing_chain_id: None, + rpc_chain_id: None, + is_valid_for_current_chain: true, + }); + } +} diff --git a/rpc/src/v1/impls/light/parity.rs b/rpc/src/v1/impls/light/parity.rs index 3d5bf78977f..c281760f3e2 100644 --- a/rpc/src/v1/impls/light/parity.rs +++ b/rpc/src/v1/impls/light/parity.rs @@ -30,18 +30,18 @@ use ethcore_logger::RotatingLogger; use jsonrpc_core::{Result, BoxFuture}; use jsonrpc_core::futures::Future; use jsonrpc_macros::Trailing; -use v1::helpers::{self, errors, ipfs, SigningQueue, SignerService, NetworkSettings}; +use v1::helpers::{self, errors, ipfs, SigningQueue, SignerService, NetworkSettings, verify_signature}; use v1::helpers::dispatch::LightDispatcher; use v1::helpers::light_fetch::{LightFetch, light_all_transactions}; use v1::metadata::Metadata; use v1::traits::Parity; use v1::types::{ - Bytes, U256, H64, H160, H256, H512, CallRequest, + Bytes, U256, U64, H64, H160, H256, H512, CallRequest, Peers, Transaction, RpcSettings, Histogram, TransactionStats, LocalTransactionStatus, - BlockNumber, LightBlockNumber, ConsensusCapability, VersionInfo, - OperationsInfo, ChainStatus, - AccountInfo, HwAccountInfo, Header, RichHeader, Receipt, + LightBlockNumber, ChainStatus, Receipt, + BlockNumber, ConsensusCapability, VersionInfo, + OperationsInfo, AccountInfo, HwAccountInfo, Header, RichHeader, RecoveredAccount }; use Host; @@ -425,4 +425,7 @@ impl Parity for ParityClient { Err(errors::status_error(has_peers)) } } + fn verify_signature(&self, is_prefixed: bool, message: Bytes, r: H256, s: H256, v: U64) -> Result { + verify_signature(is_prefixed, message, r, s, v, self.light_dispatch.client.signing_chain_id()) + } } diff --git a/rpc/src/v1/impls/parity.rs b/rpc/src/v1/impls/parity.rs index 2671a0eab51..9d73238a076 100644 --- a/rpc/src/v1/impls/parity.rs +++ b/rpc/src/v1/impls/parity.rs @@ -38,17 +38,17 @@ use jsonrpc_core::{BoxFuture, Result}; use jsonrpc_core::futures::future; use jsonrpc_macros::Trailing; -use v1::helpers::{self, errors, fake_sign, ipfs, SigningQueue, SignerService, NetworkSettings}; use v1::helpers::block_import::is_major_importing; +use v1::helpers::{self, errors, fake_sign, ipfs, SigningQueue, SignerService, NetworkSettings, verify_signature}; use v1::metadata::Metadata; use v1::traits::Parity; use v1::types::{ - Bytes, U256, H64, H160, H256, H512, CallRequest, + Bytes, U256, H64, U64, H160, H256, H512, CallRequest, Peers, Transaction, RpcSettings, Histogram, TransactionStats, LocalTransactionStatus, BlockNumber, ConsensusCapability, VersionInfo, OperationsInfo, ChainStatus, - AccountInfo, HwAccountInfo, RichHeader, Receipt, + AccountInfo, HwAccountInfo, RichHeader, Receipt, RecoveredAccount, block_number_to_id }; use Host; @@ -504,4 +504,8 @@ impl Parity for ParityClient where Err(errors::status_error(has_peers)) } } + + fn verify_signature(&self, is_prefixed: bool, message: Bytes, r: H256, s: H256, v: U64) -> Result { + verify_signature(is_prefixed, message, r, s, v, self.client.signing_chain_id()) + } } diff --git a/rpc/src/v1/tests/mocked/parity.rs b/rpc/src/v1/tests/mocked/parity.rs index 0fa623ed437..d978a4ac742 100644 --- a/rpc/src/v1/tests/mocked/parity.rs +++ b/rpc/src/v1/tests/mocked/parity.rs @@ -604,3 +604,26 @@ fn rpc_status_error_sync() { assert_eq!(io.handle_request_sync(request), Some(response.to_owned())); } + +#[test] +fn rpc_parity_verify_signature() { + let deps = Dependencies::new(); + let io = deps.default_client(); + + let request = r#"{ + "jsonrpc": "2.0", + "method": "parity_verifySignature", + "params": [ + false, + "0xe552acf4caabe9661893fd48c7b5e68af20bf007193442f8ca05ce836699d75e", + "0x2089e84151c3cdc45255c07557b349f5bf2ed3e68f6098723eaa90a0f8b2b3e5", + "0x5f70e8df7bd0c4417afb5f5a39d82e15d03adeff8796725d8b14889ed1d1aa8a", + "0x1" + ], + "id": 0 + }"#; + + let response = r#"{"jsonrpc":"2.0","result":{"address":"0x9a2a08a1170f51208c2f3cede0d29ada94481eed","isValidForCurrentChain":true,"publicKey":"0xbeec94ea24444906fe247c47841a45220f07e5718d06157fe4502fac326dab617e973e221e45746721330c2db3f63202268686378cc28b9800c1daaf0bbafeb1"},"id":0}"#; + + assert_eq!(io.handle_request_sync(request), Some(response.to_owned())); +} diff --git a/rpc/src/v1/traits/parity.rs b/rpc/src/v1/traits/parity.rs index a81e5008992..5ac46701895 100644 --- a/rpc/src/v1/traits/parity.rs +++ b/rpc/src/v1/traits/parity.rs @@ -20,10 +20,9 @@ use std::collections::BTreeMap; use jsonrpc_core::{BoxFuture, Result}; use jsonrpc_macros::Trailing; - use v1::types::{ - H64, H160, H256, H512, U256, Bytes, CallRequest, - Peers, Transaction, RpcSettings, Histogram, + H160, H256, H512, U256, U64, H64, Bytes, CallRequest, + Peers, Transaction, RpcSettings, Histogram, RecoveredAccount, TransactionStats, LocalTransactionStatus, BlockNumber, ConsensusCapability, VersionInfo, OperationsInfo, ChainStatus, @@ -237,5 +236,10 @@ build_rpc_trait! { /// Otherwise the RPC returns error. #[rpc(name = "parity_nodeStatus")] fn status(&self) -> Result<()>; + + /// Extracts Address and public key from signature using the r, s and v params. Equivalent to Solidity erecover + /// as well as checks the signature for chain replay protection + #[rpc(name = "parity_verifySignature")] + fn verify_signature(&self, bool, Bytes, H256, H256, U64) -> Result; } } diff --git a/rpc/src/v1/types/account_info.rs b/rpc/src/v1/types/account_info.rs index 487507de902..e0e2464d89b 100644 --- a/rpc/src/v1/types/account_info.rs +++ b/rpc/src/v1/types/account_info.rs @@ -13,6 +13,10 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . + +//! Return types for RPC calls + +use ethereum_types::{Public, Address}; use v1::types::{H160, H256, U256, Bytes}; /// Account information. @@ -64,3 +68,19 @@ pub struct HwAccountInfo { /// Device manufacturer. pub manufacturer: String, } + +/// account derived from a signature +/// as well as information that tells if it is valid for +/// the current chain +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all="camelCase")] +pub struct RecoveredAccount { + /// address of the recovered account + pub address: Address, + /// public key of the recovered account + pub public_key: Public, + /// If the signature contains chain replay protection, + /// And the chain_id encoded within the signature + /// matches the current chain this would be true, otherwise false. + pub is_valid_for_current_chain: bool +} diff --git a/rpc/src/v1/types/mod.rs b/rpc/src/v1/types/mod.rs index 68401fb0c15..cde98e59784 100644 --- a/rpc/src/v1/types/mod.rs +++ b/rpc/src/v1/types/mod.rs @@ -48,7 +48,7 @@ mod eip191; pub mod pubsub; pub use self::eip191::{EIP191Version, PresignedTransaction}; -pub use self::account_info::{AccountInfo, ExtAccountInfo, HwAccountInfo, EthAccount, StorageProof}; +pub use self::account_info::{AccountInfo, ExtAccountInfo, HwAccountInfo, EthAccount, StorageProof, RecoveredAccount}; pub use self::bytes::Bytes; pub use self::block::{RichBlock, Block, BlockTransactions, Header, RichHeader, Rich}; pub use self::block_number::{BlockNumber, LightBlockNumber, block_number_to_id}; diff --git a/util/fastmap/src/lib.rs b/util/fastmap/src/lib.rs index 135ce54babe..65dd9dfb4a0 100644 --- a/util/fastmap/src/lib.rs +++ b/util/fastmap/src/lib.rs @@ -21,11 +21,13 @@ extern crate plain_hasher; use ethereum_types::H256; use std::hash; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use plain_hasher::PlainHasher; /// Specialized version of `HashMap` with H256 keys and fast hashing function. pub type H256FastMap = HashMap>; +/// Specialized version of HashSet with H256 values and fast hashing function. +pub type H256FastSet = HashSet>; #[cfg(test)] mod tests { @@ -36,4 +38,4 @@ mod tests { let mut h = H256FastMap::default(); h.insert(H256::from(123), "abc"); } -} \ No newline at end of file +}