make block queue into a more generic verification queue and fix block heap size calculation

ethcore/src/client/client.rs

@@ -25,13 +25,12 @@ use time::precise_time_ns;
 use util::{Bytes, PerfTimer, Itertools, Mutex, RwLock};
 use util::journaldb::{self, JournalDB};
 use util::{U256, H256, Address, H2048, Uint};
-use util::sha3::*;
 use util::TrieFactory;
 use util::kvdb::*;
 
 // other
 use io::*;
-use views::{BlockView, HeaderView, BodyView};
+use views::{HeaderView, BodyView};
 use error::{ImportError, ExecutionError, CallError, BlockError, ImportResult, Error as EthcoreError};
 use header::BlockNumber;
 use state::State;
@@ -47,7 +46,7 @@ use transaction::{LocalizedTransaction, SignedTransaction, Action};
 use blockchain::extras::TransactionAddress;
 use types::filter::Filter;
 use log_entry::LocalizedLogEntry;
-use block_queue::{BlockQueue, BlockQueueInfo};
+use verification::queue::{BlockQueue, QueueInfo as BlockQueueInfo};
 use blockchain::{BlockChain, BlockProvider, TreeRoute, ImportRoute};
 use client::{
 	BlockID, TransactionID, UncleID, TraceId, ClientConfig, BlockChainClient,
@@ -805,7 +804,7 @@ impl BlockChainClient for Client {
 		let chain = self.chain.read();
 		match Self::block_hash(&chain, id) {
 			Some(ref hash) if chain.is_known(hash) => BlockStatus::InChain,
-			Some(hash) => self.block_queue.block_status(&hash),
+			Some(hash) => self.block_queue.status(&hash).into(),
 			None => BlockStatus::Unknown
 		}
 	}
@@ -917,16 +916,21 @@ impl BlockChainClient for Client {
 	}
 
 	fn import_block(&self, bytes: Bytes) -> Result<H256, BlockImportError> {
+		use verification::queue::kind::HasHash;
+		use verification::queue::kind::blocks::Unverified;
+
+		// create unverified block here so the `sha3` calculation can be cached.
+		let unverified = Unverified::new(bytes);
+
 		{
-			let header = BlockView::new(&bytes).header_view();
-			if self.chain.read().is_known(&header.sha3()) {
+			if self.chain.read().is_known(&unverified.hash()) {
 				return Err(BlockImportError::Import(ImportError::AlreadyInChain));
 			}
-			if self.block_status(BlockID::Hash(header.parent_hash())) == BlockStatus::Unknown {
-				return Err(BlockImportError::Block(BlockError::UnknownParent(header.parent_hash())));
+			if self.block_status(BlockID::Hash(unverified.parent_hash())) == BlockStatus::Unknown {
+				return Err(BlockImportError::Block(BlockError::UnknownParent(unverified.parent_hash())));
 			}
 		}
-		Ok(try!(self.block_queue.import_block(bytes)))
+		Ok(try!(self.block_queue.import(unverified)))
 	}
 
 	fn queue_info(&self) -> BlockQueueInfo {

ethcore/src/client/config.rs

@@ -16,11 +16,11 @@
 
 use std::str::FromStr;
 pub use std::time::Duration;
-pub use block_queue::BlockQueueConfig;
 pub use blockchain::Config as BlockChainConfig;
 pub use trace::Config as TraceConfig;
 pub use evm::VMType;
-pub use verification::VerifierType;
+
+use verification::{VerifierType, QueueConfig};
 use util::{journaldb, CompactionProfile};
 use util::trie::TrieSpec;
 
@@ -84,7 +84,7 @@ impl Default for Mode {
 #[derive(Debug, PartialEq, Default)]
 pub struct ClientConfig {
 	/// Block queue configuration.
-	pub queue: BlockQueueConfig,
+	pub queue: QueueConfig,
 	/// Blockchain configuration.
 	pub blockchain: BlockChainConfig,
 	/// Trace configuration.

ethcore/src/client/mod.rs

@@ -23,7 +23,7 @@ mod trace;
 mod client;
 
 pub use self::client::*;
-pub use self::config::{Mode, ClientConfig, DatabaseCompactionProfile, BlockQueueConfig, BlockChainConfig, VMType};
+pub use self::config::{Mode, ClientConfig, DatabaseCompactionProfile, BlockChainConfig, VMType};
 pub use self::error::Error;
 pub use types::ids::*;
 pub use self::test_client::{TestBlockChainClient, EachBlockWith};

ethcore/src/client/test_client.rs

@@ -37,7 +37,7 @@ use evm::{Factory as EvmFactory, VMType};
 use miner::{Miner, MinerService, TransactionImportResult};
 use spec::Spec;
 
-use block_queue::BlockQueueInfo;
+use verification::queue::QueueInfo;
 use block::{OpenBlock, SealedBlock};
 use executive::Executed;
 use error::CallError;
@@ -544,8 +544,8 @@ impl BlockChainClient for TestBlockChainClient {
 		Ok(h)
 	}
 
-	fn queue_info(&self) -> BlockQueueInfo {
-		BlockQueueInfo {
+	fn queue_info(&self) -> QueueInfo {
+		QueueInfo {
 			verified_queue_size: self.queue_size.load(AtomicOrder::Relaxed),
 			unverified_queue_size: 0,
 			verifying_queue_size: 0,

ethcore/src/client/traits.rs

@@ -17,7 +17,7 @@
 use std::collections::BTreeMap;
 use util::{U256, Address, H256, H2048, Bytes, Itertools};
 use blockchain::TreeRoute;
-use block_queue::BlockQueueInfo;
+use verification::queue::QueueInfo as BlockQueueInfo;
 use block::{OpenBlock, SealedBlock};
 use header::{BlockNumber};
 use transaction::{LocalizedTransaction, SignedTransaction};

ethcore/src/header.rs

@@ -295,6 +295,12 @@ impl Encodable for Header {
 	}
 }
 
+impl HeapSizeOf for Header {
+	fn heap_size_of_children(&self) -> usize {
+		self.extra_data.heap_size_of_children() + self.seal.heap_size_of_children()
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use rustc_serialize::hex::FromHex;

ethcore/src/lib.rs

@@ -119,7 +119,6 @@ pub extern crate ethstore;
 pub mod account_provider;
 pub mod engines;
 pub mod block;
-pub mod block_queue;
 pub mod client;
 pub mod error;
 pub mod ethereum;

ethcore/src/types/block_status.rs

@@ -15,6 +15,7 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Block status description module
+use verification::queue::Status as QueueStatus;
 
 /// General block status
 #[derive(Debug, Eq, PartialEq, Binary)]
@@ -28,3 +29,13 @@ pub enum BlockStatus {
 	/// Unknown.
 	Unknown,
 }
+
+impl From<QueueStatus> for BlockStatus {
+	fn from(status: QueueStatus) -> Self {
+		match status {
+			QueueStatus::Queued => BlockStatus::Queued,
+			QueueStatus::Bad => BlockStatus::Bad,
+			QueueStatus::Unknown => BlockStatus::Unknown,
+		}
+	}
+}

ethcore/src/types/mod.rs.in

@@ -25,7 +25,7 @@ pub mod executed;
 pub mod block_status;
 pub mod account_diff;
 pub mod state_diff;
-pub mod block_queue_info;
+pub mod verification_queue_info;
 pub mod filter;
 pub mod trace_filter;
 pub mod call_analytics;

ethcore/src/types/transaction.rs

@@ -20,7 +20,7 @@ use std::ops::Deref;
 use std::cell::*;
 use rlp::*;
 use util::sha3::Hashable;
-use util::{H256, Address, U256, Bytes};
+use util::{H256, Address, U256, Bytes, HeapSizeOf};
 use ethkey::{Signature, sign, Secret, Public, recover, public_to_address, Error as EthkeyError};
 use error::*;
 use evm::Schedule;
@@ -86,6 +86,12 @@ impl Transaction {
 	}
 }
 
+impl HeapSizeOf for Transaction {
+	fn heap_size_of_children(&self) -> usize {
+		self.data.heap_size_of_children()
+	}
+}
+
 impl From<ethjson::state::Transaction> for SignedTransaction {
 	fn from(t: ethjson::state::Transaction) -> Self {
 		let to: Option<ethjson::hash::Address> = t.to.into();
@@ -251,6 +257,12 @@ impl Encodable for SignedTransaction {
 	fn rlp_append(&self, s: &mut RlpStream) { self.rlp_append_sealed_transaction(s) }
 }
 
+impl HeapSizeOf for SignedTransaction {
+	fn heap_size_of_children(&self) -> usize {
+		self.unsigned.heap_size_of_children()
+	}
+}
+
 impl SignedTransaction {
 	/// Append object with a signature into RLP stream
 	pub fn rlp_append_sealed_transaction(&self, s: &mut RlpStream) {

ethcore/src/types/verification_queue_info.rs

@@ -0,0 +1,53 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Verification queue info types
+
+/// Verification queue status
+#[derive(Debug, Binary)]
+pub struct VerificationQueueInfo {
+	/// Number of queued items pending verification
+	pub unverified_queue_size: usize,
+	/// Number of verified queued items pending import
+	pub verified_queue_size: usize,
+	/// Number of items being verified
+	pub verifying_queue_size: usize,
+	/// Configured maximum number of items in the queue
+	pub max_queue_size: usize,
+	/// Configured maximum number of bytes to use
+	pub max_mem_use: usize,
+	/// Heap memory used in bytes
+	pub mem_used: usize,
+}
+
+impl VerificationQueueInfo {
+	/// The total size of the queues.
+	pub fn total_queue_size(&self) -> usize { self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size }
+
+	/// The size of the unverified and verifying queues.
+	pub fn incomplete_queue_size(&self) -> usize { self.unverified_queue_size + self.verifying_queue_size }
+
+	/// Indicates that queue is full
+	pub fn is_full(&self) -> bool {
+		self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > self.max_queue_size ||
+			self.mem_used > self.max_mem_use
+	}
+
+	/// Indicates that queue is empty
+	pub fn is_empty(&self) -> bool {
+		self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size == 0
+	}
+}

ethcore/src/verification/mod.rs

@@ -16,13 +16,15 @@
 
 pub mod verification;
 pub mod verifier;
+pub mod queue;
 mod canon_verifier;
 mod noop_verifier;
 
 pub use self::verification::*;
 pub use self::verifier::Verifier;
 pub use self::canon_verifier::CanonVerifier;
 pub use self::noop_verifier::NoopVerifier;
+pub use self::queue::{BlockQueue, Config as QueueConfig, VerificationQueue, QueueInfo};
 
 /// Verifier type.
 #[derive(Debug, PartialEq, Clone)]