[SYNC PERFORMANCE] Adjust DifficultyIterator to no longer deserialize…

… PoW proof nonces (#3671)

* replace bitvec with more efficient bitpack algorithm

* optimise proof_unpack_len

* move proof pack length calculation

* small refactor

* first pass attempt at not deserializing proof nonces in difficulty iter

* another 10 seconds gained by not deserialising the proof from the difficulty iterator

* add new deser parameters to tests where needed

* add skip_proof variants to store

* remove hash from difficulty iterator struct, rename HeaderInfo to HeaderDifficultyInfo

* replace bitvec with more efficient bitpack algorithm

* optimise proof_unpack_len

* move proof pack length calculation

* small refactor

* first pass attempt at not deserializing proof nonces in difficulty iter

* another 10 seconds gained by not deserialising the proof from the difficulty iterator

* add new deser parameters to tests where needed

* add skip_proof variants to store

* remove hash from difficulty iterator struct, rename HeaderInfo to HeaderDifficultyInfo

api/src/handlers/chain_api.rs

@@ -277,7 +277,7 @@ impl OutputHandler {
 			.context(ErrorKind::Internal("cain error".to_owned()))?;
 
 		Ok(BlockOutputs {
-			header: BlockHeaderInfo::from_header(&header),
+			header: BlockHeaderDifficultyInfo::from_header(&header),
 			outputs: outputs,
 		})
 	}

api/src/handlers/pool_api.rs

@@ -15,7 +15,7 @@
 use super::utils::w;
 use crate::core::core::hash::Hashed;
 use crate::core::core::Transaction;
-use crate::core::ser::{self, ProtocolVersion};
+use crate::core::ser::{self, DeserializationMode, ProtocolVersion};
 use crate::pool::{self, BlockChain, PoolAdapter, PoolEntry};
 use crate::rest::*;
 use crate::router::{Handler, ResponseFuture};
@@ -138,8 +138,9 @@ where
 
 	// All wallet api interaction explicitly uses protocol version 1 for now.
 	let version = ProtocolVersion(1);
-	let tx: Transaction = ser::deserialize(&mut &tx_bin[..], version)
-		.map_err(|e| ErrorKind::RequestError(format!("Bad request: {}", e)))?;
+	let tx: Transaction =
+		ser::deserialize(&mut &tx_bin[..], version, DeserializationMode::default())
+			.map_err(|e| ErrorKind::RequestError(format!("Bad request: {}", e)))?;
 
 	let source = pool::TxSource::PushApi;
 	info!(

api/src/types.rs

@@ -530,7 +530,7 @@ impl TxKernelPrintable {
 
 // Just the information required for wallet reconstruction
 #[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct BlockHeaderInfo {
+pub struct BlockHeaderDifficultyInfo {
 	// Hash
 	pub hash: String,
 	/// Height of this block since the genesis block (height 0)
@@ -539,9 +539,9 @@ pub struct BlockHeaderInfo {
 	pub previous: String,
 }
 
-impl BlockHeaderInfo {
-	pub fn from_header(header: &core::BlockHeader) -> BlockHeaderInfo {
-		BlockHeaderInfo {
+impl BlockHeaderDifficultyInfo {
+	pub fn from_header(header: &core::BlockHeader) -> BlockHeaderDifficultyInfo {
+		BlockHeaderDifficultyInfo {
 			hash: header.hash().to_hex(),
 			height: header.height,
 			previous: header.prev_hash.to_hex(),
@@ -705,7 +705,7 @@ impl CompactBlockPrintable {
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct BlockOutputs {
 	/// The block header
-	pub header: BlockHeaderInfo,
+	pub header: BlockHeaderDifficultyInfo,
 	/// A printable version of the outputs
 	pub outputs: Vec<OutputPrintable>,
 }

chain/src/linked_list.rs

@@ -84,7 +84,7 @@ pub trait ListIndex {
 	/// Key is "prefix|commit".
 	/// Note the key for an individual entry in the list is "prefix|commit|pos".
 	fn get_list(&self, batch: &Batch<'_>, commit: Commitment) -> Result<Option<Self::List>, Error> {
-		batch.db.get_ser(&self.list_key(commit))
+		batch.db.get_ser(&self.list_key(commit), None)
 	}
 
 	/// Returns one of "head", "tail" or "middle" entry variants.
@@ -95,7 +95,7 @@ pub trait ListIndex {
 		commit: Commitment,
 		pos: u64,
 	) -> Result<Option<Self::Entry>, Error> {
-		batch.db.get_ser(&self.entry_key(commit, pos))
+		batch.db.get_ser(&self.entry_key(commit, pos), None)
 	}
 
 	/// Peek the head of the list for the specified commitment.

chain/src/store.rs

@@ -14,11 +14,11 @@
 
 //! Implements storage primitives required by the chain
 
-use crate::core::consensus::HeaderInfo;
+use crate::core::consensus::HeaderDifficultyInfo;
 use crate::core::core::hash::{Hash, Hashed};
 use crate::core::core::{Block, BlockHeader, BlockSums};
 use crate::core::pow::Difficulty;
-use crate::core::ser::{ProtocolVersion, Readable, Writeable};
+use crate::core::ser::{DeserializationMode, ProtocolVersion, Readable, Writeable};
 use crate::linked_list::MultiIndex;
 use crate::types::{CommitPos, Tip};
 use crate::util::secp::pedersen::Commitment;
@@ -60,19 +60,19 @@ impl ChainStore {
 
 	/// The current chain head.
 	pub fn head(&self) -> Result<Tip, Error> {
-		option_to_not_found(self.db.get_ser(&[HEAD_PREFIX]), || "HEAD".to_owned())
+		option_to_not_found(self.db.get_ser(&[HEAD_PREFIX], None), || "HEAD".to_owned())
 	}
 
 	/// The current header head (may differ from chain head).
 	pub fn header_head(&self) -> Result<Tip, Error> {
-		option_to_not_found(self.db.get_ser(&[HEADER_HEAD_PREFIX]), || {
+		option_to_not_found(self.db.get_ser(&[HEADER_HEAD_PREFIX], None), || {
 			"HEADER_HEAD".to_owned()
 		})
 	}
 
 	/// The current chain "tail" (earliest block in the store).
 	pub fn tail(&self) -> Result<Tip, Error> {
-		option_to_not_found(self.db.get_ser(&[TAIL_PREFIX]), || "TAIL".to_owned())
+		option_to_not_found(self.db.get_ser(&[TAIL_PREFIX], None), || "TAIL".to_owned())
 	}
 
 	/// Header of the block at the head of the block chain (not the same thing as header_head).
@@ -82,7 +82,7 @@ impl ChainStore {
 
 	/// Get full block.
 	pub fn get_block(&self, h: &Hash) -> Result<Block, Error> {
-		option_to_not_found(self.db.get_ser(&to_key(BLOCK_PREFIX, h)), || {
+		option_to_not_found(self.db.get_ser(&to_key(BLOCK_PREFIX, h), None), || {
 			format!("BLOCK: {}", h)
 		})
 	}
@@ -94,7 +94,7 @@ impl ChainStore {
 
 	/// Get block_sums for the block hash.
 	pub fn get_block_sums(&self, h: &Hash) -> Result<BlockSums, Error> {
-		option_to_not_found(self.db.get_ser(&to_key(BLOCK_SUMS_PREFIX, h)), || {
+		option_to_not_found(self.db.get_ser(&to_key(BLOCK_SUMS_PREFIX, h), None), || {
 			format!("Block sums for block: {}", h)
 		})
 	}
@@ -104,11 +104,32 @@ impl ChainStore {
 		self.get_block_header(&header.prev_hash)
 	}
 
+	/// Get previous header without deserializing the proof nonces
+	pub fn get_previous_header_skip_proof(
+		&self,
+		header: &BlockHeader,
+	) -> Result<BlockHeader, Error> {
+		self.get_block_header_skip_proof(&header.prev_hash)
+	}
+
 	/// Get block header.
 	pub fn get_block_header(&self, h: &Hash) -> Result<BlockHeader, Error> {
-		option_to_not_found(self.db.get_ser(&to_key(BLOCK_HEADER_PREFIX, h)), || {
-			format!("BLOCK HEADER: {}", h)
-		})
+		option_to_not_found(
+			self.db.get_ser(&to_key(BLOCK_HEADER_PREFIX, h), None),
+			|| format!("BLOCK HEADER: {}", h),
+		)
+	}
+
+	/// Get block header without deserializing the full PoW Proof; currently used
+	/// for difficulty iterator which is called many times but doesn't need the proof
+	pub fn get_block_header_skip_proof(&self, h: &Hash) -> Result<BlockHeader, Error> {
+		option_to_not_found(
+			self.db.get_ser(
+				&to_key(BLOCK_HEADER_PREFIX, h),
+				Some(ser::DeserializationMode::SkipPow),
+			),
+			|| format!("BLOCK HEADER: {}", h),
+		)
 	}
 
 	/// Get PMMR pos for the given output commitment.
@@ -124,7 +145,7 @@ impl ChainStore {
 
 	/// Get PMMR pos and block height for the given output commitment.
 	pub fn get_output_pos_height(&self, commit: &Commitment) -> Result<Option<CommitPos>, Error> {
-		self.db.get_ser(&to_key(OUTPUT_POS_PREFIX, commit))
+		self.db.get_ser(&to_key(OUTPUT_POS_PREFIX, commit), None)
 	}
 
 	/// Builds a new batch to be used with this store.
@@ -145,17 +166,17 @@ pub struct Batch<'a> {
 impl<'a> Batch<'a> {
 	/// The head.
 	pub fn head(&self) -> Result<Tip, Error> {
-		option_to_not_found(self.db.get_ser(&[HEAD_PREFIX]), || "HEAD".to_owned())
+		option_to_not_found(self.db.get_ser(&[HEAD_PREFIX], None), || "HEAD".to_owned())
 	}
 
 	/// The tail.
 	pub fn tail(&self) -> Result<Tip, Error> {
-		option_to_not_found(self.db.get_ser(&[TAIL_PREFIX]), || "TAIL".to_owned())
+		option_to_not_found(self.db.get_ser(&[TAIL_PREFIX], None), || "TAIL".to_owned())
 	}
 
 	/// The current header head (may differ from chain head).
 	pub fn header_head(&self) -> Result<Tip, Error> {
-		option_to_not_found(self.db.get_ser(&[HEADER_HEAD_PREFIX]), || {
+		option_to_not_found(self.db.get_ser(&[HEADER_HEAD_PREFIX], None), || {
 			"HEADER_HEAD".to_owned()
 		})
 	}
@@ -182,7 +203,7 @@ impl<'a> Batch<'a> {
 
 	/// get block
 	pub fn get_block(&self, h: &Hash) -> Result<Block, Error> {
-		option_to_not_found(self.db.get_ser(&to_key(BLOCK_PREFIX, h)), || {
+		option_to_not_found(self.db.get_ser(&to_key(BLOCK_PREFIX, h), None), || {
 			format!("Block with hash: {}", h)
 		})
 	}
@@ -269,7 +290,7 @@ impl<'a> Batch<'a> {
 		let key = to_key(OUTPUT_POS_PREFIX, "");
 		let protocol_version = self.db.protocol_version();
 		self.db.iter(&key, move |k, mut v| {
-			ser::deserialize(&mut v, protocol_version)
+			ser::deserialize(&mut v, protocol_version, DeserializationMode::default())
 				.map(|pos| (k.to_vec(), pos))
 				.map_err(From::from)
 		})
@@ -288,19 +309,41 @@ impl<'a> Batch<'a> {
 
 	/// Get output_pos and block height from index.
 	pub fn get_output_pos_height(&self, commit: &Commitment) -> Result<Option<CommitPos>, Error> {
-		self.db.get_ser(&to_key(OUTPUT_POS_PREFIX, commit))
+		self.db.get_ser(&to_key(OUTPUT_POS_PREFIX, commit), None)
 	}
 
 	/// Get the previous header.
 	pub fn get_previous_header(&self, header: &BlockHeader) -> Result<BlockHeader, Error> {
 		self.get_block_header(&header.prev_hash)
 	}
 
+	/// Get the previous header, without deserializing the full PoW Proof (or the ability to derive the
+	/// block hash, this is used for the difficulty iterator).
+	pub fn get_previous_header_skip_proof(
+		&self,
+		header: &BlockHeader,
+	) -> Result<BlockHeader, Error> {
+		self.get_block_header_skip_proof(&header.prev_hash)
+	}
+
 	/// Get block header.
 	pub fn get_block_header(&self, h: &Hash) -> Result<BlockHeader, Error> {
-		option_to_not_found(self.db.get_ser(&to_key(BLOCK_HEADER_PREFIX, h)), || {
-			format!("BLOCK HEADER: {}", h)
-		})
+		option_to_not_found(
+			self.db.get_ser(&to_key(BLOCK_HEADER_PREFIX, h), None),
+			|| format!("BLOCK HEADER: {}", h),
+		)
+	}
+
+	/// Get block header without deserializing the full PoW Proof; currently used
+	/// for difficulty iterator which is called many times but doesn't need the proof
+	pub fn get_block_header_skip_proof(&self, h: &Hash) -> Result<BlockHeader, Error> {
+		option_to_not_found(
+			self.db.get_ser(
+				&to_key(BLOCK_HEADER_PREFIX, h),
+				Some(ser::DeserializationMode::SkipPow),
+			),
+			|| format!("BLOCK HEADER: {}", h),
+		)
 	}
 
 	/// Delete the block spent index.
@@ -315,7 +358,7 @@ impl<'a> Batch<'a> {
 
 	/// Get block_sums for the block.
 	pub fn get_block_sums(&self, h: &Hash) -> Result<BlockSums, Error> {
-		option_to_not_found(self.db.get_ser(&to_key(BLOCK_SUMS_PREFIX, h)), || {
+		option_to_not_found(self.db.get_ser(&to_key(BLOCK_SUMS_PREFIX, h), None), || {
 			format!("Block sums for block: {}", h)
 		})
 	}
@@ -339,9 +382,10 @@ impl<'a> Batch<'a> {
 	/// Get the "spent index" from the db for the specified block.
 	/// If we need to rewind a block then we use this to "unspend" the spent outputs.
 	pub fn get_spent_index(&self, bh: &Hash) -> Result<Vec<CommitPos>, Error> {
-		option_to_not_found(self.db.get_ser(&to_key(BLOCK_SPENT_PREFIX, bh)), || {
-			format!("spent index: {}", bh)
-		})
+		option_to_not_found(
+			self.db.get_ser(&to_key(BLOCK_SPENT_PREFIX, bh), None),
+			|| format!("spent index: {}", bh),
+		)
 	}
 
 	/// Commits this batch. If it's a child batch, it will be merged with the
@@ -364,7 +408,8 @@ impl<'a> Batch<'a> {
 		let key = to_key(BLOCK_PREFIX, "");
 		let protocol_version = self.db.protocol_version();
 		self.db.iter(&key, move |_, mut v| {
-			ser::deserialize(&mut v, protocol_version).map_err(From::from)
+			ser::deserialize(&mut v, protocol_version, DeserializationMode::default())
+				.map_err(From::from)
 		})
 	}
 
@@ -425,16 +470,20 @@ impl<'a> DifficultyIter<'a> {
 }
 
 impl<'a> Iterator for DifficultyIter<'a> {
-	type Item = HeaderInfo;
+	type Item = HeaderDifficultyInfo;
 
 	fn next(&mut self) -> Option<Self::Item> {
 		// Get both header and previous_header if this is the initial iteration.
 		// Otherwise move prev_header to header and get the next prev_header.
+		// Note that due to optimizations being called in `get_block_header_skip_proof`,
+		// Items returned by this iterator cannot be expected to correctly
+		// calculate their own hash - This iterator is purely for iterating through
+		// difficulty information
 		self.header = if self.header.is_none() {
 			if let Some(ref batch) = self.batch {
-				batch.get_block_header(&self.start).ok()
+				batch.get_block_header_skip_proof(&self.start).ok()
 			} else if let Some(ref store) = self.store {
-				store.get_block_header(&self.start).ok()
+				store.get_block_header_skip_proof(&self.start).ok()
 			} else {
 				None
 			}
@@ -446,9 +495,9 @@ impl<'a> Iterator for DifficultyIter<'a> {
 		// Otherwise we are done.
 		if let Some(header) = self.header.clone() {
 			if let Some(ref batch) = self.batch {
-				self.prev_header = batch.get_previous_header(&header).ok();
+				self.prev_header = batch.get_previous_header_skip_proof(&header).ok();
 			} else if let Some(ref store) = self.store {
-				self.prev_header = store.get_previous_header(&header).ok();
+				self.prev_header = store.get_previous_header_skip_proof(&header).ok();
 			} else {
 				self.prev_header = None;
 			}
@@ -460,8 +509,7 @@ impl<'a> Iterator for DifficultyIter<'a> {
 			let difficulty = header.total_difficulty() - prev_difficulty;
 			let scaling = header.pow.secondary_scaling;
 
-			Some(HeaderInfo::new(
-				header.hash(),
+			Some(HeaderDifficultyInfo::new(
 				header.timestamp.timestamp() as u64,
 				difficulty,
 				scaling,

chain/src/txhashset/bitmap_accumulator.rs

@@ -509,7 +509,9 @@ impl Readable for BitmapBlockSerialization {
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use crate::core::ser::{BinReader, BinWriter, ProtocolVersion, Readable, Writeable};
+	use crate::core::ser::{
+		BinReader, BinWriter, DeserializationMode, ProtocolVersion, Readable, Writeable,
+	};
 	use byteorder::ReadBytesExt;
 	use grin_util::secp::rand::Rng;
 	use rand::thread_rng;
@@ -546,7 +548,11 @@ mod tests {
 
 		// Deserialize
 		cursor.set_position(0);
-		let mut reader = BinReader::new(&mut cursor, ProtocolVersion(1));
+		let mut reader = BinReader::new(
+			&mut cursor,
+			ProtocolVersion(1),
+			DeserializationMode::default(),
+		);
 		let block2: BitmapBlock = Readable::read(&mut reader).unwrap();
 		assert_eq!(block, block2);
 	}

chain/tests/bitmap_segment.rs

@@ -1,6 +1,8 @@
 use self::chain::txhashset::{BitmapAccumulator, BitmapSegment};
 use self::core::core::pmmr::segment::{Segment, SegmentIdentifier};
-use self::core::ser::{BinReader, BinWriter, ProtocolVersion, Readable, Writeable};
+use self::core::ser::{
+	BinReader, BinWriter, DeserializationMode, ProtocolVersion, Readable, Writeable,
+};
 use croaring::Bitmap;
 use grin_chain as chain;
 use grin_core as core;
@@ -52,7 +54,11 @@ fn test_roundtrip(entries: usize) {
 
 	// Read `BitmapSegment`
 	cursor.set_position(0);
-	let mut reader = BinReader::new(&mut cursor, ProtocolVersion(1));
+	let mut reader = BinReader::new(
+		&mut cursor,
+		ProtocolVersion(1),
+		DeserializationMode::default(),
+	);
 	let bms2: BitmapSegment = Readable::read(&mut reader).unwrap();
 	assert_eq!(bms, bms2);