add segmenter for generating segments from txhashset with consistent …

…rewind (#3482)

* add segmenter for generating segments from txhashset with consistent rewind

* rework segmenter to take a txhashset wrapped in rwlock
rework our rewindable pmmr so we can convert to readonly easily

* placeholder code for rewinding readonly txhashset extension to build a rangeproof segment

* segment creation for outputs/rangeproofs/kernels/bitmaps

* placeholder segment impl

* commit

* rework segmenter to use a cached bitmap (rewind is expensive)

* cache segmenter instance based on current archive header

* integrate the real segment and segment identifier with our segmenter

* exercise the segmenter code on chain init

* wrap accumulator in an arc, no need to clone each time

chain/src/chain.rs

@@ -20,7 +20,7 @@ use crate::core::core::merkle_proof::MerkleProof;
 use crate::core::core::verifier_cache::VerifierCache;
 use crate::core::core::{
 	Block, BlockHeader, BlockSums, Committed, Inputs, KernelFeatures, Output, OutputIdentifier,
-	Transaction, TxKernel,
+	SegmentIdentifier, Transaction, TxKernel,
 };
 use crate::core::global;
 use crate::core::pow;
@@ -29,12 +29,13 @@ use crate::error::{Error, ErrorKind};
 use crate::pipe;
 use crate::store;
 use crate::txhashset;
-use crate::txhashset::{PMMRHandle, TxHashSet};
+use crate::txhashset::{PMMRHandle, Segmenter, TxHashSet};
 use crate::types::{
 	BlockStatus, ChainAdapter, CommitPos, NoStatus, Options, Tip, TxHashsetWriteStatus,
 };
 use crate::util::secp::pedersen::{Commitment, RangeProof};
-use crate::{util::RwLock, ChainStore};
+use crate::util::RwLock;
+use crate::ChainStore;
 use grin_core::ser;
 use grin_store::Error::NotFoundErr;
 use std::fs::{self, File};
@@ -152,6 +153,7 @@ pub struct Chain {
 	header_pmmr: Arc<RwLock<txhashset::PMMRHandle<BlockHeader>>>,
 	sync_pmmr: Arc<RwLock<txhashset::PMMRHandle<BlockHeader>>>,
 	verifier_cache: Arc<RwLock<dyn VerifierCache>>,
+	pibd_segmenter: Arc<RwLock<Option<Segmenter>>>,
 	// POW verification function
 	pow_verifier: fn(&BlockHeader) -> Result<(), pow::Error>,
 	archive_mode: bool,
@@ -217,6 +219,7 @@ impl Chain {
 			txhashset: Arc::new(RwLock::new(txhashset)),
 			header_pmmr: Arc::new(RwLock::new(header_pmmr)),
 			sync_pmmr: Arc::new(RwLock::new(sync_pmmr)),
+			pibd_segmenter: Arc::new(RwLock::new(None)),
 			pow_verifier,
 			verifier_cache,
 			archive_mode,
@@ -225,6 +228,22 @@ impl Chain {
 
 		chain.log_heads()?;
 
+		// Temporarily exercising the initialization process.
+		// Note: This is *really* slow because we are starting from cold.
+		//
+		// This is not required as we will lazily initialize our segmenter as required
+		// once we start receiving PIBD segment requests.
+		// In reality we will do this based on PIBD segment requests.
+		// Initialization (once per 12 hour period) will not be this slow once lmdb and PMMRs
+		// are warmed up.
+		{
+			let segmenter = chain.segmenter()?;
+			let _ = segmenter.kernel_segment(SegmentIdentifier { height: 9, idx: 0 });
+			let _ = segmenter.bitmap_segment(SegmentIdentifier { height: 9, idx: 0 });
+			let _ = segmenter.output_segment(SegmentIdentifier { height: 11, idx: 0 });
+			let _ = segmenter.rangeproof_segment(SegmentIdentifier { height: 7, idx: 0 });
+		}
+
 		Ok(chain)
 	}
 
@@ -815,6 +834,64 @@ impl Chain {
 		})
 	}
 
+	/// The segmenter is responsible for generation PIBD segments.
+	/// We cache a segmenter instance based on the current archve period (new period every 12 hours).
+	/// This allows us to efficiently generate bitmap segments for the current archive period.
+	///
+	/// It is a relatively expensive operation to initializa and cache a new segmenter instance
+	/// as this involves rewinding the txhashet by approx 720 blocks (12 hours).
+	///
+	/// Caller is responsible for only doing this when required.
+	/// Caller should verify a peer segment request is valid before calling this for example.
+	///
+	pub fn segmenter(&self) -> Result<Segmenter, Error> {
+		// The archive header corresponds to the data we will segment.
+		let ref archive_header = self.txhashset_archive_header()?;
+
+		// Use our cached segmenter if we have one and the associated header matches.
+		if let Some(x) = self.pibd_segmenter.read().as_ref() {
+			if x.header() == archive_header {
+				return Ok(x.clone());
+			}
+		}
+
+		// We have no cached segmenter or the cached segmenter is no longer useful.
+		// Initialize a new segment, cache it and return it.
+		let segmenter = self.init_segmenter(archive_header)?;
+		let mut cache = self.pibd_segmenter.write();
+		*cache = Some(segmenter.clone());
+
+		return Ok(segmenter);
+	}
+
+	/// This is an expensive rewind to recreate bitmap state but we only need to do this once.
+	/// Caller is responsible for "caching" the segmenter (per archive period) for reuse.
+	fn init_segmenter(&self, header: &BlockHeader) -> Result<Segmenter, Error> {
+		let now = Instant::now();
+		debug!(
+			"init_segmenter: initializing new segmenter for {} at {}",
+			header.hash(),
+			header.height
+		);
+
+		let mut header_pmmr = self.header_pmmr.write();
+		let mut txhashset = self.txhashset.write();
+
+		let bitmap_snapshot =
+			txhashset::extending_readonly(&mut header_pmmr, &mut txhashset, |ext, batch| {
+				ext.extension.rewind(header, batch)?;
+				Ok(ext.extension.bitmap_accumulator())
+			})?;
+
+		debug!("init_segmenter: done, took {}ms", now.elapsed().as_millis());
+
+		Ok(Segmenter::new(
+			self.txhashset(),
+			Arc::new(bitmap_snapshot),
+			header.clone(),
+		))
+	}
+
 	/// To support the ability to download the txhashset from multiple peers in parallel,
 	/// the peers must all agree on the exact binary representation of the txhashset.
 	/// This means compacting and rewinding to the exact same header.

chain/src/error.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 //! Error types for chain
+use crate::core::core::pmmr::segment;
 use crate::core::core::{block, committed, transaction};
 use crate::core::ser;
 use crate::keychain;
@@ -149,6 +150,9 @@ pub enum ErrorKind {
 	/// Error during chain sync
 	#[fail(display = "Sync error")]
 	SyncError(String),
+	/// PIBD segment related error
+	#[fail(display = "Segment error")]
+	SegmentError(segment::SegmentError),
 }
 
 impl Display for Error {
@@ -273,6 +277,14 @@ impl From<ser::Error> for Error {
 	}
 }
 
+impl From<segment::SegmentError> for Error {
+	fn from(error: segment::SegmentError) -> Error {
+		Error {
+			inner: Context::new(ErrorKind::SegmentError(error)),
+		}
+	}
+}
+
 impl From<secp::Error> for Error {
 	fn from(e: secp::Error) -> Error {
 		Error {

chain/src/txhashset.rs

@@ -17,10 +17,12 @@
 
 mod bitmap_accumulator;
 mod rewindable_kernel_view;
+mod segmenter;
 mod txhashset;
 mod utxo_view;
 
 pub use self::bitmap_accumulator::*;
 pub use self::rewindable_kernel_view::*;
+pub use self::segmenter::*;
 pub use self::txhashset::*;
 pub use self::utxo_view::*;

chain/src/txhashset/bitmap_accumulator.rs

@@ -50,7 +50,7 @@ impl BitmapAccumulator {
 	/// Crate a new empty bitmap accumulator.
 	pub fn new() -> BitmapAccumulator {
 		BitmapAccumulator {
-			backend: VecBackend::new_hash_only(),
+			backend: VecBackend::new(),
 		}
 	}
 
@@ -176,9 +176,12 @@ impl BitmapAccumulator {
 
 	/// The root hash of the bitmap accumulator MMR.
 	pub fn root(&self) -> Hash {
+		self.readonly_pmmr().root().expect("no root, invalid tree")
+	}
+
+	/// Readonly access to our internal data.
+	pub fn readonly_pmmr(&self) -> ReadonlyPMMR<BitmapChunk, VecBackend<BitmapChunk>> {
 		ReadonlyPMMR::at(&self.backend, self.backend.size())
-			.root()
-			.expect("no root, invalid tree")
 	}
 }
 

chain/src/txhashset/rewindable_kernel_view.rs

@@ -14,7 +14,7 @@
 
 //! Lightweight readonly view into kernel MMR for convenience.
 
-use crate::core::core::pmmr::RewindablePMMR;
+use crate::core::core::pmmr::{ReadablePMMR, ReadonlyPMMR, RewindablePMMR};
 use crate::core::core::{BlockHeader, TxKernel};
 use crate::error::{Error, ErrorKind};
 use grin_store::pmmr::PMMRBackend;
@@ -54,7 +54,10 @@ impl<'a> RewindableKernelView<'a> {
 	/// fast sync where a reorg past the horizon could allow a whole rewrite of
 	/// the kernel set.
 	pub fn validate_root(&self) -> Result<(), Error> {
-		let root = self.pmmr.root().map_err(|_| ErrorKind::InvalidRoot)?;
+		let root = self
+			.readonly_pmmr()
+			.root()
+			.map_err(|_| ErrorKind::InvalidRoot)?;
 		if root != self.header.kernel_root {
 			return Err(ErrorKind::InvalidTxHashSet(format!(
 				"Kernel root at {} does not match",
@@ -64,4 +67,9 @@ impl<'a> RewindableKernelView<'a> {
 		}
 		Ok(())
 	}
+
+	/// Readonly view of our internal data.
+	pub fn readonly_pmmr(&self) -> ReadonlyPMMR<TxKernel, PMMRBackend<TxKernel>> {
+		self.pmmr.as_readonly()
+	}
 }

chain/src/txhashset/segmenter.rs

@@ -0,0 +1,149 @@
+// Copyright 2020 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Generation of the various necessary segments requested during PIBD.
+
+use std::{sync::Arc, time::Instant};
+
+use crate::core::core::hash::Hash;
+use crate::core::core::pmmr::ReadablePMMR;
+use crate::core::core::{BlockHeader, OutputIdentifier, Segment, SegmentIdentifier, TxKernel};
+use crate::error::{Error, ErrorKind};
+use crate::txhashset::{BitmapAccumulator, BitmapChunk, TxHashSet};
+use crate::util::secp::pedersen::RangeProof;
+use crate::util::RwLock;
+
+/// Segmenter for generating PIBD segments.
+#[derive(Clone)]
+pub struct Segmenter {
+	txhashset: Arc<RwLock<TxHashSet>>,
+	bitmap_snapshot: Arc<BitmapAccumulator>,
+	header: BlockHeader,
+}
+
+impl Segmenter {
+	/// Create a new segmenter based on the provided txhashset.
+	pub fn new(
+		txhashset: Arc<RwLock<TxHashSet>>,
+		bitmap_snapshot: Arc<BitmapAccumulator>,
+		header: BlockHeader,
+	) -> Segmenter {
+		Segmenter {
+			txhashset,
+			bitmap_snapshot,
+			header,
+		}
+	}
+
+	/// Header associated with this segmenter instance.
+	/// The bitmap "snapshot" corresponds to rewound state at this header.
+	pub fn header(&self) -> &BlockHeader {
+		&self.header
+	}
+
+	/// Create a kernel segment.
+	pub fn kernel_segment(&self, id: SegmentIdentifier) -> Result<Segment<TxKernel>, Error> {
+		let now = Instant::now();
+		let txhashset = self.txhashset.read();
+		let kernel_pmmr = txhashset.kernel_pmmr_at(&self.header);
+		let segment = Segment::from_pmmr(id, &kernel_pmmr, false)?;
+		debug!(
+			"kernel_segment: id: ({}, {}), leaves: {}, hashes: {}, proof hashes: {}, took {}ms",
+			segment.id().height,
+			segment.id().idx,
+			segment.leaf_iter().count(),
+			segment.hash_iter().count(),
+			segment.proof().size(),
+			now.elapsed().as_millis()
+		);
+		Ok(segment)
+	}
+
+	/// The root of the output PMMR based on size from the header.
+	fn output_root(&self) -> Result<Hash, Error> {
+		let txhashset = self.txhashset.read();
+		let pmmr = txhashset.output_pmmr_at(&self.header);
+		let root = pmmr.root().map_err(&ErrorKind::TxHashSetErr)?;
+		Ok(root)
+	}
+
+	/// The root of the bitmap snapshot PMMR.
+	fn bitmap_root(&self) -> Result<Hash, Error> {
+		let pmmr = self.bitmap_snapshot.readonly_pmmr();
+		let root = pmmr.root().map_err(&ErrorKind::TxHashSetErr)?;
+		Ok(root)
+	}
+
+	/// Create a utxo bitmap segment based on our bitmap "snapshot" and return it with
+	/// the corresponding output root.
+	pub fn bitmap_segment(
+		&self,
+		id: SegmentIdentifier,
+	) -> Result<(Segment<BitmapChunk>, Hash), Error> {
+		let now = Instant::now();
+		let bitmap_pmmr = self.bitmap_snapshot.readonly_pmmr();
+		let segment = Segment::from_pmmr(id, &bitmap_pmmr, false)?;
+		let output_root = self.output_root()?;
+		debug!(
+			"bitmap_segment: id: ({}, {}), leaves: {}, hashes: {}, proof hashes: {}, took {}ms",
+			segment.id().height,
+			segment.id().idx,
+			segment.leaf_iter().count(),
+			segment.hash_iter().count(),
+			segment.proof().size(),
+			now.elapsed().as_millis()
+		);
+		Ok((segment, output_root))
+	}
+
+	/// Create an output segment and return it with the corresponding bitmap root.
+	pub fn output_segment(
+		&self,
+		id: SegmentIdentifier,
+	) -> Result<(Segment<OutputIdentifier>, Hash), Error> {
+		let now = Instant::now();
+		let txhashset = self.txhashset.read();
+		let output_pmmr = txhashset.output_pmmr_at(&self.header);
+		let segment = Segment::from_pmmr(id, &output_pmmr, true)?;
+		let bitmap_root = self.bitmap_root()?;
+		debug!(
+			"output_segment: id: ({}, {}), leaves: {}, hashes: {}, proof hashes: {}, took {}ms",
+			segment.id().height,
+			segment.id().idx,
+			segment.leaf_iter().count(),
+			segment.hash_iter().count(),
+			segment.proof().size(),
+			now.elapsed().as_millis()
+		);
+		Ok((segment, bitmap_root))
+	}
+
+	/// Create a rangeproof segment.
+	pub fn rangeproof_segment(&self, id: SegmentIdentifier) -> Result<Segment<RangeProof>, Error> {
+		let now = Instant::now();
+		let txhashset = self.txhashset.read();
+		let pmmr = txhashset.rangeproof_pmmr_at(&self.header);
+		let segment = Segment::from_pmmr(id, &pmmr, true)?;
+		debug!(
+			"rangeproof_segment: id: ({}, {}), leaves: {}, hashes: {}, proof hashes: {}, took {}ms",
+			segment.id().height,
+			segment.id().idx,
+			segment.leaf_iter().count(),
+			segment.hash_iter().count(),
+			segment.proof().size(),
+			now.elapsed().as_millis()
+		);
+		Ok(segment)
+	}
+}