diff --git a/chain/src/chain.rs b/chain/src/chain.rs index a026a6d619..ff9bcd2d62 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -980,9 +980,12 @@ impl Chain { batch.save_body_tail(&tip)?; } - // Rebuild our output_pos index in the db based on fresh UTXO set. + // Initialize output_pos index in the db based on current UTXO set. txhashset.init_output_pos_index(&header_pmmr, &batch)?; + // Initialize the kernel_pos index for recent kernel history. + txhashset.init_kernel_pos_index(&header_pmmr, &batch)?; + // Commit all the changes to the db. batch.commit()?; @@ -1082,9 +1085,12 @@ impl Chain { // current "head" and "tail" height to our cut-through horizon and // allowing an additional 60 blocks in height before allowing a further compaction. if let (Ok(tail), Ok(head)) = (self.tail(), self.head()) { - let horizon = global::cut_through_horizon() as u64; - let threshold = horizon.saturating_add(60); - let next_compact = tail.height.saturating_add(threshold); + let threshold = if global::is_production_mode() { + global::cut_through_horizon().saturating_add(60) + } else { + global::cut_through_horizon() + }; + let next_compact = tail.height.saturating_add(threshold.into()); if next_compact > head.height { debug!( "compact: skipping startup compaction (next at {})", @@ -1099,6 +1105,11 @@ impl Chain { let mut txhashset = self.txhashset.write(); let batch = self.store.batch()?; + // Remove historical blocks from the db unless we are running in archive mode. + if !self.archive_mode { + self.remove_historical_blocks(&header_pmmr, &batch)?; + } + // Compact the txhashset itself (rewriting the pruned backend files). { let head_header = batch.head_header()?; @@ -1111,11 +1122,6 @@ impl Chain { txhashset.compact(&horizon_header, &batch)?; } - // If we are not in archival mode remove historical blocks from the db. - if !self.archive_mode { - self.remove_historical_blocks(&header_pmmr, &batch)?; - } - // Make sure our output_pos index is consistent with the UTXO set. txhashset.init_output_pos_index(&header_pmmr, &batch)?; @@ -1145,6 +1151,12 @@ impl Chain { Ok(self.txhashset.read().get_output_pos(commit)?) } + /// Get the position of the kernel if it exists in the kernel_pos index. + /// The index is limited to 14 days of recent kernels. + pub fn get_kernel_pos(&self, excess: Commitment) -> Result { + self.txhashset.read().get_kernel_pos(excess) + } + /// outputs by insertion index pub fn unspent_outputs_by_pmmr_index( &self, diff --git a/chain/src/store.rs b/chain/src/store.rs index 540de65358..55b339b3a2 100644 --- a/chain/src/store.rs +++ b/chain/src/store.rs @@ -34,9 +34,11 @@ const BLOCK_PREFIX: u8 = b'b'; const HEAD_PREFIX: u8 = b'H'; const TAIL_PREFIX: u8 = b'T'; const OUTPUT_POS_PREFIX: u8 = b'p'; +const KERNEL_POS_PREFIX: u8 = b'k'; const BLOCK_INPUT_BITMAP_PREFIX: u8 = b'B'; const BLOCK_SUMS_PREFIX: u8 = b'M'; const BLOCK_SPENT_PREFIX: u8 = b'S'; +const BLOCK_KERNEL_UNDO_PREFIX: u8 = b'U'; /// All chain-related database operations pub struct ChainStore { @@ -116,7 +118,7 @@ impl ChainStore { /// Get PMMR pos for the given output commitment. pub fn get_output_pos(&self, commit: &Commitment) -> Result { match self.get_output_pos_height(commit)? { - Some((pos, _)) => Ok(pos), + Some(pos) => Ok(pos.pos), None => Err(Error::NotFoundErr(format!( "Output position for: {:?}", commit @@ -125,9 +127,19 @@ impl ChainStore { } /// Get PMMR pos and block height for the given output commitment. - pub fn get_output_pos_height(&self, commit: &Commitment) -> Result, Error> { - self.db - .get_ser(&to_key(OUTPUT_POS_PREFIX, &mut commit.as_ref().to_vec())) + pub fn get_output_pos_height(&self, commit: &Commitment) -> Result, Error> { + self.db.get_ser(&to_key(OUTPUT_POS_PREFIX, &mut commit.as_ref().to_vec())) + } + + /// Get kernel_pos and block height from index. + /// Returns a vec of possible (pos, height) entries in the MMR. + /// Returns an empty vec if no entries found. + pub fn get_kernel_pos_height(&self, excess: &Commitment) -> Result { + option_to_not_found( + self.db + .get_ser(&to_key(KERNEL_POS_PREFIX, &mut excess.as_ref().to_vec())), + || format!("Kernel position for: {:?}", excess), + ) } /// Builds a new batch to be used with this store. @@ -199,6 +211,20 @@ impl<'a> Batch<'a> { Ok(()) } + /// We maintain an "undo" index for each full block to allow the kernel_pos + /// to be easily reverted during rewind. + /// We allow duplicate kernels so we need to know what to revert the kernel_pos + /// index to if we "undo" a kernel when rewinding a block. + pub fn save_kernel_undo_list( + &self, + h: &Hash, + pos: &Vec<(Commitment, CommitPos)>, + ) -> Result<(), Error> { + self.db + .put_ser(&to_key(BLOCK_KERNEL_UNDO_PREFIX, &mut h.to_vec())[..], pos)?; + Ok(()) + } + /// Migrate a block stored in the db by serializing it using the provided protocol version. /// Block may have been read using a previous protocol version but we do not actually care. pub fn migrate_block(&self, b: &Block, version: ProtocolVersion) -> Result<(), Error> { @@ -226,6 +252,7 @@ impl<'a> Batch<'a> { { let _ = self.delete_block_sums(bh); let _ = self.delete_spent_index(bh); + let _ = self.delete_kernel_undo_list(bh); } Ok(()) @@ -243,24 +270,25 @@ impl<'a> Batch<'a> { } /// Save output_pos and block height to index. - pub fn save_output_pos_height( - &self, - commit: &Commitment, - pos: u64, - height: u64, - ) -> Result<(), Error> { + pub fn save_output_pos_height(&self, commit: &Commitment, pos: CommitPos) -> Result<(), Error> { self.db.put_ser( &to_key(OUTPUT_POS_PREFIX, &mut commit.as_ref().to_vec())[..], - &(pos, height), + &pos, ) } - /// Delete the output_pos index entry for a spent output. + /// Delete a output_pos index entry. pub fn delete_output_pos_height(&self, commit: &Commitment) -> Result<(), Error> { self.db .delete(&to_key(OUTPUT_POS_PREFIX, &mut commit.as_ref().to_vec())) } + /// Delete a kernel_pos index entry + pub fn delete_kernel_pos_height(&self, excess: &Commitment) -> Result<(), Error> { + self.db + .delete(&to_key(KERNEL_POS_PREFIX, &mut excess.as_ref().to_vec())) + } + /// When using the output_pos iterator we have access to the index keys but not the /// original commitment that the key is constructed from. So we need a way of comparing /// a key with another commitment without reconstructing the commitment from the key bytes. @@ -270,15 +298,38 @@ impl<'a> Batch<'a> { } /// Iterator over the output_pos index. - pub fn output_pos_iter(&self) -> Result, Error> { + pub fn output_pos_iter(&self) -> Result, Error> { let key = to_key(OUTPUT_POS_PREFIX, &mut "".to_string().into_bytes()); self.db.iter(&key) } - /// Get output_pos from index. + /// Save kernel_pos and block height to index. + pub fn save_kernel_pos_height(&self, excess: &Commitment, pos: CommitPos) -> Result<(), Error> { + self.db.put_ser( + &to_key(KERNEL_POS_PREFIX, &mut excess.as_ref().to_vec())[..], + &pos, + ) + } + + /// Iterator over the kernel_pos index. + pub fn kernel_pos_iter(&self) -> Result, Error> { + let key = to_key(KERNEL_POS_PREFIX, &mut "".to_string().into_bytes()); + self.db.iter(&key) + } + + /// Get kernel_pos and block height from index. + pub fn get_kernel_pos_height(&self, excess: &Commitment) -> Result { + option_to_not_found( + self.db + .get_ser(&to_key(KERNEL_POS_PREFIX, &mut excess.as_ref().to_vec())), + || format!("Kernel pos for excess: {:?}", excess), + ) + } + + /// Get PMMR pos for the given output commitment. pub fn get_output_pos(&self, commit: &Commitment) -> Result { match self.get_output_pos_height(commit)? { - Some((pos, _)) => Ok(pos), + Some(pos) => Ok(pos.pos), None => Err(Error::NotFoundErr(format!( "Output position for: {:?}", commit @@ -286,10 +337,9 @@ impl<'a> Batch<'a> { } } - /// Get output_pos and block height from index. - pub fn get_output_pos_height(&self, commit: &Commitment) -> Result, Error> { - self.db - .get_ser(&to_key(OUTPUT_POS_PREFIX, &mut commit.as_ref().to_vec())) + /// Get PMMR pos and block height for the given output commitment. + pub fn get_output_pos_height(&self, commit: &Commitment) -> Result, Error> { + self.db.get_ser(&to_key(OUTPUT_POS_PREFIX, &mut commit.as_ref().to_vec())) } /// Get the previous header. @@ -317,6 +367,11 @@ impl<'a> Batch<'a> { .delete(&to_key(BLOCK_SPENT_PREFIX, &mut bh.to_vec())) } + fn delete_kernel_undo_list(&self, bh: &Hash) -> Result<(), Error> { + self.db + .delete(&to_key(BLOCK_KERNEL_UNDO_PREFIX, &mut bh.to_vec())) + } + /// Save block_sums for the block. pub fn save_block_sums(&self, h: &Hash, sums: BlockSums) -> Result<(), Error> { self.db @@ -371,6 +426,17 @@ impl<'a> Batch<'a> { ) } + /// Get the kernel "undo list" from the db for the specified block. + /// If we need to rewind a block then we use this to revert the index to previous kernel pos + /// in the case of duplicates. + pub fn get_kernel_undo_list(&self, bh: &Hash) -> Result, Error> { + option_to_not_found( + self.db + .get_ser(&to_key(BLOCK_KERNEL_UNDO_PREFIX, &mut bh.to_vec())), + || format!("kernel undo list: {}", bh), + ) + } + /// Commits this batch. If it's a child batch, it will be merged with the /// parent, otherwise the batch is written to db. pub fn commit(self) -> Result<(), Error> { diff --git a/chain/src/txhashset/txhashset.rs b/chain/src/txhashset/txhashset.rs index 48aa0f80f9..1c9db5452b 100644 --- a/chain/src/txhashset/txhashset.rs +++ b/chain/src/txhashset/txhashset.rs @@ -20,6 +20,7 @@ use crate::core::core::hash::{Hash, Hashed}; use crate::core::core::merkle_proof::MerkleProof; use crate::core::core::pmmr::{self, Backend, ReadonlyPMMR, RewindablePMMR, PMMR}; use crate::core::core::{Block, BlockHeader, Input, Output, OutputIdentifier, TxKernel}; +use crate::core::global; use crate::core::ser::{PMMRable, ProtocolVersion}; use crate::error::{Error, ErrorKind}; use crate::store::{Batch, ChainStore}; @@ -226,12 +227,12 @@ impl TxHashSet { pub fn get_unspent(&self, output_id: &OutputIdentifier) -> Result, Error> { let commit = output_id.commit; match self.commit_index.get_output_pos_height(&commit) { - Ok(Some((pos, height))) => { + Ok(Some(pos)) => { let output_pmmr: ReadonlyPMMR<'_, Output, _> = ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos); - if let Some(out) = output_pmmr.get_data(pos) { + if let Some(out) = output_pmmr.get_data(pos.pos) { if OutputIdentifier::from(out) == *output_id { - Ok(Some(CommitPos { pos, height })) + Ok(Some(pos)) } else { Ok(None) } @@ -299,6 +300,7 @@ impl TxHashSet { .elements_from_pmmr_index(start_index, max_count, max_index) } + /// TODO - Leverage the kernel_pos as a quick way of achieving this for recent kernels. /// Find a kernel with a given excess. Work backwards from `max_index` to `min_index` pub fn find_kernel( &self, @@ -346,6 +348,15 @@ impl TxHashSet { Ok(self.commit_index.get_output_pos(&commit)?) } + /// Get the position of the kernel if it exists in the kernel_pos index. + /// The index is limited to 14 days of recent kernels. + pub fn get_kernel_pos(&self, excess: Commitment) -> Result { + Ok(self + .commit_index + .get_kernel_pos_height(&excess) + .map(|x| x.pos)?) + } + /// build a new merkle proof for the given position. pub fn merkle_proof(&mut self, commit: Commitment) -> Result { let pos = self.commit_index.get_output_pos(&commit)?; @@ -376,6 +387,12 @@ impl TxHashSet { .backend .check_compact(horizon_header.output_mmr_size, &rewind_rm_pos)?; + // TODO - Does this actually belong here? + // Note: We have not yet updated chain "tail" (updated when we remove old blocks from db). + // So technically we leave too many entries in the kernel_pos index? + debug!("txhashset: compact kernel_pos index..."); + self.compact_kernel_pos_index(batch)?; + debug!("txhashset: ... compaction finished"); Ok(()) @@ -397,9 +414,9 @@ impl TxHashSet { // Iterate over the current output_pos index, removing any entries that // do not point to to the expected output. let mut removed_count = 0; - for (key, (pos, _)) in batch.output_pos_iter()? { - if let Some(out) = output_pmmr.get_data(pos) { - if let Ok(pos_via_mmr) = batch.get_output_pos(&out.commitment()) { + for (key, pos) in batch.output_pos_iter()? { + if let Some(out) = output_pmmr.get_data(pos.pos) { + if let Some(pos_via_mmr) = batch.get_output_pos_height(&out.commitment())? { // If the pos matches and the index key matches the commitment // then keep the entry, other we want to clean it up. if pos == pos_via_mmr && batch.is_match_output_pos_key(&key, &out.commitment()) @@ -454,7 +471,8 @@ impl TxHashSet { // Note: MMR position is 1-based and not 0-based, so here must be '>' instead of '>=' break; } - batch.save_output_pos_height(&commit, pos, h.height)?; + let height = h.height; + batch.save_output_pos_height(&commit, CommitPos { pos, height })?; i += 1; } } @@ -465,6 +483,70 @@ impl TxHashSet { ); Ok(()) } + + /// Initialize the kernel_pos index. + /// Find the block header at the kernel index horizon (14 days of history). + /// Then iterate over all kernels since then and add to the index. + pub fn init_kernel_pos_index( + &self, + header_pmmr: &PMMRHandle, + batch: &Batch<'_>, + ) -> Result<(), Error> { + let now = Instant::now(); + let head_header = batch.head_header()?; + let cutoff_height = head_header + .height + .saturating_sub(global::kernel_index_horizon().into()); + let cutoff_hash = header_pmmr.get_header_hash_by_height(cutoff_height)?; + let cutoff_header = batch.get_block_header(&cutoff_hash)?; + let cutoff_pos = if cutoff_header.height == 0 { + 1 + } else { + batch.get_previous_header(&cutoff_header)?.kernel_mmr_size + 1 + }; + let mut kernel_count = 0; + let mut current_header = cutoff_header; + for pos in cutoff_pos..(self.kernel_pmmr_h.last_pos + 1) { + while pos > current_header.kernel_mmr_size { + let next_hash = header_pmmr.get_header_hash_by_height(current_header.height + 1)?; + current_header = batch.get_block_header(&next_hash)?; + } + if pmmr::is_leaf(pos) { + if let Some(kernel) = self.kernel_pmmr_h.backend.get_data(pos) { + let height = current_header.height; + batch.save_kernel_pos_height(&kernel.excess, CommitPos { pos, height })?; + kernel_count += 1; + } + } + } + debug!( + "init_kernel_pos_index: {} kernels, took {}s", + kernel_count, + now.elapsed().as_secs() + ); + Ok(()) + } + + /// Compact the kernel_pos index by removing any entries older than the kernel index horizon. + /// This will ensure we always have 14 days of kernel history in the index. + fn compact_kernel_pos_index(&self, batch: &Batch<'_>) -> Result<(), Error> { + let now = Instant::now(); + let head_header = batch.head_header()?; + let cutoff_height = head_header + .height + .saturating_sub(global::kernel_index_horizon().into()); + let deleted = batch + .kernel_pos_iter()? + .filter(|(_, pos)| pos.height < cutoff_height) + .map(|(key, _)| batch.delete(&key)) + .count(); + debug!( + "compact_kernel_pos_index: deleted {} entries from the index, took {}s", + deleted, + now.elapsed().as_secs(), + ); + Ok(()) + } } /// Starts a new unit of work to extend (or rewind) the chain with additional @@ -931,9 +1013,8 @@ impl<'a> Extension<'a> { } /// Apply a new block to the current txhashet extension (output, rangeproof, kernel MMRs). - /// Returns a vec of commit_pos representing the pos and height of the outputs spent - /// by this block. pub fn apply_block(&mut self, b: &Block, batch: &Batch<'_>) -> Result<(), Error> { + let height = b.header.height; let mut affected_pos = vec![]; // Apply the output to the output and rangeproof MMRs. @@ -942,7 +1023,7 @@ impl<'a> Extension<'a> { for out in b.outputs() { let pos = self.apply_output(out, batch)?; affected_pos.push(pos); - batch.save_output_pos_height(&out.commitment(), pos, b.header.height)?; + batch.save_output_pos_height(&out.commitment(), CommitPos { pos, height })?; } // Remove the output from the output and rangeproof MMRs. @@ -957,9 +1038,18 @@ impl<'a> Extension<'a> { } batch.save_spent_index(&b.hash(), &spent)?; + // Apply each kernel. + // Update the kernel_pos index to reflect the new kernels. + // Build the kernel "undo list" for this block. + let mut kernel_undo_list = vec![]; for kernel in b.kernels() { - self.apply_kernel(kernel)?; + if let Ok(prev_pos) = batch.get_kernel_pos_height(&kernel.excess) { + kernel_undo_list.push((kernel.excess, prev_pos)); + } + let pos = self.apply_kernel(kernel)?; + batch.save_kernel_pos_height(&kernel.excess(), CommitPos { pos, height })?; } + batch.save_kernel_undo_list(&b.hash(), &kernel_undo_list)?; // Update our BitmapAccumulator based on affected outputs (both spent and created). self.apply_to_bitmap_accumulator(&affected_pos)?; @@ -988,9 +1078,9 @@ impl<'a> Extension<'a> { fn apply_input(&mut self, input: &Input, batch: &Batch<'_>) -> Result { let commit = input.commitment(); - if let Some((pos, height)) = batch.get_output_pos_height(&commit)? { + if let Some(pos) = batch.get_output_pos_height(&commit)? { // First check this input corresponds to an existing entry in the output MMR. - if let Some(out) = self.output_pmmr.get_data(pos) { + if let Some(out) = self.output_pmmr.get_data(pos.pos) { if OutputIdentifier::from(input) != out { return Err(ErrorKind::TxHashSetErr("output pmmr mismatch".to_string()).into()); } @@ -999,12 +1089,12 @@ impl<'a> Extension<'a> { // Now prune the output_pmmr, rproof_pmmr and their storage. // Input is not valid if we cannot prune successfully (to spend an unspent // output). - match self.output_pmmr.prune(pos) { + match self.output_pmmr.prune(pos.pos) { Ok(true) => { self.rproof_pmmr - .prune(pos) + .prune(pos.pos) .map_err(ErrorKind::TxHashSetErr)?; - Ok(CommitPos { pos, height }) + Ok(pos) } Ok(false) => Err(ErrorKind::AlreadySpent(commit).into()), Err(e) => Err(ErrorKind::TxHashSetErr(e).into()), @@ -1055,11 +1145,12 @@ impl<'a> Extension<'a> { } /// Push kernel onto MMR (hash and data files). - fn apply_kernel(&mut self, kernel: &TxKernel) -> Result<(), Error> { - self.kernel_pmmr + fn apply_kernel(&mut self, kernel: &TxKernel) -> Result { + let pos = self + .kernel_pmmr .push(kernel) - .map_err(&ErrorKind::TxHashSetErr)?; - Ok(()) + .map_err(ErrorKind::TxHashSetErr)?; + Ok(pos) } /// Build a Merkle proof for the given output and the block @@ -1148,6 +1239,9 @@ impl<'a> Extension<'a> { header: &BlockHeader, batch: &Batch<'_>, ) -> Result, Error> { + // Look the full block up in the db. We can only rewind full blocks. + let block = batch.get_block(&header.hash())?; + // The spent index allows us to conveniently "unspend" everything in a block. let spent = batch.get_spent_index(&header.hash()); @@ -1176,36 +1270,43 @@ impl<'a> Extension<'a> { let mut affected_pos = spent_pos.clone(); affected_pos.push(self.output_pmmr.last_pos); - // Remove any entries from the output_pos created by the block being rewound. - let block = batch.get_block(&header.hash())?; - let mut missing_count = 0; + // Remove any output_pos entries created by the block being rewound. for out in block.outputs() { - if batch.delete_output_pos_height(&out.commitment()).is_err() { - missing_count += 1; - } - } - if missing_count > 0 { - warn!( - "rewind_single_block: {} output_pos entries missing for: {} at {}", - missing_count, - header.hash(), - header.height, - ); + let _ = batch.delete_output_pos_height(&out.commitment()); } // Update output_pos based on "unspending" all spent pos from this block. - // This is necessary to ensure the output_pos index correclty reflects a + // This is necessary to ensure the output_pos index correctly reflects a // reused output commitment. For example an output at pos 1, spent, reused at pos 2. // The output_pos index should be updated to reflect the old pos 1 when unspent. if let Ok(spent) = spent { - for (x, y) in block.inputs().into_iter().zip(spent) { - batch.save_output_pos_height(&x.commitment(), y.pos, y.height)?; + for (input, pos) in block.inputs().into_iter().zip(spent) { + batch.save_output_pos_height(&input.commitment(), pos)?; } } + // Update the kernel_pos index for the block being rewound. + self.rewind_single_block_kernels(&block, batch)?; + Ok(affected_pos) } + fn rewind_single_block_kernels(&self, block: &Block, batch: &Batch<'_>) -> Result<(), Error> { + // Remove any kernel_pos entries created by the block being rewound. + for kern in block.kernels() { + let _ = batch.delete_kernel_pos_height(&kern.excess()); + } + + // Add back any previous kernel instances replaced by the block being rewound. + if let Ok(undo_list) = batch.get_kernel_undo_list(&block.hash()) { + for (excess, pos) in undo_list { + batch.save_kernel_pos_height(&excess, pos)?; + } + } + + Ok(()) + } + /// Rewinds the MMRs to the provided positions, given the output and /// kernel pos we want to rewind to. fn rewind_mmrs_to_pos( diff --git a/chain/src/types.rs b/chain/src/types.rs index b07e100fdb..e19a4286e6 100644 --- a/chain/src/types.rs +++ b/chain/src/types.rs @@ -259,7 +259,7 @@ impl OutputRoots { } /// Minimal struct representing a known MMR position and associated block height. -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct CommitPos { /// MMR position pub pos: u64, diff --git a/chain/tests/mine_simple_chain.rs b/chain/tests/mine_simple_chain.rs index 567f110cd4..d96c73afdb 100644 --- a/chain/tests/mine_simple_chain.rs +++ b/chain/tests/mine_simple_chain.rs @@ -730,14 +730,8 @@ fn spend_in_fork_and_compact() { prev = next.header.clone(); chain.process_block(next, chain::Options::SKIP_POW).unwrap(); } - chain.validate(false).unwrap(); - if let Err(e) = chain.compact() { - panic!("Error compacting chain: {:?}", e); - } - if let Err(e) = chain.validate(false) { - panic!("Validation error after compacting chain: {:?}", e); - } + chain.validate(false).expect("chain validation error"); } // Cleanup chain directory clean_output_dir(".grin6"); diff --git a/chain/tests/test_kernel_index.rs b/chain/tests/test_kernel_index.rs new file mode 100644 index 0000000000..eca0c984c6 --- /dev/null +++ b/chain/tests/test_kernel_index.rs @@ -0,0 +1,115 @@ +// Copyright 2020 The Grin Developers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use self::chain::Chain; +use self::core::core::hash::Hashed; +use self::core::core::{Block, BlockHeader, Transaction}; +use self::core::global::ChainTypes; +use self::core::libtx; +use self::core::pow::Difficulty; +use self::core::{global, pow}; +use self::keychain::{ExtKeychain, ExtKeychainPath, Keychain}; +use chrono::Duration; +use grin_chain as chain; +use grin_core as core; +use grin_keychain as keychain; +use grin_util as util; + +mod chain_test_helper; + +use self::chain_test_helper::{clean_output_dir, init_chain}; + +#[test] +fn kernel_index_after_compaction() { + global::set_mining_mode(ChainTypes::AutomatedTesting); + util::init_test_logger(); + // Cleanup chain directory + let chain_dir = ".grin_kernel_idx"; + clean_output_dir(chain_dir); + + let chain = init_chain(chain_dir, pow::mine_genesis_block().unwrap()); + let mut prev = chain.head_header().unwrap(); + let kc = ExtKeychain::from_random_seed(false).unwrap(); + + // mine some blocks + for n in 0..30 { + let next = prepare_block(&kc, &prev, &chain, 10 + n); + prev = next.header.clone(); + chain.process_block(next, chain::Options::SKIP_POW).unwrap(); + } + + chain.validate(false).unwrap(); + + { + let head = chain.head().unwrap(); + let header_at_horizon = chain + .get_header_by_height( + head.height + .saturating_sub(global::kernel_index_horizon() as u64), + ) + .unwrap(); + let block_at_horizon = chain.get_block(&header_at_horizon.hash()).unwrap(); + let block_pre_horizon = chain.get_block(&header_at_horizon.prev_hash).unwrap(); + + // Chain compaction will remove all blocks earlier than the horizon. + chain.compact().expect("chain compaction error"); + + // Kernels up to and including the horizon must be in the kernel index. + let kernel = block_at_horizon.kernels().first().unwrap(); + chain.get_kernel_pos(kernel.excess).unwrap(); + + // Kernels beyond the horizon are no longer in the kernel index. + let kernel = block_pre_horizon.kernels().first().unwrap(); + chain + .get_kernel_pos(kernel.excess) + .expect_err("kernel_pos should be compacted"); + } + + // Cleanup chain directory + clean_output_dir(chain_dir); +} + +fn prepare_block(kc: &K, prev: &BlockHeader, chain: &Chain, diff: u64) -> Block +where + K: Keychain, +{ + let mut b = prepare_block_nosum(kc, prev, diff, vec![]); + chain.set_txhashset_roots(&mut b).unwrap(); + b +} + +fn prepare_block_nosum(kc: &K, prev: &BlockHeader, diff: u64, txs: Vec<&Transaction>) -> Block +where + K: Keychain, +{ + let proof_size = global::proofsize(); + let key_id = ExtKeychainPath::new(1, diff as u32, 0, 0, 0).to_identifier(); + + let fees = txs.iter().map(|tx| tx.fee()).sum(); + let reward = + libtx::reward::output(kc, &libtx::ProofBuilder::new(kc), &key_id, fees, false).unwrap(); + let mut b = match core::core::Block::new( + prev, + txs.into_iter().cloned().collect(), + Difficulty::from_num(diff), + reward, + ) { + Err(e) => panic!("{:?}", e), + Ok(b) => b, + }; + b.header.timestamp = prev.timestamp + Duration::seconds(60); + b.header.pow.total_difficulty = prev.total_difficulty() + Difficulty::from_num(diff); + b.header.pow.proof = pow::Proof::random(proof_size); + b +} diff --git a/core/src/consensus.rs b/core/src/consensus.rs index 94218e4af1..4da85c5324 100644 --- a/core/src/consensus.rs +++ b/core/src/consensus.rs @@ -93,6 +93,11 @@ pub const BASE_EDGE_BITS: u8 = 24; /// easier to reason about. pub const CUT_THROUGH_HORIZON: u32 = WEEK_HEIGHT as u32; +/// A relative kernel lock is only applicable within a limited number of recent blocks. +/// This is consensus critical as the lock condition will be met once this number of blocks +/// has been exceeded and the referenced kernel "ages out". +pub const KERNEL_RELATIVE_HEIGHT_LIMIT: u32 = WEEK_HEIGHT as u32; + /// Default number of blocks in the past to determine the height where we request /// a txhashset (and full blocks from). Needs to be long enough to not overlap with /// a long reorg. diff --git a/core/src/global.rs b/core/src/global.rs index 339c3f7f40..b6fc408596 100644 --- a/core/src/global.rs +++ b/core/src/global.rs @@ -19,8 +19,8 @@ use crate::consensus::{ graph_weight, valid_header_version, HeaderInfo, BASE_EDGE_BITS, BLOCK_TIME_SEC, COINBASE_MATURITY, CUT_THROUGH_HORIZON, DAY_HEIGHT, DEFAULT_MIN_EDGE_BITS, - DIFFICULTY_ADJUST_WINDOW, INITIAL_DIFFICULTY, MAX_BLOCK_WEIGHT, PROOFSIZE, - SECOND_POW_EDGE_BITS, STATE_SYNC_THRESHOLD, + DIFFICULTY_ADJUST_WINDOW, INITIAL_DIFFICULTY, KERNEL_RELATIVE_HEIGHT_LIMIT, MAX_BLOCK_WEIGHT, + PROOFSIZE, SECOND_POW_EDGE_BITS, STATE_SYNC_THRESHOLD, }; use crate::core::block::HeaderVersion; use crate::pow::{ @@ -67,6 +67,12 @@ pub const AUTOMATED_TESTING_CUT_THROUGH_HORIZON: u32 = 20; /// Testing cut through horizon in blocks pub const USER_TESTING_CUT_THROUGH_HORIZON: u32 = 70; +/// Kernel index horizon for automated tests +pub const AUTOMATED_TESTING_KERNEL_INDEX_HORIZON: u32 = 25; + +/// Kernel index horizon for user testing +pub const USER_TESTING_KERNEL_INDEX_HORIZON: u32 = 2 * USER_TESTING_CUT_THROUGH_HORIZON; + /// Testing state sync threshold in blocks pub const TESTING_STATE_SYNC_THRESHOLD: u32 = 20; @@ -283,6 +289,17 @@ pub fn cut_through_horizon() -> u32 { } } +/// We maintain an index of "recent" kernels back to this horizon. +/// This must be sufficient for validating an NSKR lock under rewind scenario. +pub fn kernel_index_horizon() -> u32 { + let param_ref = CHAIN_TYPE.read(); + match *param_ref { + ChainTypes::AutomatedTesting => AUTOMATED_TESTING_KERNEL_INDEX_HORIZON, + ChainTypes::UserTesting => USER_TESTING_KERNEL_INDEX_HORIZON, + _ => KERNEL_RELATIVE_HEIGHT_LIMIT + CUT_THROUGH_HORIZON, + } +} + /// Threshold at which we can request a txhashset (and full blocks from) pub fn state_sync_threshold() -> u32 { let param_ref = CHAIN_TYPE.read();