From 57f459249904a5e4ed2ed4bc26448dfcbf8aabeb Mon Sep 17 00:00:00 2001 From: Antioch Peverell Date: Tue, 23 Feb 2021 19:34:32 +0000 Subject: [PATCH] Retire pruned cache (#3573) * use range beneath subtree for efficient is_pruned check --- core/src/core/pmmr/pmmr.rs | 22 +++++++++++++---- core/tests/pmmr.rs | 12 ++++++++++ store/src/pmmr.rs | 3 +++ store/src/prune_list.rs | 48 ++++++++++++++++---------------------- 4 files changed, 53 insertions(+), 32 deletions(-) diff --git a/core/src/core/pmmr/pmmr.rs b/core/src/core/pmmr/pmmr.rs index c0591c9b9c..082c182235 100644 --- a/core/src/core/pmmr/pmmr.rs +++ b/core/src/core/pmmr/pmmr.rs @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::marker; -use std::u64; +use std::{marker, ops::Range, u64}; use croaring::Bitmap; @@ -541,10 +540,18 @@ pub fn peak_map_height(mut pos: u64) -> (u64, u64) { /// index. This function is the base on which all others, as well as the MMR, /// are built. pub fn bintree_postorder_height(num: u64) -> u64 { - if num == 0 { + let mut pos = num.saturating_sub(1); + if pos == 0 { return 0; } - peak_map_height(num - 1).1 + let mut peak_size = ALL_ONES >> pos.leading_zeros(); + while peak_size != 0 { + if pos >= peak_size { + pos -= peak_size; + } + peak_size >>= 1; + } + pos } /// Is this position a leaf in the MMR? @@ -658,3 +665,10 @@ pub fn bintree_leftmost(num: u64) -> u64 { let height = bintree_postorder_height(num); num + 2 - (2 << height) } + +/// All pos in the subtree beneath the provided root, including root itself. +pub fn bintree_range(num: u64) -> Range { + let height = bintree_postorder_height(num); + let leftmost = num + 2 - (2 << height); + leftmost..(num + 1) +} diff --git a/core/tests/pmmr.rs b/core/tests/pmmr.rs index c903a2dd50..46a862f73f 100644 --- a/core/tests/pmmr.rs +++ b/core/tests/pmmr.rs @@ -90,6 +90,18 @@ fn first_100_mmr_heights() { } } +#[test] +fn test_bintree_range() { + assert_eq!(pmmr::bintree_range(0), 0..1); + assert_eq!(pmmr::bintree_range(1), 1..2); + assert_eq!(pmmr::bintree_range(2), 2..3); + assert_eq!(pmmr::bintree_range(3), 1..4); + assert_eq!(pmmr::bintree_range(4), 4..5); + assert_eq!(pmmr::bintree_range(5), 5..6); + assert_eq!(pmmr::bintree_range(6), 4..7); + assert_eq!(pmmr::bintree_range(7), 1..8); +} + // The pos of the rightmost leaf for the provided MMR size (last leaf in subtree). #[test] fn test_bintree_rightmost() { diff --git a/store/src/pmmr.rs b/store/src/pmmr.rs index 908d59b8ee..6063e3e85e 100644 --- a/store/src/pmmr.rs +++ b/store/src/pmmr.rs @@ -285,6 +285,9 @@ impl PMMRBackend { } fn is_compacted(&self, pos: u64) -> bool { + if self.leaf_set.includes(pos) { + return false; + } self.is_pruned(pos) && !self.is_pruned_root(pos) } diff --git a/store/src/prune_list.rs b/store/src/prune_list.rs index 122700a1f3..0acb6d9faf 100644 --- a/store/src/prune_list.rs +++ b/store/src/prune_list.rs @@ -25,8 +25,9 @@ use std::io::{self, Write}; use std::path::{Path, PathBuf}; use croaring::Bitmap; +use grin_core::core::pmmr; -use crate::core::core::pmmr::{bintree_postorder_height, family, path}; +use crate::core::core::pmmr::{bintree_postorder_height, family}; use crate::{read_bitmap, save_via_temp_file}; /// Maintains a list of previously pruned nodes in PMMR, compacting the list as @@ -44,8 +45,6 @@ pub struct PruneList { path: Option, /// Bitmap representing pruned root node positions. bitmap: Bitmap, - /// Bitmap representing all pruned node positions (everything under the pruned roots). - pruned_cache: Bitmap, shift_cache: Vec, leaf_shift_cache: Vec, } @@ -59,7 +58,6 @@ impl PruneList { PruneList { path, bitmap, - pruned_cache: Bitmap::create(), shift_cache: vec![], leaf_shift_cache: vec![], } @@ -85,11 +83,10 @@ impl PruneList { prune_list.init_caches(); if !prune_list.bitmap.is_empty() { - debug!("bitmap {} pos ({} bytes), pruned_cache {} pos ({} bytes), shift_cache {}, leaf_shift_cache {}", + debug!( + "bitmap {} pos ({} bytes), shift_cache {}, leaf_shift_cache {}", prune_list.bitmap.cardinality(), prune_list.bitmap.get_serialized_size_in_bytes(), - prune_list.pruned_cache.cardinality(), - prune_list.pruned_cache.get_serialized_size_in_bytes(), prune_list.shift_cache.len(), prune_list.leaf_shift_cache.len(), ); @@ -102,7 +99,6 @@ impl PruneList { pub fn init_caches(&mut self) { self.build_shift_cache(); self.build_leaf_shift_cache(); - self.build_pruned_cache(); } /// Save the prune_list to disk. @@ -232,16 +228,17 @@ impl PruneList { pub fn add(&mut self, pos: u64) { assert!(pos > 0, "prune list 1-indexed, 0 not valid pos"); + if self.is_pruned(pos) { + return; + } + let mut current = pos; loop { let (parent, sibling) = family(current); - - if self.bitmap.contains(sibling as u32) || self.pruned_cache.contains(sibling as u32) { - self.pruned_cache.add(current as u32); + if self.is_pruned_root(sibling) { self.bitmap.remove(sibling as u32); current = parent; } else { - self.pruned_cache.add(current as u32); self.bitmap.add(current as u32); break; } @@ -263,26 +260,21 @@ impl PruneList { self.bitmap.iter().map(|x| x as u64).collect() } - /// Is the pos pruned? - /// Assumes the pruned_cache is fully built and up to date. + /// A pos is pruned if it is a pruned root directly or if it is + /// beneath the "next" pruned subtree. + /// We only need to consider the "next" subtree due to the append-only MMR structure. pub fn is_pruned(&self, pos: u64) -> bool { assert!(pos > 0, "prune list 1-indexed, 0 not valid pos"); - self.pruned_cache.contains(pos as u32) - } - - fn build_pruned_cache(&mut self) { - if self.bitmap.is_empty() { - return; + if self.is_pruned_root(pos) { + return true; } - let maximum = self.bitmap.maximum().unwrap_or(0); - self.pruned_cache = Bitmap::create_with_capacity(maximum); - for pos in 1..(maximum + 1) { - let pruned = path(pos as u64, maximum as u64).any(|x| self.bitmap.contains(x as u32)); - if pruned { - self.pruned_cache.add(pos as u32) - } + let rank = self.bitmap.rank(pos as u32); + if let Some(root) = self.bitmap.select(rank as u32) { + let range = pmmr::bintree_range(root as u64); + range.contains(&pos) + } else { + false } - self.pruned_cache.run_optimize(); } /// Is the specified position a root of a pruned subtree?