Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions grovedb-bulk-append-tree/src/proof/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use bincode::{Decode, Encode};
use grovedb_dense_fixed_sized_merkle_tree::DenseTreeProof;
use grovedb_merkle_mountain_range::MmrTreeProof;
#[cfg(feature = "storage")]
use grovedb_merkle_mountain_range::{MMRStoreReadOps, MmrKeySize, MmrNode, MmrStore};
use grovedb_merkle_mountain_range::{MmrKeySize, MmrNode, MmrStore, MMR};
use grovedb_query::{Query, QueryItem};
#[cfg(feature = "storage")]
use grovedb_storage::StorageContext;
Expand Down Expand Up @@ -239,8 +239,9 @@ impl BulkAppendTreeProof {
let chunk_indices: Vec<u64> = chunk_indices_set.into_iter().collect();

let mmr_store = MmrStore::with_key_size(&tree.dense_tree.storage, MmrKeySize::U32);
let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, tree.mmr_overlay.clone());
let get_node = |pos: u64| -> grovedb_merkle_mountain_range::Result<Option<MmrNode>> {
(&mmr_store).element_at_position(pos).unwrap() // unwrap CostResult
mmr.batch.element_at_position(pos).unwrap() // unwrap CostResult
};

MmrTreeProof::generate(mmr_size, &chunk_indices, get_node).map_err(|e| {
Expand Down
39 changes: 32 additions & 7 deletions grovedb-bulk-append-tree/src/tree/append.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
Ok(Self {
total_count: 0,
dense_tree,
mmr_overlay: Vec::new(),
})
}

Expand All @@ -42,6 +43,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
Ok(Self {
total_count,
dense_tree,
mmr_overlay: Vec::new(),
})
}

Expand Down Expand Up @@ -136,24 +138,26 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
let leaf_count = mmr_size_to_leaf_count(mmr_size);
hash_count += hash_count_for_push(leaf_count);

// Create MmrStore on the fly from the dense tree's storage
// Create MmrStore on the fly from the dense tree's storage.
// Use the overlay from previous compactions so cross-compaction
// reads work without a storage round-trip. After push+get_root,
// take the overlay back (don't commit — that happens at session end).
let mmr_root = {
let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
let mut mmr = MMR::new(mmr_size, &mmr_store);
let mut mmr =
MMR::new_with_overlay(mmr_size, &mmr_store, std::mem::take(&mut self.mmr_overlay));
mmr.push(leaf)
.unwrap()
.map_err(|e| BulkAppendError::MmrError(format!("MMR push failed: {}", e)))?;

// Get root BEFORE commit — data is still in the MMRBatch overlay
let root_node = mmr
.get_root()
.unwrap()
.map_err(|e| BulkAppendError::MmrError(format!("MMR get_root failed: {}", e)))?;
let root = root_node.hash();

mmr.commit()
.unwrap()
.map_err(|e| BulkAppendError::MmrError(format!("MMR commit failed: {}", e)))?;
// Take overlay back instead of committing
self.mmr_overlay = mmr.batch.take_overlay();

root
};
Expand All @@ -171,11 +175,32 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
return Ok([0u8; 32]);
}
let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
let mmr = MMR::new(mmr_size, &mmr_store);
let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, self.mmr_overlay.clone());
let root_node = mmr
.get_root()
.unwrap()
.map_err(|e| BulkAppendError::MmrError(format!("MMR get_root failed: {}", e)))?;
Ok(root_node.hash())
}

/// Flush the MMR overlay to storage.
///
/// Call this at the end of a session to persist all MMR nodes that were
/// buffered during compaction cycles. This is a no-op if no compactions
/// occurred.
pub fn commit_mmr(&mut self) -> Result<(), BulkAppendError> {
if self.mmr_overlay.is_empty() {
return Ok(());
}
let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
let mut mmr = MMR::new_with_overlay(
self.mmr_size(),
&mmr_store,
std::mem::take(&mut self.mmr_overlay),
);
mmr.commit()
.unwrap()
.map_err(|e| BulkAppendError::MmrError(format!("MMR commit failed: {}", e)))?;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
Ok(())
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
11 changes: 8 additions & 3 deletions grovedb-bulk-append-tree/src/tree/fetch.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Read operations for BulkAppendTree.

use grovedb_dense_fixed_sized_merkle_tree::DenseTreeProof;
use grovedb_merkle_mountain_range::{leaf_to_pos, MMRStoreReadOps, MmrKeySize, MmrStore, MMR};
use grovedb_merkle_mountain_range::{leaf_to_pos, MmrKeySize, MmrStore, MMR};
use grovedb_query::Query;
use grovedb_storage::StorageContext;

Expand Down Expand Up @@ -69,13 +69,18 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
///
/// This reads from the **chunk MMR**, which stores immutable epoch blobs.
/// Returns `None` if the chunk hasn't been completed yet.
///
/// Uses the MMR overlay to find nodes that were pushed during this session
/// but not yet committed to storage.
pub fn get_chunk_value(&self, chunk_index: u64) -> Result<Option<Vec<u8>>, BulkAppendError> {
if chunk_index >= self.chunk_count() {
return Ok(None);
}
let mmr_pos = leaf_to_pos(chunk_index);
let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
let node = (&mmr_store)
let mmr = MMR::new_with_overlay(self.mmr_size(), &mmr_store, self.mmr_overlay.clone());
let node = mmr
.batch
.element_at_position(mmr_pos)
.unwrap()
.map_err(|e| {
Expand Down Expand Up @@ -130,7 +135,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
(Vec::new(), [0u8; 32])
} else {
let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
let mmr = MMR::new(mmr_size, &mmr_store);
let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, self.mmr_overlay.clone());

let positions: Vec<u64> = chunk_indices.iter().map(|&idx| leaf_to_pos(idx)).collect();
let proof = mmr.gen_proof(positions).unwrap().map_err(|e| {
Expand Down
6 changes: 6 additions & 0 deletions grovedb-bulk-append-tree/src/tree/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ pub use fetch::{BufferQueryResult, ChunkQueryResult};
mod tests;

use grovedb_dense_fixed_sized_merkle_tree::DenseFixedSizedMerkleTree;
use grovedb_merkle_mountain_range::MmrNode;

#[cfg(feature = "storage")]
use crate::BulkAppendError;
Expand Down Expand Up @@ -69,6 +70,11 @@ pub struct BulkAppendTree<S> {
/// the MMR size and dense tree state.
pub total_count: u64,
pub dense_tree: DenseFixedSizedMerkleTree<S>,
/// MMR node overlay: holds nodes pushed during this session that have
/// not yet been committed to storage. Persists across MMR instance
/// lifetimes (compaction cycles) so that reads can find recently-pushed
/// nodes without a storage round-trip.
pub(crate) mmr_overlay: Vec<(u64, Vec<MmrNode>)>,
}

impl<S> BulkAppendTree<S> {
Expand Down
10 changes: 10 additions & 0 deletions grovedb-commitment-tree/src/commitment_tree/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,16 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree<S, M> {

// ── BulkAppendTree delegates ──────────────────────────────────────

/// Flush the MMR overlay to storage.
///
/// Delegates to [`BulkAppendTree::commit_mmr`]. Call this at the end of a
/// session to persist MMR nodes buffered during compaction cycles.
pub fn commit_mmr(&mut self) -> Result<(), CommitmentTreeError> {
self.bulk_tree
.commit_mmr()
.map_err(|e| CommitmentTreeError::InvalidData(format!("MMR commit: {}", e)))
}

/// Get the total count of items appended (from the BulkAppendTree).
pub fn total_count(&self) -> u64 {
self.bulk_tree.total_count
Expand Down
10 changes: 8 additions & 2 deletions grovedb-dense-fixed-sized-merkle-tree/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -681,8 +681,14 @@ fn test_get_store_inconsistency_errors() {

#[test]
fn test_proof_generate_store_failure() {
let mut tree = DenseFixedSizedMerkleTree::new(3, MemStorageContext::new()).expect("height 3");
tree.insert(b"val").unwrap().expect("insert should succeed");
// Use from_state (not insert) so the write-through cache is empty.
// Then corrupt the store to simulate missing data.
let store = MemStorageContext::new();
store
.data
.borrow_mut()
.insert(position_key(0).to_vec(), b"val".to_vec());
let tree = DenseFixedSizedMerkleTree::from_state(3, 1, store).expect("from_state");

// Corrupt the store by removing the value
tree.storage
Expand Down
53 changes: 47 additions & 6 deletions grovedb-dense-fixed-sized-merkle-tree/src/tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ pub fn position_key(pos: u16) -> [u8; 2] {
///
/// Storage is embedded directly on the struct (like Merk).
///
/// A write-through cache (`cache`) holds values written during this session.
/// Reads check the cache first, falling back to storage. This enables use
/// with transactional storage contexts where writes are deferred to a batch
/// and not yet visible through reads.
///
/// Note: root hash computation is O(n) per insert where n = count, since no
/// intermediate hashes are cached. Suitable for small trees (epoch sizes
/// typically 16-256).
Expand All @@ -41,6 +46,10 @@ pub struct DenseFixedSizedMerkleTree<S> {
count: u16,
/// The underlying storage context.
pub storage: S,
/// Write-through cache: holds values written in this session.
/// Indexed by position. `None` means the value has not been written
/// in this session (fall back to storage).
cache: Vec<Option<Vec<u8>>>,
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

// ── Pure accessors (no storage bounds needed) ─────────────────────────
Expand Down Expand Up @@ -77,14 +86,20 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree<S> {
/// Height must be between 1 and 16 inclusive.
pub fn new(height: u8, storage: S) -> Result<Self, DenseMerkleError> {
validate_height(height)?;
let capacity = Self::capacity_for_height(height);
Ok(Self {
height,
count: 0,
storage,
cache: vec![None; capacity as usize],
})
}

/// Reconstitute a tree from stored state.
///
/// The cache starts empty — pre-existing values are loaded from storage
/// on demand. Only values written via [`insert`] or [`try_insert`] in
/// this session are cached.
pub fn from_state(height: u8, count: u16, storage: S) -> Result<Self, DenseMerkleError> {
validate_height(height)?;
let capacity = Self::capacity_for_height(height);
Expand All @@ -98,6 +113,7 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree<S> {
height,
count,
storage,
cache: vec![None; capacity as usize],
})
}

Expand Down Expand Up @@ -204,16 +220,31 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree<S> {

/// Reset the tree to empty state.
///
/// Sets count to 0. Old values remain in the underlying storage (they
/// will be overwritten on the next cycle).
/// Sets count to 0 and clears the write-through cache. Old values
/// remain in the underlying storage (they will be overwritten on the
/// next cycle).
pub fn reset(&mut self) {
self.count = 0;
self.cache.fill(None);
}

// ── Internal storage helpers ──────────────────────────────────────

/// Read a value by position from storage.
/// Read a value by position, checking the write-through cache first.
///
/// Cache hits return deterministic costs (seek_count=1,
/// storage_loaded_bytes=len) matching the MMRBatch pattern, so fee
/// estimates are consistent regardless of cache state.
pub(crate) fn get_value(&self, position: u16) -> CostResult<Option<Vec<u8>>, DenseMerkleError> {
// Check write-through cache first
if let Some(Some(cached)) = self.cache.get(position as usize) {
return Ok(Some(cached.clone())).wrap_with_cost(OperationCost {
seek_count: 1,
storage_loaded_bytes: cached.len() as u64,
..Default::default()
});
}
// Fall back to storage
let mut cost = OperationCost::default();
let key = position_key(position);
let result = self.storage.get(key).unwrap_add_cost(&mut cost);
Expand All @@ -227,16 +258,26 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree<S> {
}
}

/// Write a value by position to storage.
fn put_value(&self, position: u16, value: &[u8]) -> CostResult<(), DenseMerkleError> {
/// Write a value by position to storage and cache.
///
/// On success, the value is stored in the write-through cache so that
/// subsequent reads (e.g., during root hash computation) can be served
/// from memory even when the storage context defers writes.
fn put_value(&mut self, position: u16, value: &[u8]) -> CostResult<(), DenseMerkleError> {
let mut cost = OperationCost::default();
let key = position_key(position);
let result = self
.storage
.put(key, value, None, None)
.unwrap_add_cost(&mut cost);
match result {
Ok(()) => Ok(()).wrap_with_cost(cost),
Ok(()) => {
// Cache on successful write
if let Some(slot) = self.cache.get_mut(position as usize) {
*slot = Some(value.to_vec());
}
Ok(()).wrap_with_cost(cost)
}
Err(e) => Err(DenseMerkleError::StoreError(format!(
"put at pos {}: {}",
position, e
Expand Down
12 changes: 12 additions & 0 deletions grovedb-merkle-mountain-range/src/mmr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,18 @@ impl<S> MMR<S> {
}
}

/// Create an MMR with a pre-populated overlay.
///
/// The overlay contains nodes from previous push operations that have
/// not yet been committed to storage. This allows an MMR instance to
/// read nodes pushed by a previous instance without a storage round-trip.
pub fn new_with_overlay(mmr_size: u64, store: S, overlay: Vec<(u64, Vec<MmrNode>)>) -> Self {
MMR {
mmr_size,
batch: MMRBatch::with_overlay(store, overlay),
}
}

/// Returns `true` if the MMR contains no elements.
pub fn is_empty(&self) -> bool {
self.mmr_size == 0
Expand Down
20 changes: 20 additions & 0 deletions grovedb-merkle-mountain-range/src/mmr_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,26 @@ impl<Store> MMRBatch<Store> {
}
}

/// Create a batch with pre-populated overlay data.
///
/// Used to restore an overlay from a previous session (e.g., across
/// multiple compaction cycles in BulkAppendTree) so that reads can
/// find nodes that were pushed but not yet committed to storage.
pub(crate) fn with_overlay(store: Store, overlay: Vec<(u64, Vec<MmrNode>)>) -> Self {
MMRBatch {
memory_batch: overlay,
store,
}
}

/// Extract the in-memory overlay data, leaving the batch empty.
///
/// Used to persist the overlay across MMR instance lifetimes without
/// committing to storage.
pub fn take_overlay(&mut self) -> Vec<(u64, Vec<MmrNode>)> {
std::mem::take(&mut self.memory_batch)
}

/// Buffer a contiguous run of elements starting at `pos`.
///
/// Entries must be appended with monotonically increasing positions
Expand Down
Loading
Loading