From 56926538240df3217be163ac2d709d3c0fd00f5d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:05:30 +0700 Subject: [PATCH 1/8] feat(bulk-append-tree): chunk-aligned paginated range reads with proof helpers Add the position-range read seam at the shared BulkAppendTree layer so every BulkAppendTree-backed element type (CommitmentTree today, the planned PrivateDocumentStore / DataCommitmentTree) inherits it: - BulkAppendTree::get_range(start, limit): fetch entries for [start, start + limit) clamped to total_count, returned as a RangePage (entries + total_count). Chunk-aligned: each completed chunk overlapping the range is read and deserialized exactly once, so a page costs O(chunks touched) blob reads instead of O(entries) random reads. - position_range_query(start, limit): the canonical Query (8-byte big-endian position keys) shared by prover and verifier. - BulkAppendTreeProof::generate_for_range / verify_range: paginated proof round-trip over the existing chunk-MMR + dense-buffer proof, with completeness enforced; absence past the end falls out of the authenticated total_count (position >= count), not per-position absence proofs. - CommitmentTree::get_range pass-through for the shielded-pool scanning path. Part of work-plan item 4 of dashpay/grovedb#784. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/lib.rs | 4 +- grovedb-bulk-append-tree/src/proof/mod.rs | 67 +++++++ grovedb-bulk-append-tree/src/proof/tests.rs | 166 ++++++++++++++++++ grovedb-bulk-append-tree/src/tree/fetch.rs | 71 +++++++- grovedb-bulk-append-tree/src/tree/mod.rs | 15 ++ grovedb-bulk-append-tree/src/tree/tests.rs | 139 +++++++++++++++ .../src/commitment_tree/mod.rs | 16 +- grovedb-commitment-tree/src/lib.rs | 2 +- 8 files changed, 475 insertions(+), 5 deletions(-) diff --git a/grovedb-bulk-append-tree/src/lib.rs b/grovedb-bulk-append-tree/src/lib.rs index f46436b85..36a18f7b1 100644 --- a/grovedb-bulk-append-tree/src/lib.rs +++ b/grovedb-bulk-append-tree/src/lib.rs @@ -22,7 +22,7 @@ pub use error::BulkAppendError; pub use grovedb_dense_fixed_sized_merkle_tree::{DenseFixedSizedMerkleTree, DenseTreeProof}; #[cfg(feature = "storage")] pub use grovedb_merkle_mountain_range::{MmrKeySize, MmrStore}; -pub use proof::{BulkAppendTreeProof, BulkAppendTreeProofResult}; -pub use tree::{hash::compute_state_root, leaf_count_to_mmr_size, BulkAppendTree}; +pub use proof::{position_range_query, BulkAppendTreeProof, BulkAppendTreeProofResult}; +pub use tree::{hash::compute_state_root, leaf_count_to_mmr_size, BulkAppendTree, RangePage}; #[cfg(feature = "storage")] pub use tree::{AppendResult, BufferQueryResult, ChunkQueryResult}; diff --git a/grovedb-bulk-append-tree/src/proof/mod.rs b/grovedb-bulk-append-tree/src/proof/mod.rs index a8dd6186d..4dbcab404 100644 --- a/grovedb-bulk-append-tree/src/proof/mod.rs +++ b/grovedb-bulk-append-tree/src/proof/mod.rs @@ -176,6 +176,25 @@ fn query_to_ranges(query: &Query, total_count: u64) -> Result, B Ok(merged) } +/// Build the canonical [`Query`] selecting the position range +/// `[start, start + limit)`, with positions encoded as 8-byte big-endian +/// keys. +/// +/// This is the query shape used by the paginated-scan pattern: prover and +/// verifier both derive it from `(start, limit)`, so a client only needs its +/// cursor and page size. `start + limit` saturates at `u64::MAX`, and +/// verification clamps the range to the tree's provable total count. +pub fn position_range_query(start: u64, limit: u16) -> Query { + let end = start.saturating_add(limit as u64); + Query { + items: vec![QueryItem::Range( + start.to_be_bytes().to_vec()..end.to_be_bytes().to_vec(), + )], + left_to_right: true, + ..Query::default() + } +} + /// Check whether `pos` falls inside any of the sorted, non-overlapping ranges. fn in_ranges(pos: u64, ranges: &[(u64, u64)]) -> bool { ranges @@ -325,6 +344,54 @@ impl BulkAppendTreeProof { }) } + /// Generate a proof for the paginated position range + /// `[start, start + limit)`. + /// + /// Convenience wrapper over [`generate`](Self::generate) using the + /// canonical [`position_range_query`]. The proof is chunk-aligned: it + /// carries each completed chunk blob overlapping the range plus the + /// buffer entries in range, so proof size is O(chunks touched). + /// + /// Ranges past the end of the tree are valid and produce a proof of the + /// (empty) result: absence of positions `>= total_count` falls out of + /// the authenticated element's total count, not out of per-position + /// absence proofs. + #[cfg(feature = "storage")] + pub fn generate_for_range<'db, S: StorageContext<'db>>( + tree: &BulkAppendTree, + start: u64, + limit: u16, + ) -> Result { + Self::generate(&position_range_query(start, limit), tree) + } + + /// Verify this proof against the paginated position range + /// `[start, start + limit)`. + /// + /// Convenience wrapper over + /// [`verify_against_query`](Self::verify_against_query) using the + /// canonical [`position_range_query`]. Returns the `(global_position, + /// value)` pairs in the range, ascending and contiguous, clamped to + /// `total_count`. Completeness is enforced: a proof missing any + /// requested position below `total_count` is rejected. Positions + /// `>= total_count` are provably absent by `total_count` itself, which + /// callers must take from the authenticated BulkAppendTree element. + pub fn verify_range( + &self, + expected_state_root: &[u8; 32], + height: u8, + total_count: u64, + start: u64, + limit: u16, + ) -> Result)>, BulkAppendError> { + self.verify_against_query( + expected_state_root, + height, + total_count, + &position_range_query(start, limit), + ) + } + /// Verify this proof against an expected state root. /// /// `height` and `total_count` come from the authenticated BulkAppendTree diff --git a/grovedb-bulk-append-tree/src/proof/tests.rs b/grovedb-bulk-append-tree/src/proof/tests.rs index 83b6f30d3..ec6a77e44 100644 --- a/grovedb-bulk-append-tree/src/proof/tests.rs +++ b/grovedb-bulk-append-tree/src/proof/tests.rs @@ -940,4 +940,170 @@ mod proof_tests { ); } } + + // ── generate_for_range / verify_range (paginated scan pattern) ─────── + + /// Helper: build a tree of `n` values "val_0".."val_{n-1}" and return + /// (state_root, tree). + fn build_indexed_tree(height: u8, n: u32) -> ([u8; 32], BulkAppendTree) { + let values: Vec> = (0..n).map(|i| format!("val_{}", i).into_bytes()).collect(); + build_test_tree(height, &values) + } + + /// Helper: round-trip a range proof and assert the returned page is + /// exactly positions `expected_start..expected_end`. + fn assert_range_roundtrip( + state_root: &[u8; 32], + tree: &BulkAppendTree, + start: u64, + limit: u16, + expected_start: u64, + expected_end: u64, + ) { + let proof = + BulkAppendTreeProof::generate_for_range(tree, start, limit).expect("generate range"); + + // Wire round-trip: encode + decode like a real client + let bytes = proof.encode_to_vec().expect("encode"); + let decoded = BulkAppendTreeProof::decode_from_slice(&bytes).expect("decode"); + + let entries = decoded + .verify_range(state_root, tree.height(), tree.total_count, start, limit) + .expect("verify range"); + + assert_eq!(entries.len(), (expected_end - expected_start) as usize); + for (i, (pos, value)) in entries.iter().enumerate() { + assert_eq!(*pos, expected_start + i as u64); + assert_eq!(value, format!("val_{}", pos).as_bytes()); + } + } + + #[test] + fn test_range_roundtrip_buffer_only() { + // height=3, capacity=7: 5 values all in buffer + let (root, tree) = build_indexed_tree(3, 5); + assert_range_roundtrip(&root, &tree, 1, 3, 1, 4); + } + + #[test] + fn test_range_roundtrip_across_chunk_boundary() { + // height=2, epoch_size=4: 10 values = 2 chunks + 2 buffered + let (root, tree) = build_indexed_tree(2, 10); + // spans chunk 0 / chunk 1 + assert_range_roundtrip(&root, &tree, 3, 3, 3, 6); + // spans chunk 1 / buffer + assert_range_roundtrip(&root, &tree, 6, 4, 6, 10); + } + + #[test] + fn test_range_roundtrip_single_entry_pages() { + let (root, tree) = build_indexed_tree(2, 10); + for pos in 0..10u64 { + assert_range_roundtrip(&root, &tree, pos, 1, pos, pos + 1); + } + } + + #[test] + fn test_range_roundtrip_empty_range() { + let (root, tree) = build_indexed_tree(2, 10); + // limit 0: proof still verifies against the root, returns nothing + assert_range_roundtrip(&root, &tree, 3, 0, 3, 3); + } + + #[test] + fn test_range_roundtrip_past_end() { + let (root, tree) = build_indexed_tree(2, 10); + // starts exactly at total_count + assert_range_roundtrip(&root, &tree, 10, 5, 10, 10); + // starts far past total_count + assert_range_roundtrip(&root, &tree, 1000, 5, 1000, 1000); + // clamped at the end + assert_range_roundtrip(&root, &tree, 8, 100, 8, 10); + } + + #[test] + fn test_range_roundtrip_large_multi_chunk_page() { + // height=4, epoch_size=16: 100 values = 6 chunks + 4 buffered. + // One page covering everything touches all chunks and the buffer. + let (root, tree) = build_indexed_tree(4, 100); + assert_range_roundtrip(&root, &tree, 0, 100, 0, 100); + // A large page crossing several chunk boundaries mid-tree + assert_range_roundtrip(&root, &tree, 10, 70, 10, 80); + } + + #[test] + fn test_range_roundtrip_empty_tree() { + let (_, tree) = build_indexed_tree(2, 0); + // For an empty tree the state root is blake3("bulk_state" || 0*32 || 0*32) + let root = crate::compute_state_root(&[0u8; 32], &[0u8; 32]); + assert_range_roundtrip(&root, &tree, 0, 10, 0, 0); + } + + #[test] + fn test_range_paged_scan_covers_everything() { + // The client scan pattern: page through the whole tree with + // limit=7 (deliberately not aligned to epoch_size=4). + let (root, tree) = build_indexed_tree(2, 30); + let mut cursor = 0u64; + let mut seen = Vec::new(); + while cursor < tree.total_count { + let proof = + BulkAppendTreeProof::generate_for_range(&tree, cursor, 7).expect("generate page"); + let entries = proof + .verify_range(&root, tree.height(), tree.total_count, cursor, 7) + .expect("verify page"); + assert!(!entries.is_empty()); + cursor += entries.len() as u64; + seen.extend(entries); + } + assert_eq!(seen.len(), 30); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, format!("val_{}", i).as_bytes()); + } + } + + #[test] + fn test_range_proof_wrong_root_rejected() { + let (root, tree) = build_indexed_tree(2, 10); + let proof = BulkAppendTreeProof::generate_for_range(&tree, 0, 5).expect("generate"); + let mut bad_root = root; + bad_root[0] ^= 1; + proof + .verify_range(&bad_root, tree.height(), tree.total_count, 0, 5) + .expect_err("tampered root must be rejected"); + } + + #[test] + fn test_range_proof_missing_chunk_rejected() { + // Proof generated for [0, 2) (chunk 0 only) must not verify a + // request for [0, 6) which also needs chunk 1. + let (root, tree) = build_indexed_tree(2, 10); + let narrow = BulkAppendTreeProof::generate_for_range(&tree, 0, 2).expect("generate"); + narrow + .verify_range(&root, tree.height(), tree.total_count, 0, 6) + .expect_err("proof missing chunk 1 must be rejected for the wider range"); + } + + #[test] + fn test_position_range_query_shape() { + let q = super::super::position_range_query(5, 3); + assert_eq!(q.items.len(), 1); + match &q.items[0] { + QueryItem::Range(r) => { + assert_eq!(r.start, 5u64.to_be_bytes().to_vec()); + assert_eq!(r.end, 8u64.to_be_bytes().to_vec()); + } + other => panic!("expected Range item, got {:?}", other), + } + + // start + limit saturates instead of wrapping + let q = super::super::position_range_query(u64::MAX - 1, 100); + match &q.items[0] { + QueryItem::Range(r) => { + assert_eq!(r.end, u64::MAX.to_be_bytes().to_vec()); + } + other => panic!("expected Range item, got {:?}", other), + } + } } diff --git a/grovedb-bulk-append-tree/src/tree/fetch.rs b/grovedb-bulk-append-tree/src/tree/fetch.rs index 3ad8bed63..4d9ea342d 100644 --- a/grovedb-bulk-append-tree/src/tree/fetch.rs +++ b/grovedb-bulk-append-tree/src/tree/fetch.rs @@ -5,7 +5,7 @@ use grovedb_merkle_mountain_range::{leaf_to_pos, MmrKeySize, MmrStore, MMR}; use grovedb_query::Query; use grovedb_storage::StorageContext; -use super::BulkAppendTree; +use super::{BulkAppendTree, RangePage}; use crate::{chunk::deserialize_chunk_blob, BulkAppendError}; /// Result of querying the dense tree buffer. @@ -63,6 +63,75 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { Ok(BufferQueryResult { entries, proof }) } + // ── Range operations (chunks + buffer) ─────────────────────────── + + /// Fetch entries for the position range `[start, start + limit)`, + /// clamped to the tree's total count. + /// + /// This is the paginated-scan read path: clients walking "all entries + /// since my cursor" call it with their cursor as `start` and advance by + /// `entries.len()`. The read is chunk-aligned — each completed chunk + /// overlapping the range is read and deserialized exactly once, so a + /// page costs O(chunks touched) blob reads plus one read per buffer + /// entry, not O(entries) random reads. + /// + /// Absence needs no lookup: positions `>= total_count` do not exist, so + /// a page shorter than `limit` means the end of the tree was reached. + pub fn get_range(&self, start: u64, limit: u16) -> Result { + let total_count = self.total_count; + let end = start.saturating_add(limit as u64).min(total_count); + if start >= end { + return Ok(RangePage { + entries: Vec::new(), + total_count, + }); + } + let mut entries = Vec::with_capacity((end - start) as usize); + + let epoch_size = self.epoch_size(); + let buffer_start = self.chunk_count() * epoch_size; + + // Completed chunks overlapping [start, min(end, buffer_start)) + let chunk_end = end.min(buffer_start); + if start < chunk_end { + let first_chunk = start / epoch_size; + let last_chunk = (chunk_end - 1) / epoch_size; + for chunk_idx in first_chunk..=last_chunk { + let blob = self.get_chunk_value(chunk_idx)?.ok_or_else(|| { + BulkAppendError::CorruptedData(format!( + "missing chunk blob for index {}", + chunk_idx + )) + })?; + let chunk_entries = deserialize_chunk_blob(&blob)?; + let chunk_start = chunk_idx * epoch_size; + for (i, value) in chunk_entries.into_iter().enumerate() { + let pos = chunk_start + i as u64; + if pos >= start && pos < chunk_end { + entries.push((pos, value)); + } + } + } + } + + // Buffer tail: positions in [max(start, buffer_start), end) + for pos in start.max(buffer_start)..end { + let buffer_pos = (pos - buffer_start) as u16; + let value = self.get_buffer_value(buffer_pos)?.ok_or_else(|| { + BulkAppendError::CorruptedData(format!( + "missing buffer value at position {}", + buffer_pos + )) + })?; + entries.push((pos, value)); + } + + Ok(RangePage { + entries, + total_count, + }) + } + // ── Chunk operations (MMR) ─────────────────────────────────────── /// Get a single completed chunk's raw blob by chunk index. diff --git a/grovedb-bulk-append-tree/src/tree/mod.rs b/grovedb-bulk-append-tree/src/tree/mod.rs index bcea9a521..e41f2e6b8 100644 --- a/grovedb-bulk-append-tree/src/tree/mod.rs +++ b/grovedb-bulk-append-tree/src/tree/mod.rs @@ -57,6 +57,21 @@ pub struct AppendNoStateRootResult { pub compacted: bool, } +/// A contiguous page of entries returned by a position-range read. +/// +/// Produced by [`BulkAppendTree::get_range`], which fetches the entries for +/// `[start, start + limit)` clamped to the tree's total count. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RangePage { + /// `(global_position, value)` pairs, ascending and contiguous from the + /// requested start position (clamped to `total_count`). + pub entries: Vec<(u64, Vec)>, + /// Total number of entries in the tree at read time. Positions + /// `>= total_count` do not exist, so a page that ends before + /// `start + limit` is complete — there is nothing further to fetch. + pub total_count: u64, +} + /// Compute MMR size from leaf count: `2 * n - popcount(n)`. /// /// This is a well-known MMR property: the total number of nodes (leaves + diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index 3c3f2e490..e5987e8d2 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -390,3 +390,142 @@ fn query_chunks_empty_indices_returns_empty_proof() { assert!(result.mmr_proof_items.is_empty()); assert_eq!(result.mmr_root, [0u8; 32]); } + +// ── get_range (paginated position-range reads) ─────────────────────── + +/// Helper: build a tree with `n` single-byte values `[0], [1], ...`. +fn build_range_tree(height: u8, n: u8) -> BulkAppendTree { + let mut tree = BulkAppendTree::new(height, MemStorageContext::new()).expect("create tree"); + for i in 0..n { + tree.append(&[i]).expect("append"); + } + tree +} + +/// Helper: assert a page holds exactly positions `start..end` with value +/// `[pos as u8]` at each. +fn assert_page(page: &super::RangePage, start: u64, end: u64, total_count: u64) { + assert_eq!(page.total_count, total_count); + assert_eq!(page.entries.len(), (end - start) as usize); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, start + i as u64); + assert_eq!(value, &vec![*pos as u8]); + } +} + +#[test] +fn get_range_buffer_only() { + // height=3, capacity=7: 5 values all in buffer + let tree = build_range_tree(3, 5); + assert_eq!(tree.chunk_count(), 0); + + let page = tree.get_range(1, 3).expect("get range"); + assert_page(&page, 1, 4, 5); +} + +#[test] +fn get_range_single_chunk() { + // height=2, epoch_size=4: 8 values = 2 full chunks + let tree = build_range_tree(2, 8); + assert_eq!(tree.chunk_count(), 2); + assert_eq!(tree.buffer_count(), 0); + + // Page entirely inside chunk 0 + let page = tree.get_range(1, 2).expect("get range"); + assert_page(&page, 1, 3, 8); +} + +#[test] +fn get_range_across_chunk_boundary() { + // height=2, epoch_size=4: 10 values = 2 chunks + 2 buffered + let tree = build_range_tree(2, 10); + assert_eq!(tree.chunk_count(), 2); + assert_eq!(tree.buffer_count(), 2); + + // Page [3, 6) spans the chunk 0 / chunk 1 boundary + let page = tree.get_range(3, 3).expect("get range"); + assert_page(&page, 3, 6, 10); + + // Page [6, 10) spans the chunk 1 / buffer boundary + let page = tree.get_range(6, 4).expect("get range"); + assert_page(&page, 6, 10, 10); +} + +#[test] +fn get_range_whole_tree() { + let tree = build_range_tree(2, 10); + let page = tree.get_range(0, 100).expect("get range"); + assert_page(&page, 0, 10, 10); +} + +#[test] +fn get_range_empty_limit() { + let tree = build_range_tree(2, 10); + let page = tree.get_range(3, 0).expect("get range"); + assert_page(&page, 3, 3, 10); +} + +#[test] +fn get_range_past_end() { + let tree = build_range_tree(2, 10); + + // Start exactly at total_count + let page = tree.get_range(10, 5).expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Start far past total_count + let page = tree.get_range(1000, 5).expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Range that starts inside but extends past the end is clamped + let page = tree.get_range(8, 100).expect("get range"); + assert_page(&page, 8, 10, 10); +} + +#[test] +fn get_range_single_entry() { + let tree = build_range_tree(2, 10); + let page = tree.get_range(7, 1).expect("get range"); + assert_page(&page, 7, 8, 10); +} + +#[test] +fn get_range_empty_tree() { + let tree = build_range_tree(2, 0); + let page = tree.get_range(0, 10).expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 0); +} + +#[test] +fn get_range_start_saturating_overflow() { + let tree = build_range_tree(2, 10); + let page = tree.get_range(u64::MAX, u16::MAX).expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); +} + +#[test] +fn get_range_paged_scan_covers_everything() { + // The scanning pattern: walk the whole tree in pages of 3 and check the + // concatenation matches per-position reads. + let tree = build_range_tree(2, 11); // 2 chunks + 3 buffered + let mut cursor = 0u64; + let mut seen = Vec::new(); + loop { + let page = tree.get_range(cursor, 3).expect("get range"); + if page.entries.is_empty() { + assert!(cursor >= page.total_count, "empty page only at the end"); + break; + } + cursor += page.entries.len() as u64; + seen.extend(page.entries); + } + assert_eq!(seen.len(), 11); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, &vec![i as u8]); + } +} diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 8649211d0..4521207d4 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -11,7 +11,7 @@ use std::marker::PhantomData; -use grovedb_bulk_append_tree::BulkAppendTree; +use grovedb_bulk_append_tree::{BulkAppendTree, RangePage}; use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_storage::StorageContext; use orchard::{ @@ -636,6 +636,20 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { .map_err(|e| CommitmentTreeError::InvalidData(format!("chunk value: {}", e))) } + /// Fetch entries for the position range `[start, start + limit)`, + /// clamped to the tree's total count. + /// + /// This is the shielded-pool scanning read path: each returned value is + /// the raw `cmx || rho || cv_net || payload` bytes at that position. + /// Delegates to [`BulkAppendTree::get_range`], so the read is + /// chunk-aligned — O(chunks touched) blob reads, not O(entries) random + /// reads. + pub fn get_range(&self, start: u64, limit: u16) -> Result { + self.bulk_tree + .get_range(start, limit) + .map_err(|e| CommitmentTreeError::InvalidData(format!("range read: {}", e))) + } + /// The number of entries per completed chunk (epoch). pub fn epoch_size(&self) -> u64 { self.bulk_tree.epoch_size() diff --git a/grovedb-commitment-tree/src/lib.rs b/grovedb-commitment-tree/src/lib.rs index 2792576b2..f94285c9c 100644 --- a/grovedb-commitment-tree/src/lib.rs +++ b/grovedb-commitment-tree/src/lib.rs @@ -71,7 +71,7 @@ pub use commitment_tree::{ pub use error::CommitmentTreeError; #[cfg(feature = "server")] pub use grovedb_bulk_append_tree::{ - deserialize_chunk_blob, serialize_chunk_blob, BulkAppendError, BulkAppendTree, + deserialize_chunk_blob, serialize_chunk_blob, BulkAppendError, BulkAppendTree, RangePage, }; pub use grovedb_costs::{self}; pub use incrementalmerkletree::{Hashable, Level, Position, Retention}; From b4734315004c91f42013e535ecdb93fda431f7d0 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:09:48 +0700 Subject: [PATCH 2/8] fix(grovedb): chain empty BulkAppendTree V1 lower layers through NULL_HASH A V1 proof descending into an empty BulkAppendTree could never verify: insert commits the element with a NULL_HASH child hash (there is no bulk state until the first append) and verify_grovedb's integrity walk mirrors that, but verify_bulk_append_lower_layer returned the domain-tagged empty state root blake3("bulk_state" || 0*32 || 0*32), so the combine_hash chain check always failed. Return NULL_HASH for a zero-count BulkAppendTree element instead, matching what the writer commits. This is sound because verify_and_compute_root has already rejected any proof carrying chunk or buffer data for a zero-count tree. CommitmentTree keeps the computed value: its insert commits EMPTY_COMMITMENT_TREE_STATE_ROOT, which folds in the computed empty bulk root. Verification-only change on an element type not reachable in any shipped grove version's state; no wire format or state root changes, so no GROVE_V* gate. Co-Authored-By: Claude Fable 5 --- grovedb/src/operations/proof/verify.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index ab3db34cd..a20f12d19 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -1645,11 +1645,27 @@ impl GroveDb { .verify_and_compute_root(element_height, element_total_count) .map_err(|e| Error::InvalidProof(query.clone(), format!("{}", e)))?; + // An empty `BulkAppendTree` element chains through NULL_HASH, not the + // domain-tagged empty state root: insert commits the element with a + // NULL_HASH child hash (there is no bulk state until the first + // append), and `verify_grovedb`'s integrity walk mirrors that. This + // is sound because `verify_and_compute_root` above already rejected + // any proof carrying data for a zero-count tree. `CommitmentTree` is + // different — its insert commits `EMPTY_COMMITMENT_TREE_STATE_ROOT`, + // which folds in the *computed* empty bulk root, so the CT wrapper + // (our caller) keeps the computed value. + let child_hash = + if element_total_count == 0 && matches!(element, Element::BulkAppendTree(..)) { + NULL_HASH + } else { + bulk_state_root + }; + // Root only: the caller is binding the parent element and does not // report this layer's entries, so there is no query at this path to // extract a position range from. if !report_contents { - return Ok(bulk_state_root); + return Ok(child_hash); } // Get the query range from the path query to extract matching values @@ -1722,8 +1738,8 @@ impl GroveDb { } } - // Return computed state_root as child Merk hash - Ok(bulk_state_root) + // Return the derived child Merk hash (see the empty-tree note above) + Ok(child_hash) } /// Verify a CommitmentTree lower layer proof and add results. From 9ae0377ed42fd5674e4f8eae3aa07fd145d6b562 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:10:10 +0700 Subject: [PATCH 3/8] feat(grovedb): paginated position-range reads and proofs for append-only trees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the BulkAppendTree range-read seam through GroveDB, per work-plan item 4 of dashpay/grovedb#784 — clients of append-only stores walk "all entries since my cursor" in pages: - GroveDb::bulk_get_range / GroveDb::commitment_tree_get_range: chunk-aligned page fetch returning a RangePage (entries + total_count), O(chunks touched) instead of O(entries) random reads. - PathQuery::new_bulk_position_range: the canonical query shape for a page — element key plus an 8-byte big-endian position-range subquery — derived identically by prover and verifier from (start, limit). - GroveDb::prove_bulk_position_range: proves one page via the existing V1 proof layering (ProofBytes::BulkAppendTree / CommitmentTree); no wire-format change, so no new GROVE_V* gate. - GroveDb::verify_bulk_position_range_proof: verifies the page entries (ascending, contiguous, complete) and extracts the authenticated total_count from the same proof bytes by subset-verifying the element itself. Absence beyond the end falls out of the provable count (position >= total_count) — no per-position absence proofs. The seam lives at the shared BulkAppendTree layer and dispatches on the element type, so the planned PrivateDocumentStore (#784) and DataCommitmentTree (#783) pick it up by adding their element variants to the prove/verify match arms. Tests cover round-trips across chunk boundaries, empty ranges, ranges past the end, single-entry pages, large multi-chunk pages, empty trees, nested paths, wrong-element errors, narrower-proof rejection, and the cursor-walk scan pattern for both BulkAppendTree and CommitmentTree. Co-Authored-By: Claude Fable 5 --- grovedb/src/lib.rs | 2 + grovedb/src/operations/bulk_append_tree.rs | 66 +++- grovedb/src/operations/commitment_tree.rs | 67 +++- grovedb/src/operations/proof/generate.rs | 56 +++ grovedb/src/operations/proof/verify.rs | 138 ++++++++ grovedb/src/query/mod.rs | 42 +++ grovedb/src/tests/bulk_append_tree_tests.rs | 365 ++++++++++++++++++++ grovedb/src/tests/commitment_tree_tests.rs | 241 +++++++++++++ 8 files changed, 975 insertions(+), 2 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a3f509a8a..ec3690ab1 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -180,6 +180,8 @@ pub use element::aggregate_sum_query::{AggregateSumQueryOptions, AggregateSumQue pub use element::Element; #[cfg(any(feature = "minimal", feature = "verify"))] pub use element::ElementFlags; +#[cfg(any(feature = "minimal", feature = "verify"))] +pub use grovedb_bulk_append_tree::RangePage; #[cfg(feature = "minimal")] use grovedb_costs::cost_return_on_error_into; #[cfg(feature = "minimal")] diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index 50d13cccb..e87ef35fd 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; -use grovedb_bulk_append_tree::{deserialize_chunk_blob, BulkAppendTree}; +use grovedb_bulk_append_tree::{deserialize_chunk_blob, BulkAppendTree, RangePage}; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_into, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, @@ -250,6 +250,70 @@ impl GroveDb { } } + /// Fetch entries for the position range `[start, start + limit)` from a + /// BulkAppendTree, clamped to its total count. + /// + /// This is the paginated-scan read path: clients walking "all entries + /// since my cursor" call it with their cursor as `start` and advance by + /// `entries.len()`. The read is chunk-aligned — each completed chunk + /// overlapping the range is read and deserialized exactly once, so a + /// page costs O(chunks touched) blob reads plus one read per buffer + /// entry, not O(entries) random reads. + /// + /// The returned [`RangePage`] also carries the tree's `total_count`: + /// positions `>= total_count` do not exist, so a page shorter than + /// `limit` means the end of the tree was reached. + pub fn bulk_get_range<'b, B, P>( + &self, + path: P, + key: &[u8], + start: u64, + limit: u16, + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path.clone(), key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped BulkAppendTree is still one. + let (total_count, chunk_power) = match element.underlying() { + Element::BulkAppendTree(tc, cp, _) => (*tc, *cp), + _ => { + return Err(Error::InvalidInput("element is not a BulkAppendTree")) + .wrap_with_cost(cost); + } + }; + + let subtree_path_vec = self.build_subtree_path_for_bulk(&path, key); + let subtree_path_refs: Vec<&[u8]> = subtree_path_vec.iter().map(|v| v.as_slice()).collect(); + let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); + + let storage_ctx = self + .db + .get_transactional_storage_context(subtree_path, None, tx.as_ref()) + .unwrap_add_cost(&mut cost); + + let tree = cost_return_on_error_no_add!( + cost, + BulkAppendTree::from_state(total_count, chunk_power, storage_ctx).map_err(map_bulk_err) + ); + + let page = + cost_return_on_error_no_add!(cost, tree.get_range(start, limit).map_err(map_bulk_err)); + + Ok(page).wrap_with_cost(cost) + } + /// Get a completed chunk blob from a BulkAppendTree. /// /// Returns the raw serialized blob (length-prefixed entries) for the given diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index b55a2f4e9..a8e08aa78 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use grovedb_commitment_tree::{ deserialize_chunk_blob, serialize_ciphertext, Anchor, CommitmentTree, DashMemo, MemoSize, - TransmittedNoteCiphertext, + RangePage, TransmittedNoteCiphertext, }; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_into, cost_return_on_error_no_add, CostResult, @@ -388,6 +388,71 @@ impl GroveDb { } } + /// Fetch entries for the position range `[start, start + limit)` from a + /// CommitmentTree, clamped to its total count. + /// + /// This is the shielded-pool scanning read path: clients walk "all notes + /// since my cursor" in pages, trial-decrypting each returned + /// `cmx || rho || cv_net || payload` value. The read is chunk-aligned — + /// each completed chunk overlapping the range is read and deserialized + /// exactly once, so a page costs O(chunks touched) blob reads plus one + /// read per buffer entry, not O(entries) random reads. + /// + /// The returned [`RangePage`] also carries the tree's `total_count`: + /// positions `>= total_count` do not exist, so a page shorter than + /// `limit` means the scan caught up with the tip. + pub fn commitment_tree_get_range<'b, B, P>( + &self, + path: P, + key: &[u8], + start: u64, + limit: u16, + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path.clone(), key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped CommitmentTree is still one. + let (total_count, chunk_power) = match element.underlying() { + Element::CommitmentTree(tc, cp, _) => (*tc, *cp), + _ => { + return Err(Error::InvalidInput("element is not a commitment tree")) + .wrap_with_cost(cost); + } + }; + + let ct_path_vec = self.build_ct_path(&path, key); + let ct_path_refs: Vec<&[u8]> = ct_path_vec.iter().map(|v| v.as_slice()).collect(); + let ct_path = SubtreePath::from(ct_path_refs.as_slice()); + + let storage_ctx = self + .db + .get_transactional_storage_context(ct_path, None, tx.as_ref()) + .unwrap_add_cost(&mut cost); + + let ct = cost_return_on_error!( + &mut cost, + CommitmentTree::<_, DashMemo>::open(total_count, chunk_power, storage_ctx) + .map(|r| r.map_err(map_ct_err)) + ); + + let page = + cost_return_on_error_no_add!(cost, ct.get_range(start, limit).map_err(map_ct_err)); + + Ok(page).wrap_with_cost(cost) + } + /// Get the total count of items in a CommitmentTree. pub fn commitment_tree_count<'b, B, P>( &self, diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index b095ee0f5..6f982564d 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -205,6 +205,62 @@ impl GroveDb { } } + /// Prove the paginated position range `[start, start + limit)` of the + /// append-only, BulkAppendTree-backed element at `path`/`key` + /// (`Element::BulkAppendTree` or `Element::CommitmentTree`). + /// + /// This is the scanning hot path: clients walking "all entries since my + /// cursor" prove one page per call, passing their cursor as `start`. The + /// proof reuses the existing V1 layering (`ProofBytes::BulkAppendTree` / + /// `ProofBytes::CommitmentTree`) over the canonical + /// [`PathQuery::new_bulk_position_range`] query, so it is chunk-aligned: + /// it carries each completed chunk blob overlapping the range plus the + /// in-range buffer entries — O(chunks touched), not O(entries). It also + /// binds the element itself, whose authenticated `total_count` makes + /// absence beyond the end provable (`position >= total_count`), so + /// ranges past the end need no per-position absence proofs. + /// + /// Verify with [`GroveDb::verify_bulk_position_range_proof`], which + /// derives the same canonical query from `(path, key, start, limit)`. + pub fn prove_bulk_position_range( + &self, + path: Vec>, + key: &[u8], + start: u64, + limit: u16, + prove_options: Option, + grove_version: &GroveVersion, + ) -> CostResult, Error> { + let mut cost = OperationCost::default(); + + // Fail fast with a clear error when the target is not an append-only + // store — a generic proof for some other element type would only be + // rejected later, at verification time. + let path_refs: Vec<&[u8]> = path.iter().map(|p| p.as_slice()).collect(); + let subtree_path = grovedb_path::SubtreePath::from(path_refs.as_slice()); + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(subtree_path, key, true, None, grove_version) + ); + match element.underlying() { + Element::BulkAppendTree(..) | Element::CommitmentTree(..) => {} + _ => { + return Err(Error::InvalidInput( + "prove_bulk_position_range requires a BulkAppendTree or CommitmentTree \ + element", + )) + .wrap_with_cost(cost); + } + } + + let path_query = PathQuery::new_bulk_position_range(path, key.to_vec(), start, limit); + let proof = cost_return_on_error!( + &mut cost, + self.prove_query(&path_query, prove_options, grove_version) + ); + Ok(proof).wrap_with_cost(cost) + } + /// Helper for the top-level count-offset gate in /// `prove_query_non_serialized_v{0,1}`. Opens the merk at /// `path_query.path` and confirms its `tree_type` is one of the diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index a20f12d19..bf0a562d3 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -2596,6 +2596,144 @@ impl GroveDb { ) } + /// Verify a proof produced by [`GroveDb::prove_bulk_position_range`], + /// returning `(root_hash, page)` for the position range + /// `[start, start + limit)` of the append-only, BulkAppendTree-backed + /// element at `path`/`key` (`Element::BulkAppendTree` or + /// `Element::CommitmentTree`). + /// + /// The page's entries are the `(position, value)` pairs, ascending and + /// contiguous from `start`, clamped to the element's authenticated + /// `total_count` (returned in + /// [`RangePage::total_count`](grovedb_bulk_append_tree::RangePage::total_count)). + /// Completeness is enforced: a proof missing any requested position below + /// `total_count` is rejected. Absence beyond the end falls out of the + /// provable count — positions `>= total_count` do not exist — so a page + /// shorter than `limit` means the scan caught up with the tip; no + /// per-position absence proofs are involved. + /// + /// Both the entries and `total_count` are extracted from the same proof + /// bytes: the entries by verifying the canonical + /// [`PathQuery::new_bulk_position_range`] query, and `total_count` by + /// subset-verifying the element itself, whose serialized bytes are bound + /// to the parent Merk (and through it to `root_hash`). + pub fn verify_bulk_position_range_proof( + proof: &[u8], + path: Vec>, + key: &[u8], + start: u64, + limit: u16, + grove_version: &GroveVersion, + ) -> Result<(CryptoHash, grovedb_bulk_append_tree::RangePage), Error> { + // 1. Verify the range entries against the canonical range query. + // Succinctness cannot be required: range proofs are chunk-aligned + // and intentionally carry whole chunk blobs — a superset of the + // queried positions. + let range_query = + PathQuery::new_bulk_position_range(path.clone(), key.to_vec(), start, limit); + let (root_hash, results) = Self::verify_query_with_options( + proof, + &range_query, + VerifyOptions { + absence_proofs_for_non_existing_searched_keys: false, + verify_proof_succinctness: false, + include_empty_trees_in_result: false, + }, + grove_version, + )?; + + // 2. Extract the element's authenticated total_count from the same + // proof bytes via a single-key subset query on the element itself. + let element_query = PathQuery::new_single_key(path, key.to_vec()); + let (element_root_hash, element_results) = + Self::verify_subset_query(proof, &element_query, grove_version)?; + if element_root_hash != root_hash { + return Err(Error::InvalidProof( + range_query, + "range and element sub-proofs derived different root hashes".to_string(), + )); + } + let total_count = match element_results.as_slice() { + [(_, element_key, Some(element))] if element_key.as_slice() == key => { + match element.underlying() { + Element::BulkAppendTree(total_count, _, _) + | Element::CommitmentTree(total_count, _, _) => *total_count, + _ => { + return Err(Error::InvalidProof( + range_query, + "element at path/key is not a BulkAppendTree or CommitmentTree" + .to_string(), + )); + } + } + } + _ => { + return Err(Error::InvalidProof( + range_query, + "proof does not bind the append-only element at path/key".to_string(), + )); + } + }; + + // 3. Map the results to (position, value) entries and require the + // page to be exactly [start, min(start + limit, total_count)) — + // ascending, contiguous, complete. The lower-layer verification + // already enforces completeness; this re-check keeps the helper's + // guarantee independent of that layer's internals. + let end = start + .saturating_add(limit as u64) + .min(total_count) + .max(start); + let mut entries = Vec::with_capacity((end - start) as usize); + let mut expected_position = start; + for (_, position_key, element) in results { + let position_bytes: [u8; 8] = position_key.as_slice().try_into().map_err(|_| { + Error::InvalidProof( + range_query.clone(), + "range entry key is not an 8-byte big-endian position".to_string(), + ) + })?; + let position = u64::from_be_bytes(position_bytes); + let value = match element { + Some(Element::Item(value, _)) => value, + _ => { + return Err(Error::InvalidProof( + range_query, + format!("range entry at position {} is not an item", position), + )); + } + }; + if position != expected_position { + return Err(Error::InvalidProof( + range_query, + format!( + "range entries not contiguous: expected position {}, got {}", + expected_position, position + ), + )); + } + expected_position += 1; + entries.push((position, value)); + } + if expected_position != end { + return Err(Error::InvalidProof( + range_query, + format!( + "range proof incomplete: expected positions [{}, {}), got up to {}", + start, end, expected_position + ), + )); + } + + Ok(( + root_hash, + grovedb_bulk_append_tree::RangePage { + entries, + total_count, + }, + )) + } + /// The point of this query is to get the parent tree information which will /// be present because we are querying in a subtree pub fn verify_query_get_parent_tree_info( diff --git a/grovedb/src/query/mod.rs b/grovedb/src/query/mod.rs index 7d102b581..2a1dee7fa 100644 --- a/grovedb/src/query/mod.rs +++ b/grovedb/src/query/mod.rs @@ -526,6 +526,48 @@ impl PathQuery { Self { path, query } } + /// Canonical `PathQuery` for a paginated position-range read of an + /// append-only, BulkAppendTree-backed element (`BulkAppendTree` or + /// `CommitmentTree`) at `path`/`key`. + /// + /// Selects the element at `key` and subqueries positions + /// `[start, start + limit)`, encoded as 8-byte big-endian keys + /// (`start + limit` saturates at `u64::MAX`). Prover and verifier both + /// derive this query from `(start, limit)`, so a scanning client only + /// needs its cursor and page size — see + /// [`GroveDb::prove_bulk_position_range`] and + /// [`GroveDb::verify_bulk_position_range_proof`]. + /// + /// [`GroveDb::prove_bulk_position_range`]: + /// crate::GroveDb::prove_bulk_position_range + /// [`GroveDb::verify_bulk_position_range_proof`]: + /// crate::GroveDb::verify_bulk_position_range_proof + pub fn new_bulk_position_range( + path: Vec>, + key: Vec, + start: u64, + limit: u16, + ) -> Self { + let position_query = grovedb_bulk_append_tree::position_range_query(start, limit); + Self { + path, + query: SizedQuery { + query: Query { + items: vec![QueryItem::Key(key)], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(position_query.into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + add_parent_tree_on_subquery: false, + }, + limit: None, + offset: None, + }, + } + } + /// Construct a `PathQuery` for an aggregate-count-on-range query against /// the subtree at `path`. `range` is the inner `QueryItem` describing the /// keys to count over; see [`Query::new_aggregate_count_on_range`] for the diff --git a/grovedb/src/tests/bulk_append_tree_tests.rs b/grovedb/src/tests/bulk_append_tree_tests.rs index 7028e8e0d..2e7cfcd38 100644 --- a/grovedb/src/tests/bulk_append_tree_tests.rs +++ b/grovedb/src/tests/bulk_append_tree_tests.rs @@ -2125,3 +2125,368 @@ fn test_bulk_batch_multi_compaction_transaction_rollback() { "bulk tree should be empty after multi-compaction tx rollback" ); } + +// =========================================================================== +// Paginated position-range reads (get_range) and range proofs +// =========================================================================== + +/// Helper: create a DB with a BulkAppendTree at root key `b"bulk"` holding +/// `n` values `b"value_0"`, `b"value_1"`, ... (chunk_power = 2 → chunk size +/// 4). +fn make_bulk_db_with_values(n: u32) -> crate::tests::TempGroveDb { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"bulk", + Element::empty_bulk_append_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert bulk append tree"); + + for i in 0..n { + db.bulk_append( + EMPTY_PATH, + b"bulk", + format!("value_{}", i).into_bytes(), + None, + grove_version, + ) + .unwrap() + .expect("bulk append"); + } + + db +} + +/// Helper: assert a page holds exactly positions `start..end` with the +/// values written by [`make_bulk_db_with_values`]. +fn assert_bulk_page(page: &crate::RangePage, start: u64, end: u64, total_count: u64) { + assert_eq!(page.total_count, total_count, "page total_count"); + assert_eq!(page.entries.len(), (end - start) as usize, "page length"); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, start + i as u64, "page position"); + assert_eq!(value, format!("value_{}", pos).as_bytes(), "page value"); + } +} + +#[test] +fn test_bulk_get_range_matches_get_value() { + let grove_version = GroveVersion::latest(); + // 11 values = 2 full chunks (8) + 3 buffered + let db = make_bulk_db_with_values(11); + + // Every possible page of size 4 must match per-position reads + for start in 0..12u64 { + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", start, 4, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_eq!(page.total_count, 11); + + let end = (start + 4).min(11); + let expected_len = end.saturating_sub(start.min(end)); + assert_eq!(page.entries.len(), expected_len as usize); + + for (pos, value) in &page.entries { + let expected = db + .bulk_get_value(EMPTY_PATH, b"bulk", *pos, None, grove_version) + .unwrap() + .expect("bulk get value") + .expect("value exists"); + assert_eq!(value, &expected); + } + } +} + +#[test] +fn test_bulk_get_range_empty_and_past_end() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(10); + + // limit = 0 + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 3, 0, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_bulk_page(&page, 3, 3, 10); + + // start exactly at total_count + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 10, 5, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_bulk_page(&page, 10, 10, 10); + + // start far past total_count + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 1000, 5, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // clamped at the end + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 8, 100, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_bulk_page(&page, 8, 10, 10); +} + +#[test] +fn test_bulk_get_range_wrong_element_type() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"normal", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + + let result = db + .bulk_get_range(EMPTY_PATH, b"normal", 0, 4, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidInput(_)))); +} + +/// Helper: prove and verify a position range page, asserting the returned +/// root hash matches the database root hash, and return the page. +fn roundtrip_bulk_range_proof( + db: &crate::tests::TempGroveDb, + start: u64, + limit: u16, +) -> crate::RangePage { + let grove_version = GroveVersion::latest(); + + let proof = db + .prove_bulk_position_range(vec![], b"bulk", start, limit, None, grove_version) + .unwrap() + .expect("prove bulk position range"); + + let (root_hash, page) = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"bulk", + start, + limit, + grove_version, + ) + .expect("verify bulk position range proof"); + + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root, "proof root must match db root"); + + page +} + +#[test] +fn test_bulk_position_range_proof_across_chunk_boundary() { + // 10 values = 2 chunks (8) + 2 buffered, chunk size 4 + let db = make_bulk_db_with_values(10); + + // Page [3, 6) spans the chunk 0 / chunk 1 boundary + let page = roundtrip_bulk_range_proof(&db, 3, 3); + assert_bulk_page(&page, 3, 6, 10); + + // Page [6, 10) spans the chunk 1 / buffer boundary + let page = roundtrip_bulk_range_proof(&db, 6, 4); + assert_bulk_page(&page, 6, 10, 10); +} + +#[test] +fn test_bulk_position_range_proof_empty_range() { + let db = make_bulk_db_with_values(10); + let page = roundtrip_bulk_range_proof(&db, 3, 0); + assert_bulk_page(&page, 3, 3, 10); +} + +#[test] +fn test_bulk_position_range_proof_past_end() { + let db = make_bulk_db_with_values(10); + + // Start exactly at total_count: provably nothing there — the verified + // total_count is the absence proof. + let page = roundtrip_bulk_range_proof(&db, 10, 5); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Start far past total_count + let page = roundtrip_bulk_range_proof(&db, 1000, 5); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Range starting inside but extending past the end is clamped + let page = roundtrip_bulk_range_proof(&db, 8, 100); + assert_bulk_page(&page, 8, 10, 10); +} + +#[test] +fn test_bulk_position_range_proof_single_entry_pages() { + let db = make_bulk_db_with_values(10); + for pos in 0..10u64 { + let page = roundtrip_bulk_range_proof(&db, pos, 1); + assert_bulk_page(&page, pos, pos + 1, 10); + } +} + +#[test] +fn test_bulk_position_range_proof_large_multi_chunk_page() { + // 30 values = 7 full chunks (28) + 2 buffered, chunk size 4. + let db = make_bulk_db_with_values(30); + + // One page covering everything touches all chunks and the buffer + let page = roundtrip_bulk_range_proof(&db, 0, 30); + assert_bulk_page(&page, 0, 30, 30); + + // A large page crossing several chunk boundaries mid-tree + let page = roundtrip_bulk_range_proof(&db, 5, 20); + assert_bulk_page(&page, 5, 25, 30); +} + +#[test] +fn test_bulk_position_range_proof_empty_tree() { + let db = make_bulk_db_with_values(0); + let page = roundtrip_bulk_range_proof(&db, 0, 10); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 0); +} + +#[test] +fn test_bulk_position_range_paged_scan() { + // The scanning pattern: walk the whole tree in proved pages of 7 + // (deliberately not aligned to the chunk size of 4). + let db = make_bulk_db_with_values(30); + + let mut cursor = 0u64; + let mut seen = Vec::new(); + loop { + let page = roundtrip_bulk_range_proof(&db, cursor, 7); + if page.entries.is_empty() { + assert!(cursor >= page.total_count, "empty page only at the end"); + break; + } + cursor += page.entries.len() as u64; + seen.extend(page.entries); + } + assert_eq!(seen.len(), 30); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, format!("value_{}", i).as_bytes()); + } +} + +#[test] +fn test_bulk_position_range_prove_wrong_element_type() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"normal", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + + let result = db + .prove_bulk_position_range(vec![], b"normal", 0, 4, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidInput(_)))); +} + +#[test] +fn test_bulk_position_range_proof_wrong_range_rejected() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(10); + + // Proof generated for [0, 2) (inside chunk 0) must not verify a request + // for [0, 6), which also needs chunk 1. + let narrow_proof = db + .prove_bulk_position_range(vec![], b"bulk", 0, 2, None, grove_version) + .unwrap() + .expect("prove narrow range"); + + crate::GroveDb::verify_bulk_position_range_proof( + &narrow_proof, + vec![], + b"bulk", + 0, + 6, + grove_version, + ) + .expect_err("proof for a narrower range must be rejected"); +} + +#[test] +fn test_bulk_position_range_proof_deep_path() { + // Same flow but with the BulkAppendTree nested under a normal tree. + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"deep", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert parent tree"); + + db.insert( + &[b"deep"], + b"bulk", + Element::empty_bulk_append_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert bulk append tree"); + + for i in 0..10u32 { + db.bulk_append( + &[b"deep"], + b"bulk", + format!("value_{}", i).into_bytes(), + None, + grove_version, + ) + .unwrap() + .expect("bulk append"); + } + + let proof = db + .prove_bulk_position_range(vec![b"deep".to_vec()], b"bulk", 2, 5, None, grove_version) + .unwrap() + .expect("prove nested bulk position range"); + + let (root_hash, page) = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![b"deep".to_vec()], + b"bulk", + 2, + 5, + grove_version, + ) + .expect("verify nested bulk position range proof"); + + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root); + assert_bulk_page(&page, 2, 7, 10); +} diff --git a/grovedb/src/tests/commitment_tree_tests.rs b/grovedb/src/tests/commitment_tree_tests.rs index 2d76f08b4..660ad12d7 100644 --- a/grovedb/src/tests/commitment_tree_tests.rs +++ b/grovedb/src/tests/commitment_tree_tests.rs @@ -2858,3 +2858,244 @@ fn test_commitment_tree_element_count_subset_query_against_note_fetch_proof() { other => panic!("expected CommitmentTree element, got {:?}", other), } } + +// =========================================================================== +// Paginated position-range reads (get_range) and range proofs +// =========================================================================== + +/// Helper: create a DB with a CommitmentTree at [b"root"]/b"pool" +/// (chunk_power = 2 → chunk size 4) holding `n` notes, returning the +/// expected `cmx || rho || cv_net || payload` bytes per position. +fn make_ct_db_with_notes(n: u8) -> (crate::tests::TempGroveDb, Vec>) { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"root", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert root tree"); + + db.insert( + &[b"root"], + b"pool", + Element::empty_commitment_tree(2).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert commitment tree"); + + let mut expected_values = Vec::new(); + for i in 0..n { + let cmx = test_cmx(i); + let rho = test_rho(i); + let cv_net = test_cv_net(i); + let payload = serialize_ciphertext(&test_ciphertext(i)); + db.commitment_tree_insert( + &[b"root"], + b"pool", + cmx, + rho, + cv_net, + test_ciphertext(i), + None, + grove_version, + ) + .unwrap() + .expect("commitment tree insert"); + + let mut expected = Vec::with_capacity(32 + 32 + 32 + payload.len()); + expected.extend_from_slice(&cmx); + expected.extend_from_slice(&rho); + expected.extend_from_slice(&cv_net); + expected.extend_from_slice(&payload); + expected_values.push(expected); + } + + (db, expected_values) +} + +#[test] +fn test_commitment_tree_get_range_matches_get_value() { + let grove_version = GroveVersion::latest(); + // 10 notes = 2 full chunks (8) + 2 buffered, chunk size 4 + let (db, expected_values) = make_ct_db_with_notes(10); + + for start in 0..11u64 { + let page = db + .commitment_tree_get_range(&[b"root"], b"pool", start, 4, None, grove_version) + .unwrap() + .expect("commitment tree get range"); + assert_eq!(page.total_count, 10); + + let end = (start + 4).min(10); + let expected_len = end.saturating_sub(start.min(end)); + assert_eq!(page.entries.len(), expected_len as usize); + + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, start + i as u64); + assert_eq!(value, &expected_values[*pos as usize]); + + let per_position = db + .commitment_tree_get_value(&[b"root"], b"pool", *pos, None, grove_version) + .unwrap() + .expect("commitment tree get value") + .expect("value exists"); + assert_eq!(value, &per_position); + } + } +} + +#[test] +fn test_commitment_tree_get_range_wrong_element_type() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"normal", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + + let result = db + .commitment_tree_get_range(EMPTY_PATH, b"normal", 0, 4, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidInput(_)))); +} + +/// Helper: prove and verify a CommitmentTree position range page, asserting +/// the returned root hash matches the database root hash. +fn roundtrip_ct_range_proof( + db: &crate::tests::TempGroveDb, + start: u64, + limit: u16, +) -> crate::RangePage { + let grove_version = GroveVersion::latest(); + + let proof = db + .prove_bulk_position_range( + vec![b"root".to_vec()], + b"pool", + start, + limit, + None, + grove_version, + ) + .unwrap() + .expect("prove commitment tree position range"); + + let (root_hash, page) = GroveDb::verify_bulk_position_range_proof( + &proof, + vec![b"root".to_vec()], + b"pool", + start, + limit, + grove_version, + ) + .expect("verify commitment tree position range proof"); + + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root, "proof root must match db root"); + + page +} + +#[test] +fn test_commitment_tree_position_range_proof_across_chunk_boundary() { + let (db, expected_values) = make_ct_db_with_notes(10); + + // Page [3, 7) spans the chunk 0 / chunk 1 boundary + let page = roundtrip_ct_range_proof(&db, 3, 4); + assert_eq!(page.total_count, 10); + assert_eq!(page.entries.len(), 4); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, 3 + i as u64); + assert_eq!(value, &expected_values[*pos as usize]); + } + + // Page [6, 10) spans the chunk 1 / buffer boundary + let page = roundtrip_ct_range_proof(&db, 6, 4); + assert_eq!(page.entries.len(), 4); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, 6 + i as u64); + assert_eq!(value, &expected_values[*pos as usize]); + } +} + +#[test] +fn test_commitment_tree_position_range_proof_empty_and_past_end() { + let (db, _) = make_ct_db_with_notes(10); + + // Empty range + let page = roundtrip_ct_range_proof(&db, 3, 0); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Past the end: the verified total_count is the absence proof for + // positions >= 10. + let page = roundtrip_ct_range_proof(&db, 10, 5); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Clamped at the end + let page = roundtrip_ct_range_proof(&db, 8, 100); + assert_eq!(page.entries.len(), 2); + assert_eq!(page.total_count, 10); +} + +#[test] +fn test_commitment_tree_position_range_proof_single_entry_pages() { + let (db, expected_values) = make_ct_db_with_notes(6); + + for pos in 0..6u64 { + let page = roundtrip_ct_range_proof(&db, pos, 1); + assert_eq!(page.total_count, 6); + assert_eq!(page.entries.len(), 1); + assert_eq!(page.entries[0].0, pos); + assert_eq!(page.entries[0].1, expected_values[pos as usize]); + } +} + +#[test] +fn test_commitment_tree_position_range_paged_scan() { + // The shielded-pool scanning pattern: walk all notes since cursor 0 in + // proved pages of 3 (not aligned to the chunk size of 4). + let (db, expected_values) = make_ct_db_with_notes(10); + + let mut cursor = 0u64; + let mut seen = Vec::new(); + loop { + let page = roundtrip_ct_range_proof(&db, cursor, 3); + if page.entries.is_empty() { + assert!(cursor >= page.total_count, "empty page only at the end"); + break; + } + cursor += page.entries.len() as u64; + seen.extend(page.entries); + } + assert_eq!(seen.len(), 10); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, &expected_values[i]); + } +} + +#[test] +fn test_commitment_tree_position_range_proof_empty_tree() { + let (db, _) = make_ct_db_with_notes(0); + let page = roundtrip_ct_range_proof(&db, 0, 10); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 0); +} From 3f3efe2e0ee9ac15b572b6e994237a2be11a9dfd Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:10:28 +0700 Subject: [PATCH 4/8] test(grovedb): add paged-scan benchmark for bulk position-range reads Benchmark the append-only scanning hot path on a 4096-entry BulkAppendTree (chunk size 64, 96-byte entries): single-page reads, per-page proof generation and verification at page sizes 16/256/1024, and a full proved cursor-walk of the tree at page size 256. Co-Authored-By: Claude Fable 5 --- grovedb/Cargo.toml | 4 + grovedb/benches/bulk_range_scan_benchmark.rs | 203 +++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 grovedb/benches/bulk_range_scan_benchmark.rs diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 6358fbba8..6097a721d 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -80,6 +80,10 @@ harness = false name = "cidx_benchmark" harness = false +[[bench]] +name = "bulk_range_scan_benchmark" +harness = false + [features] default = ["full", "estimated_costs"] proof_debug = ["grovedb-merk/proof_debug"] diff --git a/grovedb/benches/bulk_range_scan_benchmark.rs b/grovedb/benches/bulk_range_scan_benchmark.rs new file mode 100644 index 000000000..0d3206852 --- /dev/null +++ b/grovedb/benches/bulk_range_scan_benchmark.rs @@ -0,0 +1,203 @@ +//! Benchmark for the BulkAppendTree paged-scan pattern. +//! +//! Clients of append-only stores walk "all entries since my cursor" in +//! pages. This benchmark measures that pattern end to end on a +//! BulkAppendTree populated with fixed-size entries: +//! +//! 1. **Paged read** (`bulk_get_range`): fetching one page of entries, +//! chunk-aligned, at various page sizes. +//! 2. **Paged proof generation** (`prove_bulk_position_range`): proving one +//! page. +//! 3. **Paged proof verification** (`verify_bulk_position_range_proof`): +//! verifying one page. +//! 4. **Full scan**: walking the entire tree page by page with proofs, the +//! shielded-pool-style sync flow. + +#[cfg(feature = "minimal")] +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +#[cfg(feature = "minimal")] +use grovedb::{Element, GroveDb}; +#[cfg(feature = "minimal")] +use grovedb_version::version::GroveVersion; +#[cfg(feature = "minimal")] +use tempfile::TempDir; + +/// Total number of entries appended to the tree. +#[cfg(feature = "minimal")] +const N_ENTRIES: u32 = 4096; + +/// Chunk power of the tree: chunk size = 2^6 = 64 entries, so the tree holds +/// 64 completed chunks with the buffer empty. +#[cfg(feature = "minimal")] +const CHUNK_POWER: u8 = 6; + +/// Size of each entry in bytes (a 32-byte commitment plus a small payload). +#[cfg(feature = "minimal")] +const ENTRY_SIZE: usize = 96; + +#[cfg(feature = "minimal")] +fn setup_db() -> (TempDir, GroveDb) { + let grove_version = GroveVersion::latest(); + let dir = TempDir::new().expect("cannot create temp dir"); + let db = GroveDb::open(dir.path()).expect("cannot open grovedb"); + + db.insert( + &[] as &[&[u8]], + b"bulk", + Element::empty_bulk_append_tree(CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert bulk append tree"); + + for i in 0..N_ENTRIES { + let mut entry = vec![0u8; ENTRY_SIZE]; + entry[..4].copy_from_slice(&i.to_be_bytes()); + db.bulk_append(&[] as &[&[u8]], b"bulk", entry, None, grove_version) + .unwrap() + .expect("bulk append"); + } + + (dir, db) +} + +/// Read one page of entries at various page sizes, starting mid-tree so the +/// page is not chunk-aligned. +#[cfg(feature = "minimal")] +pub fn paged_read(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + let mut group = c.benchmark_group("bulk_range_paged_read"); + + for &page_size in &[16u16, 256, 1024] { + group.bench_function(BenchmarkId::from_parameter(page_size), |b| { + b.iter(|| { + let page = db + .bulk_get_range( + &[] as &[&[u8]], + b"bulk", + (N_ENTRIES / 3) as u64, + page_size, + None, + grove_version, + ) + .unwrap() + .expect("bulk get range"); + assert_eq!(page.entries.len(), page_size as usize); + }); + }); + } + group.finish(); +} + +/// Prove one page at various page sizes. +#[cfg(feature = "minimal")] +pub fn paged_proof_generation(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + let mut group = c.benchmark_group("bulk_range_paged_prove"); + + for &page_size in &[16u16, 256, 1024] { + group.bench_function(BenchmarkId::from_parameter(page_size), |b| { + b.iter(|| { + let _proof = db + .prove_bulk_position_range( + vec![], + b"bulk", + (N_ENTRIES / 3) as u64, + page_size, + None, + grove_version, + ) + .unwrap() + .expect("prove bulk position range"); + }); + }); + } + group.finish(); +} + +/// Verify one page proof at various page sizes. +#[cfg(feature = "minimal")] +pub fn paged_proof_verification(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + let mut group = c.benchmark_group("bulk_range_paged_verify"); + + for &page_size in &[16u16, 256, 1024] { + let start = (N_ENTRIES / 3) as u64; + let proof = db + .prove_bulk_position_range(vec![], b"bulk", start, page_size, None, grove_version) + .unwrap() + .expect("prove bulk position range"); + + group.bench_function(BenchmarkId::from_parameter(page_size), |b| { + b.iter(|| { + let (_root, page) = GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"bulk", + start, + page_size, + grove_version, + ) + .expect("verify bulk position range proof"); + assert_eq!(page.entries.len(), page_size as usize); + }); + }); + } + group.finish(); +} + +/// Walk the entire tree with proved pages of 256 — the client sync flow. +#[cfg(feature = "minimal")] +pub fn full_paged_scan_with_proofs(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + const PAGE: u16 = 256; + + c.bench_function("bulk_range_full_scan_with_proofs", |b| { + b.iter(|| { + let mut cursor = 0u64; + let mut total = 0usize; + loop { + let proof = db + .prove_bulk_position_range(vec![], b"bulk", cursor, PAGE, None, grove_version) + .unwrap() + .expect("prove page"); + let (_root, page) = GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"bulk", + cursor, + PAGE, + grove_version, + ) + .expect("verify page"); + if page.entries.is_empty() { + break; + } + cursor += page.entries.len() as u64; + total += page.entries.len(); + } + assert_eq!(total, N_ENTRIES as usize); + }); + }); +} + +#[cfg(feature = "minimal")] +criterion_group!( + name = benches; + config = Criterion::default().sample_size(10); + targets = paged_read, + paged_proof_generation, + paged_proof_verification, + full_paged_scan_with_proofs +); +#[cfg(feature = "minimal")] +criterion_main!(benches); + +#[cfg(not(feature = "minimal"))] +fn main() {} From ad8c8a7520aebbff6b8b175d8718ab7155dcf25b Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:40:27 +0700 Subject: [PATCH 5/8] refactor: address CodeRabbit review on paginated range reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Build the chunk MMR once per get_range call instead of once per chunk: going through get_chunk_value re-cloned the MMR overlay for every chunk in the page, exactly the repeated work chunk-alignment is meant to avoid. Reuses the same single-MMR pattern as BulkAppendTreeProof::generate. - Add explicit version gates (prove_bulk_position_range, verify_bulk_position_range_proof) matching the other public proof entry points; 0 across GROVE_V1..V4, so no behavior change — the delegated prove_query/verify_query gates still apply transitively. - Pin the shared-chunk disjoint-range semantics with a test: a proof generated for [0, 2) verifies a request for [2, 4) inside the same chunk, because chunk-aligned proofs carry the whole authenticated blob. Rejection of ranges needing an unproved chunk is unchanged. - Add the CommitmentTree-envelope wrong-range rejection test mirroring the BulkAppendTree one (different proof envelope and child-hash derivation, so the bulk test did not cover it). Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/fetch.rs | 19 ++++++++++++-- .../src/version/grovedb_versions.rs | 2 ++ grovedb-version/src/version/v1.rs | 2 ++ grovedb-version/src/version/v2.rs | 2 ++ grovedb-version/src/version/v3.rs | 2 ++ grovedb-version/src/version/v4.rs | 2 ++ grovedb/src/operations/proof/generate.rs | 8 ++++++ grovedb/src/operations/proof/verify.rs | 8 ++++++ grovedb/src/tests/bulk_append_tree_tests.rs | 18 +++++++++++++ grovedb/src/tests/commitment_tree_tests.rs | 26 +++++++++++++++++++ 10 files changed, 87 insertions(+), 2 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/fetch.rs b/grovedb-bulk-append-tree/src/tree/fetch.rs index 4d9ea342d..1f85b89a0 100644 --- a/grovedb-bulk-append-tree/src/tree/fetch.rs +++ b/grovedb-bulk-append-tree/src/tree/fetch.rs @@ -91,13 +91,28 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let epoch_size = self.epoch_size(); let buffer_start = self.chunk_count() * epoch_size; - // Completed chunks overlapping [start, min(end, buffer_start)) + // Completed chunks overlapping [start, min(end, buffer_start)). + // The MMR (with its overlay clone) is built once and reused for every + // chunk in the page — going through `get_chunk_value` would rebuild + // it, and re-clone the overlay, per chunk. let chunk_end = end.min(buffer_start); if start < chunk_end { let first_chunk = start / epoch_size; let last_chunk = (chunk_end - 1) / epoch_size; + let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); + let mmr = MMR::new_with_overlay(self.mmr_size(), &mmr_store, self.mmr_overlay.clone()); for chunk_idx in first_chunk..=last_chunk { - let blob = self.get_chunk_value(chunk_idx)?.ok_or_else(|| { + let node = mmr + .batch + .element_at_position(leaf_to_pos(chunk_idx)) + .unwrap() + .map_err(|e| { + BulkAppendError::MmrError(format!( + "failed to read MMR node for chunk {}: {}", + chunk_idx, e + )) + })?; + let blob = node.and_then(|n| n.into_value()).ok_or_else(|| { BulkAppendError::CorruptedData(format!( "missing chunk blob for index {}", chunk_idx diff --git a/grovedb-version/src/version/grovedb_versions.rs b/grovedb-version/src/version/grovedb_versions.rs index b63af3694..119492680 100644 --- a/grovedb-version/src/version/grovedb_versions.rs +++ b/grovedb-version/src/version/grovedb_versions.rs @@ -130,6 +130,8 @@ pub struct GroveDBOperationsProofVersions { pub prove_trunk_chunk_non_serialized: FeatureVersion, pub prove_branch_chunk: FeatureVersion, pub prove_branch_chunk_non_serialized: FeatureVersion, + pub prove_bulk_position_range: FeatureVersion, + pub verify_bulk_position_range_proof: FeatureVersion, pub verify_query_with_options: FeatureVersion, pub verify_query_raw: FeatureVersion, pub verify_layer_proof: FeatureVersion, diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index be2b1d472..945c44d58 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -157,6 +157,8 @@ pub const GROVE_V1: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 0, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 14f8d0936..677918d7c 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -157,6 +157,8 @@ pub const GROVE_V2: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 0, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb-version/src/version/v3.rs b/grovedb-version/src/version/v3.rs index c65b8518f..4e2d9b974 100644 --- a/grovedb-version/src/version/v3.rs +++ b/grovedb-version/src/version/v3.rs @@ -161,6 +161,8 @@ pub const GROVE_V3: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 1, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb-version/src/version/v4.rs b/grovedb-version/src/version/v4.rs index 7b53a2033..f8e74454c 100644 --- a/grovedb-version/src/version/v4.rs +++ b/grovedb-version/src/version/v4.rs @@ -207,6 +207,8 @@ pub const GROVE_V4: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 1, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 6f982564d..6fa88c4f5 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -231,6 +231,14 @@ impl GroveDb { prove_options: Option, grove_version: &GroveVersion, ) -> CostResult, Error> { + check_grovedb_v0_with_cost!( + "prove_bulk_position_range", + grove_version + .grovedb_versions + .operations + .proof + .prove_bulk_position_range + ); let mut cost = OperationCost::default(); // Fail fast with a clear error when the target is not an append-only diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index bf0a562d3..cfaf54a06 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -2625,6 +2625,14 @@ impl GroveDb { limit: u16, grove_version: &GroveVersion, ) -> Result<(CryptoHash, grovedb_bulk_append_tree::RangePage), Error> { + check_grovedb_v0!( + "verify_bulk_position_range_proof", + grove_version + .grovedb_versions + .operations + .proof + .verify_bulk_position_range_proof + ); // 1. Verify the range entries against the canonical range query. // Succinctness cannot be required: range proofs are chunk-aligned // and intentionally carry whole chunk blobs — a superset of the diff --git a/grovedb/src/tests/bulk_append_tree_tests.rs b/grovedb/src/tests/bulk_append_tree_tests.rs index 2e7cfcd38..00e92e046 100644 --- a/grovedb/src/tests/bulk_append_tree_tests.rs +++ b/grovedb/src/tests/bulk_append_tree_tests.rs @@ -2429,6 +2429,24 @@ fn test_bulk_position_range_proof_wrong_range_rejected() { grove_version, ) .expect_err("proof for a narrower range must be rejected"); + + // Intended semantics pin: a DISJOINT range within the SAME chunk + // verifies. The proof for [0, 2) carries the whole chunk 0 blob + // (positions 0..4) because proofs are chunk-aligned, and every entry in + // it is authenticated against the root — so a verifier asking [2, 4) + // legitimately gets those entries from the same bytes. + let (root_hash, page) = crate::GroveDb::verify_bulk_position_range_proof( + &narrow_proof, + vec![], + b"bulk", + 2, + 2, + grove_version, + ) + .expect("disjoint range within the proved chunk must verify"); + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root); + assert_bulk_page(&page, 2, 4, 10); } #[test] diff --git a/grovedb/src/tests/commitment_tree_tests.rs b/grovedb/src/tests/commitment_tree_tests.rs index 660ad12d7..f4732dd9d 100644 --- a/grovedb/src/tests/commitment_tree_tests.rs +++ b/grovedb/src/tests/commitment_tree_tests.rs @@ -3099,3 +3099,29 @@ fn test_commitment_tree_position_range_proof_empty_tree() { assert!(page.entries.is_empty()); assert_eq!(page.total_count, 0); } + +/// The CommitmentTree envelope (`ProofBytes::CommitmentTree`, sinsemilla +/// prefix + bulk proof) must reject a wider range than was proved, same as +/// the plain BulkAppendTree envelope. +#[test] +fn test_commitment_tree_position_range_proof_wrong_range_rejected() { + let grove_version = GroveVersion::latest(); + let (db, _) = make_ct_db_with_notes(10); + + // Proof generated for [0, 2) (inside chunk 0) must not verify a request + // for [0, 6), which also needs chunk 1. + let narrow_proof = db + .prove_bulk_position_range(vec![b"root".to_vec()], b"pool", 0, 2, None, grove_version) + .unwrap() + .expect("prove narrow range"); + + GroveDb::verify_bulk_position_range_proof( + &narrow_proof, + vec![b"root".to_vec()], + b"pool", + 0, + 6, + grove_version, + ) + .expect_err("proof for a narrower range must be rejected"); +} From 9326581f1d996e61133487c38f1bb2186939e688 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:51:39 +0700 Subject: [PATCH 6/8] test: cover reachable error paths in paginated range reads codecov/patch flagged the new code at 80.55% against the 90% bar. Cover the arms that are reachable without contrived proofs: - get_range corruption paths: storage claiming chunks / buffered entries it does not hold must error, not silently skip entries. - Version-gate rejections for prove_bulk_position_range and verify_bulk_position_range_proof under an unknown feature version. - verify_bulk_position_range_proof adversarial inputs: a canonical range proof over a plain Tree with 8-byte item keys (element type must be rejected when extracting total_count) and over a nonexistent key (a proof binding no element must not invent a total_count). The remaining uncovered lines are defense-in-depth InvalidProof arms (root-mismatch between sub-proofs of the same bytes, non-8-byte / non-item / non-contiguous rows under a genuine bulk element) that cannot be reached through any proof that survives the lower layer's own checks. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/tests.rs | 23 ++++ grovedb/src/tests/bulk_append_tree_tests.rs | 110 ++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index e5987e8d2..1a7413386 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -529,3 +529,26 @@ fn get_range_paged_scan_covers_everything() { assert_eq!(value, &vec![i as u8]); } } + +#[test] +fn get_range_missing_chunk_is_corruption() { + // Storage claims 2 completed chunks (via from_state) but holds no data: + // the chunk MMR leaf lookup comes back empty and the read must surface + // corruption, not silently skip entries. + let tree = BulkAppendTree::from_state(4, 1, MemStorageContext::new()).expect("from_state"); + assert_eq!(tree.chunk_count(), 2); + let err = tree + .get_range(0, 4) + .expect_err("missing chunk blob must error"); + assert!(matches!(err, crate::BulkAppendError::CorruptedData(_))); +} + +#[test] +fn get_range_missing_buffer_value_is_corruption() { + // Storage claims 1 buffered entry (via from_state) but holds no data: + // the buffer read must surface an error, not silently skip entries. + let tree = BulkAppendTree::from_state(1, 2, MemStorageContext::new()).expect("from_state"); + assert_eq!(tree.buffer_count(), 1); + tree.get_range(0, 1) + .expect_err("missing buffer value must error"); +} diff --git a/grovedb/src/tests/bulk_append_tree_tests.rs b/grovedb/src/tests/bulk_append_tree_tests.rs index 00e92e046..e4a69742c 100644 --- a/grovedb/src/tests/bulk_append_tree_tests.rs +++ b/grovedb/src/tests/bulk_append_tree_tests.rs @@ -2508,3 +2508,113 @@ fn test_bulk_position_range_proof_deep_path() { assert_eq!(root_hash, expected_root); assert_bulk_page(&page, 2, 7, 10); } + +#[test] +fn test_bulk_position_range_version_gates() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(4); + let proof = db + .prove_bulk_position_range(vec![], b"bulk", 0, 4, None, grove_version) + .unwrap() + .expect("prove under latest version"); + + // Unknown prove version → VersionError before any work happens + let mut gated = grove_version.clone(); + gated + .grovedb_versions + .operations + .proof + .prove_bulk_position_range = 99; + let result = db + .prove_bulk_position_range(vec![], b"bulk", 0, 4, None, &gated) + .unwrap(); + assert!(matches!(result, Err(Error::VersionError(_)))); + + // Unknown verify version → VersionError before any work happens + let mut gated = grove_version.clone(); + gated + .grovedb_versions + .operations + .proof + .verify_bulk_position_range_proof = 99; + let result = + crate::GroveDb::verify_bulk_position_range_proof(&proof, vec![], b"bulk", 0, 4, &gated); + assert!(matches!(result, Err(Error::VersionError(_)))); +} + +/// A proof over a NORMAL tree holding items at 8-byte keys: the range +/// entries verify as plain Merk rows, but total_count extraction must +/// reject the element type instead of trusting a non-append-only element. +#[test] +fn test_bulk_position_range_verify_rejects_non_append_element() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"plain", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + for i in 0..4u64 { + db.insert( + &[b"plain"], + &i.to_be_bytes(), + Element::new_item(vec![i as u8]), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert item"); + } + + // Bypass prove_bulk_position_range's own element check by proving the + // canonical query directly, as a crafted prover would. + let query = crate::PathQuery::new_bulk_position_range(vec![], b"plain".to_vec(), 0, 4); + let proof = db + .prove_query(&query, None, grove_version) + .unwrap() + .expect("prove canonical query over normal tree"); + + let err = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"plain", + 0, + 4, + grove_version, + ) + .expect_err("normal tree element must be rejected"); + assert!(matches!(err, Error::InvalidProof(..))); +} + +/// A proof for a key that does not exist: the range query verifies (as an +/// absence), but the element subset query binds nothing, so the verifier +/// must refuse to invent a total_count. +#[test] +fn test_bulk_position_range_verify_rejects_unbound_element() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(4); + + let query = crate::PathQuery::new_bulk_position_range(vec![], b"ghost".to_vec(), 0, 4); + let proof = db + .prove_query(&query, None, grove_version) + .unwrap() + .expect("prove canonical query for missing key"); + + let err = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"ghost", + 0, + 4, + grove_version, + ) + .expect_err("proof that binds no element must be rejected"); + assert!(matches!(err, Error::InvalidProof(..))); +} From eac3480b8b8f4c75c7b009a7f87bd9ad957b0b8b Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 08:04:58 +0700 Subject: [PATCH 7/8] test: cover the MMR storage-error path in get_range MemStorageContext (codecov-ignored test util) grows a fail_gets switch that makes every read error, so the chunk-lookup MmrError arm in get_range is exercised: a broken backing store must surface as an error, not a panic or a silent empty page. The buffer-side ok_or_else arm stays uncovered by design: the dense tree's get() errors (never returns Ok(None)) for a missing value below its count, so that arm is unreachable defensive depth, same as the remaining InvalidProof arms in verify_bulk_position_range_proof. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/test_utils.rs | 9 +++++++++ grovedb-bulk-append-tree/src/tree/tests.rs | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/grovedb-bulk-append-tree/src/test_utils.rs b/grovedb-bulk-append-tree/src/test_utils.rs index fbf79a715..732dd934d 100644 --- a/grovedb-bulk-append-tree/src/test_utils.rs +++ b/grovedb-bulk-append-tree/src/test_utils.rs @@ -16,6 +16,9 @@ use grovedb_storage::{Batch, RawIterator, StorageContext}; #[derive(Default)] pub(crate) struct MemStorageContext { pub data: RefCell, Vec>>, + /// When set, every `get` fails — simulates a broken backing store for + /// exercising storage-error paths. + pub fail_gets: std::cell::Cell, } impl MemStorageContext { @@ -29,6 +32,12 @@ impl<'db> StorageContext<'db> for MemStorageContext { type RawIterator = MemRawIterator; fn get>(&self, key: K) -> CostResult>, grovedb_storage::Error> { + if self.fail_gets.get() { + return Err(grovedb_storage::Error::StorageError( + "simulated read failure".to_string(), + )) + .wrap_with_cost(OperationCost::default()); + } Ok(self.data.borrow().get(key.as_ref()).cloned()).wrap_with_cost(OperationCost::default()) } diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index 1a7413386..648ec2fa1 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -552,3 +552,17 @@ fn get_range_missing_buffer_value_is_corruption() { tree.get_range(0, 1) .expect_err("missing buffer value must error"); } + +#[test] +fn get_range_storage_read_failure_is_mmr_error() { + // A backing store that fails reads must surface as an MMR error from the + // chunk lookup, not a panic or a silent empty page. + let ctx = MemStorageContext::new(); + ctx.fail_gets.set(true); + let tree = BulkAppendTree::from_state(4, 1, ctx).expect("from_state"); + assert_eq!(tree.chunk_count(), 2); + let err = tree + .get_range(0, 4) + .expect_err("failing storage must error"); + assert!(matches!(err, crate::BulkAppendError::MmrError(_))); +} From acb99bdbdfea5508f09ece4bfe4f0433a4da3863 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 08:54:37 +0700 Subject: [PATCH 8/8] fix: charge range-read storage costs and validate completed chunk length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review on paginated range reads: - get_range now returns a CostResult: the chunk-MMR node reads and each dense-buffer read charge their seeks and loaded bytes, aggregated through CommitmentTree::get_range, bulk_get_range, and commitment_tree_get_range, so cost limits reflect a page's actual work instead of treating up to 65,535 buffer reads as free. The buffer reads go through dense_tree.get directly so their costs are captured. - get_range rejects a completed chunk as corrupted unless it holds exactly epoch_size entries: a short blob would silently omit positions and an oversized one would overlap the next chunk, breaking the contiguous-page contract and stalling cursor scans. Unlike proof verification — where chunk bytes are bound to the state root and a length check is redundant (see the NOTE in proof/mod.rs) — this raw read path has no root comparison backing it. Tests: a tampered-overlay chunk with the wrong entry count (short and oversized) is rejected with CorruptedData, and bulk_get_range reports nonzero seek/loaded-byte costs that grow with page size. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/fetch.rs | 85 ++++++++++++++----- grovedb-bulk-append-tree/src/tree/tests.rs | 70 ++++++++++++--- .../src/commitment_tree/mod.rs | 6 +- grovedb/src/operations/bulk_append_tree.rs | 7 +- grovedb/src/operations/commitment_tree.rs | 6 +- grovedb/src/tests/bulk_append_tree_tests.rs | 37 ++++++++ 6 files changed, 171 insertions(+), 40 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/fetch.rs b/grovedb-bulk-append-tree/src/tree/fetch.rs index 1f85b89a0..d1974e4fc 100644 --- a/grovedb-bulk-append-tree/src/tree/fetch.rs +++ b/grovedb-bulk-append-tree/src/tree/fetch.rs @@ -1,5 +1,6 @@ //! Read operations for BulkAppendTree. +use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_dense_fixed_sized_merkle_tree::DenseTreeProof; use grovedb_merkle_mountain_range::{leaf_to_pos, MmrKeySize, MmrStore, MMR}; use grovedb_query::Query; @@ -77,14 +78,20 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// /// Absence needs no lookup: positions `>= total_count` do not exist, so /// a page shorter than `limit` means the end of the tree was reached. - pub fn get_range(&self, start: u64, limit: u16) -> Result { + /// + /// Returns a [`CostResult`] so callers can charge the page's actual + /// storage work (chunk MMR seeks and buffer reads) against cost limits. + pub fn get_range(&self, start: u64, limit: u16) -> CostResult { + let mut cost = OperationCost::default(); + let total_count = self.total_count; let end = start.saturating_add(limit as u64).min(total_count); if start >= end { return Ok(RangePage { entries: Vec::new(), total_count, - }); + }) + .wrap_with_cost(cost); } let mut entries = Vec::with_capacity((end - start) as usize); @@ -102,23 +109,48 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); let mmr = MMR::new_with_overlay(self.mmr_size(), &mmr_store, self.mmr_overlay.clone()); for chunk_idx in first_chunk..=last_chunk { - let node = mmr + let node = match mmr .batch .element_at_position(leaf_to_pos(chunk_idx)) - .unwrap() - .map_err(|e| { - BulkAppendError::MmrError(format!( + .unwrap_add_cost(&mut cost) + { + Ok(node) => node, + Err(e) => { + return Err(BulkAppendError::MmrError(format!( "failed to read MMR node for chunk {}: {}", chunk_idx, e - )) - })?; - let blob = node.and_then(|n| n.into_value()).ok_or_else(|| { - BulkAppendError::CorruptedData(format!( + ))) + .wrap_with_cost(cost); + } + }; + let Some(blob) = node.and_then(|n| n.into_value()) else { + return Err(BulkAppendError::CorruptedData(format!( "missing chunk blob for index {}", chunk_idx - )) - })?; - let chunk_entries = deserialize_chunk_blob(&blob)?; + ))) + .wrap_with_cost(cost); + }; + let chunk_entries = match deserialize_chunk_blob(&blob) { + Ok(chunk_entries) => chunk_entries, + Err(e) => return Err(e).wrap_with_cost(cost), + }; + // A completed chunk holds exactly `epoch_size` entries — a + // short blob would silently omit positions and an oversized + // one would overlap the next chunk, breaking the contiguous + // page contract. Unlike proof verification (where chunk + // bytes are bound to the state root and a length check is + // redundant — see the NOTE in proof/mod.rs), this raw read + // path has no root comparison backing it, so the length is + // validated here. + if chunk_entries.len() as u64 != epoch_size { + return Err(BulkAppendError::CorruptedData(format!( + "chunk {} holds {} entries, expected {}", + chunk_idx, + chunk_entries.len(), + epoch_size + ))) + .wrap_with_cost(cost); + } let chunk_start = chunk_idx * epoch_size; for (i, value) in chunk_entries.into_iter().enumerate() { let pos = chunk_start + i as u64; @@ -129,15 +161,27 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { } } - // Buffer tail: positions in [max(start, buffer_start), end) + // Buffer tail: positions in [max(start, buffer_start), end). Read + // through the dense tree directly so each read's cost is charged. for pos in start.max(buffer_start)..end { let buffer_pos = (pos - buffer_start) as u16; - let value = self.get_buffer_value(buffer_pos)?.ok_or_else(|| { - BulkAppendError::CorruptedData(format!( - "missing buffer value at position {}", - buffer_pos - )) - })?; + let value = match self.dense_tree.get(buffer_pos).unwrap_add_cost(&mut cost) { + Ok(Some(value)) => value, + Ok(None) => { + return Err(BulkAppendError::CorruptedData(format!( + "missing buffer value at position {}", + buffer_pos + ))) + .wrap_with_cost(cost); + } + Err(e) => { + return Err(BulkAppendError::StorageError(format!( + "dense tree get at {} failed: {}", + buffer_pos, e + ))) + .wrap_with_cost(cost); + } + }; entries.push((pos, value)); } @@ -145,6 +189,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { entries, total_count, }) + .wrap_with_cost(cost) } // ── Chunk operations (MMR) ─────────────────────────────────────── diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index 648ec2fa1..0f5c7c847 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -419,7 +419,7 @@ fn get_range_buffer_only() { let tree = build_range_tree(3, 5); assert_eq!(tree.chunk_count(), 0); - let page = tree.get_range(1, 3).expect("get range"); + let page = tree.get_range(1, 3).unwrap().expect("get range"); assert_page(&page, 1, 4, 5); } @@ -431,7 +431,7 @@ fn get_range_single_chunk() { assert_eq!(tree.buffer_count(), 0); // Page entirely inside chunk 0 - let page = tree.get_range(1, 2).expect("get range"); + let page = tree.get_range(1, 2).unwrap().expect("get range"); assert_page(&page, 1, 3, 8); } @@ -443,25 +443,25 @@ fn get_range_across_chunk_boundary() { assert_eq!(tree.buffer_count(), 2); // Page [3, 6) spans the chunk 0 / chunk 1 boundary - let page = tree.get_range(3, 3).expect("get range"); + let page = tree.get_range(3, 3).unwrap().expect("get range"); assert_page(&page, 3, 6, 10); // Page [6, 10) spans the chunk 1 / buffer boundary - let page = tree.get_range(6, 4).expect("get range"); + let page = tree.get_range(6, 4).unwrap().expect("get range"); assert_page(&page, 6, 10, 10); } #[test] fn get_range_whole_tree() { let tree = build_range_tree(2, 10); - let page = tree.get_range(0, 100).expect("get range"); + let page = tree.get_range(0, 100).unwrap().expect("get range"); assert_page(&page, 0, 10, 10); } #[test] fn get_range_empty_limit() { let tree = build_range_tree(2, 10); - let page = tree.get_range(3, 0).expect("get range"); + let page = tree.get_range(3, 0).unwrap().expect("get range"); assert_page(&page, 3, 3, 10); } @@ -470,31 +470,31 @@ fn get_range_past_end() { let tree = build_range_tree(2, 10); // Start exactly at total_count - let page = tree.get_range(10, 5).expect("get range"); + let page = tree.get_range(10, 5).unwrap().expect("get range"); assert!(page.entries.is_empty()); assert_eq!(page.total_count, 10); // Start far past total_count - let page = tree.get_range(1000, 5).expect("get range"); + let page = tree.get_range(1000, 5).unwrap().expect("get range"); assert!(page.entries.is_empty()); assert_eq!(page.total_count, 10); // Range that starts inside but extends past the end is clamped - let page = tree.get_range(8, 100).expect("get range"); + let page = tree.get_range(8, 100).unwrap().expect("get range"); assert_page(&page, 8, 10, 10); } #[test] fn get_range_single_entry() { let tree = build_range_tree(2, 10); - let page = tree.get_range(7, 1).expect("get range"); + let page = tree.get_range(7, 1).unwrap().expect("get range"); assert_page(&page, 7, 8, 10); } #[test] fn get_range_empty_tree() { let tree = build_range_tree(2, 0); - let page = tree.get_range(0, 10).expect("get range"); + let page = tree.get_range(0, 10).unwrap().expect("get range"); assert!(page.entries.is_empty()); assert_eq!(page.total_count, 0); } @@ -502,7 +502,10 @@ fn get_range_empty_tree() { #[test] fn get_range_start_saturating_overflow() { let tree = build_range_tree(2, 10); - let page = tree.get_range(u64::MAX, u16::MAX).expect("get range"); + let page = tree + .get_range(u64::MAX, u16::MAX) + .unwrap() + .expect("get range"); assert!(page.entries.is_empty()); assert_eq!(page.total_count, 10); } @@ -515,7 +518,7 @@ fn get_range_paged_scan_covers_everything() { let mut cursor = 0u64; let mut seen = Vec::new(); loop { - let page = tree.get_range(cursor, 3).expect("get range"); + let page = tree.get_range(cursor, 3).unwrap().expect("get range"); if page.entries.is_empty() { assert!(cursor >= page.total_count, "empty page only at the end"); break; @@ -539,6 +542,7 @@ fn get_range_missing_chunk_is_corruption() { assert_eq!(tree.chunk_count(), 2); let err = tree .get_range(0, 4) + .unwrap() .expect_err("missing chunk blob must error"); assert!(matches!(err, crate::BulkAppendError::CorruptedData(_))); } @@ -550,6 +554,7 @@ fn get_range_missing_buffer_value_is_corruption() { let tree = BulkAppendTree::from_state(1, 2, MemStorageContext::new()).expect("from_state"); assert_eq!(tree.buffer_count(), 1); tree.get_range(0, 1) + .unwrap() .expect_err("missing buffer value must error"); } @@ -563,6 +568,45 @@ fn get_range_storage_read_failure_is_mmr_error() { assert_eq!(tree.chunk_count(), 2); let err = tree .get_range(0, 4) + .unwrap() .expect_err("failing storage must error"); assert!(matches!(err, crate::BulkAppendError::MmrError(_))); } + +#[test] +fn get_range_wrong_chunk_entry_count_is_corruption() { + // A completed chunk must hold exactly epoch_size entries: a short blob + // would silently omit positions and an oversized one would overlap the + // next chunk. Tamper the MMR overlay so chunk 0's blob deserializes to + // the wrong entry count and verify the read rejects it. + for bad_count in [1usize, 3] { + // epoch_size = 2: append 2 values to complete one genuine chunk. + let mut tree = BulkAppendTree::new(1u8, MemStorageContext::new()).expect("create tree"); + tree.append(&[0]).expect("append"); + tree.append(&[1]).expect("append"); + assert_eq!(tree.chunk_count(), 1); + + let bad_blob = + crate::serialize_chunk_blob(&(0..bad_count).map(|i| vec![i as u8]).collect::>()) + .expect("serialize bad blob"); + tree.mmr_overlay = vec![( + 0, + vec![grovedb_merkle_mountain_range::MmrNode::leaf(bad_blob)], + )]; + + let err = tree + .get_range(0, 2) + .unwrap() + .expect_err("wrong chunk entry count must error"); + match err { + crate::BulkAppendError::CorruptedData(msg) => { + assert!( + msg.contains(&format!("holds {} entries, expected 2", bad_count)), + "unexpected message: {}", + msg + ); + } + other => panic!("expected CorruptedData, got {:?}", other), + } + } +} diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 4521207d4..04c15b128 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -643,11 +643,11 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { /// the raw `cmx || rho || cv_net || payload` bytes at that position. /// Delegates to [`BulkAppendTree::get_range`], so the read is /// chunk-aligned — O(chunks touched) blob reads, not O(entries) random - /// reads. - pub fn get_range(&self, start: u64, limit: u16) -> Result { + /// reads — and carries the page's storage costs. + pub fn get_range(&self, start: u64, limit: u16) -> CostResult { self.bulk_tree .get_range(start, limit) - .map_err(|e| CommitmentTreeError::InvalidData(format!("range read: {}", e))) + .map(|r| r.map_err(|e| CommitmentTreeError::InvalidData(format!("range read: {}", e)))) } /// The number of entries per completed chunk (epoch). diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index e87ef35fd..955c27ef4 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -308,8 +308,11 @@ impl GroveDb { BulkAppendTree::from_state(total_count, chunk_power, storage_ctx).map_err(map_bulk_err) ); - let page = - cost_return_on_error_no_add!(cost, tree.get_range(start, limit).map_err(map_bulk_err)); + let page = cost_return_on_error!( + &mut cost, + tree.get_range(start, limit) + .map(|r| r.map_err(map_bulk_err)) + ); Ok(page).wrap_with_cost(cost) } diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index a8e08aa78..6fa337406 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -447,8 +447,10 @@ impl GroveDb { .map(|r| r.map_err(map_ct_err)) ); - let page = - cost_return_on_error_no_add!(cost, ct.get_range(start, limit).map_err(map_ct_err)); + let page = cost_return_on_error!( + &mut cost, + ct.get_range(start, limit).map(|r| r.map_err(map_ct_err)) + ); Ok(page).wrap_with_cost(cost) } diff --git a/grovedb/src/tests/bulk_append_tree_tests.rs b/grovedb/src/tests/bulk_append_tree_tests.rs index e4a69742c..bc27a1123 100644 --- a/grovedb/src/tests/bulk_append_tree_tests.rs +++ b/grovedb/src/tests/bulk_append_tree_tests.rs @@ -2618,3 +2618,40 @@ fn test_bulk_position_range_verify_rejects_unbound_element() { .expect_err("proof that binds no element must be rejected"); assert!(matches!(err, Error::InvalidProof(..))); } + +/// Range reads must charge their storage work: the page's chunk-MMR seeks +/// and buffer reads all show up in the returned cost, so cost limits +/// reflect the actual work instead of treating a 65k-entry page as free. +#[test] +fn test_bulk_get_range_reports_storage_costs() { + let grove_version = GroveVersion::latest(); + // 11 values = 2 full chunks + 3 buffered (chunk size 4) + let db = make_bulk_db_with_values(11); + + // A page spanning both chunks and the buffer + let result = db.bulk_get_range(EMPTY_PATH, b"bulk", 0, 11, None, grove_version); + let cost = result.cost.clone(); + let page = result.unwrap().expect("bulk get range"); + assert_eq!(page.entries.len(), 11); + assert!( + cost.storage_loaded_bytes > 0, + "range read must charge loaded bytes, got {:?}", + cost + ); + assert!( + cost.seek_count > 0, + "range read must charge seeks, got {:?}", + cost + ); + + // A wider page must not cost less than a narrower one + let narrow_cost = db + .bulk_get_range(EMPTY_PATH, b"bulk", 8, 1, None, grove_version) + .cost; + assert!( + cost.storage_loaded_bytes > narrow_cost.storage_loaded_bytes, + "an 11-entry page must load more bytes than a 1-entry page ({} vs {})", + cost.storage_loaded_bytes, + narrow_cost.storage_loaded_bytes + ); +}